From 63721408f44d3adee5f292260958f5c14eade95c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 14:55:47 +0100 Subject: [PATCH 001/255] test(live): a child's closed result must stay off the family page Red first: the demo family's compound-het pair in KCNH2 names the son's planted result to act on, and the family page shows it open. The new browser checks also plant a de novo candidate and a grandparent stretch over it, and check the son's copy-number dot in a closed gene. The unit checks want results.json to carry the closed results (veil.spans), facts and drawing items tagged, and core.js veilId() to answer as veil_id(). Bead: pgp-iqon.24 --- tests/demo/make_demo_sample.py | 3 +- .../live/demo/DEMO-FAMILY/compound_het.tsv | 1 + tests/fixtures/live/demo/README.md | 2 + tests/test_live_demo.py | 219 +++++++++++++++++- tests/test_live_results.py | 93 +++++++- 5 files changed, 312 insertions(+), 6 deletions(-) diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 9def678d8..352eb0b93 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -107,7 +107,8 @@ def own_chroms(): FINDINGS = {"DEMO-002": {"fmr1", "chip", "mocha"}, "DEMO-003": {"annotsv", "act"}} # The son's results to act on ("act"): a pathogenic ClinVar hit in KCNH2, an ACMG secondary-findings gene that # is on no carrier list, and an AnnotSV class 5 deletion over it. Public gene name, invented positions and -# alleles. tests/test_live_demo.py checks that his page names neither until each is opened on purpose. +# alleles. tests/test_live_demo.py checks that his page names neither until each is opened on purpose, and that the +# family page keeps closed the compound-het pair in KCNH2 of tests/fixtures/live/demo/DEMO-FAMILY too. PLANTED = {"gene": "KCNH2", "hit": ("chr7", 150960000, "C", "T"), "sv": ("chr7", 150955000, 150965000)} # Stranger's thresholds per locus (normal up to, pathologic from), invented for the demo's catalog. STR_THRESHOLDS = {"HTT": (26, 40), "FMR1": (54, 200), "C9ORF72": (20, 30), "ATXN1": (35, 39), "DMPK": (34, 50), diff --git a/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv b/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv index a5927bf30..5a68b490d 100644 --- a/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv +++ b/tests/fixtures/live/demo/DEMO-FAMILY/compound_het.tsv @@ -1,3 +1,4 @@ child gene variant_a variant_b a_from b_from phase DEMO-003 G0412 chr2:73481166:A>G chr2:73492310:C>T mother . unknown DEMO-003 G0187 chr7:94371020:G>A chr7:94398114:T>C father mother in trans +DEMO-003 KCNH2 chr7:150960000:C>T chr7:150962400:G>A father mother in trans diff --git a/tests/fixtures/live/demo/README.md b/tests/fixtures/live/demo/README.md index 62ca22167..d08c2eade 100644 --- a/tests/fixtures/live/demo/README.md +++ b/tests/fixtures/live/demo/README.md @@ -14,5 +14,7 @@ sample DEMO-001 that `tests/demo/make_demo_sample.py` writes. No real sample, pe - `DEMO-FAMILY/family_plan.tsv`, `relationships.tsv`, `ibd_pairs.tsv`, `DEMO-003_segments.tsv`, `de_novo.tsv`, `compound_het.tsv`: step 38's files for the trio, invented, in the columns step 38 writes, so the family page draws its grid, the son's painted stretches and his tables. + `compound_het.tsv` also holds a pair in KCNH2 whose first variant is the son's planted result to act on + (`make_demo_sample.py` PLANTED): the family page keeps that row closed until the result is opened there. The run itself is `../demo-001.weblog.jsonl`, written by `tests/demo/make_demo_events.py`. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 805606cfb..8f10fe575 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -53,7 +53,15 @@ marks and ranking only, and a reload closes them again; the home page and every finding page of the son (an open finding's rail and related list too) name them nowhere, and the symptom ranking's own page shows the gene - once opened on purpose; DEMO-001's page + once opened on purpose; on the family page the son's rows that name a planted + result (the compound-het pair in KCNH2 of the fixture, and a de novo + candidate and a stretch painted from a grandparent planted in family.json) + name its gene, positions and that grandparent in no row, link, title or + count until "Show the result" is clicked there, the click brings each row + back, and a reload closes them again; with the son's results.json + unavailable every row of his that could name one stays closed; on his page + a CHIP call and a copy-number dot in the gene of a closed result stay out + of their drawings until it is opened; DEMO-001's page mid-replay and at the end leads with the report (a short run band above the lead while it runs, none once finished), ends with the step timeline folded, keeps the wire to at most eight one-sentence lines with the plain @@ -64,7 +72,8 @@ the veil, person.js and finding.js that read a missing state as a standalone adult, and search.js without its filter; for the planted results, results.json without its veil tags, person.js whose table rows ignore them, person.js whose findings ignore them, charts.js that keeps every mark, search.js that ignores a hit's tag -and finding.js whose findings ignore their tags), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +and finding.js whose findings ignore their tags; for the family page, family-page.js whose rows ignore the veil +and core.js whose veilId() tags nothing; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -314,6 +323,8 @@ def browser_checks(out): fold_checks(b, base, out) veil_checks(b, base, out) planted_checks(b, base, out) + family_veil_checks(b, base, out) + widget_veil_checks(b, base, out) finally: b.close() srv.shutdown() @@ -408,7 +419,8 @@ def family_depth(b, base): b.go("about:blank") b.wait("location.href === 'about:blank'") b.go(f"{base}/runs/{FAM}/") - return b.wait("document.documentElement.dataset.pgpStatus === 'ready' && JSON.stringify([" + return b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " + "(document.documentElement.dataset.pgpKids || '').indexOf('DEMO-003:known') >= 0 && JSON.stringify([" "document.querySelectorAll('table.rel-grid td:not(.self):not(.none)').length, " "document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg').length, " "document.querySelectorAll('details[data-child=\"DEMO-003\"] table.rows').length, " @@ -716,6 +728,207 @@ def write(p, text): write(p, text) +# The family page: the son's rows that name his planted result to act on. +FAMILY_SHOWN_JS = ("[document.body.textContent].concat(Array.prototype.map.call(document.querySelectorAll('[href],[title],[aria-label]," + "[alt]'), function (e) { return ['href', 'title', 'aria-label', 'alt'].map(function (k) { " + "return e.getAttribute(k) || ''; }).join(' '); })).join(' ')") +GRANDPARENT = "DEMO-GF" +PAIR_B = "150962400" # the second variant of the fixture's compound-het pair in KCNH2 + + +def no_cache(b): + """Chrome keeps a script it fetched a moment ago without asking again: the controls below swap a script, so + every request goes to the server from here on.""" + b.call("Network.enable", {}, b.session) + b.call("Network.setCacheDisabled", {"cacheDisabled": True}, b.session) + + +def family_page(b, base): + """The family page once it knows the son's results (or after 10 s, for a page that never learns them).""" + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{FAM}/") + b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " + "/DEMO-003:(known|failed)/.test(document.documentElement.dataset.pgpKids || '') && 'ready'", 10) + time.sleep(0.3) + + +def family_closed(b, base): + """[problems] of the family page before any click: the son's planted gene, its positions or the grandparent of + the stretch over it named anywhere; the rows that name nothing closed gone too.""" + gene, hit, sv = planted_words() + family_page(b, base) + shown = b.eval(FAMILY_SHOWN_JS) or "" + out = [f"{w} shown on the family page while closed" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] + rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " + "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") + closed = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") + segs = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg.closed').length") + # The stretches' table, the 2 de novo rows of the fixture and its 2 other pairs stay; one closed box each. + if not rows or json.loads(rows)[1:] != [2, 2] or closed != 3 or segs != 1: + out.append(f"the open rows and the closed boxes: tables {rows}, {closed} closed boxes, {segs} closed stretches") + return out + + +def family_problems(b, base): + """[problems] of the family page: closed, then opened with one click, then reloaded.""" + gene, hit, _ = planted_words() + out = family_closed(b, base) + if out: + return out + b.eval("document.querySelector('details[data-child=\"DEMO-003\"] .veil button').click(); true") + time.sleep(0.5) + shown = b.eval(FAMILY_SHOWN_JS) or "" + rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " + "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") + left = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") + if not all(w in shown for w in [gene, GRANDPARENT, hit[0]]) or json.loads(rows or "[0,0,0]")[1:] != [3, 3] or left: + out.append(f"one click on 'Show the result' did not bring back each row: tables {rows}, {left} closed boxes left") + out += [f"after a reload: {x}" for x in family_closed(b, base)] + return out + + +def family_unknown(b, base): + """[problems] of the family page with the son's results.json unavailable: a row of his named, or any of his de + novo or compound-het rows open.""" + gene, hit, sv = planted_words() + family_page(b, base) + shown = b.eval(FAMILY_SHOWN_JS) or "" + out = [f"{w} shown with the son's results unavailable" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] + rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " + "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") + if rows is None or len(json.loads(rows)) != 1: + out.append(f"the son's de novo or compound-het rows open with his results unavailable: tables {rows}") + return out + + +def family_veil_checks(b, base, out): + """On the family page a child's rows that name a closed result stay closed as on the child's page, until that + result is opened there; a page that cannot read the child's results keeps every row of his that could closed.""" + no_cache(b) + import make_demo_sample + fj = os.path.join(out, "runs", FAM, "family.json") + rj = os.path.join(out, "runs", OTHERS[1], "results.json") + fp, cj = (os.path.join(out, "assets", x) for x in ("family-page.js", "core.js")) + keep = {} + for p in (fj, fp, cj): + with open(p) as f: + keep[p] = f.read() + + def write(p, text): + with open(p, "w") as f: + f.write(text) + try: + # A de novo candidate at the planted hit, and the father's copy over it painted from a grandparent. + fam = json.loads(keep[fj]) + c, pos, ref, alt = make_demo_sample.PLANTED["hit"] + row = dict(fam["de_novo"][0], chrom=c, pos=str(pos), ref=ref, alt=alt) + fam["de_novo"].append(row) + segs = fam["segments"]["DEMO-003"] + over = [s_ for s_ in segs if s_["chrom"] == c and s_["side"] == "paternal" and int(s_["start"]) <= pos <= int(s_["end"])] + if len(over) != 1: + check("browser: the son's stretches hold one paternal stretch over the planted hit", False, over) + return + over[0]["ancestor"] = GRANDPARENT + write(fj, json.dumps(fam)) + keep["planted"] = json.dumps(fam) + bad = family_problems(b, base) + check("browser: on the family page the son's rows that name his planted result (a compound-het pair, a de novo " + "candidate, a stretch from a grandparent) name its gene, positions and grandparent nowhere until 'Show the " + "result' is clicked there, one click brings each row back, and a reload closes them", not bad, bad) + for p, a, z, desc in ( + (fp, "function isClosed(child, tag) { return !!tag && ", "function isClosed(child, tag) { return false && ", + "family-page.js whose rows ignore the veil"), + (cj, " if (!spans || !spans.length) { return null; }", " return null;", "core.js whose veilId() tags nothing")): + if a not in keep[p]: + check(f"control: {desc} (the planted line is in {os.path.basename(p)})", False, a) + continue + write(p, keep[p].replace(a, z)) + control(desc, not family_closed(b, base)) + write(p, keep[p]) + os.replace(rj, rj + ".away") + try: + bad = family_unknown(b, base) + finally: + os.replace(rj + ".away", rj) + check("browser: with the son's results.json unavailable the family page keeps every row of his that could name " + "a result to act on closed", not bad, bad) + finally: + keep.pop("planted", None) + for p, text in keep.items(): + write(p, text) + + +def widget_veil_checks(b, base, out): + """On the son's page a CHIP call and a copy-number dot in the gene of a closed result stay out of sight until that + result is opened; the other dot stays drawn. A CHIP call is also a finding of its card that names its gene, so + the CHIP card closes whole with the result; SMN1 at two copies is no finding, so only its dot's tag closes it.""" + no_cache(b) + rj = os.path.join(out, "runs", OTHERS[1], "results.json") + js = os.path.join(out, "assets", "person.js") + with open(rj) as f: + keep = f.read() + with open(js) as f: + keep_js = f.read() + + def drawn(): + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{OTHERS[1]}/") + b.wait("document.documentElement.dataset.pgpResults && document.documentElement.dataset.pgpStatus === 'ready' " + "&& document.querySelector('[data-widget=copies]') && document.querySelector('.veil') && 'drawn'", 15) + return shown() + + def shown(): + return b.eval("JSON.stringify([" + SHOWN_JS + ", (document.querySelector('[data-widget=copies]') || {}).textContent || ''])") + + def problems(text, opened): + page, cops = json.loads(text) if text else ["", ""] + out_ = [] + if "SMN2" not in cops: + out_.append(f"the open dot is not drawn: {cops[:120]}") + if ("p.R175H" in page) != opened or ("SMN1" in cops) != opened: + out_.append(f"the CHIP call or the SMN1 dot of the closed results {'not ' if opened else ''}shown: " + f"{'p.R175H' in page} | {cops[:200]}") + return out_ + try: + with open(os.path.join(os.path.dirname(out), "root", OTHERS[1], "summary.json")) as f: + summary = json.load(f) + S = summary.setdefault("sections", {}) + S["cpsr"] = {"state": "ok", "data": {"call_floor_dp": 10, "low_confidence": 0, "plp": [ + {"gene": "TP53", "variant": "c.524G>A", "final": "Pathogenic", "clinvar": "Pathogenic"}, + {"gene": "SMN1", "variant": "c.1A>G", "final": "Pathogenic", "clinvar": "Pathogenic"}]}} + S["chip"] = {"state": "ok", "data": {"median_limit": 0.05, "median_depth": 72, "calls": [ + {"gene": g, "protein_change": ch, "vaf": v, "alt_reads": 6, "depth": 75, "status": "pass", "note": "a clone"} + for g, ch, v in (("DNMT3A", "p.R882H", 0.12), ("TP53", "p.R175H", 0.08))], + "genes": [{"gene": g, "detection_limit_vaf": 0.05, "mean_depth": 75, "status": "ok"} for g in ("DNMT3A", "TP53")]}} + S["paralogs"] = {"state": "ok", "data": {"genes": [{"gene": g, "copy_number": n, "status": "called", "detail": "."} + for g, n in (("SMN1", 2), ("SMN2", 2))]}} + res = live_results.build(OTHERS[1], summary) + with open(rj, "w") as f: + json.dump(res, f) + bad = problems(drawn(), False) + if not bad: + for _ in range(2): # the CPSR card holds both results: one click opens one + b.eval("document.querySelector('#card-cpsr .veil button').click(); true") + time.sleep(0.5) + bad = problems(shown(), True) + check("browser: on the son's page a CHIP call and a copy-number dot in the gene of a closed result stay out of " + "sight until it is opened", not bad, bad) + a = "add(text, widget(of[1]));" + if a not in keep_js: + check("control: person.js that draws every item (the planted line is in person.js)", False, a) + else: + with open(js, "w") as f: + f.write(keep_js.replace(a, "add(text, widget(c.widget));")) + control("person.js that draws every item", not problems(drawn(), False)) + finally: + with open(rj, "w") as f: + f.write(keep) + with open(js, "w") as f: + f.write(keep_js) + + # finding.js as it was: a missing state or one with no plugins read as a standalone adult. OLD_FINDING = ("if (!st || !st.plugins) { return Promise.resolve(UNKNOWN); }\n var fam = st.plugins.family;", "var fam = st && st.plugins && st.plugins.family;") diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 8213596a2..7a403b68c 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -42,12 +42,17 @@ the ids and no other row does, the ranking's finding carries the secondary finding's id, and so does a segment of the lead that names the gene; the home page's view of the run (overview) lists no such finding among its - first ones and its closed lead names none. + first ones and its closed lead names none; a CHIP call, a copy-number dot + and a fact that name a closed result's gene carry its id (the CHIP chart's + panel of genes never does), results.json carries the closed results as + veil.spans, and core.js veilId(), which tags the family page's rows of a + child, answers each row as veil_id() does, under node. Each check runs again on a planted mistake and must fail there: 1 with VNTYPER's entry dropped, 2 with one card's caveat blanked, 3 with "ok" planted in a card and "All clear" in a script, 4 with the hits emptied, 5 with the results route registered without the login, 6 with a row printed as its Python dict, 7 with the Slivar adapter returning nothing, -8 with a veil rule that tags nothing (rows and lines) and one that tags every row. A check +8 with a veil rule that tags nothing (rows, lines, facts and drawings) and one that tags every row, and +veilId() with its gene rule or its overlap rule dropped. A check that cannot fail is no check. Run: python3 tests/test_live_results.py @@ -402,6 +407,75 @@ def veil_lines_problems(): return out +def widget_problems(): + """What is wrong with the veil's tags of a card's facts and drawing: a CHIP call and a copy-number dot, and a + fact, that name a closed result's gene carry its id; the CHIP chart's list of genes, the same panel for + everyone, is never tagged; nothing else is.""" + found = [{"id": "cpsr-tp53-test", "level": "act", "gene": "TP53", "detail": "TP53 c.524G>A"}, + {"id": "two-hits-smn1-test", "level": "act", "gene": "SMN1"}] + S = {"chip": {"state": "ok", "data": {"median_limit": 0.05, "median_depth": 72, "calls": [ + {"gene": "DNMT3A", "protein_change": "p.R882H", "vaf": 0.12, "alt_reads": 9, "depth": 75, "status": "pass", "note": "a clone"}, + {"gene": "TP53", "protein_change": "p.R175H", "vaf": 0.08, "alt_reads": 6, "depth": 75, "status": "pass", "note": "a clone"}], + "genes": [{"gene": "DNMT3A", "detection_limit_vaf": 0.05, "mean_depth": 75, "status": "ok"}, + {"gene": "TP53", "detection_limit_vaf": 0.06, "mean_depth": 70, "status": "ok"}]}}, + "paralogs": {"state": "ok", "data": {"genes": [ + {"gene": "SMN1", "copy_number": 1, "status": "called", "detail": "."}, + {"gene": "SMN2", "copy_number": 2, "status": "called", "detail": "."}]}}} + summ = {"sections": S} + chip = dict(live_results.a_chip(S, summ), id="chip") + par = dict(live_results.a_paralogs(S, summ), id="paralogs") + par["facts"] = [["SMN1 copies", "1"], ["Genes read", "2"]] + v = live_results.veil(summ, found, [{"sections": [chip, par]}]) + out = [] + want = {"chip": [None, "cpsr-tp53-test"], "paralogs": ["two-hits-smn1-test", None]} + if v.get("widget") != want: + out.append(f"drawings tagged {v.get('widget')}, want {want}") + if (v.get("facts") or {}) != {"paralogs": ["two-hits-smn1-test", None]}: + out.append(f"facts tagged {v.get('facts')}") + if v.get("spans") != [list(x) for x in live_results.closable(found)]: + out.append(f"spans {v.get('spans')}") + return out + + +# Rows as the family page tags them (family-page.js): [text, chrom, pos, end], against two closed results. +VEIL_SPANS = [["sf-kcnh2", "KCNH2", "chr7", 150960000, 150960000], ["sv-del", "KCNH2", "chr7", 150955000, 150965000], + ["two-hits-hla", "HLA-A", None, None, None]] +VEIL_ROWS = [["chr7:150960000 C>T", "chr7", "150960000", None], ["chr7:150961000 G>A", "chr7", "150961000", None], + ["chr7:150970000 G>A", "chr7", "150970000", None], ["chr3:48211904 C>T", "chr3", "48211904", None], + ["KCNH2 chr7:94371020:G>A chr7:94398114:T>C", None, None, None], + ["G0187 chr7:150,962,000:G>A chr7:1:T>C", None, None, None], ["G0412 chr2:1:A>G", None, None, None], + ["chr7:61600000-159285973", "chr7", "61600000", "159285973"], ["in KCNH2.", None, None, None], + ["HLA-A*02:01", None, None, None], ["HLA-AB", None, None, None], ["kcnh2 lower case", None, None, None], + ["7:150960000", "7", 150960000, None], ["chrX:1-2", "chrX", "x", None], ["", "chr7", "150955000", "150965000"]] +CORE_JS = os.path.join(ASSETS, "core.js") +NODE_VEIL = ("const vm=require('vm'),fs=require('fs');const w={};" + "const doc={documentElement:{setAttribute(){}},addEventListener(){}};" + "vm.runInNewContext(fs.readFileSync(process.argv[1],'utf8'),{window:w,document:doc});" + "const sp=JSON.parse(process.argv[2]);" + "console.log(JSON.stringify(JSON.parse(process.argv[3]).map(r=>w.PGPUI.veilId(sp,r[0],r[1],r[2],r[3]))))") + + +def node_veil(js_path): + """veilId() of core.js at JS_PATH on VEIL_ROWS, under node: [ids] or the error.""" + node = shutil.which("node") + if not node: + return "no node" + p = subprocess.run([node, "-e", NODE_VEIL, js_path, json.dumps(VEIL_SPANS), json.dumps(VEIL_ROWS)], + capture_output=True, text=True) + return json.loads(p.stdout) if p.returncode == 0 else p.stderr + + +def veil_js_problems(js_path): + """What veilId() of core.js answers differently from veil_id() on VEIL_ROWS.""" + want = [live_results.veil_id([tuple(x) for x in VEIL_SPANS], *r) for r in VEIL_ROWS] + got = node_veil(js_path) + if got != want: + return [f"veilId {got} != veil_id {want}"] + if len({w for w in want}) < 4: + return [f"the rows exercise too few answers: {want}"] + return [] + + def overview_problems(): """What is wrong with the home page's view of a child's run (overview): a finding that names a closed result (a symptom ranking of its gene) is listed among the first findings, or the closed lead still names it.""" @@ -519,6 +593,20 @@ def tagged(sample_dir): bad = overview_problems() check("8. a child's home tile lists no finding that names a closed result, and its closed lead names none", not bad, bad) + bad = widget_problems() + check("8. a CHIP call, a copy-number dot and a fact that name a closed result's gene carry its id, the CHIP " + "chart's gene panel none, and results.json carries the closed results as veil.spans", not bad, bad) + check("8. node is on the PATH (ubuntu-latest has it)", shutil.which("node") is not None) + bad = veil_js_problems(CORE_JS) + check("8. core.js veilId() tags each family-page row as veil_id() does (under node)", not bad, bad) + with open(CORE_JS) as f: + core = f.read() + for a, z, desc in (("if (s[1] && words[s[1]]) { return s[0]; }", "", "veilId() without its gene rule"), + ("l[1] <= s[4] && l[2] >= s[3]", "false", "veilId() without its overlap rule")): + planted = os.path.join(work, "core-planted.js") + with open(planted, "w") as f: + f.write(core.replace(a, z)) + control(desc, a in core and not veil_js_problems(planted)) bad = veil_problems(*tagged(d3)) check("8. the planted ClinVar hit's row, viewer link, tick and star bar carry its secondary finding's id, the " "deletion's span its own, every other card's row of the gene and the symptom ranking's finding one of them, " @@ -529,6 +617,7 @@ def tagged(sample_dir): live_results.veil_id = lambda spans, *a, **k: None control("a veil rule that tags nothing", not veil_problems(*tagged(d3))) control("a veil rule that tags no line", not veil_lines_problems()) + control("a veil rule that tags no fact or drawing", not widget_problems()) live_results.veil_id = lambda spans, *a, **k: spans[0][0] if spans else None control("a veil rule that tags every row", not veil_problems(*tagged(d3))) finally: From 76354c4481375a2f84625e34d4ab4957006596be Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:05:41 +0100 Subject: [PATCH 002/255] fix(live): tag the closed results for the family page, and close a child's dots and facts with them results.json carries the closed results as veil.spans, and tags each card's facts and the items of its drawing (CHIP calls, copy-number dots). core.js gets veilId(), a mirror of veil_id() for the family page's rows that a unit check runs against the Python rule under node, and the closed box both pages draw. Bead: pgp-iqon.24 --- assets/live/core.js | 55 ++++++++++++++++++++++++++++++++++++++++++- assets/live/person.js | 43 +++++++++++++++++++-------------- bin/live_results.py | 19 +++++++++++++-- 3 files changed, 96 insertions(+), 21 deletions(-) diff --git a/assets/live/core.js b/assets/live/core.js index b6e4ff071..e0aa05e32 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -208,8 +208,61 @@ return isFinite(x) && isFinite(y) && y > x ? (y - x) / 3600e3 : null; } + // --- the veil: a child's results to act on stay closed until someone opens them on purpose --------- + // The closed box, the same on a child's page (person.js) and on the family page (family-page.js): what stays + // closed, and the button that opens it. + function veil(text, button, onOpen) { + var v = el('div', { 'class': 'veil' }); + add(v, add(el('div', { 'class': 'vt' }), el('b', {}, 'A result about a child. '), text)); + var b = el('button', { type: 'button', 'class': 'btn' }, button || 'Show the result'); + b.addEventListener('click', onOpen); + add(v, b); + return v; + } + // The id of the closed result a row belongs to, or null: bin/live_results.py veil_id() word for word, for the + // rows only the family page holds (a child's de novo and compound-het candidates). spans: results.json's + // veil.spans, [[id, gene, chrom, start, end]]. text: what the row shows (its gene names; its loci are read + // too); chrom, pos and end its own position. tests/test_live_results.py runs it under node against veil_id(). + var LOCUS_RE = /(?= s[3]; })) { return s[0]; } + } + return null; + } + window.PGPUI = { el: el, svg: svg, add: add, getJSON: getJSON, avatar: avatar, day: day, caveat: caveat, facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, glyph: glyph, karyogram: karyogram, locus: locus, LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, - hm: hm, dm: dm, hours: hours }; + hm: hm, dm: dm, hours: hours, veil: veil, veilId: veilId }; }()); diff --git a/assets/live/person.js b/assets/live/person.js index 6dcd7490f..c0b582ac8 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -344,6 +344,22 @@ out.rows = rows; return [rows.length ? out : null, t.rows.length - rows.length]; } + // A card's facts and drawing without their closed items (a fact, a CHIP call or a copy-number dot that names a + // closed result's gene): [facts, widget, how many items they left out]. + function openFacts(c) { + var ft = veilTags('facts', c.id), facts = (c.facts || []).filter(function (f, i) { return !(ft[i] && tagClosed(ft[i])); }); + var w = c.widget, key = w && (w.kind === 'vaf' ? 'calls' : w.kind === 'copies' ? 'genes' : null); + var wt = veilTags('widget', c.id), shut = (c.facts || []).length - facts.length; + if (key && wt.length && w[key]) { + var items = w[key].filter(function (x, i) { return !(wt[i] && tagClosed(wt[i])); }); + shut += w[key].length - items.length; + var copy = {}; + Object.keys(w).forEach(function (k) { copy[k] = w[k]; }); + copy[key] = items; + w = copy; + } + return [facts, w, shut]; + } function veiledOf(c) { return (c.findings || []).map(findingById).filter(function (f) { return f && veiled(f); }); } // The search box (plugins/search.js) asks the same question for each hit, by the summary key and the anchor // of the section it comes from: a hit from a card this page keeps closed is left out. A closed page with @@ -398,20 +414,16 @@ } else if (g === 'empty') { add(text, el('p', { 'class': 'notassessed' }, 'Not assessed: its step ended without a result for this section.')); } else if (closedFs.length) { - var v = el('div', { 'class': 'veil' }); // One button opens one result: a card with two results to act on asks twice. - add(v, add(el('div', { 'class': 'vt' }), el('b', {}, 'A result about a child. '), - (closedFs.length === 1 ? 'This section holds a result' : 'This section holds ' + closedFs.length + ' results') + - ' to act on in adult life. Its rows stay closed until ' + (closedFs.length === 1 ? 'it is' : 'each is') + ' opened.')); - var vb = el('button', { type: 'button', 'class': 'btn' }, 'Show the result'); - vb.addEventListener('click', function () { openFinding(closedFs[0]); redraw(true); var x = document.getElementById('f-' + closedFs[0].id) || document.getElementById('card-' + c.id); if (x) { x.scrollIntoView({ block: 'start' }); } }); - add(v, vb); - add(text, v); + add(text, U.veil((closedFs.length === 1 ? 'This section holds a result' : 'This section holds ' + closedFs.length + ' results') + + ' to act on in adult life. Its rows stay closed until ' + (closedFs.length === 1 ? 'it is' : 'each is') + ' opened.', null, + function () { openFinding(closedFs[0]); redraw(true); var x = document.getElementById('f-' + closedFs[0].id) || document.getElementById('card-' + c.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); } else { (c.notes || []).forEach(function (n) { add(text, el('p', { 'class': 'note-stale' }, n)); }); if (c.headline) { add(text, el('p', { 'class': 'headline' }, c.headline)); } - add(margin, U.facts(c.facts)); - add(text, widget(c.widget)); + var of = openFacts(c); + add(margin, U.facts(of[0])); + add(text, widget(of[1])); Object.keys((results && results.charts) || {}).forEach(function (chart) { if (results.charts[chart] === c.id) { add(text, el('div', { 'class': 'chart-slot', 'data-chart-slot': chart })); } }); @@ -419,7 +431,7 @@ var ot = openTable(c), tb = U.table(ot[0]); var lt = veilTags('lines', c.id); var lines = (c.lines || []).filter(function (l, i) { return !(lt[i] && tagClosed(lt[i])); }); - var shut = ot[1] + (c.lines || []).length - lines.length; + var shut = ot[1] + (c.lines || []).length - lines.length + of[2]; if (tb) { add(text, tb); } else if (c.empty_text && !ot[1]) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } if (shut) { add(text, el('p', { 'class': 'dim veil-rows' }, (shut === 1 ? 'One row stays' : shut + ' rows stay') + @@ -534,13 +546,8 @@ if (at) { art.classList.add('new'); add(art, el('span', { 'class': 'newband' }, 'New at ' + U.hm(at))); add(art, el('br', {})); } add(art, U.findingLevel(f.level, f.level_words)); if (veiled(f)) { - var v = el('div', { 'class': 'veil' }); - add(v, add(el('div', { 'class': 'vt' }), el('b', {}, 'A result about a child. '), - 'A result to act on in adult life is usually not looked at before the child can choose. It stays closed until you open it.')); - var b = el('button', { type: 'button', 'class': 'btn' }, 'Show the result'); - b.addEventListener('click', function () { openFinding(f); redraw(true); var x = document.getElementById('f-' + f.id); if (x) { x.scrollIntoView({ block: 'start' }); } }); - add(v, b); - add(art, v); + add(art, U.veil('A result to act on in adult life is usually not looked at before the child can choose. It stays closed until you open it.', + null, function () { openFinding(f); redraw(true); var x = document.getElementById('f-' + f.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); add(row, art, el('aside', {})); return row; } diff --git a/bin/live_results.py b/bin/live_results.py index ef5d93eb3..a2202011f 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -1210,11 +1210,16 @@ def veil(summary, found, chapters, lead_parts=()): intronic site is printed whether hit or not), the marks of the report's charts (marks: chart -> {mark id: id}), every other finding that names a closed result (findings: finding id -> id; a symptom ranking can rank the gene) and the lead's segments that name one (lead: one per segment of LEAD_PARTS, the "act" - names left out: the page closes those itself). Empty when nothing can be closed.""" + names left out: the page closes those itself), each card's facts (facts: card -> one per fact) and the items + of its drawing (widget: card -> one per CHIP call or copy-number dot). spans is closable() itself, so the + family page (assets/live/family-page.js) can tag its own rows of a child, the de novo and compound-het + candidates, by this rule (core.js veilId(), checked against veil_id() under node). Empty when nothing can + be closed.""" spans = closable(found) if not spans: return {} - out = {"rows": {}, "viewer": {}, "lines": {}, "marks": {}, "findings": {}, "lead": []} + out = {"rows": {}, "viewer": {}, "lines": {}, "facts": {}, "widget": {}, "marks": {}, "findings": {}, "lead": [], + "spans": [list(s) for s in spans]} S = summary.get("sections") or {} hits = [h for h in _items(_data(S, "clinvar").get("hits"))[:50]] tags = [veil_id(spans, str(h.get("gene") or ""), h.get("chrom"), h.get("pos")) if isinstance(h, dict) else None @@ -1238,6 +1243,16 @@ def veil(summary, found, chapters, lead_parts=()): tags = [veil_id(spans, ln) for ln in c.get("lines") or []] if any(tags): out["lines"][c["id"]] = tags + tags = [veil_id(spans, " ".join(str(x) for x in f)) for f in c.get("facts") or []] + if any(tags): + out["facts"][c["id"]] = tags + # A drawing's items: the CHIP calls (gene and change) and the copy-number dots (gene). The CHIP + # chart's list of genes with their detection limits is the same panel for everyone: never tagged. + w = c.get("widget") or {} + items = w.get("calls") if w.get("kind") == "vaf" else w.get("genes") if w.get("kind") == "copies" else None + tags = [veil_id(spans, f"{x.get('gene') or ''} {x.get('change') or ''}") for x in items or []] + if any(tags): + out["widget"][c["id"]] = tags try: specs = rr.chart_specs(summary) except (KeyError, TypeError, ValueError, AttributeError, IndexError, ZeroDivisionError): From 1bf0623a0a529e1b19842d8ab2c98c22e1c1e46b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:05:41 +0100 Subject: [PATCH 003/255] fix(live): keep a child's closed result off the family page The family page reads each child's results.json and keeps closed every de novo candidate, compound-het pair and grandparent stretch that names a closed result, until it is opened there; while it cannot read them, every such row of that child stays closed. Bead: pgp-iqon.24 --- CHANGELOG.md | 2 + assets/live/family-page.js | 131 ++++++++++++++++++++++++++++++++++--- assets/live/live.css | 1 + docs/live-site.md | 6 +- tests/test_live_demo.py | 2 +- 5 files changed, 130 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4fcb5480..1ff9e66f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The family page now keeps a child's results to act on closed, as the child's own page does. A de novo candidate or a compound-het pair that names a closed result, and a stretch painted from a grandparent over one, stay behind "Show the result" until that result is opened there; a reload closes them again. While the page cannot read the child's results, every such row of that child stays closed. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: the files behind the pages hold every row. + - VEP consequence ranking (steps 23 and 31) and the gnomAD constraint join now run in the Rust `og` binary, inside `BCFTOOLS_IMAGE`: `og vcf csq-rank` replaces `bin/csq_rank.awk` and `og table constraint-join` replaces `bin/constraint_join.awk`, which are gone. Neither awk program runs on the host anymore (step 23 still uses the host's `awk` for small table edits), and the ranker is no longer the slow stage of step 23's main pipe. Steps 23 and 31 now need `og`: `scripts/setup.sh --og` builds it. Their output is the same, byte for byte, except for these changes. - The seven `##INFO=` header descriptions end in `(og vcf csq-rank)`, not `(bin/csq_rank.awk)`. No tag, value or column changes. - A ranking floor that is not a number, or a `--keep` other than `all` or `noncommon`, is a usage error (exit 2). awk read such a floor as 0. diff --git a/assets/live/family-page.js b/assets/live/family-page.js index ed5421947..2cbf03b6a 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -6,6 +6,9 @@ // phone), each child's stretches painted by parent, the de novo and compound-het candidates, and the // members side by side. It draws what the steps wrote and computes nothing about genetics. Text through textContent, // the drawing through createElementNS; images from this server only. +// A child's results to act on stay closed here as on the child's own page (person.js): each child's results.json +// gives the closed results (veil.spans), and a row of the child that names one (a de novo or compound-het +// candidate, a stretch painted from a grandparent) stays closed until that result is opened on purpose here. (function () { 'use strict'; var U = window.PGPUI; @@ -70,6 +73,73 @@ } function memberUrl(f, s) { return f.runs && f.runs[s] ? f.runs[s].url : null; } + // --- the veil --------------------------------------------------------------------------------------- + // What this page knows of each child's results: { status: 'pending', 'known' or 'failed', spans, acts }. A child + // with no run here, or whose results.json has not answered or could not be read, cannot say which rows name a + // result to act on, so every row of theirs that could stays closed (UNKNOWN) until opened on purpose. + // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. + var UNKNOWN = '?'; + var kids = {}, opened = {}, openedAll = {}, last = null; + function loadKids(f) { + (f.state.children || []).forEach(function (c) { + var url = memberUrl(f, c.child); + var k = kids[c.child]; + if (!url) { kids[c.child] = { status: 'failed' }; return; } + if (k && k.url === url && k.busy) { return; } + kids[c.child] = k && k.url === url ? k : { status: 'pending', url: url }; + kids[c.child].busy = true; + U.getJSON(url + 'results.json').then(function (r) { + var cur = kids[c.child]; + if (!cur || cur.url !== url) { return; } + cur.busy = false; + var v = (r && r.veil) || {}; + var acts = ((r && r.findings) || []).filter(function (x) { return x.level === 'act'; }).map(function (x) { return x.id; }); + var next = r ? { status: 'known', url: url, spans: v.spans || null, acts: acts } : { status: 'failed', url: url }; + // An answer that cannot be read keeps what was known: the page never opens on a failed request. + if (!r && cur.status === 'known') { return; } + var key = JSON.stringify([next.status, next.spans, next.acts]); + if (key === cur.key) { return; } + next.key = key; + kids[c.child] = next; + if (last) { redraw(); } + }).catch(function () { + var cur = kids[c.child]; + if (!cur || cur.url !== url) { return; } + cur.busy = false; + if (cur.status !== 'known' && cur.status !== 'failed') { cur.status = 'failed'; cur.key = 'failed'; if (last) { redraw(); } } + }); + }); + } + // The closed result a row of CHILD belongs to: its id, UNKNOWN when this page cannot tell, or null. + function tagOf(child, text, chrom, pos, end) { + var k = kids[child]; + if (!k || k.status !== 'known') { return UNKNOWN; } + if (!k.spans) { return k.acts.length ? UNKNOWN : null; } // results without the tags: cannot tell + return U.veilId(k.spans, text, chrom, pos, end); + } + function isClosed(child, tag) { return !!tag && !(tag === UNKNOWN ? openedAll[child] : opened[tag]); } + // The rows of CHILD split by the veil: { open: rows, closed: [tags of the closed rows] }. + function veiled(child, rows, tagFn) { + var out = { open: [], closed: [] }; + rows.forEach(function (r) { + var t = tagFn(r); + if (isClosed(child, t)) { out.closed.push(t); } else { out.open.push(r); } + }); + return out; + } + // The closed box under a table of CHILD: how many rows stay closed, and one button that opens one result. + function veilBox(child, closed, what, whats) { + if (!closed.length) { return null; } + var n = closed.length === 1 ? 'One ' + what + ' stays' : closed.length + ' ' + whats + ' stay'; + var first = closed[0]; + if (first === UNKNOWN) { + return U.veil(n + ' closed: this page cannot read ' + child + '’s results yet, so it cannot tell which ' + whats + + ' name a result to act on in adult life.', 'Show the ' + whats, function () { openedAll[child] = true; redraw(); }); + } + return U.veil(n + ' closed with the result to act on in adult life ' + (closed.length === 1 ? 'it belongs' : 'they belong') + + ' to, until that result is opened.', null, function () { opened[first] = true; redraw(); }); + } + // The pedigree drawn from the layout the server gives: a sequenced person with a run here is a link. function pedigree(f) { var L = f.layout, st = f.state; @@ -286,7 +356,13 @@ // A child's stretches, painted: per chromosome the father's copy above the mother's, each stretch // coloured by the grandparent it came from (or by the parent when no grandparent was sequenced). var SIDE = { paternal: 'father', maternal: 'mother' }; - function painting(f, child, rows) { + function painting(f, child, all) { + // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a + // closed result says whether the child got it: it stays closed. A stretch from a parent alone says nothing. + var split = veiled(child, all, function (r) { + return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end) : null; + }); + var rows = split.open; var lengths = f.chrom_lengths || []; var longest = Math.max.apply(null, lengths.map(function (x) { return x[1]; }).concat([1])); var colours = Object.create(null), next = 1; @@ -305,6 +381,14 @@ ['paternal', 'maternal'].forEach(function (side) { var track = el('span', { 'class': 'paint-track ' + side }); track.style.setProperty('--len', (100 * cl[1] / longest).toFixed(2) + '%'); + all.filter(function (r) { return r.chrom === chrom && r.side === side && rows.indexOf(r) < 0; }).forEach(function (r) { + var a = parseInt(r.start, 10), b = parseInt(r.end, 10); + if (!isFinite(a) || !isFinite(b) || b < a) { return; } + var seg = el('span', { 'class': 'paint-seg closed', title: 'A closed stretch' }); + seg.style.setProperty('--a', (100 * a / cl[1]).toFixed(3) + '%'); + seg.style.setProperty('--w', Math.max(0.4, 100 * (b - a) / cl[1]).toFixed(3) + '%'); + add(track, seg); + }); rows.filter(function (r) { return r.chrom === chrom && r.side === side; }).forEach(function (r) { var a = parseInt(r.start, 10), b = parseInt(r.end, 10); if (!isFinite(a) || !isFinite(b) || b < a) { return; } @@ -329,8 +413,10 @@ who + (colours[who].basis === 'other copy' ? ' (the copy not from the sequenced parent)' : '')); add(leg, it); }); - add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).')); + add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).' + + (split.closed.length ? ' Hatched: a closed stretch.' : ''))); add(fig, leg); + add(fig, veilBox(child, split.closed, 'stretch', 'stretches')); var d = el('details', { 'class': 'w-fallback' }); add(d, el('summary', {}, 'Every stretch (' + rows.length + ')')); add(d, U.table({ columns: ['Copy', 'Chromosome', 'Start', 'End', 'From', 'Sites', 'Discordant', 'Basis'], @@ -361,27 +447,41 @@ if (rows && rows.length) { add(d, painting(f, c.child, rows)); } else if (c.segments) { - add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); + // The drawing as an image cannot close one stretch: while this child has a result to act on that is + // closed (or this page cannot tell), it stays closed whole. + var k = kids[c.child] || {}; + var shut = k.status !== 'known' ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }).filter(function (id) { return !opened[id]; }); + if (!k.spans && k.status === 'known' && k.acts.length) { shut = [UNKNOWN]; } + if (shut.length && !(shut[0] === UNKNOWN && openedAll[c.child])) { + add(d, U.veil('The chromosome map of ' + c.child + ' stays closed while a result to act on in adult life is: ' + + 'it could show which copy the child got there.', 'Show the map', function () { shut.forEach(function (t) { if (t === UNKNOWN) { openedAll[c.child] = true; } else { opened[t] = true; } }); redraw(); })); + } else { + add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); + } } else { add(d, el('p', { 'class': 'dim' }, 'No chromosome map yet: step 38 draws one for a child with a sequenced parent.')); } var dn = (f.de_novo || []).filter(function (r) { return r.child === c.child; }); add(d, el('h3', {}, 'De novo candidates')); if (dn.length) { + var dv = veiled(c.child, dn, function (r) { return tagOf(c.child, r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.chrom, r.pos); }); add(d, el('p', { 'class': 'dim' }, 'In the child and in neither parent. Most candidates in a genome are genotype errors: ' + 'check the reads before reading anything into one.')); add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], - rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, + rows: dv.open.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); + add(d, veilBox(c.child, dv.closed, 'row', 'rows')); } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'de_novo' }, empty(c.child, 'de_novo', 'de_novo.tsv'))); } var ch = (f.compound_het || []).filter(function (r) { return r.child === c.child; }); add(d, el('h3', {}, 'Compound-het pairs')); if (ch.length) { + var cv = veiled(c.child, ch, function (r) { return tagOf(c.child, [r.gene, r.variant_a, r.variant_b].join(' ')); }); add(d, el('p', { 'class': 'dim' }, 'Two candidate variants in one gene, in trans first: one from each parent.')); add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], - rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); + rows: cv.open.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); + add(d, veilBox(c.child, cv.closed, 'row', 'rows')); } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'compound_het' }, empty(c.child, 'compound_het', 'compound_het.tsv'))); } @@ -431,7 +531,12 @@ } function draw(f) { + last = f; var main = document.getElementById('family'); + // A redraw (a result opened, a child's results answered) keeps each child's fold as the reader left it. + var folds = {}; + Array.prototype.forEach.call(main.querySelectorAll('details[data-child]'), function (x) { folds[x.getAttribute('data-child')] = x.open; }); + var y = window.scrollY; main.textContent = ''; var page = el('section', { 'class': 'family-page', 'data-plugin': 'family', 'aria-label': 'Family ' + f.state.name }); if (f.state.error) { @@ -462,14 +567,22 @@ add(page, related(f), couples(f), children(f), sideBySide(f)); if ((f.state.plan || []).length || f.state.reports.length) { add(page, more(f)); } add(main, page); + Array.prototype.forEach.call(main.querySelectorAll('details[data-child]'), function (x) { + var k = x.getAttribute('data-child'); + if (k in folds) { x.open = folds[k]; } + }); + window.scrollTo(0, y); + // What the page knows of each child's results (for the tests): "DEMO-003:known". + root.setAttribute('data-pgp-kids', Object.keys(kids).sort().map(function (c) { return c + ':' + kids[c].status; }).join(' ')); root.setAttribute('data-pgp-status', 'ready'); } + function redraw() { if (last) { draw(last); } } // The page asks again every 30 s and redraws only when family.json changed (a member's run moved on, // step 39 finished). A failed first request says so, with a retry; a later one keeps what is shown. - var shown = null; + var drawnKey = null; function failed() { - if (shown !== null) { return; } + if (drawnKey !== null) { return; } var main = document.getElementById('family'); main.textContent = ''; var b = el('button', { type: 'button', 'class': 'more' }, 'Try again'); @@ -481,7 +594,9 @@ return U.getJSON('family.json').then(function (f) { if (!f) { failed(); return; } var k = JSON.stringify(f); - if (k !== shown) { shown = k; draw(f); } + // Each child's results are asked again with the family: a result to act on can appear while a run goes. + if (!f.state.error) { loadKids(f); } + if (k !== drawnKey) { drawnKey = k; draw(f); } }).catch(failed); } function start() { diff --git a/assets/live/live.css b/assets/live/live.css index 5dba9f176..4adff6602 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -607,6 +607,7 @@ ul.rel-pairs .chip { white-space: normal; } .anc1 { background: var(--anc1); } .anc2 { background: var(--anc2); } .anc3 { background: var(--anc3); } .anc4 { background: var(--anc4); } .anc5 { background: var(--anc5); } .paint-seg.other, .paint-swatch.other { background-image: repeating-linear-gradient(45deg, transparent 0 2px, rgba(0, 0, 0, .35) 2px 4px); } +.paint-seg.closed { background: repeating-linear-gradient(-45deg, rgba(143, 141, 134, .45) 0 2px, transparent 2px 6px); } .paint-legend { display: flex; flex-wrap: wrap; gap: 4px 14px; align-items: center; font-size: 12.5px; margin-top: 8px; } .paint-key { display: inline-flex; gap: 6px; align-items: center; } .paint-swatch { display: inline-block; width: 14px; height: 10px; border-radius: 3px; } diff --git a/docs/live-site.md b/docs/live-site.md index 11b784bc2..0f6935941 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,7 +56,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the chromosome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the chromosome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a copy-number dot (a CHIP call is a finding of its own card, so the CHIP card closes whole), a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. @@ -72,7 +72,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, and a reload closes it again. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, or a request that failed), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. @@ -88,7 +88,7 @@ The reads cross the network to the browser that logged in: only the byte ranges `GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. -The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words` and `databases`. It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. +The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the rows outside a result's own card that close with it, and `spans`, the closed results with their gene and position, which the family page tags a child's rows with by the same rule). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. ## Privacy and what is stored diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 8f10fe575..54431709f 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -408,7 +408,7 @@ def widget_checks(b, base, out): keep = f.read() try: with open(js, "w") as f: - f.write(keep.replace("add(text, widget(c.widget));", "")) + f.write(keep.replace("add(text, widget(of[1]));", "")) control("person.js without its widget line", not widget_problems(widget_counts(b, base))) finally: with open(js, "w") as f: From 0d715a074a53d84149d9ea6b3627610ef7399afc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:42:09 +0100 Subject: [PATCH 004/255] fix(live): keep a family row closed until every closed result it names is opened A compound-het pair or a painted stretch can name two closed results: core.js veilIds() lists each one by veil_id()'s rule, and the row stays closed until all are opened. Opened results are kept per child and run, since two children can carry the same result id. Results the server could not build count as unreadable. On a child's page the phone's card line takes only an open fact, and the closed count names rows and items apart. Bead: pgp-iqon.24 --- assets/live/core.js | 7 +++++- assets/live/family-page.js | 45 ++++++++++++++++++++++---------------- assets/live/person.js | 14 +++++++----- docs/live-site.md | 2 +- tests/test_live_demo.py | 33 ++++++++++++++++++++++------ tests/test_live_results.py | 24 ++++++++++++-------- 6 files changed, 83 insertions(+), 42 deletions(-) diff --git a/assets/live/core.js b/assets/live/core.js index e0aa05e32..fff06e955 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -260,9 +260,14 @@ } return null; } + // Every closed result a row belongs to, by the same rule taken one result at a time: a compound-het pair can + // name two, and a painted stretch can hold two, so the family page keeps such a row closed until each is opened. + function veilIds(spans, text, chrom, pos, end) { + return (spans || []).filter(function (s) { return veilId([s], text, chrom, pos, end); }).map(function (s) { return s[0]; }); + } window.PGPUI = { el: el, svg: svg, add: add, getJSON: getJSON, avatar: avatar, day: day, caveat: caveat, facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, glyph: glyph, karyogram: karyogram, locus: locus, LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, - hm: hm, dm: dm, hours: hours, veil: veil, veilId: veilId }; + hm: hm, dm: dm, hours: hours, veil: veil, veilId: veilId, veilIds: veilIds }; }()); diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 2cbf03b6a..1253b4d59 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -77,9 +77,11 @@ // What this page knows of each child's results: { status: 'pending', 'known' or 'failed', spans, acts }. A child // with no run here, or whose results.json has not answered or could not be read, cannot say which rows name a // result to act on, so every row of theirs that could stays closed (UNKNOWN) until opened on purpose. - // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. + // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. They + // are kept per child and per run (two children can carry the same result, under the same id). var UNKNOWN = '?'; var kids = {}, opened = {}, openedAll = {}, last = null; + function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; } function loadKids(f) { (f.state.children || []).forEach(function (c) { var url = memberUrl(f, c.child); @@ -93,10 +95,12 @@ if (!cur || cur.url !== url) { return; } cur.busy = false; var v = (r && r.veil) || {}; - var acts = ((r && r.findings) || []).filter(function (x) { return x.level === 'act'; }).map(function (x) { return x.id; }); - var next = r ? { status: 'known', url: url, spans: v.spans || null, acts: acts } : { status: 'failed', url: url }; + // Results the server could not build come back without their findings: as unreadable as no answer. + var usable = !!(r && Array.isArray(r.findings)); + var acts = usable ? r.findings.filter(function (x) { return x.level === 'act'; }).map(function (x) { return x.id; }) : []; + var next = usable ? { status: 'known', url: url, spans: v.spans || null, acts: acts } : { status: 'failed', url: url }; // An answer that cannot be read keeps what was known: the page never opens on a failed request. - if (!r && cur.status === 'known') { return; } + if (!usable && cur.status === 'known') { return; } var key = JSON.stringify([next.status, next.spans, next.acts]); if (key === cur.key) { return; } next.key = key; @@ -110,20 +114,23 @@ }); }); } - // The closed result a row of CHILD belongs to: its id, UNKNOWN when this page cannot tell, or null. + // The closed results a row of CHILD belongs to: their ids, [UNKNOWN] when this page cannot tell, or []. function tagOf(child, text, chrom, pos, end) { var k = kids[child]; - if (!k || k.status !== 'known') { return UNKNOWN; } - if (!k.spans) { return k.acts.length ? UNKNOWN : null; } // results without the tags: cannot tell - return U.veilId(k.spans, text, chrom, pos, end); + if (!k || k.status !== 'known') { return [UNKNOWN]; } + if (!k.spans) { return k.acts.length ? [UNKNOWN] : []; } // results without the tags: cannot tell + return U.veilIds(k.spans, text, chrom, pos, end); } - function isClosed(child, tag) { return !!tag && !(tag === UNKNOWN ? openedAll[child] : opened[tag]); } - // The rows of CHILD split by the veil: { open: rows, closed: [tags of the closed rows] }. + // The results of TAGS still closed for CHILD: a row stays closed while one of them is. + function stillClosed(child, tags) { + return tags.filter(function (t) { return t === UNKNOWN ? !openedAll[child] : !opened[openKey(child, t)]; }); + } + // The rows of CHILD split by the veil: { open: rows, closed: [the still closed results of each closed row] }. function veiled(child, rows, tagFn) { var out = { open: [], closed: [] }; rows.forEach(function (r) { - var t = tagFn(r); - if (isClosed(child, t)) { out.closed.push(t); } else { out.open.push(r); } + var t = stillClosed(child, tagFn(r)); + if (t.length) { out.closed.push(t); } else { out.open.push(r); } }); return out; } @@ -131,13 +138,13 @@ function veilBox(child, closed, what, whats) { if (!closed.length) { return null; } var n = closed.length === 1 ? 'One ' + what + ' stays' : closed.length + ' ' + whats + ' stay'; - var first = closed[0]; + var first = closed[0][0]; if (first === UNKNOWN) { return U.veil(n + ' closed: this page cannot read ' + child + '’s results yet, so it cannot tell which ' + whats + ' name a result to act on in adult life.', 'Show the ' + whats, function () { openedAll[child] = true; redraw(); }); } return U.veil(n + ' closed with the result to act on in adult life ' + (closed.length === 1 ? 'it belongs' : 'they belong') + - ' to, until that result is opened.', null, function () { opened[first] = true; redraw(); }); + ' to, until that result is opened.', null, function () { opened[openKey(child, first)] = true; redraw(); }); } // The pedigree drawn from the layout the server gives: a sequenced person with a run here is a link. @@ -360,7 +367,7 @@ // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a // closed result says whether the child got it: it stays closed. A stretch from a parent alone says nothing. var split = veiled(child, all, function (r) { - return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end) : null; + return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end) : []; }); var rows = split.open; var lengths = f.chrom_lengths || []; @@ -450,11 +457,11 @@ // The drawing as an image cannot close one stretch: while this child has a result to act on that is // closed (or this page cannot tell), it stays closed whole. var k = kids[c.child] || {}; - var shut = k.status !== 'known' ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }).filter(function (id) { return !opened[id]; }); - if (!k.spans && k.status === 'known' && k.acts.length) { shut = [UNKNOWN]; } - if (shut.length && !(shut[0] === UNKNOWN && openedAll[c.child])) { + var all = k.status !== 'known' || (!k.spans && k.acts.length) ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }); + var shut = stillClosed(c.child, all); + if (shut.length) { add(d, U.veil('The chromosome map of ' + c.child + ' stays closed while a result to act on in adult life is: ' + - 'it could show which copy the child got there.', 'Show the map', function () { shut.forEach(function (t) { if (t === UNKNOWN) { openedAll[c.child] = true; } else { opened[t] = true; } }); redraw(); })); + 'it could show which copy the child got there.', 'Show the map', function () { shut.forEach(function (t) { if (t === UNKNOWN) { openedAll[c.child] = true; } else { opened[openKey(c.child, t)] = true; } }); redraw(); })); } else { add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); } diff --git a/assets/live/person.js b/assets/live/person.js index c0b582ac8..4a45383d6 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -431,11 +431,15 @@ var ot = openTable(c), tb = U.table(ot[0]); var lt = veilTags('lines', c.id); var lines = (c.lines || []).filter(function (l, i) { return !(lt[i] && tagClosed(lt[i])); }); - var shut = ot[1] + (c.lines || []).length - lines.length + of[2]; + var shut = ot[1] + (c.lines || []).length - lines.length, items = of[2]; if (tb) { add(text, tb); } else if (c.empty_text && !ot[1]) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } - if (shut) { - add(text, el('p', { 'class': 'dim veil-rows' }, (shut === 1 ? 'One row stays' : shut + ' rows stay') + - ' closed with the result to act on ' + (shut === 1 ? 'it belongs' : 'they belong') + ' to, until that result is opened.')); + if (shut || items) { + // Rows of its table and lines, and items of its facts and drawing, counted apart. + var what = [shut ? (shut === 1 ? 'One row' : shut + ' rows') : '', items ? (items === 1 ? (shut ? 'one item' : 'One item') : items + ' items') + + ' of its numbers or drawing' : ''].filter(Boolean).join(' and '); + var many = shut + items > 1; + add(text, el('p', { 'class': 'dim veil-rows' }, what + (many ? ' stay' : ' stays') + + ' closed with the result to act on ' + (many ? 'they belong' : 'it belongs') + ' to, until that result is opened.')); } add(text, lineList(lines)); if (c.hidden) { @@ -711,7 +715,7 @@ function cardLine(c) { var g = gate(c); return g === 'show' && veiledOf(c).length ? 'A result about a child, closed until opened.' - : g === 'show' ? (c.headline || (c.facts && c.facts[0] ? c.facts[0][0] + ': ' + c.facts[0][1] : NOT_ALL_CLEAR)) + : g === 'show' ? (c.headline || (openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)) : pending(g) ? 'Waiting for ' + stepWords(c) + '.' : g === 'failed' ? 'Not assessed: its step failed.' : 'Not assessed in this run.'; } // A card the report flags that has no finding of its own: on a phone it is the only place that flag diff --git a/docs/live-site.md b/docs/live-site.md index 0f6935941..4634555ab 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -72,7 +72,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, and a reload closes it again. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, or a request that failed), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 54431709f..2fa60be1a 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -57,8 +57,9 @@ result (the compound-het pair in KCNH2 of the fixture, and a de novo candidate and a stretch painted from a grandparent planted in family.json) name its gene, positions and that grandparent in no row, link, title or - count until "Show the result" is clicked there, the click brings each row - back, and a reload closes them again; with the son's results.json + count until "Show the result" is clicked there, a row that names both + planted results stays closed until both are opened (one per click), each + row then comes back, and a reload closes them again; with the son's results.json unavailable every row of his that could name one stays closed; on his page a CHIP call and a copy-number dot in the gene of a closed result stay out of their drawings until it is opened; DEMO-001's page @@ -771,19 +772,28 @@ def family_closed(b, base): def family_problems(b, base): - """[problems] of the family page: closed, then opened with one click, then reloaded.""" + """[problems] of the family page: closed, then opened one result per click, then reloaded. Each planted row + names both of the son's planted results (the secondary finding's position, and the deletion's span around it, + both in KCNH2), so the first click opens one result and every row stays closed for the other.""" gene, hit, _ = planted_words() out = family_closed(b, base) if out: return out - b.eval("document.querySelector('details[data-child=\"DEMO-003\"] .veil button').click(); true") + click = "document.querySelector('details[data-child=\"DEMO-003\"] .veil button').click(); true" + b.eval(click) + time.sleep(0.5) + shown = b.eval(FAMILY_SHOWN_JS) or "" + left = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") + if any(w in shown for w in [gene, GRANDPARENT, PAIR_B] + hit) or left != 3: + out.append(f"one click opened a row that also names the other closed result ({left} closed boxes left)") + b.eval(click) time.sleep(0.5) shown = b.eval(FAMILY_SHOWN_JS) or "" rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") left = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") if not all(w in shown for w in [gene, GRANDPARENT, hit[0]]) or json.loads(rows or "[0,0,0]")[1:] != [3, 3] or left: - out.append(f"one click on 'Show the result' did not bring back each row: tables {rows}, {left} closed boxes left") + out.append(f"two clicks on 'Show the result' did not bring back each row: tables {rows}, {left} closed boxes left") out += [f"after a reload: {x}" for x in family_closed(b, base)] return out @@ -835,9 +845,10 @@ def write(p, text): bad = family_problems(b, base) check("browser: on the family page the son's rows that name his planted result (a compound-het pair, a de novo " "candidate, a stretch from a grandparent) name its gene, positions and grandparent nowhere until 'Show the " - "result' is clicked there, one click brings each row back, and a reload closes them", not bad, bad) + "result' is clicked there, a row naming two closed results stays closed until both are opened, then each " + "row comes back, and a reload closes them", not bad, bad) for p, a, z, desc in ( - (fp, "function isClosed(child, tag) { return !!tag && ", "function isClosed(child, tag) { return false && ", + (fp, "return tags.filter(function (t) {", "return [].filter(function (t) {", "family-page.js whose rows ignore the veil"), (cj, " if (!spans || !spans.length) { return null; }", " return null;", "core.js whose veilId() tags nothing")): if a not in keep[p]: @@ -853,6 +864,14 @@ def write(p, text): os.replace(rj + ".away", rj) check("browser: with the son's results.json unavailable the family page keeps every row of his that could name " "a result to act on closed", not bad, bad) + # The results route answers a summary it cannot build with a note and no findings (owngenome_live_results.py). + with open(rj) as f: + keep[rj] = f.read() + write(rj, json.dumps({"schema_version": 1, "sample": OTHERS[1], "chapters": [], + "note": "The results could not be read (KeyError)."})) + bad = family_unknown(b, base) + check("browser: with the son's results answered as unreadable (a note, no findings) the family page keeps every " + "row of his that could name a result to act on closed", not bad, bad) finally: keep.pop("planted", None) for p, text in keep.items(): diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 7a403b68c..becc5e182 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -52,7 +52,7 @@ and "All clear" in a script, 4 with the hits emptied, 5 with the results route registered without the login, 6 with a row printed as its Python dict, 7 with the Slivar adapter returning nothing, 8 with a veil rule that tags nothing (rows, lines, facts and drawings) and one that tags every row, and -veilId() with its gene rule or its overlap rule dropped. A check +veilId() with its gene rule or its overlap rule dropped and veilIds() keeping only the first result. A check that cannot fail is no check. Run: python3 tests/test_live_results.py @@ -452,11 +452,12 @@ def widget_problems(): "const doc={documentElement:{setAttribute(){}},addEventListener(){}};" "vm.runInNewContext(fs.readFileSync(process.argv[1],'utf8'),{window:w,document:doc});" "const sp=JSON.parse(process.argv[2]);" - "console.log(JSON.stringify(JSON.parse(process.argv[3]).map(r=>w.PGPUI.veilId(sp,r[0],r[1],r[2],r[3]))))") + "console.log(JSON.stringify(JSON.parse(process.argv[3]).map(r=>[w.PGPUI.veilId(sp,r[0],r[1],r[2],r[3])," + "w.PGPUI.veilIds(sp,r[0],r[1],r[2],r[3])])))") def node_veil(js_path): - """veilId() of core.js at JS_PATH on VEIL_ROWS, under node: [ids] or the error.""" + """[veilId(), veilIds()] of core.js at JS_PATH for each of VEIL_ROWS, under node, or the error.""" node = shutil.which("node") if not node: return "no node" @@ -466,12 +467,14 @@ def node_veil(js_path): def veil_js_problems(js_path): - """What veilId() of core.js answers differently from veil_id() on VEIL_ROWS.""" - want = [live_results.veil_id([tuple(x) for x in VEIL_SPANS], *r) for r in VEIL_ROWS] + """What veilId() of core.js answers differently from veil_id() on VEIL_ROWS, and veilIds() from veil_id() taken + one closed result at a time (every result the row belongs to).""" + spans = [tuple(x) for x in VEIL_SPANS] + want = [[live_results.veil_id(spans, *r), [s_[0] for s_ in spans if live_results.veil_id([s_], *r)]] for r in VEIL_ROWS] got = node_veil(js_path) if got != want: - return [f"veilId {got} != veil_id {want}"] - if len({w for w in want}) < 4: + return [f"veilId, veilIds {got} != veil_id {want}"] + if len({w[0] for w in want}) < 4 or not any(len(w[1]) > 1 for w in want): return [f"the rows exercise too few answers: {want}"] return [] @@ -598,11 +601,14 @@ def tagged(sample_dir): "chart's gene panel none, and results.json carries the closed results as veil.spans", not bad, bad) check("8. node is on the PATH (ubuntu-latest has it)", shutil.which("node") is not None) bad = veil_js_problems(CORE_JS) - check("8. core.js veilId() tags each family-page row as veil_id() does (under node)", not bad, bad) + check("8. core.js veilId() tags each family-page row as veil_id() does, and veilIds() with every result " + "veil_id() finds one result at a time (under node)", not bad, bad) with open(CORE_JS) as f: core = f.read() for a, z, desc in (("if (s[1] && words[s[1]]) { return s[0]; }", "", "veilId() without its gene rule"), - ("l[1] <= s[4] && l[2] >= s[3]", "false", "veilId() without its overlap rule")): + ("l[1] <= s[4] && l[2] >= s[3]", "false", "veilId() without its overlap rule"), + (".map(function (s) { return s[0]; });\n }", ".map(function (s) { return s[0]; }).slice(0, 1);\n }", + "veilIds() keeping only the first result")): planted = os.path.join(work, "core-planted.js") with open(planted, "w") as f: f.write(core.replace(a, z)) From 40c41109ae6653c3276944cb058d7aff21e27a23 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:38:48 +0100 Subject: [PATCH 005/255] fix(live): close position-only family rows with a gene-only result, and open the map alone A closed result known by its gene alone (two changes in one gene) could sit at any de novo candidate or painted stretch, which name no gene, so those rows stay closed with it. 'Show the map' now opens the drawn map only, not the child's results. Bead: pgp-iqon.24 --- assets/live/family-page.js | 22 ++++++---- docs/live-site.md | 2 +- tests/test_live_demo.py | 88 +++++++++++++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 11 deletions(-) diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 1253b4d59..99b5b3237 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -80,7 +80,7 @@ // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. They // are kept per child and per run (two children can carry the same result, under the same id). var UNKNOWN = '?'; - var kids = {}, opened = {}, openedAll = {}, last = null; + var kids = {}, opened = {}, openedAll = {}, mapShown = {}, last = null; function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; } function loadKids(f) { (f.state.children || []).forEach(function (c) { @@ -115,11 +115,16 @@ }); } // The closed results a row of CHILD belongs to: their ids, [UNKNOWN] when this page cannot tell, or []. - function tagOf(child, text, chrom, pos, end) { + // COORDS: the row has a position and no gene name (a de novo candidate, a stretch). A closed result known by its + // gene alone (two changes in one gene, with no position) could sit at that position, and only the gene would + // tell: such a row stays closed with every result of that kind too. + function tagOf(child, text, chrom, pos, end, coords) { var k = kids[child]; if (!k || k.status !== 'known') { return [UNKNOWN]; } if (!k.spans) { return k.acts.length ? [UNKNOWN] : []; } // results without the tags: cannot tell - return U.veilIds(k.spans, text, chrom, pos, end); + var ids = U.veilIds(k.spans, text, chrom, pos, end); + if (coords) { k.spans.forEach(function (s) { if (!s[2] && ids.indexOf(s[0]) < 0) { ids.push(s[0]); } }); } + return ids; } // The results of TAGS still closed for CHILD: a row stays closed while one of them is. function stillClosed(child, tags) { @@ -367,7 +372,7 @@ // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a // closed result says whether the child got it: it stays closed. A stretch from a parent alone says nothing. var split = veiled(child, all, function (r) { - return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end) : []; + return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end, true) : []; }); var rows = split.open; var lengths = f.chrom_lengths || []; @@ -458,10 +463,11 @@ // closed (or this page cannot tell), it stays closed whole. var k = kids[c.child] || {}; var all = k.status !== 'known' || (!k.spans && k.acts.length) ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }); - var shut = stillClosed(c.child, all); - if (shut.length) { + var shut = stillClosed(c.child, all), mk = openKey(c.child, 'map'); + // "Show the map" opens the map only: each result keeps its own closed rows until it is opened. + if (shut.length && !mapShown[mk]) { add(d, U.veil('The chromosome map of ' + c.child + ' stays closed while a result to act on in adult life is: ' + - 'it could show which copy the child got there.', 'Show the map', function () { shut.forEach(function (t) { if (t === UNKNOWN) { openedAll[c.child] = true; } else { opened[openKey(c.child, t)] = true; } }); redraw(); })); + 'it could show which copy the child got there.', 'Show the map', function () { mapShown[mk] = true; redraw(); })); } else { add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); } @@ -471,7 +477,7 @@ var dn = (f.de_novo || []).filter(function (r) { return r.child === c.child; }); add(d, el('h3', {}, 'De novo candidates')); if (dn.length) { - var dv = veiled(c.child, dn, function (r) { return tagOf(c.child, r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.chrom, r.pos); }); + var dv = veiled(c.child, dn, function (r) { return tagOf(c.child, r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.chrom, r.pos, null, true); }); add(d, el('p', { 'class': 'dim' }, 'In the child and in neither parent. Most candidates in a genome are genotype errors: ' + 'check the reads before reading anything into one.')); add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], diff --git a/docs/live-site.md b/docs/live-site.md index 4634555ab..a06b94f3a 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -72,7 +72,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. A result known by its gene alone, with no position, could sit at any de novo candidate or painted stretch, which name no gene: those rows stay closed with it. When the stretches come only as the drawn image, the image stays closed while a result is, and "Show the map" opens the image and no result. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 2fa60be1a..36e0f5559 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -60,7 +60,10 @@ count until "Show the result" is clicked there, a row that names both planted results stays closed until both are opened (one per click), each row then comes back, and a reload closes them again; with the son's results.json - unavailable every row of his that could name one stays closed; on his page + unavailable, or answered as unreadable, every row of his that could name one + stays closed; with a closed result known by its gene alone his de novo rows + (a position, no gene) stay closed with it; his map as an image stays closed, + and "Show the map" opens the map and no result; on his page a CHIP call and a copy-number dot in the gene of a closed result stay out of their drawings until it is opened; DEMO-001's page mid-replay and at the end leads with the report (a short run band above @@ -74,7 +77,8 @@ without its filter; for the planted results, results.json without its veil tags, person.js whose table rows ignore them, person.js whose findings ignore them, charts.js that keeps every mark, search.js that ignores a hit's tag and finding.js whose findings ignore their tags; for the family page, family-page.js whose rows ignore the veil -and core.js whose veilId() tags nothing; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +core.js whose veilId() tags nothing, family-page.js without the gene-only fallback and a "Show the map" that opens +the results too; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -812,6 +816,52 @@ def family_unknown(b, base): return out +def son_tables(b): + """[rows of each table] in the son's fold of the family page.""" + rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " + "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") + return json.loads(rows) if rows else None + + +def gene_only_problems(b, base): + """[problems] of the family page when the son also has a closed result known by its gene alone: a de novo row + shown before it is opened, a pair in another gene closed, or the rows not back once each result is opened.""" + family_page(b, base) + out = [] + t = son_tables(b) + shown = b.eval(FAMILY_SHOWN_JS) or "" + # Open: the stretches' table and the two pairs in other genes; closed: both de novo rows. + if t != [80, 2] or "48211904" in shown: + out.append(f"before any click: tables {t}") + for _ in range(3): # the gene-only result, then the two planted ones (the pair in KCNH2) + b.eval("var x = document.querySelector('details[data-child=\"DEMO-003\"] .veil button'); x && x.click(); true") + time.sleep(0.4) + t = son_tables(b) + if t != [80, 2, 3]: + out.append(f"after opening each result: tables {t}") + return out + + +def map_problems(b, base): + """[problems] of the family page whose son has his map as an image: the image shown while a result is closed, + or 'Show the map' opening a result's rows.""" + family_page(b, base) + out = [] + img = b.eval("!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')") + if img: + out.append("the map shown while a result is closed") + b.eval("var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil'), " + "function (x) { return x.textContent.indexOf('Show the map') >= 0; })[0]; v && v.querySelector('button').click(); true") + time.sleep(0.4) + img = b.eval("!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')") + shown = b.eval(FAMILY_SHOWN_JS) or "" + if not img: + out.append("'Show the map' did not show the map") + if "KCNH2" in shown or PAIR_B in shown: + out.append("'Show the map' opened the closed pair too") + return out + + def family_veil_checks(b, base, out): """On the family page a child's rows that name a closed result stay closed as on the child's page, until that result is opened there; a page that cannot read the child's results keeps every row of his that could closed.""" @@ -872,6 +922,40 @@ def write(p, text): bad = family_unknown(b, base) check("browser: with the son's results answered as unreadable (a note, no findings) the family page keeps every " "row of his that could name a result to act on closed", not bad, bad) + + # A closed result known by its gene alone: a row with only a position (a de novo candidate) could be it. + res = json.loads(keep[rj]) + res["veil"]["spans"].append(["two-hits-g0999", "G0999", None, None, None]) + write(rj, json.dumps(res)) + write(fj, keep[fj]) # the fixture's family: two de novo rows, three pairs (one of them in KCNH2) + bad = gene_only_problems(b, base) + check("browser: with a closed result known by its gene alone, the son's de novo rows (a position, no gene) stay " + "closed until it is opened, and his pairs in other genes stay open", not bad, bad) + a = "if (coords) {" + if a not in keep[fp]: + check("control: family-page.js without the gene-only fallback (the planted line is in family-page.js)", False, a) + else: + write(fp, keep[fp].replace(a, "if (false) {")) + control("family-page.js without the gene-only fallback", not gene_only_problems(b, base)) + write(fp, keep[fp]) + write(rj, keep[rj]) + + # The drawing as an image (no stretches in family.json): "Show the map" opens the map and no result. + fam = json.loads(keep[fj]) + fam["segments"] = {} + fam["state"]["children"][0]["segments"] = fam["state"]["pedigree"] + write(fj, json.dumps(fam)) + bad = map_problems(b, base) + check("browser: the son's map as an image stays closed while a result is, and 'Show the map' opens the map " + "and leaves each result's rows closed", not bad, bad) + a = "function () { mapShown[mk] = true; redraw(); }" + if a not in keep[fp]: + check("control: a 'Show the map' that opens the results too (the planted line is in family-page.js)", False, a) + else: + write(fp, keep[fp].replace(a, "function () { mapShown[mk] = true; shut.forEach(function (t) { " + "opened[openKey(c.child, t)] = true; }); redraw(); }")) + control("a 'Show the map' that opens the results too", not map_problems(b, base)) + write(fp, keep[fp]) finally: keep.pop("planted", None) for p, text in keep.items(): From 996de9d7dc485876302fd370a50acb43456ac650 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:42:29 +0100 Subject: [PATCH 006/255] fix(live): close a child's side-by-side rows and the grandparents of the painting with a closed result The side by side showed a child's CPIC row (RYR1, CACNA1S) for a gene whose result is closed on the child's page, and the painting drew a closed stretch hatched at its own place, which pointed at the result. A side-by-side row that names a child's closed result is now left out until opened there (every medicines and HLA row while the child's results cannot be read), and while a grandparent stretch lies over a closed result every stretch of that child is drawn from its parent alone. --- CHANGELOG.md | 2 +- assets/live/family-page.js | 59 +++++++++++++++++------ assets/live/live.css | 1 - docs/live-site.md | 2 +- tests/test_live_demo.py | 75 +++++++++++++++++++++++++---- tests/test_owngenome_live_family.py | 6 +++ 6 files changed, 118 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83143e76b..69de5401e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The family page now keeps a child's results to act on closed, as the child's own page does. A de novo candidate or a compound-het pair that names a closed result, and a stretch painted from a grandparent over one, stay behind "Show the result" until that result is opened there; a reload closes them again. While the page cannot read the child's results, every such row of that child stays closed. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: the files behind the pages hold every row. +- The family page now keeps a child's results to act on closed, as the child's own page does. A de novo candidate or a compound-het pair that names a closed result, and a side-by-side row whose value in the child's column names one (a CPIC gene such as RYR1), stay behind "Show the result" until that result is opened there; a reload closes them again. While a stretch painted from a grandparent lies over a closed result, the child's painting shows every stretch from its parent alone. While the page cannot read the child's results, every such row of that child stays closed (in the side by side, every medicines and HLA row). On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: the files behind the pages hold every row. - Step 49 (clonal haematopoiesis) now prepares the CHIP gene models and writes its two tables with the Rust `og` binary: `og chip prepare`, `og chip report` and `og chip check-table` replace the commands of `bin/chip_report.py`, which keeps only the detection limit and the column names the reports import. On Ensembl 116's full GFF3, prepare takes 1.7 s and 6 MB instead of 15 s and 20 MB. The two tables, `chip.gff3` and `chip.bed` are the same, byte for byte. What changes: a missing input file or a position that is not a whole number stops with an `ERROR:` line and exit 1, not a Python traceback, and each output is written whole or not at all. Step 49 now needs `og` (`scripts/setup.sh --og`). - VEP consequence ranking (steps 23 and 31) and the gnomAD constraint join now run in the Rust `og` binary, inside `BCFTOOLS_IMAGE`: `og vcf csq-rank` replaces `bin/csq_rank.awk` and `og table constraint-join` replaces `bin/constraint_join.awk`, which are gone. Neither awk program runs on the host anymore (step 23 still uses the host's `awk` for small table edits), and the ranker is no longer the slow stage of step 23's main pipe. Steps 23 and 31 now need `og`: `scripts/setup.sh --og` builds it. Their output is the same, byte for byte, except for these changes. diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 99b5b3237..2bbef5a13 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -8,7 +8,8 @@ // the drawing through createElementNS; images from this server only. // A child's results to act on stay closed here as on the child's own page (person.js): each child's results.json // gives the closed results (veil.spans), and a row of the child that names one (a de novo or compound-het -// candidate, a stretch painted from a grandparent) stays closed until that result is opened on purpose here. +// candidate, a side-by-side row) stays closed until that result is opened on purpose here; while a stretch +// painted from a grandparent lies over one, the child's painting shows each stretch's parent only. (function () { 'use strict'; var U = window.PGPUI; @@ -80,6 +81,7 @@ // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. They // are kept per child and per run (two children can carry the same result, under the same id). var UNKNOWN = '?'; + var GENE_GROUPS = { Medicines: true, HLA: true }; // the side-by-side groups whose rows name a gene var kids = {}, opened = {}, openedAll = {}, mapShown = {}, last = null; function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; } function loadKids(f) { @@ -370,11 +372,18 @@ var SIDE = { paternal: 'father', maternal: 'mother' }; function painting(f, child, all) { // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a - // closed result says whether the child got it: it stays closed. A stretch from a parent alone says nothing. + // closed result says whether the child got it. A stretch from a parent alone says nothing. Closing only that + // stretch would point at the closed result (a hatch, a gap in the table), so while one is closed every stretch + // of this child is drawn from its parent alone, and no grandparent, basis or closed mark is shown. var split = veiled(child, all, function (r) { return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end, true) : []; }); - var rows = split.open; + var shut = []; + split.closed.forEach(function (t) { t.forEach(function (x) { if (shut.indexOf(x) < 0) { shut.push(x); } }); }); + var rows = !shut.length ? all : all.map(function (r) { + return { side: r.side, parent: r.parent, chrom: r.chrom, start: r.start, end: r.end, sites: r.sites, + discordant: r.discordant, ancestor: '.', basis: 'parent' }; + }); var lengths = f.chrom_lengths || []; var longest = Math.max.apply(null, lengths.map(function (x) { return x[1]; }).concat([1])); var colours = Object.create(null), next = 1; @@ -393,14 +402,6 @@ ['paternal', 'maternal'].forEach(function (side) { var track = el('span', { 'class': 'paint-track ' + side }); track.style.setProperty('--len', (100 * cl[1] / longest).toFixed(2) + '%'); - all.filter(function (r) { return r.chrom === chrom && r.side === side && rows.indexOf(r) < 0; }).forEach(function (r) { - var a = parseInt(r.start, 10), b = parseInt(r.end, 10); - if (!isFinite(a) || !isFinite(b) || b < a) { return; } - var seg = el('span', { 'class': 'paint-seg closed', title: 'A closed stretch' }); - seg.style.setProperty('--a', (100 * a / cl[1]).toFixed(3) + '%'); - seg.style.setProperty('--w', Math.max(0.4, 100 * (b - a) / cl[1]).toFixed(3) + '%'); - add(track, seg); - }); rows.filter(function (r) { return r.chrom === chrom && r.side === side; }).forEach(function (r) { var a = parseInt(r.start, 10), b = parseInt(r.end, 10); if (!isFinite(a) || !isFinite(b) || b < a) { return; } @@ -425,10 +426,18 @@ who + (colours[who].basis === 'other copy' ? ' (the copy not from the sequenced parent)' : '')); add(leg, it); }); - add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).' + - (split.closed.length ? ' Hatched: a closed stretch.' : ''))); + add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).')); add(fig, leg); - add(fig, veilBox(child, split.closed, 'stretch', 'stretches')); + if (shut.length) { + // One box for the whole painting: a count of closed stretches would say how many sit over the result. + add(fig, shut[0] === UNKNOWN + ? U.veil('The grandparents of ' + child + '’s stretches stay hidden: this page cannot read ' + child + '’s results yet, ' + + 'so it cannot tell which stretch lies over a result to act on in adult life. Each stretch shows its parent only.', + 'Show the grandparents', function () { openedAll[child] = true; redraw(); }) + : U.veil('The grandparents of ' + child + '’s stretches stay hidden while a result to act on in adult life is ' + + 'closed: they could show which copy the child got there. Each stretch shows its parent only.', null, + function () { opened[openKey(child, shut[0])] = true; redraw(); })); + } var d = el('details', { 'class': 'w-fallback' }); add(d, el('summary', {}, 'Every stretch (' + rows.length + ')')); add(d, U.table({ columns: ['Copy', 'Chromosome', 'Start', 'End', 'From', 'Sites', 'Discordant', 'Basis'], @@ -510,9 +519,25 @@ if (!sb || !sb.rows.length) { add(s, el('p', { 'class': 'dim' }, 'No member has a summary yet, or no summary lists these results.')); return s; } add(s, el('p', { 'class': 'dim' }, 'Each member’s own report, one column per person: CYP2D6, the CPIC genes ' + '(phenotypes in CPIC’s words), HLA, the haplogroups and the blood groups.')); + // A child's value that names a closed result (a CPIC gene that is also the gene of a result to act on, RYR1 or + // CACNA1S) closes its whole row, as the child's own page closes that row of its medicines card: a closed cell + // under the gene's name would say which gene it is. While this page cannot read a child's results, every row + // that names a gene (the medicines and HLA rows) stays closed; the haplogroups and blood groups name none. + var kidsHere = (f.state.children || []).map(function (c) { return c.child; }); + var closedBy = {}, shown = []; + sb.rows.forEach(function (r) { + var open = true; + sb.members.forEach(function (m, i) { + if (kidsHere.indexOf(m) < 0) { return; } + var t = stillClosed(m, tagOf(m, r.label + ' ' + (r.values[i] || ''))); + if (t[0] === UNKNOWN && !GENE_GROUPS[r.group]) { return; } + if (t.length) { (closedBy[m] = closedBy[m] || []).push(t); open = false; } + }); + if (open) { shown.push(r); } + }); // A value from a section older than its input (state "stale") says so in its own cell. var group = null, rows = [], old = 0; - sb.rows.forEach(function (r) { + shown.forEach(function (r) { rows.push([r.group === group ? '' : r.group, r.label].concat(r.values.map(function (v, i) { if (!(r.stale || [])[i]) { return v; } old++; @@ -521,6 +546,10 @@ group = r.group; }); add(s, U.table({ columns: ['', 'Gene or system'].concat(sb.members), rows: rows }, 14)); + sb.members.forEach(function (m) { + var box = closedBy[m] && veilBox(m, closedBy[m], 'row', 'rows'); + if (box) { box.setAttribute('data-veil-child', m); add(s, box); } + }); if (old) { add(s, el('p', { 'class': 'dim', 'data-stale': String(old) }, 'Marked' + STALE + ': the member’s result file is from an ' + 'earlier run, and the latest run did not write it again; that member’s page says why.')); diff --git a/assets/live/live.css b/assets/live/live.css index 4adff6602..5dba9f176 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -607,7 +607,6 @@ ul.rel-pairs .chip { white-space: normal; } .anc1 { background: var(--anc1); } .anc2 { background: var(--anc2); } .anc3 { background: var(--anc3); } .anc4 { background: var(--anc4); } .anc5 { background: var(--anc5); } .paint-seg.other, .paint-swatch.other { background-image: repeating-linear-gradient(45deg, transparent 0 2px, rgba(0, 0, 0, .35) 2px 4px); } -.paint-seg.closed { background: repeating-linear-gradient(-45deg, rgba(143, 141, 134, .45) 0 2px, transparent 2px 6px); } .paint-legend { display: flex; flex-wrap: wrap; gap: 4px 14px; align-items: center; font-size: 12.5px; margin-top: 8px; } .paint-key { display: inline-flex; gap: 6px; align-items: center; } .paint-swatch { display: inline-block; width: 14px; height: 10px; border-radius: 3px; } diff --git a/docs/live-site.md b/docs/live-site.md index a06b94f3a..d2f6ecf85 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -72,7 +72,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one, and a stretch painted from a grandparent over one (it says which grandparent's copy the child got there), stay closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. A result known by its gene alone, with no position, could sit at any de novo candidate or painted stretch, which name no gene: those rows stay closed with it. When the stretches come only as the drawn image, the image stays closed while a result is, and "Show the map" opens the image and no result. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one stays closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. A stretch painted from a grandparent over one says which grandparent's copy the child got there: while such a stretch exists, every stretch of that child is drawn and listed from its parent alone, with no grandparent, so the painting does not point at where the result lies, and "Show the result" brings the grandparents back. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. A result known by its gene alone, with no position, could sit at any de novo candidate or painted stretch, which name no gene: those rows stay closed with it. When the stretches come only as the drawn image, the image stays closed while a result is, and "Show the map" opens the image and no result. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. A row whose value in a child's column names a closed result of that child (a CPIC gene such as RYR1 that is also the gene of a result to act on) is left out whole, for every member, until that result is opened here: a closed cell under the gene's name would say which gene it is. While the page cannot read a child's results, every medicines and HLA row stays closed; the haplogroups and blood groups name no gene. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 36e0f5559..db9b5f30b 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -59,9 +59,12 @@ name its gene, positions and that grandparent in no row, link, title or count until "Show the result" is clicked there, a row that names both planted results stays closed until both are opened (one per click), each - row then comes back, and a reload closes them again; with the son's results.json + row then comes back, and a reload closes them again; while a stretch painted + from a grandparent lies over one, every stretch of his is drawn and listed from + its parent alone, none marked; with a closed result of his in RYR1 the side by + side's RYR1 row stays closed until opened there; with the son's results.json unavailable, or answered as unreadable, every row of his that could name one - stays closed; with a closed result known by its gene alone his de novo rows + stays closed (in the side by side every medicines and HLA row); with a closed result known by its gene alone his de novo rows (a position, no gene) stay closed with it; his map as an image stays closed, and "Show the map" opens the map and no result; on his page a CHIP call and a copy-number dot in the gene of a closed result stay out @@ -77,8 +80,8 @@ without its filter; for the planted results, results.json without its veil tags, person.js whose table rows ignore them, person.js whose findings ignore them, charts.js that keeps every mark, search.js that ignores a hit's tag and finding.js whose findings ignore their tags; for the family page, family-page.js whose rows ignore the veil -core.js whose veilId() tags nothing, family-page.js without the gene-only fallback and a "Show the map" that opens -the results too; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +core.js whose veilId() tags nothing, family-page.js without the gene-only fallback, a "Show the map" that opens +the results too, a closed painting that keeps each stretch's grandparent and a side by side that ignores the veil; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -768,10 +771,14 @@ def family_closed(b, base): rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") closed = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") - segs = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg.closed').length") - # The stretches' table, the 2 de novo rows of the fixture and its 2 other pairs stay; one closed box each. - if not rows or json.loads(rows)[1:] != [2, 2] or closed != 3 or segs != 1: - out.append(f"the open rows and the closed boxes: tables {rows}, {closed} closed boxes, {segs} closed stretches") + segs = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg').length") + marked = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg.closed, " + "[data-painting=\"DEMO-003\"] .paint-seg.other').length") + # The 2 de novo rows of the fixture and its 2 other pairs stay, one closed box each; the painting closes whole + # (one box): every stretch drawn and listed from its parent, none marked, so none points at the closed result. + if not rows or json.loads(rows)[1:] != [2, 2] or closed != 3 or json.loads(rows)[0] != segs or marked: + out.append(f"the open rows and the closed boxes: tables {rows}, {closed} closed boxes, {segs} stretches drawn, " + f"{marked} marked") return out @@ -813,6 +820,38 @@ def family_unknown(b, base): "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") if rows is None or len(json.loads(rows)) != 1: out.append(f"the son's de novo or compound-het rows open with his results unavailable: tables {rows}") + side = side_labels(b) + if any(g in side for g in SIDE_GENES) or "Mitochondrial" not in side: + out.append(f"side by side with the son's results unavailable: rows {sorted(side)[:8]}") + return out + + +SIDE_GENES = ("RYR1", "CACNA1S", "CYP2D6", "HLA-A") + + +def side_labels(b): + """The 'Gene or system' column of the family page's side by side.""" + got = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows tbody tr'), " + "function (r) { return r.children[1] ? r.children[1].textContent : ''; }))") + return json.loads(got) if got else [] + + +def side_problems(b, base): + """[problems] of the family page's side by side with a closed result of the son's in RYR1: the RYR1 row (his + diplotype and phenotype under the gene's name) shown before it is opened there, the other rows closed with it, + or the row not back once opened.""" + family_page(b, base) + out = [] + labels = side_labels(b) + shown = b.eval("document.getElementById('ch-side-by-side').textContent") or "" + boxes = b.eval("document.querySelectorAll('#ch-side-by-side .veil[data-veil-child=\"DEMO-003\"]').length") + if "RYR1" in shown or "CACNA1S" not in labels or boxes != 1: + out.append(f"before any click: RYR1 {'shown' if 'RYR1' in shown else 'closed'}, {len(labels)} rows, {boxes} closed boxes") + b.eval("var x = document.querySelector('#ch-side-by-side .veil button'); x && x.click(); true") + time.sleep(0.4) + labels2 = side_labels(b) + if "RYR1" not in labels2 or len(labels2) != len(labels) + 1: + out.append(f"after 'Show the result': {len(labels2)} rows, RYR1 {'back' if 'RYR1' in labels2 else 'still closed'}") return out @@ -900,7 +939,9 @@ def write(p, text): for p, a, z, desc in ( (fp, "return tags.filter(function (t) {", "return [].filter(function (t) {", "family-page.js whose rows ignore the veil"), - (cj, " if (!spans || !spans.length) { return null; }", " return null;", "core.js whose veilId() tags nothing")): + (cj, " if (!spans || !spans.length) { return null; }", " return null;", "core.js whose veilId() tags nothing"), + (fp, "ancestor: '.', basis: 'parent' };", "ancestor: r.ancestor, basis: r.basis };", + "a closed painting that keeps each stretch's grandparent")): if a not in keep[p]: check(f"control: {desc} (the planted line is in {os.path.basename(p)})", False, a) continue @@ -940,6 +981,22 @@ def write(p, text): write(fp, keep[fp]) write(rj, keep[rj]) + # A closed result in a CPIC gene: the side by side lists that gene with the son's diplotype in his column. + res = json.loads(keep[rj]) + res["veil"]["spans"].append(["sf-ryr1-demo", "RYR1", "chr19", 38440000, 38440000]) + write(rj, json.dumps(res)) + bad = side_problems(b, base) + check("browser: with a closed result of the son's in RYR1, the side by side's RYR1 row (his diplotype and " + "phenotype) stays closed until it is opened there, and the other rows stay", not bad, bad) + a = "var t = stillClosed(m, tagOf(m, r.label + ' ' + (r.values[i] || '')));" + if a not in keep[fp]: + check("control: a side by side that ignores the veil (the planted line is in family-page.js)", False, a) + else: + write(fp, keep[fp].replace(a, "var t = [];")) + control("a side by side that ignores the veil", not side_problems(b, base)) + write(fp, keep[fp]) + write(rj, keep[rj]) + # The drawing as an image (no stretches in family.json): "Show the map" opens the map and no result. fam = json.loads(keep[fj]) fam["segments"] = {} diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index 8cb7c8678..5965c1045 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -349,6 +349,12 @@ def browser_checks(port, work): couple and "1 gene(s) where both carry a finding, 1 X-linked" in couple and b.eval("!!document.querySelector('p[data-couple=\\'father_a mother_a\\'] a')"), couple) b.eval("document.querySelector('details[data-child=child_a]').open = true; true") + # child_a's results cannot be built here (its summary is cut short), so the page cannot tell whether a + # result to act on is closed: the map stays closed until "Show the map" is clicked. + shut = b.wait("var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=child_a] .veil'), " + "function (x) { return x.textContent.indexOf('Show the map') >= 0; })[0]; " + "v && !document.querySelector('details[data-child=child_a] img') && (v.querySelector('button').click(), true)", 10) + check("browser: child_a's chromosome map stays closed while its results cannot be read", shut, shut) seg = b.wait("var i = document.querySelector('details[data-child=child_a] img'); " "i && i.complete && i.naturalWidth > 0 && i.getAttribute('src')", 10) check("browser: child_a's chromosome map loads", seg == "/family/family_a/child_a_segments.svg", seg) From 0bb490670b33a768a9b83e25125ff7a951e0000a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:48:27 +0100 Subject: [PATCH 007/255] fix(live): open a child's drawing only by its own button while the page cannot read the child's results 'Show the rows' of a table opened the child's map as an image, and the painting's grandparents, too. --- assets/live/family-page.js | 18 +++++++++----- tests/test_live_demo.py | 49 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 2bbef5a13..174352935 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -132,6 +132,11 @@ function stillClosed(child, tags) { return tags.filter(function (t) { return t === UNKNOWN ? !openedAll[child] : !opened[openKey(child, t)]; }); } + // The same for a whole drawing of CHILD (its painting, its map as an image): while this page cannot read the + // child's results, only the drawing's own button (KEY) opens it, never "Show the rows" of a table. + function drawingClosed(child, tags, key) { + return tags.filter(function (t) { return t === UNKNOWN ? !mapShown[openKey(child, key)] : !opened[openKey(child, t)]; }); + } // The rows of CHILD split by the veil: { open: rows, closed: [the still closed results of each closed row] }. function veiled(child, rows, tagFn) { var out = { open: [], closed: [] }; @@ -375,11 +380,12 @@ // closed result says whether the child got it. A stretch from a parent alone says nothing. Closing only that // stretch would point at the closed result (a hatch, a gap in the table), so while one is closed every stretch // of this child is drawn from its parent alone, and no grandparent, basis or closed mark is shown. - var split = veiled(child, all, function (r) { - return r.ancestor && r.ancestor !== '.' ? tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end, true) : []; + var shut = [], pk = openKey(child, 'paint'); + all.forEach(function (r) { + if (!r.ancestor || r.ancestor === '.') { return; } + drawingClosed(child, tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end, true), 'paint') + .forEach(function (x) { if (shut.indexOf(x) < 0) { shut.push(x); } }); }); - var shut = []; - split.closed.forEach(function (t) { t.forEach(function (x) { if (shut.indexOf(x) < 0) { shut.push(x); } }); }); var rows = !shut.length ? all : all.map(function (r) { return { side: r.side, parent: r.parent, chrom: r.chrom, start: r.start, end: r.end, sites: r.sites, discordant: r.discordant, ancestor: '.', basis: 'parent' }; @@ -433,7 +439,7 @@ add(fig, shut[0] === UNKNOWN ? U.veil('The grandparents of ' + child + '’s stretches stay hidden: this page cannot read ' + child + '’s results yet, ' + 'so it cannot tell which stretch lies over a result to act on in adult life. Each stretch shows its parent only.', - 'Show the grandparents', function () { openedAll[child] = true; redraw(); }) + 'Show the grandparents', function () { mapShown[pk] = true; redraw(); }) : U.veil('The grandparents of ' + child + '’s stretches stay hidden while a result to act on in adult life is ' + 'closed: they could show which copy the child got there. Each stretch shows its parent only.', null, function () { opened[openKey(child, shut[0])] = true; redraw(); })); @@ -472,7 +478,7 @@ // closed (or this page cannot tell), it stays closed whole. var k = kids[c.child] || {}; var all = k.status !== 'known' || (!k.spans && k.acts.length) ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }); - var shut = stillClosed(c.child, all), mk = openKey(c.child, 'map'); + var shut = drawingClosed(c.child, all, 'map'), mk = openKey(c.child, 'map'); // "Show the map" opens the map only: each result keeps its own closed rows until it is opened. if (shut.length && !mapShown[mk]) { add(d, U.veil('The chromosome map of ' + c.child + ' stays closed while a result to act on in adult life is: ' + diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index db9b5f30b..968eeaaa9 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -64,7 +64,9 @@ its parent alone, none marked; with a closed result of his in RYR1 the side by side's RYR1 row stays closed until opened there; with the son's results.json unavailable, or answered as unreadable, every row of his that could name one - stays closed (in the side by side every medicines and HLA row); with a closed result known by its gene alone his de novo rows + stays closed (in the side by side every medicines and HLA row), and "Show the + rows" opens neither his painting's grandparents nor his map as an image, which + open only by their own button; with a closed result known by its gene alone his de novo rows (a position, no gene) stay closed with it; his map as an image stays closed, and "Show the map" opens the map and no result; on his page a CHIP call and a copy-number dot in the gene of a closed result stay out @@ -81,7 +83,8 @@ rows ignore them, person.js whose findings ignore them, charts.js that keeps every mark, search.js that ignores a hit's tag and finding.js whose findings ignore their tags; for the family page, family-page.js whose rows ignore the veil core.js whose veilId() tags nothing, family-page.js without the gene-only fallback, a "Show the map" that opens -the results too, a closed painting that keeps each stretch's grandparent and a side by side that ignores the veil; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +the results too, a closed painting that keeps each stretch's grandparent, a side by side that ignores the veil and a +drawing that "Show the rows" opens; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -829,6 +832,33 @@ def family_unknown(b, base): SIDE_GENES = ("RYR1", "CACNA1S", "CYP2D6", "HLA-A") +def click_veil(b, words): + """Click the son's closed box whose button says WORDS; whether there was one.""" + return b.eval("var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil " + "button'), function (x) { return x.textContent === " + json.dumps(words) + "; })[0]; " + "if (v) { v.click(); } !!v") + + +def unknown_drawing_problems(b, base, image): + """[problems] of the family page with the son's results unavailable: "Show the rows" of a table opening his + drawing (the map as an image, or the painting's grandparents), or the drawing's own button not opening it.""" + family_page(b, base) + out = [] + if not click_veil(b, "Show the rows"): + out.append("no 'Show the rows' box") + time.sleep(0.4) + img = "!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')" + seen = (lambda: b.eval(img)) if image else (lambda: GRANDPARENT in (b.eval(FAMILY_SHOWN_JS) or "")) + if seen(): + out.append("'Show the rows' opened the drawing too") + if not click_veil(b, "Show the map" if image else "Show the grandparents"): + out.append("no closed box of the drawing") + time.sleep(0.4) + if not seen(): + out.append("the drawing's own button did not open it") + return out + + def side_labels(b): """The 'Gene or system' column of the family page's side by side.""" got = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows tbody tr'), " @@ -951,6 +981,7 @@ def write(p, text): os.replace(rj, rj + ".away") try: bad = family_unknown(b, base) + bad += unknown_drawing_problems(b, base, False) finally: os.replace(rj + ".away", rj) check("browser: with the son's results.json unavailable the family page keeps every row of his that could name " @@ -1005,6 +1036,20 @@ def write(p, text): bad = map_problems(b, base) check("browser: the son's map as an image stays closed while a result is, and 'Show the map' opens the map " "and leaves each result's rows closed", not bad, bad) + os.replace(rj, rj + ".away") + try: + bad = unknown_drawing_problems(b, base, True) + check("browser: with the son's results unavailable, 'Show the rows' leaves his map as an image closed, and " + "'Show the map' opens it", not bad, bad) + a = "return t === UNKNOWN ? !mapShown[openKey(child, key)] :" + if a not in keep[fp]: + check("control: a drawing that 'Show the rows' opens (the planted line is in family-page.js)", False, a) + else: + write(fp, keep[fp].replace(a, "return t === UNKNOWN ? !mapShown[openKey(child, key)] && !openedAll[child] :")) + control("a drawing that 'Show the rows' opens", not unknown_drawing_problems(b, base, True)) + write(fp, keep[fp]) + finally: + os.replace(rj + ".away", rj) a = "function () { mapShown[mk] = true; redraw(); }" if a not in keep[fp]: check("control: a 'Show the map' that opens the results too (the planted line is in family-page.js)", False, a) From 097f6c13bc4ad68d8b6c835662ed60c86165e5ec Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:26:40 +0100 Subject: [PATCH 008/255] test(live): check that each demo person page draws the genome map The demo pages must draw Figure 1 from the og map document, place every finding the old karyogram placed, keep closed results off the map, name missing layers, and fall back to a table without a map. The checks run against main first, so they fail until the map lands. Bead: pgp-iqon.8 --- tests/demo/build_live_demo.py | 72 ++++- tests/fixtures/map/karyogram-marks.tsv | 16 + tests/test_live_demo.py | 415 ++++++++++++++++++++++++- tests/test_live_results.py | 50 ++- 4 files changed, 541 insertions(+), 12 deletions(-) create mode 100644 tests/fixtures/map/karyogram-marks.tsv diff --git a/tests/demo/build_live_demo.py b/tests/demo/build_live_demo.py index f30a624b8..bee9cdb78 100644 --- a/tests/demo/build_live_demo.py +++ b/tests/demo/build_live_demo.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """build_live_demo.py: the live site's public demo, a static folder replaying an invented run. - build_live_demo.py --out DIR [--genome-dir GDIR] [--records N] [--work WDIR] + build_live_demo.py --out DIR [--genome-dir GDIR] [--records N] [--work WDIR] [--og OG] writes DIR, which the docs workflow publishes at /demo/ of the docs site: @@ -13,6 +13,11 @@ runs/DEMO-001/results.json the person's results (bin/live_results.py), as the results plugin serves them at the end of the run; each card shows once the replay reaches the end of its steps + runs/DEMO-001/map.json the genome map of Figure 1: `og map build` with those results on + stdin, as the map plugin (bin/owngenome_live_map.py) sends it + (gzip bytes); the same for each person + map/genes/ `og map genes` of the public GENCODE slice + (tests/fixtures/map/gencode-slice), as /map/genes/ sends them runs/DEMO-001/finding// one page per finding (assets/live/finding.html), as the results plugin's finding route serves it runs/DEMO-001/... the files the plugins point at (the charts, the @@ -56,7 +61,12 @@ meta tag cannot set, left out), and the site's absolute links (/runs/..., /family/...) become relative ones; - the genome viewer opens the reader's own reads on the reader's own - computer, so the demo's viewer/ page says it is not in the demo. + computer, so the demo's viewer/ page says it is not in the demo; + - the map's genes come from the GENCODE slice copied into the run folder's + reference/, since setup has not installed GENCODE there. The map needs og + (OG, default bin/og, which scripts/setup.sh --og builds): without it the + build writes no map.json, says so, and the page lists the findings with a + place in a table instead. Standard library only. tests/test_live_demo.py builds it and checks it. """ @@ -102,6 +112,8 @@ DEMO_RAM = 64 << 30 DEMO_DISK = os.statvfs_result((4096, 4096, 512 << 20, 230 << 20, 230 << 20, 128 << 20, 100 << 20, 100 << 20, 0, 255)) INTERVAL_MS = 600 +OG = os.path.join(BIN, "og") +GENCODE_SLICE = os.path.join(REPO, "tests", "fixtures", "map", "gencode-slice", "reference") # The server's policy; a meta tag cannot set frame-ancestors (the browser ignores it there and warns). CSP = "; ".join(p for p in owngenome_live.CSP.split("; ") if not p.startswith("frame-ancestors")) @@ -167,6 +179,9 @@ def prepare(root, genome_dir=None, records=20000): other_person(root, name, sex, seed, max(2000, records // 4)) fam = os.path.join(root, FAMILY) shutil.copytree(os.path.join(EXTRA, FAMILY), fam) + # The genes the map places a gene-level finding on (and draws later): setup's GENCODE files, here the slice. + if not os.path.isdir(os.path.join(root, "reference")): + shutil.copytree(GENCODE_SLICE, os.path.join(root, "reference")) couple = os.path.join(d, "couple", f"{SAMPLE}_couple_carrier.tsv") if os.path.isfile(couple): a, b = sorted((SAMPLE, PARTNER)) @@ -383,7 +398,44 @@ def person_site(out, root, name, plugin_js, frames, results): "../../../assets/live.css")) -def write_site(out, root, frames, fam): +def og_map(og, args, stdin=b""): + """og map ARGS: its stdout, or None when og is not built on this machine (exit 127).""" + try: + p = subprocess.run([og, "map"] + args, input=stdin, capture_output=True, check=False) + except FileNotFoundError: + return None + if p.returncode == 127: + return None + if p.returncode != 0: + raise SystemExit(f"ERROR: og map {args[0]} failed ({p.returncode}): {p.stderr.decode(errors='replace').strip()}") + return p.stdout + + +def map_files(out, root, people, og): + """runs//map.json for each (name, results) of PEOPLE and map/genes/, as the map plugin sends + them; False (and a note) when og is not built here.""" + cache = os.path.join(root, ".owngenome-live", "map") + assets = os.path.join(REPO, "assets") + for name, results in people: + doc = og_map(og, ["build", "--root", root, "--run", name, "--assets", assets, "--cache-dir", cache], + json.dumps({"results": results}).encode()) + if doc is None: + print(f"NOTE: og is not built here ({og}): the demo has no genome map; scripts/setup.sh --og builds it", + file=sys.stderr) + return False + with open(os.path.join(out, "runs", name, "map.json"), "wb") as f: + f.write(doc) + gd = os.path.join(out, "map", "genes") + os.makedirs(gd) + chroms = [f"chr{c}" for c in list(range(1, 23)) + ["X", "Y"]] + for which in chroms + ["names"]: + data = og_map(og, ["genes", "--root", root, "--cache-dir", cache] + (["--names"] if which == "names" else ["--chrom", which])) + with open(os.path.join(gd, f"{which}.json"), "wb") as f: + f.write(data) + return True + + +def write_site(out, root, frames, fam, og=OG): if os.path.exists(out): shutil.rmtree(out) shutil.copytree(ASSETS, os.path.join(out, "assets"), @@ -400,6 +452,7 @@ def write_site(out, root, frames, fam): for name, _, _ in OTHERS: o = fam["others"][name] person_site(out, root, name, plugin_js, o["frames"], o["results"]) + drawn = map_files(out, root, [(SAMPLE, fam["results"])] + [(n, fam["others"][n]["results"]) for n, _, _ in OTHERS], og) fd = os.path.join(out, "runs", FAMILY) os.makedirs(fd) @@ -414,10 +467,12 @@ def write_site(out, root, frames, fam): f.write(pedigree_svg.draw(order, members, FAMILY)) for name in (fam["state"]["plugins"].get("family") or {}).get("files") or []: shutil.copy(os.path.join(ffiles, name), os.path.join(famdir, name)) + return drawn -def build(out, genome_dir=None, records=20000, work=None, drop_hook=None): - """Write the demo into out; returns {frames, hooks, family} for the tests.""" +def build(out, genome_dir=None, records=20000, work=None, drop_hook=None, og=OG): + """Write the demo into out; returns {frames, hooks, family, map} for the tests (map: the map + documents were written).""" tmp = None if work: os.makedirs(work, exist_ok=True) @@ -430,8 +485,8 @@ def build(out, genome_dir=None, records=20000, work=None, drop_hook=None): try: held = prepare(root, genome_dir, records) frames, fam, app = replay(root, held, drop_hook) - write_site(out, root, frames, fam) - return {"frames": frames, "family": fam, "hooks": [n for n, _ in app.state_hooks]} + drawn = write_site(out, root, frames, fam, og) + return {"frames": frames, "family": fam, "hooks": [n for n, _ in app.state_hooks], "map": drawn} finally: if tmp: shutil.rmtree(tmp, ignore_errors=True) @@ -443,8 +498,9 @@ def main(argv=None): ap.add_argument("--genome-dir", help="a folder holding DEMO-001 (default: write one now)") ap.add_argument("--records", type=int, default=20000, help="variant records of a DEMO-001 written now") ap.add_argument("--work", help="an empty folder for the run folder, kept afterwards (default: a temporary one)") + ap.add_argument("--og", default=OG, help="the og that builds the map documents (default: bin/og)") a = ap.parse_args(argv) - r = build(a.out, a.genome_dir, a.records, a.work) + r = build(a.out, a.genome_dir, a.records, a.work, og=a.og) size = sum(os.path.getsize(os.path.join(p, f)) for p, _, fs in os.walk(a.out) for f in fs) print(f"Demo written: {a.out} ({len(r['frames'])} frames, {size // 1024} KB; plugins {', '.join(r['hooks'])})") return 0 diff --git a/tests/fixtures/map/karyogram-marks.tsv b/tests/fixtures/map/karyogram-marks.tsv new file mode 100644 index 000000000..6104ef186 --- /dev/null +++ b/tests/fixtures/map/karyogram-marks.tsv @@ -0,0 +1,16 @@ +# The marks the findings karyogram (core.js karyogram(), person.js Figure 1) drew on the demo's person pages, +# recorded before the genome map replaced it: one row per mark, from each page at the end of its run. A mark is a +# finding with a viewer place (chrN:pos or chrN:start-end) that the page did not keep closed (DEMO-003 is a child, +# so its results to act on and the findings tagged with them were not drawn). mb is the place's first +# position in Mb, rounded to 0.01. tests/test_live_demo.py checks the map places each one. +run id chr mb level +DEMO-001 clinvar-democ-chr16-2087914-ct-c 16 2.09 review +DEMO-001 clinvar-demoa-chr2-47403191-c-t 2 47.40 review +DEMO-001 clinvar-demob-chr11-108259012-g-a 11 108.26 review +DEMO-002 clinvar-democ-chr16-2087914-ct-c 16 2.09 review +DEMO-002 clinvar-demoa-chr2-47403191-c-t 2 47.40 review +DEMO-002 clinvar-demob-chr11-108259012-g-a 11 108.26 review +DEMO-003 clinvar-democ-chr16-2087914-ct-c 16 2.09 review +DEMO-003 clinvar-demoa-chr2-47403191-c-t 2 47.40 review +DEMO-003 clinvar-demob-chr11-108259012-g-a 11 108.26 review +DEMO-003 sv-chr16-2084000-2096000 16 2.08 review diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 805606cfb..0fedd27d5 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -72,10 +72,13 @@ Run: python3 tests/test_live_demo.py """ +import base64 import copy +import gzip import json import os import shutil +import subprocess import sys import tempfile import threading @@ -314,6 +317,7 @@ def browser_checks(out): fold_checks(b, base, out) veil_checks(b, base, out) planted_checks(b, base, out) + map_checks(b, base, out) finally: b.close() srv.shutdown() @@ -1175,6 +1179,412 @@ def fold_checks(b, base, out): f.write(keep_js) +# --- the genome map (Figure 1, assets/live/map.js) ------------------------------------------ + +MARKS_TSV = os.path.join(REPO, "tests", "fixtures", "map", "karyogram-marks.tsv") + + +def og_binary(): + """An og that runs here: bin/og when scripts/setup.sh --og built it, else one cargo builds now from rust/ + without the BAM feature (the map reads no BAM), kept in CARGO_TARGET_DIR or a folder of the temp dir. + None when neither works.""" + og = os.path.join(REPO, "bin", "og") + if subprocess.run([og, "map", "--help"], capture_output=True, check=False).returncode == 0: + return og + cargo = shutil.which("cargo") or os.path.expanduser("~/.cargo/bin/cargo") + if not os.path.isfile(cargo): + return None + target = os.environ.get("CARGO_TARGET_DIR") or os.path.join(tempfile.gettempdir(), "owngenome-test-og") + p = subprocess.run([cargo, "build", "--locked", "--quiet", "-p", "og", "--no-default-features"], capture_output=True, + text=True, cwd=os.path.join(REPO, "rust"), env=dict(os.environ, CARGO_TARGET_DIR=target), check=False) + exe = os.path.join(target, "debug", "og") + if p.returncode != 0 or not os.path.isfile(exe): + print(f"cargo could not build og ({p.returncode}): {p.stderr[-2000:]}", file=sys.stderr) + return None + return exe + + +def karyogram_marks(): + """(run, id, chrom, mb, level) of each mark the karyogram drew, recorded before the map replaced it.""" + rows = [] + with open(MARKS_TSV) as f: + for line in f: + if line.startswith(("#", "run\t")) or not line.strip(): + continue + run, fid, c, m, lv = line.rstrip("\n").split("\t") + rows.append((run, fid, "chr" + c, float(m), lv)) + return rows + + +def read_map(path): + with open(path, "rb") as f: + return json.loads(gzip.decompress(f.read())) + + +def write_map(path, doc): + with open(path, "wb") as f: + f.write(gzip.compress(json.dumps(doc, separators=(",", ":")).encode(), mtime=0)) + + +# What the map shows: its state, the findings it draws (its list of marks), its timings, level and place, +# its chips, its margin's words and every text and title it holds, and the address. +MAP_JS = r"""JSON.stringify((function () { + var f = document.getElementById('genome-map'); + if (!f) { return { state: 'absent', marks: [] }; } + var t = document.querySelector('.gm-tip'); + return { state: f.dataset.pgpMap || '', level: f.dataset.level || '', chrom: f.dataset.chrom || '', view: f.dataset.view || '', + parse: Number(f.dataset.pgpMapParseMs), paint: Number(f.dataset.pgpMapPaintMs), + marks: Array.prototype.map.call(f.querySelectorAll('.gm-marks a'), function (a) { return { id: a.dataset.id, + level: a.dataset.level, chrom: a.dataset.chrom, s: Number(a.dataset.s), e: Number(a.dataset.e) }; }), + chips: Array.prototype.map.call(f.querySelectorAll('.gm-chip'), function (c) { return c.textContent; }), + title: (document.getElementById('gm-title') || {}).textContent || '', + layers: Array.prototype.map.call(f.querySelectorAll('.gm-layer'), function (l) { return [l.dataset.layer, l.dataset.state, + l.querySelector('dd').textContent]; }), + toast: (f.querySelector('.gm-toast') || { hidden: true }).hidden ? '' : f.querySelector('.gm-toast').textContent, + tip: t && !t.hidden ? t.textContent : '', + text: [f.textContent].concat(Array.prototype.map.call(f.querySelectorAll('[href],[title],[aria-label]'), function (e) { + return (e.getAttribute('href') || '') + ' ' + (e.getAttribute('title') || '') + ' ' + (e.getAttribute('aria-label') || ''); })).join(' '), + hash: location.hash }; +}()))""" +MAP_READY = ("document.documentElement.dataset.pgpResults && (function () { var f = document.getElementById('genome-map'); " + "return f && /^(drawn|failed|nodecompress)$/.test(f.dataset.pgpMap || '') && f.dataset.level !== undefined " + "&& (f.dataset.pgpMap !== 'drawn' || !!f.dataset.level); }())") + + +def map_page(b, base, run, hash_=""): + """A fresh load of RUN's page with HASH_; the map's facts once it settled (None when it never drew).""" + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{run}/{hash_}") + if not b.wait(MAP_READY): + return json.loads(b.eval(MAP_JS)) + time.sleep(0.6) # the margin follows the camera 140 ms after it settles + return json.loads(b.eval(MAP_JS)) + + +def map_now(b): + time.sleep(1.5) # the camera eases for about 0.6 s, then the margin follows 140 ms after it settles + return json.loads(b.eval(MAP_JS)) + + +def parity_problems(b, base, rows): + """The karyogram's marks the map does not place with the same level and chromosome within 0.01 Mb.""" + bad = [] + for run in sorted({r[0] for r in rows}): + got = map_page(b, base, run) + if got["state"] != "drawn": + bad.append(f"{run}: map {got['state']}") + continue + by = {m["id"]: m for m in got["marks"]} + for _, fid, c, m, lv in (r for r in rows if r[0] == run): + g = by.get(fid) + if not g: + bad.append(f"{run}: {fid} not on the map") + elif g["level"] != lv or g["chrom"] != c or abs(g["s"] / 1e6 - m) > 0.01: + bad.append(f"{run}: {fid} at {g['chrom']} {g['s'] / 1e6:.3f} Mb {g['level']}, want {c} {m} {lv}") + return bad + + +def click_canvas(b, x, y, move_only=False): + """A mouse at (x, y) of the map's canvas, in CSS px from its corner: moved there, and clicked unless MOVE_ONLY.""" + b.eval("document.querySelector('.gm-canvas').scrollIntoView({block: 'center'}); true") + time.sleep(0.3) + r = json.loads(b.eval("JSON.stringify(document.querySelector('.gm-canvas').getBoundingClientRect())")) + px, py = r["left"] + x, r["top"] + y + b.call("Input.dispatchMouseEvent", {"type": "mouseMoved", "x": px, "y": py}, b.session) + if not move_only: + for t in ("mousePressed", "mouseReleased"): + b.call("Input.dispatchMouseEvent", {"type": t, "x": px, "y": py, "button": "left", "clickCount": 1}, b.session) + time.sleep(0.6) + + +def genome_xy(b, doc, chrom, bp): + """Where BP of CHROM sits on the genome level of the canvas (map.js gx and rowY, at the desktop width).""" + cw = b.eval("document.querySelector('.gm-canvas').clientWidth") + rows = [c for c in doc["ref"]["chroms"] if not (c["n"] == "chrY" and doc["people"][0].get("has_y") is False)] + i = [c["n"] for c in rows].index(chrom) + ch = b.eval("document.querySelector('.gm-canvas').clientHeight") + rh = max(22, min(44, (ch - 18) / len(rows))) + return 40 + bp * (cw - 40 - 22) / rows[0]["len"], 9 + i * rh + rh / 2 - 1 + + +def state_problems(b, base): + """The genome and chromosome levels, the links that open each, a click on a row, Esc, and a place that + cannot be read.""" + bad = [] + g = map_page(b, base, S) + if not (g["level"] == "genome" and "all 24 chromosomes" in g["title"] and g["view"] == "genome"): + bad.append(f"genome: {g['level']} {g['title']!r} {g['view']}") + for link, chrom, view in (("#chr16", "chr16", "chr16"), ("#chr16:10.0-40.0M", "chr16", "chr16:10.0-40.0M"), + ("#view=chromosome&chr=11", "chr11", "chr11"), ("#chr2:20000001-60000000", "chr2", "chr2:20.0-60.0M")): + c = map_page(b, base, S, link) + if not (c["level"] == "chromosome" and c["chrom"] == chrom and c["view"] == view): + bad.append(f"{link}: {c['level']} {c['chrom']} {c['view']}") + elif any(m["chrom"] != chrom for m in c["marks"]): + bad.append(f"{link}: marks of another chromosome listed: {c['marks']}") + c = map_page(b, base, S, "#chr16") + b.eval("location.hash = '#genome'; true") + c = map_now(b) + if c["level"] != "genome": + bad.append(f"#genome after #chr16: {c['level']}") + b.eval("location.hash = '#chrQ'; true") + c = map_now(b) + if "Could not read the place" not in c["toast"] or c["level"] != "genome": + bad.append(f"#chrQ: toast {c['toast']!r}, level {c['level']}") + g = map_page(b, base, S) + click_canvas(b, 200, 9 + 2 * 22 + 11) + c = map_now(b) + if not (c["level"] == "chromosome" and c["chrom"] == "chr3" and c["hash"] == "#chr3"): + bad.append(f"a click on chromosome 3's row: {c['level']} {c['chrom']} {c['hash']}") + b.eval("document.querySelector('.gm-canvas').dispatchEvent(new KeyboardEvent('keydown', {key: 'Escape', bubbles: true})); true") + c = map_now(b) + if c["level"] != "genome" or c["hash"] != "#genome": + bad.append(f"Esc: {c['level']} {c['hash']}") + return bad + + +def map_veil_problems(b, base, out): + """On the son's closed page: his closed result to act on (the planted deletion) is in no mark, tooltip, + chip, sentence, link or address of the map until opened; the chip's Open (after its confirm) and the card's + "Show the result" each bring back exactly its mark, on the map and in the card; Close it again and a reload + close it. On DEMO-001, an adult's page, APOE stays closed until opened, from the chip or its card's fold.""" + bad = [] + son = OTHERS[1] + doc = read_map(os.path.join(out, "runs", son, "map.json")) + f = doc["per"][son]["findings"] + acts = [i for i, lv in enumerate(f["level"]) if lv == "act" and not f["gate"][i]] + if not acts: + return ["the son's map has no result to act on to close"] + i = acts[0] + fid, gene = f["id"][i], f["gene"][i] + words = [gene, f"{f['chrom'][i]}:{f['s'][i] + 1}", f"{f['s'][i] + 1:,}", fid] + x, y = genome_xy(b, doc, f["chrom"][i], (f["s"][i] + f["e"][i]) / 2) + + def leaks(c): + return [w for w in words if w in c["text"] or w in c["tip"] or w in c["hash"]] + \ + [m["id"] for m in c["marks"] if m["id"] == fid] + c = map_page(b, base, son) + click_canvas(b, x, y, move_only=True) + c = map_now(b) + if c["state"] != "drawn": + return [f"the son's map: {c['state']}"] + if leaks(c): + bad.append(f"closed: the map shows {leaks(c)}") + if not any("1 result closed" in t for t in c["chips"]): + bad.append(f"closed: no chip counting it: {c['chips']}") + b.eval("document.querySelector('.gm-chip[data-closed=act] button').click(); true") + c = map_now(b) + if not b.eval("!!document.querySelector('.gm-chip[data-closed=act] .gm-confirm')") or leaks(c): + bad.append("Open: no confirm step, or it opened at once") + b.eval("document.querySelector('.gm-chip[data-closed=act] .gm-yes').click(); true") + c = map_now(b) + card = b.eval("!!document.querySelector('#f-%s h3') && !document.querySelector('#f-%s .veil')" % (fid, fid)) + if [m["id"] for m in c["marks"] if m["id"] == fid] != [fid] or not card: + bad.append(f"Open: mark {[m['id'] for m in c['marks']]}, card shown {card}") + if not any(t.startswith("Opened: " + f["title"][i]) for t in c["chips"]): + bad.append(f"Open: chips {c['chips']}") + if fid in c["hash"] or gene in c["hash"]: + bad.append(f"Open: the address names it: {c['hash']}") + click_canvas(b, x, y, move_only=True) + c = map_now(b) + if gene not in c["tip"]: + bad.append(f"Open: its tooltip at its place does not name it: {c['tip']!r}") + b.eval("document.querySelector('.gm-chip.opened button').click(); true") + c = map_now(b) + card = b.eval("!!document.querySelector('#f-%s .veil')" % fid) + if leaks(c) or not card: + bad.append(f"Close it again: map shows {leaks(c)}, card closed {card}") + b.eval("document.querySelector('#f-%s .veil button').click(); true" % fid) + c = map_now(b) + if [m["id"] for m in c["marks"] if m["id"] == fid] != [fid]: + bad.append("the card's Show the result does not open it on the map") + c = map_page(b, base, son) + if leaks(c): + bad.append(f"a reload does not close it: {leaks(c)}") + # APOE on an adult's page. + c = map_page(b, base, S) + if any(m["id"] == "apoe" for m in c["marks"]) or not any(t.startswith("APOE closed") for t in c["chips"]): + bad.append(f"APOE on DEMO-001: marks {[m['id'] for m in c['marks']]}, chips {c['chips']}") + b.eval("document.querySelector('.gm-chip[data-closed=gate] button').click(); true") + time.sleep(0.3) + b.eval("document.querySelector('.gm-chip[data-closed=gate] .gm-yes').click(); true") + c = map_now(b) + fold = b.eval("(document.querySelector('details[data-gate=apoe]') || {}).open") + if not any(m["id"] == "apoe" for m in c["marks"]) or fold is not True: + bad.append(f"Open APOE: marks {[m['id'] for m in c['marks']]}, its card's fold open {fold}") + b.eval("document.querySelector('details[data-gate=apoe] summary').click(); true") + c = map_now(b) + if any(m["id"] == "apoe" for m in c["marks"]): + bad.append("closing APOE's card fold leaves it on the map") + b.eval("document.querySelector('details[data-gate=apoe] summary').click(); true") + c = map_now(b) + if not any(m["id"] == "apoe" for m in c["marks"]): + bad.append("opening APOE's card fold does not open it on the map") + c = map_page(b, base, S) + if any(m["id"] == "apoe" for m in c["marks"]): + bad.append("a reload does not close APOE") + return bad + + +def missing_layer_problems(b, base, out): + """A map whose ROH layer is missing says so in words, in the margin, never a blank.""" + path = os.path.join(out, "runs", S, "map.json") + with open(path, "rb") as fh: + keep = fh.read() + try: + doc = read_map(path) + doc["layers"]["roh"] = dict(doc["layers"]["roh"], state="missing", count=0, note="planted: no ROH text") + doc["per"][S].pop("roh", None) + write_map(path, doc) + c = map_page(b, base, S, "#chr1") + row = [x for x in c["layers"] if x[0] == "roh"] + if not row or row[0][1] != "missing" or "Not on the map: planted: no ROH text" not in row[0][2]: + return [f"the missing ROH layer reads {row}"] + return [] + finally: + with open(path, "wb") as fh: + fh.write(keep) + + +def fallback_problems(b, base, out): + """A browser with no DecompressionStream: the sentence and the findings with a place, each through the veil.""" + son = OTHERS[1] + doc = read_map(os.path.join(out, "runs", son, "map.json")) + f = doc["per"][son]["findings"] + gene = [f["gene"][i] for i, lv in enumerate(f["level"]) if lv == "act"][0] + b.call("Page.enable", session=b.session) # a script for each new document needs the Page domain on + sid = b.call("Page.addScriptToEvaluateOnNewDocument", {"source": "delete window.DecompressionStream;"}, b.session)["identifier"] + try: + c = map_page(b, base, son) + rows = b.eval("document.querySelectorAll('.gm-fallback table tbody tr').length") + finally: + b.call("Page.removeScriptToEvaluateOnNewDocument", {"identifier": sid}, b.session) + bad = [] + if c["state"] != "nodecompress" or "This browser cannot open the map file" not in c["text"]: + bad.append(f"no DecompressionStream: {c['state']}") + if not rows: + bad.append("no DecompressionStream: no findings table") + if gene in c["text"]: + bad.append(f"no DecompressionStream: the table names the closed {gene}") + return bad + + +def map_screens(b, base, dest): + """Screenshots of the map, dark, at 1440 x 900 and 390 x 844: genome and chromosome, a closed page.""" + os.makedirs(dest, exist_ok=True) + taken = [] + try: + for w, h in ((1440, 900), (390, 844)): + b.call("Emulation.setDeviceMetricsOverride", {"width": w, "height": h, "deviceScaleFactor": 1, "mobile": False}, + b.session) + for run, link, name in ((S, "", "genome"), (S, "#chr16", "chromosome"), (OTHERS[1], "", "closed")): + map_page(b, base, run, link) + b.eval("document.getElementById('genome-map').scrollIntoView({block: 'start'}); true") + time.sleep(0.5) + png = b.call("Page.captureScreenshot", {"format": "png"}, b.session)["data"] + p = os.path.join(dest, f"map-{name}-{w}x{h}.png") + with open(p, "wb") as fh: + fh.write(base64.b64decode(png)) + taken.append(p) + finally: + b.call("Emulation.clearDeviceMetricsOverride", {}, b.session) + return taken + + +def map_checks(b, base, out): + """Figure 1, the genome map: drawn from map.json with every mark the karyogram drew, in time; its levels, + links and a place it cannot read; the veil and opening on purpose; a missing layer; no DecompressionStream; + no sideways scroll at 390 px.""" + if not os.path.isfile(os.path.join(out, "runs", S, "map.json")): + ci = os.environ.get("CI") == "true" + print(("[FAIL] " if ci else "[SKIP] ") + "the demo has no map.json: no og to build it") + if ci: + FAILS.append("no map.json in CI") + return + b.call("Network.enable", session=b.session) # the controls below change files: never serve them from cache + b.call("Network.setCacheDisabled", {"cacheDisabled": True}, b.session) + # A desktop window, so the whole canvas fits in view for the clicks. + b.call("Emulation.setDeviceMetricsOverride", {"width": 1440, "height": 900, "deviceScaleFactor": 1, "mobile": False}, + b.session) + rows = karyogram_marks() + bad = parity_problems(b, base, rows) + check(f"map: the DEMO-001 to DEMO-003 person pages draw the map from map.json and every karyogram mark " + f"({len(rows)} marks: same level and chromosome, within 0.01 Mb)", not bad and len(rows) >= 9, bad) + path = os.path.join(out, "runs", S, "map.json") + with open(path, "rb") as fh: + keep = fh.read() + try: + doc = read_map(path) + f = doc["per"][S]["findings"] + drop = f["id"].index(next(r[1] for r in rows if r[0] == S)) + for k in f: + f[k] = [v for j, v in enumerate(f[k]) if j != drop] + write_map(path, doc) + control("a map.json without one of the karyogram's findings", not parity_problems(b, base, rows)) + finally: + with open(path, "wb") as fh: + fh.write(keep) + g = map_page(b, base, S) + check(f"map: first paint within 150 ms of the document arriving, parsed within 30 ms ({g.get('paint')} ms, " + f"{g.get('parse')} ms)", g.get("paint", 999) <= 150 and g.get("parse", 999) <= 30, g) + bad = state_problems(b, base) + check("map: genome and chromosome levels; #chr16, #chr16:10.0-40.0M, chr=11 and a 1-based span open their view; " + "a click on a row opens it, Esc closes it; an unknown place gives a toast", not bad, bad) + js = os.path.join(out, "assets", "person.js") + with open(js) as fh: + keep_js = fh.read() + bad = map_veil_problems(b, base, out) + check("map: the son's closed result is in no mark, tooltip, chip, sentence, link or address until opened; Open " + "(after its confirm) and the card's Show the result each bring back exactly its mark there and in the card; " + "Close it again and a reload close it; APOE stays closed on an adult's page until opened, from the chip " + "or its card's fold", not bad, bad) + try: + with open(js, "w") as fh: + fh.write(keep_js.replace("function mapVeil(id, gated) {", "function mapVeil(id, gated) { return 'open';")) + control("person.js whose map veil keeps nothing closed", not map_veil_problems(b, base, out)) + finally: + with open(js, "w") as fh: + fh.write(keep_js) + bad = missing_layer_problems(b, base, out) + check("map: a missing layer says why in words, never a blank", not bad, bad) + bad = fallback_problems(b, base, out) + check("map: a browser with no DecompressionStream gets the sentence and a findings table through the veil", not bad, bad) + b.call("Emulation.setDeviceMetricsOverride", {"width": 390, "height": 844, "deviceScaleFactor": 2, "mobile": False}, b.session) + try: + wide = {} + for link in ("", "#chr16"): + map_page(b, base, S, link) + wide[link or "genome"] = too_wide(b) + finally: + b.call("Emulation.clearDeviceMetricsOverride", {}, b.session) + check("map: at 390 px the map, open or not, scrolls nothing sideways", not any(wide.values()), wide) + if os.environ.get("MAP_SCREENS"): + print("map screenshots: " + ", ".join(map_screens(b, base, os.environ["MAP_SCREENS"]))) + + +def screens_main(dest, site): + """--map-screens DEST SITE: the map's screenshots of an already built demo (the docs workflow's artifact).""" + chrome = find_chrome() + if not chrome: + print("ERROR: no Chrome or Chromium to take the screenshots") + return 1 + handler = type("H", (Static,), {"__init__": lambda self, *a, **k: SimpleHTTPRequestHandler.__init__( + self, *a, directory=site, **k)}) + srv = ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=srv.serve_forever, daemon=True).start() + work = tempfile.mkdtemp(prefix="map-screens.") + b = Chrome(chrome, os.path.join(work, "chrome")) + try: + taken = map_screens(b, f"http://127.0.0.1:{srv.server_address[1]}", dest) + finally: + b.close() + srv.shutdown() + srv.server_close() + shutil.rmtree(work, ignore_errors=True) + print("\n".join(taken)) + return 0 if len(taken) == 6 else 1 + + def main(): work = tempfile.mkdtemp(prefix="test_live_demo.") try: @@ -1188,7 +1598,8 @@ def main(): control("a fixture missing its last message", make_demo_events.main(["--check", "--out", changed]) == 0) out = os.path.join(work, "demo") - built = build_live_demo.build(out, work=os.path.join(work, "root")) + og = og_binary() or build_live_demo.OG + built = build_live_demo.build(out, work=os.path.join(work, "root"), og=og) frames = built["frames"] print(f"built: {len(frames)} frames, hooks {built['hooks']}") @@ -1267,4 +1678,6 @@ def main(): if __name__ == "__main__": + if sys.argv[1:2] == ["--map-screens"] and len(sys.argv) == 4: + sys.exit(screens_main(sys.argv[2], sys.argv[3])) sys.exit(main()) diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 8213596a2..3e304b9aa 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -10,9 +10,12 @@ its SECTION_NOTES constants, word for word, never empty; 3. the wording gate: no string of results.json, of family.json (tests/fixtures/families_live family_a) or of the page assets - (assets/live: the text of the pages, the string literals of the scripts) - says a result is fine (live_results.FORBIDDEN), a quoted tool term and a - caveat aside; + (assets/live: the text of the pages, the string literals of the scripts, + map.js among them) says a result is fine (live_results.FORBIDDEN), a quoted + tool term and a caveat aside; nor does any string of the genome map + documents `og map build` writes for DEMO-001 and DEMO-003 (Figure 1 shows + their sentences), a layer's state code aside (skipped without og or + cargo, except in CI); 4. ClinVar: with hits the card lists them in a table; with none it has no table and its empty text says ClinVar lists only part of what causes disease (never an all-clear); @@ -145,6 +148,32 @@ def asset_problems(texts): return out +def map_documents(root, people): + """[(run, document)] of `og map build` for each (run, results) of PEOPLE, as the map plugin builds them; + None without an og (tests/test_live_demo.py og_binary).""" + import gzip + from test_live_demo import og_binary + og = og_binary() + if not og: + return None + ref = os.path.join(root, "reference") + if not os.path.isdir(ref): + shutil.copytree(os.path.join(REPO, "tests", "fixtures", "map", "gencode-slice", "reference"), ref) + out = [] + for run, results in people: + body = {k: v for k, v in results.items() if not k.startswith("_")} + p = subprocess.run([og, "map", "build", "--root", root, "--run", run, "--assets", os.path.join(REPO, "assets"), + "--cache-dir", os.path.join(root, ".map-cache")], input=json.dumps({"results": body}).encode(), + capture_output=True, check=True) + out.append((run, json.loads(gzip.decompress(p.stdout)))) + return out + + +def map_wording(doc): + """The strings of a map document that read as an all-clear; a layer's state is a code, not words.""" + return live_results.wording_problems(doc, allow=("state",)) + + # --- 4. ClinVar with and without hits ------------------------------------------------------- def clinvar_card(summary): @@ -534,6 +563,21 @@ def tagged(sample_dir): finally: live_results.veil_id = keep + # 3, the map documents: og map build of DEMO-001 and DEMO-003, with the results each page gets. + map_docs = map_documents(root, [(SAMPLE, res), ("DEMO-003", tagged(d3)[0])]) + if map_docs is None: + ci = os.environ.get("CI") == "true" + print(("[FAIL] " if ci else "[SKIP] ") + "3. no og (and no cargo to build one) for the map documents") + if ci: + FAILS.append("no og in CI") + else: + bad = [(run, x) for run, doc in map_docs for x in map_wording(doc)] + check(f"3. no string of the genome map documents reads as an all-clear ({len(map_docs)} documents)", + not bad and len(map_docs) == 2, bad) + planted = copy.deepcopy(map_docs[0][1]) + planted["text"]["genome"] = [{"t": "All clear on the map.", "b": False}] + control("'All clear' planted in a map document's sentence", not map_wording(planted)) + # 5. pwfile = os.path.join(work, "password") with open(pwfile, "w") as f: From 2e3e9d85fa2cb5a31b8d5acb4ba059f693f7cbbc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:28:26 +0100 Subject: [PATCH 009/255] feat(live): replace the findings karyogram with a genome map Figure 1 of a person page now draws the og map document: every chromosome at genome level, and one chromosome opened in place down to about 10 Mb, with the findings lane, runs of homozygosity and the bands. Closed results stay off the map until opened on purpose, with the same switch as the card. Without the document the page shows a table. Bead: pgp-iqon.8 --- assets/live/core.js | 70 +-- assets/live/index.html | 2 + assets/live/live.css | 6 - assets/live/map.css | 57 +++ assets/live/map.js | 1074 ++++++++++++++++++++++++++++++++++++++++ assets/live/person.js | 67 ++- 6 files changed, 1186 insertions(+), 90 deletions(-) create mode 100644 assets/live/map.css create mode 100644 assets/live/map.js diff --git a/assets/live/core.js b/assets/live/core.js index b6e4ff071..0d1db0fe9 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -1,6 +1,6 @@ // core.js: what the live site's pages share (home.html, index.html, finding.html, family.html). Element builders, // a fetch that sends the session and goes to the login on a 401, the pedigree symbol, dates, the -// caveat block, the facts list, the table, and the Journal's level marks and chromosome map. Text only through textContent and createElementNS: +// caveat block, the facts list, the table, and the Journal's level marks. Text only through textContent and createElementNS: // no innerHTML, no inline script, no eval. Everything hangs on window.PGPUI. (function () { 'use strict'; @@ -124,7 +124,7 @@ }); } - // --- the Journal's shared marks: the level glyph and word, and the chromosome map of findings --- + // --- the Journal's shared marks: the level glyph and word --- var LEVEL_WORDS = { act: 'Take to a doctor', medicine: 'Tell your prescriber', review: 'Worth a look', carrier: 'For family planning', trust: 'Data quality', info: 'For information' }; var LEVEL_SHORT = { act: 'doctor', medicine: 'prescriber', review: 'look', carrier: 'family', trust: 'quality', info: 'info' }; @@ -137,68 +137,6 @@ } // A finding's level: a data-quality finding is always a problem with the data, so its mark is red. function findingLevel(lv, words) { return level(lv, words, lv === 'trust' ? 'problem' : undefined); } - // GRCh38: [name, length Mb, centromere Mb]. - var CHR = [['1', 248.96, 123.4], ['2', 242.19, 93.9], ['3', 198.3, 90.9], ['4', 190.21, 50], ['5', 181.54, 48.8], - ['6', 170.81, 59.8], ['7', 159.35, 60.1], ['8', 145.14, 45.2], ['9', 138.39, 43], ['10', 133.8, 39.8], - ['11', 135.09, 53.4], ['12', 133.28, 35.5], ['13', 114.36, 17.7], ['14', 107.04, 17.2], ['15', 101.99, 19], - ['16', 90.34, 36.8], ['17', 83.26, 25.1], ['18', 80.37, 18.5], ['19', 58.62, 26.2], ['20', 64.44, 28.1], - ['21', 46.71, 12], ['22', 50.82, 15], ['X', 156.04, 60.6], ['Y', 57.23, 10.4]]; - var LV_COLOUR = { act: '#d03b3b', medicine: '#ec835a', review: '#fab219', carrier: '#9085e9', trust: '#c3c2b7', info: '#c3c2b7' }; - function glyph(g, lv, x, y) { - var c = LV_COLOUR[lv] || LV_COLOUR.info; - if (lv === 'act') { g.appendChild(svg('rect', { x: x - 4, y: y - 4, width: 8, height: 8, fill: c, transform: 'rotate(45 ' + x + ' ' + y + ')' })); } - else if (lv === 'medicine') { g.appendChild(svg('rect', { x: x - 5.5, y: y - 3, width: 11, height: 6, rx: 3, fill: c })); } - else if (lv === 'review') { g.appendChild(svg('path', { d: 'M' + x + ' ' + (y - 5) + 'L' + (x + 5) + ' ' + (y + 4) + 'L' + (x - 5) + ' ' + (y + 4) + 'Z', fill: c })); } - else if (lv === 'carrier') { - g.appendChild(svg('circle', { cx: x - 2.6, cy: y, r: 3.4, fill: 'none', stroke: c, 'stroke-width': 1.6 })); - g.appendChild(svg('circle', { cx: x + 2.6, cy: y, r: 3.4, fill: 'none', stroke: c, 'stroke-width': 1.6 })); - } else { g.appendChild(svg('circle', { cx: x, cy: y, r: 4, fill: 'none', stroke: c, 'stroke-width': 1.5 })); } - } - // "chr16:2087914" or "chr16:2084000-2096000" -> {chr: '16', mb: 2.08}; null for anything else. - function locus(v) { - var m = /^(?:chr)?([0-9]{1,2}|X|Y):([0-9]+)/.exec(String(v || '')); - return m ? { chr: m[1], mb: Number(m[2]) / 1e6 } : null; - } - // The chromosome map: the 24 chromosomes in a row, each finding a glyph on its chromosome, with its - // name beside it and the same words in a title (and in the list under the figure). - function karyogram(marks, opts) { - opts = opts || {}; - var W = opts.width || 680, H = opts.height || 190, top = 22, bottom = 22; - var slot = (W - 56) / CHR.length, scale = (H - top - bottom) / 249; - var s = svg('svg', { viewBox: '0 0 ' + W + ' ' + H, role: 'img', 'aria-label': opts.aria || 'Where the findings sit, by chromosome' }); - CHR.forEach(function (c, i) { - var x = i * slot + slot / 2 - 3.5; - s.appendChild(svg('rect', { 'class': 'chr', x: x, y: top, width: 7, height: Math.max(1, c[2] * scale - 1.2), rx: 3.5 })); - s.appendChild(svg('rect', { 'class': 'chr', x: x, y: top + c[2] * scale + 1.2, width: 7, height: Math.max(1, (c[1] - c[2]) * scale - 1.2), rx: 3.5 })); - var t = svg('text', { 'class': 'lab', x: x + 3.5, y: H - 6, 'text-anchor': 'middle' }); - t.textContent = c[0]; - s.appendChild(t); - }); - (marks || []).forEach(function (m) { - var i = -1; - CHR.forEach(function (c, j) { if (c[0] === m.chr) { i = j; } }); - if (i < 0) { return; } - var x = i * slot + slot / 2 + 7, y = top + Math.min(m.mb, CHR[i][1]) * scale; - var g = svg('g', { 'class': 'mk', tabindex: '0' }); - var tt = svg('title', {}); - tt.textContent = (LEVEL_WORDS[m.lv] || '') + ': ' + (m.tip || m.label); - g.appendChild(tt); - g.appendChild(svg('line', { x1: x - 10.5, x2: x - 1, y1: y, y2: y, stroke: LV_COLOUR[m.lv] || LV_COLOUR.info, 'stroke-width': 2 })); - glyph(g, m.lv, x + 4, y); - if (m.label) { - var t = svg('text', { x: x + 11, y: y + 3.5, stroke: '#0e0f10', 'stroke-width': 4, 'paint-order': 'stroke', 'stroke-linejoin': 'round' }); - t.textContent = m.label; - g.appendChild(t); - } - if (m.href) { - g.setAttribute('role', 'link'); - g.addEventListener('click', function () { window.location.href = m.href; }); - g.addEventListener('keydown', function (e) { if (e.key === 'Enter') { window.location.href = m.href; } }); - } - s.appendChild(g); - }); - return s; - } // A plain time: "14:22" in UTC, "4 Nov" for a day. var MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; function hm(utc) { var d = new Date(utc); return isFinite(d) ? ('0' + d.getUTCHours()).slice(-2) + ':' + ('0' + d.getUTCMinutes()).slice(-2) : ''; } @@ -209,7 +147,7 @@ } window.PGPUI = { el: el, svg: svg, add: add, getJSON: getJSON, avatar: avatar, day: day, caveat: caveat, - facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, glyph: glyph, - karyogram: karyogram, locus: locus, LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, + facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, + LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, hm: hm, dm: dm, hours: hours }; }()); diff --git a/assets/live/index.html b/assets/live/index.html index b272cd44f..d740c1eda 100644 --- a/assets/live/index.html +++ b/assets/live/index.html @@ -5,8 +5,10 @@ {{run}}: OwnGenome + + {{plugins}} diff --git a/assets/live/live.css b/assets/live/live.css index 5dba9f176..6c0959048 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -321,12 +321,6 @@ ol.w-path li.w-low { border-style: dashed; border-color: var(--car); } .w-count { color: var(--ink-3); font-size: 12px; } /* the chromosome map of the findings (person.js) and the charts plugin */ -.kary { margin-top: 12px; } -.kary svg { width: 100%; height: auto; display: block; } -.kary .chr { fill: rgba(255, 255, 255, .07); } -.kary .lab { fill: var(--ink-3); font-size: 10px; } -.kary .mk { cursor: pointer; } -.kary .mk text { fill: var(--ink-2); font-size: 10.5px; } .chart-slot:empty { display: none; } figure.chart { margin: 0 0 14px; } figure.chart h3 { font: 600 10.5px var(--sans); letter-spacing: .12em; text-transform: uppercase; color: var(--ink-3); margin-bottom: 8px; } diff --git a/assets/live/map.css b/assets/live/map.css new file mode 100644 index 000000000..cc3e42e00 --- /dev/null +++ b/assets/live/map.css @@ -0,0 +1,57 @@ +/* map.css: Figure 1 of a person's page, the genome map (assets/live/map.js). Dark only, the Journal tokens of + live.css. Brass marks where you are, never a level; the levels' colours mark findings only. */ +.gmap { margin-top: 34px; scroll-margin-top: 72px; } +.gm-main { min-width: 0; } +.gm-head { display: grid; gap: 6px; margin-bottom: 10px; } +.gm-titlerow { display: flex; align-items: baseline; justify-content: space-between; gap: 16px; flex-wrap: wrap; } +.gm-title { font: 400 27px/1.15 var(--serif); color: var(--ink); } +.gm-gauge { display: inline-flex; border: 1px solid var(--hair-2); border-radius: 999px; overflow: hidden; } +.gm-gauge button { border: 0; border-radius: 0; background: transparent; color: var(--ink-3); padding: 6px 12px; font-size: 12px; } +.gm-gauge button[aria-pressed="true"] { color: var(--page); background: var(--brass); } +.gm-crumb { color: var(--ink-3); font-size: 12px; } +.gm-crumb button { background: none; border: 0; padding: 0; color: var(--ink-2); font: inherit; text-decoration: underline; + text-decoration-color: var(--brass-dim); text-underline-offset: 3px; } +.gm-crumb [aria-current] { color: var(--ink); } +.gm-chips { display: flex; flex-wrap: wrap; gap: 8px; } +.gm-chips:empty { display: none; } +.gm-chip { display: flex; flex-wrap: wrap; align-items: center; gap: 8px 10px; font: 12.5px/1.4 var(--sans); color: var(--ink-2); + border: 1px dashed var(--hair-2); border-radius: 8px; padding: 6px 10px; max-width: 100%; } +.gm-chip.opened { border-style: solid; border-color: var(--brass-dim); } +.gm-chip button { padding: 5px 10px; font-size: 12px; } +.gm-confirm { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; color: var(--ink); } +.gm-stage { position: relative; } +.gm-canvas { display: block; width: 100%; touch-action: pan-y; border-radius: 6px; } +.gm-canvas:focus-visible { outline-offset: 4px; } +.gm-tip { position: absolute; z-index: 5; max-width: 260px; pointer-events: none; background: var(--paper-2); border: 1px solid var(--hair-2); + border-radius: 6px; padding: 7px 9px; font: 12px/1.4 var(--sans); color: var(--ink-2); display: grid; gap: 2px; } +.gm-tip b { color: var(--ink); font-weight: 500; } +.gm-toast { position: absolute; left: 50%; top: 8px; transform: translateX(-50%); background: var(--paper-2); border: 1px solid var(--brass-dim); + border-radius: 6px; padding: 7px 12px; font: 12.5px/1.4 var(--sans); color: var(--ink); max-width: 90%; } +/* The marks as links: the path for the keyboard and a screen reader, out of sight until focused. */ +.gm-marks { list-style: none; margin: 0; padding: 0; position: relative; } +.gm-marks li { position: absolute; width: 1px; height: 1px; overflow: hidden; clip-path: inset(50%); white-space: nowrap; } +.gm-marks li:focus-within { position: static; width: auto; height: auto; clip-path: none; white-space: normal; font-size: 12.5px; margin-top: 6px; } +.gm-fallback .gm-fail { color: var(--ink-2); } +.gm-margin { font-size: 13px; } +.gm-mline { color: var(--ink-3); font-size: 12px; margin: 6px 0 4px; } +.gm-narration { font: 400 15.5px/1.5 var(--serif); color: var(--ink-2); margin-top: 8px; } +.gm-narration b { color: var(--ink); font-weight: 600; } +.gm-narration p { margin: 0 0 10px; } +.gm-narration .dim { font: 12.5px/1.45 var(--sans); } +.gm-more { margin-top: 12px; } +.gm-more > summary { font: 12.5px var(--sans); color: var(--ink-3); cursor: pointer; } +.gm-layers { margin: 10px 0 0; display: grid; gap: 8px; } +.gm-layer dt { font: 600 10.5px/1.3 var(--sans); letter-spacing: .08em; text-transform: uppercase; color: var(--ink-3); } +.gm-layer dd { margin: 2px 0 0; color: var(--ink-2); font-size: 12.5px; line-height: 1.45; } +.gm-layer[data-state="missing"] dd, .gm-layer[data-state="unreadable"] dd { color: var(--ink); } +.gm-caveat summary { color: var(--ink-3); cursor: pointer; font-size: 12px; } +.gm-key { display: flex; flex-wrap: wrap; gap: 6px 12px; margin-top: 14px; font-size: 12px; color: var(--ink-2); } +.gm-key .fig-label { flex-basis: 100%; } +.gm-k-roh, .gm-k-mask { display: inline-flex; align-items: center; gap: 6px; } +.gm-k-roh .sw { width: 16px; height: 3px; background: var(--ink); border-radius: 2px; } +.gm-k-mask .sw { width: 16px; height: 8px; border-radius: 3px; background: repeating-linear-gradient(135deg, #26272c 0 2px, #4a4b50 2px 3px); } +@media (max-width: 700px) { + .gm-title { font-size: 22px; } + .gm-margin { margin-top: 12px; } + .gm-narration { font-size: 14px; } +} diff --git a/assets/live/map.js b/assets/live/map.js new file mode 100644 index 000000000..94ffbaee0 --- /dev/null +++ b/assets/live/map.js @@ -0,0 +1,1074 @@ +// map.js: Figure 1 of a person's page, the genome map. It draws the map document the server builds with +// `og map build` (GET map.json beside the page: gzip-wrapped columnar JSON, rust/crates/og-map/README.md): +// the 24 chromosomes as rows with their bands, the masked spans hatched, each finding with a place as its +// level's glyph, and the runs of homozygosity; a click opens a chromosome in place, where the bands, the +// findings and the runs get lanes of their own under a ruler. Every mark comes from the document, never from +// results.json, and every sentence the margin shows is one the document wrote (text, phrase, empty, note), +// with the counts filled in here. +// +// The veil. The document is the same for every reader: each row a closed result could reveal carries the id +// of that result (veil). Whether it is closed is the page's question (person.js veil()): a closed row is not +// drawn, has no hit target, no tooltip, no line in the margin and never reaches the address. The closed +// results are counted in a dashed chip that names no gene and no place, with an Open button that asks once, +// then opens the result through the page's own function, so the map and the card open together. A reload +// closes them again: the open state lives in the page's memory only. +// +// No library and no build step: one canvas, the camera {ci, c, ls, k} (open chromosome, centre bp, natural +// log of the span, 0 genome .. 1 open) eased toward its target each frame, one level() by bp per CSS pixel. +// Text only through textContent: no innerHTML, no style attribute. Everything hangs on window.PGPMap. +(function () { + 'use strict'; + var U = window.PGPUI; + if (!U) { return; } + var el = U.el, add = U.add; + var LV = { act: '#d03b3b', medicine: '#ec835a', review: '#fab219', carrier: '#9085e9', trust: '#c3c2b7', info: '#c3c2b7' }; + var PRI = { act: 0, medicine: 1, review: 2, carrier: 3, trust: 4, info: 5 }; + var STAIN = { gneg: '#26272c', gpos25: '#323337', gpos50: '#3e3f42', gpos75: '#4c4c4d', gpos100: '#5a5955', + gvar: '#25262a', stalk: '#25262a', acen: '#2e2f34' }; + var BRASS = '#c8ab78', INK = '#f0efec', INK2 = '#c3c2b7', INK3 = '#8f8d86', PAGE = '#0e0f10'; + var SANS = 'system-ui, -apple-system, "Segoe UI", sans-serif'; + var MONO = 'ui-monospace, "SF Mono", Menlo, monospace'; + // The chromosome level ends at 16,667 bp per CSS pixel: closer views (region, gene, reads) come later. + var CHROM_BPP = 16667; + // A genome row's least height: 22 CSS px, 14 on a phone (the page there is long enough already). + var ROW_MIN = 22, ROW_PHONE = 14; + var reduced = window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches; + + function lerp(a, b, t) { return a + (b - a) * t; } + function clamp(v, a, b) { return Math.max(a, Math.min(b, v)); } + function mb(bp, d) { return (bp / 1e6).toFixed(d === undefined ? 1 : d); } + function comma(n) { return String(Math.round(n)).replace(/\B(?=(\d{3})+(?!\d))/g, ','); } + // A sentence of the document: pieces of text, the bold ones in . + function runs(list) { + var p = el('p', {}); + (list || []).forEach(function (x) { add(p, x.b ? el('b', {}, x.t) : document.createTextNode(x.t)); }); + return p; + } + function short(n) { return String(n).replace(/^chr/, ''); } + // A layer's states that put nothing on the map (the document's layer entry); any other state has landed. + var NOT_ON_MAP = { missing: 1, unreadable: 1, not_run: 1, not_in_this_run: 1, running: 1, waiting: 1, failed: 1, skipped: 1 }; + function landed(lay) { return !!lay && !NOT_ON_MAP[lay.state]; } + + // --- the document: gzip -> JSON -> typed columns ------------------------------------------------------ + // Starts are gaps from the previous row (ds) and ends are lengths (l): s, e and the running maximum of e + // are rebuilt here in one pass, so a view [a, b) is a binary search on that maximum. + function columns(o) { + var n = (o.ds || []).length, s = new Float64Array(n), e = new Float64Array(n), me = new Float64Array(n); + var at = 0, m = 0; + for (var i = 0; i < n; i++) { + at += o.ds[i]; + s[i] = at; + e[i] = at + o.l[i]; + m = Math.max(m, e[i]); + me[i] = m; + } + return { n: n, s: s, e: e, me: me, o: o }; + } + // The first row whose running maximum end passes a: every row from there with s < b meets [a, b). + function first(c, a) { + var lo = 0, hi = c.n; + while (lo < hi) { var mid = (lo + hi) >> 1; if (c.me[mid] > a) { hi = mid; } else { lo = mid + 1; } } + return lo; + } + function prepare(d, who) { + var person = (d.people || [])[0] || {}; + var id = who || person.id; + var per = (d.per || {})[id] || {}; + var chroms = ((d.ref && d.ref.chroms) || []).filter(function (c) { return !(c.n === 'chrY' && person.has_y === false); }); + var by = {}; + chroms.forEach(function (c, i) { by[c.n] = i; }); + var bands = {}, roh = {}, bins = {}; + chroms.forEach(function (c) { + var b = d.ref.bands && d.ref.bands[c.n]; + if (b) { bands[c.n] = columns(b); } + if (per.roh && per.roh[c.n]) { roh[c.n] = columns(per.roh[c.n]); } + if (per.roh_bins && per.roh_bins[c.n]) { bins[c.n] = new Float64Array(per.roh_bins[c.n]); } + }); + var f = per.findings || {}, veilIds = (d.dict && d.dict.veil) || []; + var rows = []; + (f.id || []).forEach(function (fid, i) { + if (!(f.chrom[i] in by)) { return; } + rows.push({ id: fid, level: f.level[i], gate: f.gate[i], chrom: f.chrom[i], pos: f.pos[i], s: f.s[i], e: f.e[i], + gene: f.gene[i], title: f.title[i], href: f.href[i], phrase: f.phrase[i], veil: f.veil[i] >= 0 ? veilIds[f.veil[i]] : null }); + }); + var gated = {}; + rows.forEach(function (r) { if (r.gate) { gated[r.id] = r; } }); + return { doc: d, id: id, label: person.label || id, chroms: chroms, by: by, bands: bands, roh: roh, bins: bins, + binSize: (per.roh_bins && per.roh_bins.size) || 1e6, findings: rows, gated: gated, veilIds: veilIds, + layers: d.layers || {}, text: d.text || {}, art: (d.dict && d.dict.art) || [] }; + } + + // --- the figure --------------------------------------------------------------------------------------- + // create({url, veil(id, gated) -> 'open'|'closed'|'opened', now(id) -> shown at this moment, open(id), + // close(id), fallback() -> rows}): one figure per page. The page calls refresh(key) on each of its redraws; a new key + // fetches the document again and keeps the camera. + function create(opts) { + var M = null, key = null, failure = null, t0 = 0; + var fig = el('figure', { 'class': 'row fig1 gmap', id: 'genome-map', 'aria-labelledby': 'gm-title' }); + var main = el('div', { 'class': 'gm-main' }); + var head = el('div', { 'class': 'gm-head' }); + var kicker = el('div', { 'class': 'fig-label' }, 'Figure 1 · Genome map'); + var title = el('h2', { 'class': 'gm-title', id: 'gm-title' }, 'The genome map'); + var gauge = el('div', { 'class': 'gm-gauge', role: 'group', 'aria-label': 'Zoom level' }); + var stopG = el('button', { type: 'button', 'data-stop': 'genome' }, 'Genome'); + var stopC = el('button', { type: 'button', 'data-stop': 'chromosome' }, 'Chromosome'); + add(gauge, stopG, stopC); + var crumb = el('nav', { 'class': 'gm-crumb mono', 'aria-label': 'Where the map is' }); + var chips = el('div', { 'class': 'gm-chips' }); + add(head, kicker, add(el('div', { 'class': 'gm-titlerow' }), title, gauge), crumb, chips); + var stage = el('div', { 'class': 'gm-stage' }); + var canvas = el('canvas', { 'class': 'gm-canvas', tabindex: '0', role: 'img', 'aria-describedby': 'gm-narration' }); + var tip = el('div', { 'class': 'gm-tip', role: 'tooltip', hidden: 'hidden' }); + var toast = el('div', { 'class': 'gm-toast', role: 'status', hidden: 'hidden' }); + add(stage, canvas, tip, toast); + // The marks drawn, as links: the keyboard and screen-reader path to each finding (Tab steps through them). + var marks = el('ul', { 'class': 'gm-marks', 'aria-label': 'Findings on the map' }); + var fallback = el('div', { 'class': 'gm-fallback', hidden: 'hidden' }); + add(main, head, stage, marks, fallback); + var margin = el('aside', { 'class': 'gm-margin' }); + var mkick = el('div', { 'class': 'fig-label' }, 'The whole genome'); + var mline = el('div', { 'class': 'gm-mline mono' }); + var narr = el('div', { 'class': 'gm-narration', id: 'gm-narration', 'aria-live': 'polite' }); + var layersBox = el('dl', { 'class': 'gm-layers' }); + var keyBox = el('div', { 'class': 'gm-key', 'aria-label': 'Key' }); + // The layers and the key: open beside the map, folded under it on a phone. + var more = el('details', { 'class': 'gm-more' }); + more.open = !(window.matchMedia && window.matchMedia('(max-width: 700px)').matches); + add(more, el('summary', {}, 'Layers and key'), layersBox, keyBox); + add(margin, mkick, mline, narr, more); + add(fig, main, margin); + + var ctx = canvas.getContext('2d'); + var CW = 0, CH = 0, DPR = 1, phone = false; + var cam = { ci: 0, c: 0, ls: 0, k: 0 }, tgt = { ci: 0, c: 0, ls: 0, k: 0 }; + var hover = { row: -1, hit: null }, hits = [], raf = 0, settleT = 0, hashT = 0, confirming = null, touched = false; + var cache = null, cacheKey = ''; + var hatch = null; + + // --- the veil: what this page keeps closed at this moment --- + function stateOf(id) { return opts.veil(id, !!(M && M.gated[id])); } + // A row shows when its result is not closed and the page shows its finding at this moment (a replay + // before the finding's step ended does not). + function shownRow(r) { return (!r.veil || stateOf(r.veil) !== 'closed') && (!opts.now || opts.now(r.id)); } + function drawn() { return M ? M.findings.filter(shownRow) : []; } + + // --- geometry --- + function chrom() { return M.chroms[cam.ci]; } + function rowH() { var lo = phone ? ROW_PHONE : ROW_MIN; return M ? clamp((CH - 18) / M.chroms.length, lo, 44) : lo; } + function gutG() { return phone ? 30 : 40; } + function pad() { return phone ? 10 : 22; } + function gut() { return lerp(gutG(), phone ? 64 : 128, cam.k); } + function plotW() { return CW - gut() - pad(); } + function minSpan(len) { return Math.min(len, CHROM_BPP * Math.max(200, CW - (phone ? 64 : 128) - pad())); } + function view() { + var span = Math.exp(cam.ls); + return [cam.c - span / 2, cam.c + span / 2, span]; + } + function gx(bp) { return gutG() + bp * (CW - gutG() - pad()) / M.chroms[0].len; } + function ox(bp) { var v = view(); return gut() + (bp - v[0]) / v[2] * plotW(); } + function xAt(bp, open) { return open ? lerp(gx(bp), ox(bp), cam.k) : gx(bp); } + function rowY(i) { return 9 + i * rowH() + rowH() / 2; } + function level() { + if (cam.k < 0.5) { return 'genome'; } + return 'chromosome'; + } + // The lanes of an open chromosome, top to bottom: y of each, stretched to the canvas. + function lanes() { + var free = Math.max(0, CH - 230), st = 1 + Math.min(0.75, free / 400); + var fy = 118, fh = 50 * Math.min(1.2, st), ry = fy + fh + 14, rh = 22 * st; + return { over: 10, ruler: 46, bands: 72, bh: 38, fy: fy, fh: fh, ry: ry, rh: rh }; + } + + // --- the canvas size, the hatch pattern --- + function size() { + var w = Math.max(280, Math.floor(stage.clientWidth || 680)); + phone = w < 520; + var n = M ? M.chroms.length : 24; + var h = 18 + n * (phone ? ROW_PHONE : ROW_MIN); + DPR = Math.min(2, window.devicePixelRatio || 1); + if (w !== CW || h !== CH) { + CW = w; CH = h; + canvas.width = Math.round(w * DPR); + canvas.height = Math.round(h * DPR); + canvas.style.setProperty('width', w + 'px'); + canvas.style.setProperty('height', h + 'px'); + cacheKey = ''; + } + } + function pattern(alpha) { + var p = document.createElement('canvas'); + p.width = p.height = 6; + var c = p.getContext('2d'); + c.strokeStyle = 'rgba(255,255,255,' + alpha + ')'; + c.lineWidth = 1; + c.beginPath(); c.moveTo(-1, 7); c.lineTo(7, -1); c.moveTo(5, 7); c.lineTo(7, 5); c.moveTo(-1, 1); c.lineTo(1, -1); c.stroke(); + return ctx.createPattern(p, 'repeat'); + } + + // --- drawing --- + function rounded(c, x, y, w, h, r) { + r = Math.max(0, Math.min(r, w / 2, h / 2)); + c.moveTo(x + r, y); c.lineTo(x + w - r, y); c.arcTo(x + w, y, x + w, y + r, r); c.lineTo(x + w, y + h - r); + c.arcTo(x + w, y + h, x + w - r, y + h, r); c.lineTo(x + r, y + h); c.arcTo(x, y + h, x, y + h - r, r); + c.lineTo(x, y + r); c.arcTo(x, y, x + r, y, r); + } + // One chromosome's body: two rounded arms pinched at the centromere, filled with its bands, the masked + // spans hatched. X maps a bp to a canvas x. + function body(c, ch, X, yc, h, alpha) { + var cen = ch.cen ? (ch.cen[0] + ch.cen[1]) / 2 : null; + c.save(); + c.globalAlpha = alpha; + c.beginPath(); + var x0 = X(0), x1 = X(ch.len), xm = cen === null ? null : X(cen); + if (xm === null) { rounded(c, x0, yc - h / 2, x1 - x0, h, h / 2); } else { + rounded(c, x0, yc - h / 2, Math.max(1, xm - x0 - 0.6), h, h / 2); + rounded(c, xm + 0.6, yc - h / 2, Math.max(1, x1 - xm - 0.6), h, h / 2); + } + c.clip(); + c.fillStyle = STAIN.gneg; + c.fillRect(x0, yc - h / 2, x1 - x0, h); + var b = M.bands[ch.n]; + if (b) { + for (var i = 0; i < b.n; i++) { + var st = b.o.stain[i]; + if (st === 'gneg') { continue; } + var a = X(b.s[i]), z = X(b.e[i]); + if (z < 0 || a > CW) { continue; } + c.fillStyle = STAIN[st] || STAIN.gneg; + c.fillRect(a, yc - h / 2, Math.max(0.5, z - a), h); + } + } + c.fillStyle = hatch; + (ch.masked || []).forEach(function (m) { var a = X(m[0]), z = X(m[1]); c.fillRect(a, yc - h / 2, Math.max(1, z - a), h); }); + c.restore(); + } + function glyph(c, lv, x, y, r) { + c.save(); + c.fillStyle = LV[lv] || LV.info; + c.strokeStyle = PAGE; + c.lineWidth = 3; + c.beginPath(); + if (lv === 'act') { c.moveTo(x, y - r); c.lineTo(x + r, y); c.lineTo(x, y + r); c.lineTo(x - r, y); c.closePath(); c.stroke(); c.fill(); } + else if (lv === 'medicine') { rounded(c, x - r * 1.3, y - r * 0.7, r * 2.6, r * 1.4, r * 0.7); c.stroke(); c.fill(); } + else if (lv === 'review') { c.moveTo(x, y - r); c.lineTo(x + r, y + r * 0.8); c.lineTo(x - r, y + r * 0.8); c.closePath(); c.stroke(); c.fill(); } + else { + var two = lv === 'carrier', rr = two ? r * 0.75 : r * 0.9; + var at = two ? [x - rr * 0.75, x + rr * 0.75] : [x]; + at.forEach(function (cx) { c.moveTo(cx + rr, y); c.arc(cx, y, rr, 0, Math.PI * 2); }); + c.stroke(); + c.strokeStyle = LV[lv] || LV.info; + c.lineWidth = 1.5; + c.beginPath(); + at.forEach(function (cx) { c.moveTo(cx + rr, y); c.arc(cx, y, rr, 0, Math.PI * 2); }); + c.stroke(); + } + c.restore(); + } + function label(c, text, x, y, font, colour) { + c.font = font; + c.lineJoin = 'round'; + c.strokeStyle = PAGE; + c.lineWidth = 4; + c.strokeText(text, x, y); + c.fillStyle = colour; + c.fillText(text, x, y); + } + var widths = {}; + function measure(font, text) { + var k = font + '\u0000' + text; + if (!(k in widths)) { ctx.font = font; widths[k] = ctx.measureText(text).width; } + return widths[k]; + } + + // The genome rows, drawn once into an offscreen canvas until the data, the open results or the size change. + function genomeRows() { + var k = [CW, CH, DPR, drawn().map(function (r) { return r.id; }).join(',')].join('|'); + if (cache && k === cacheKey) { return cache; } + cache = cache || document.createElement('canvas'); + cache.width = canvas.width; + cache.height = canvas.height; + var c = cache.getContext('2d'); + c.setTransform(DPR, 0, 0, DPR, 0, 0); + c.clearRect(0, 0, CW, CH); + var fs = drawn(); + var occupied = {}; + M.chroms.forEach(function (ch, i) { + var yc = rowY(i); + c.fillStyle = INK3; + c.font = '11px ' + SANS; + c.textAlign = 'right'; + c.textBaseline = 'middle'; + c.fillText(short(ch.n), gutG() - 10, yc); + c.textAlign = 'left'; + body(c, ch, gx, yc, 8, 1); + // Runs of homozygosity under the body: the long ones solid, the rest as 1 Mb coverage bins. + var bins = M.bins[ch.n], ry = yc + 6.5; + if (bins) { + c.fillStyle = INK2; + for (var b = 0; b < bins.length; b++) { + if (!bins[b]) { continue; } + c.globalAlpha = 0.22 * Math.min(1, bins[b] / M.binSize); + c.fillRect(gx(b * M.binSize), ry - 1, Math.max(1, gx((b + 1) * M.binSize) - gx(b * M.binSize)), 2); + } + c.globalAlpha = 1; + } + var rr = M.roh[ch.n]; + if (rr) { + c.strokeStyle = INK; + c.lineWidth = 2.5; + for (var j = 0; j < rr.n; j++) { + if (!rr.o.long[j]) { continue; } + c.beginPath(); c.moveTo(gx(rr.s[j]), ry); c.lineTo(gx(rr.e[j]), ry); c.stroke(); + } + } + occupied[i] = []; + }); + // Findings: glyphs pinned to their row, names to the right by level (act first), never over another + // glyph or name of the row: a name that does not fit is left to the margin and the list. + var byLevel = fs.slice().sort(function (a, b) { return PRI[a.level] - PRI[b.level] || a.s - b.s; }); + byLevel.forEach(function (r) { + var i = M.by[r.chrom], yc = rowY(i), x = gx((r.s + r.e) / 2); + if (gx(r.e) - gx(r.s) > 3) { + c.fillStyle = LV[r.level] || LV.info; + c.fillRect(gx(r.s), yc + 4.5, gx(r.e) - gx(r.s), 2); + } + glyph(c, r.level, x, yc - 1, phone ? 3.5 : 4.5); + occupied[i].push([x - 6, x + 6]); + }); + byLevel.forEach(function (r) { + var i = M.by[r.chrom], yc = rowY(i), x = gx((r.s + r.e) / 2); + var name = r.gene || ''; + if (!name) { return; } + var font = '500 10.5px ' + SANS, w = measure(font, name), lx = x + 8; + var clash = occupied[i].some(function (o) { return !(lx + w < o[0] || lx > o[1]) && !(o[0] === x - 6 && o[1] === x + 6); }); + if (lx + w > CW - 2 || clash) { return; } + occupied[i].push([lx - 2, lx + w + 4]); + c.textBaseline = 'middle'; + label(c, name, lx, yc - 1, font, INK2); + }); + cacheKey = k; + return cache; + } + + function draw() { + raf = 0; + if (!M) { return; } + var t = performance.now(); + size(); // the page may have changed width since the last frame (a phone turned, a fold opened) + step(); + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.setTransform(DPR, 0, 0, DPR, 0, 0); + hits = []; + if (cam.k < 0.999) { drawGenome(); } + if (cam.k > 0.001) { drawOpen(); } + if (!fig.hasAttribute('data-pgp-map-paint-ms') && t0) { + fig.setAttribute('data-pgp-map-paint-ms', String(Math.round(performance.now() - t0))); + } + if (moving()) { schedule(); } else { settle(); } + return t; + } + function drawGenome() { + var k = cam.k, n = M.chroms.length; + if (k < 0.001) { + if (hover.row >= 0 && level() === 'genome') { + ctx.fillStyle = 'rgba(255,255,255,.035)'; + ctx.fillRect(0, rowY(hover.row) - rowH() / 2, CW, rowH()); + var ch = M.chroms[hover.row], words = 'Open chromosome ' + short(ch.n) + ' ›'; + var font = '500 11px ' + SANS, w = measure(font, words), x = gx(ch.len) + 12; + if (x + w < CW - 4) { ctx.textBaseline = 'middle'; label(ctx, words, x, rowY(hover.row), font, BRASS); } + } + ctx.drawImage(genomeRows(), 0, 0, CW, CH); + M.chroms.forEach(function (c, i) { + hits.push({ kind: 'row', i: i, x0: 0, x1: CW, y0: rowY(i) - rowH() / 2, y1: rowY(i) + rowH() / 2 }); + }); + drawn().forEach(function (r) { + hits.push({ kind: 'finding', r: r, x: gx((r.s + r.e) / 2), y: rowY(M.by[r.chrom]) - 1, rad: 7 }); + }); + return; + } + // The motion: the other rows slide away from the one opening and fade. + var a = Math.pow(1 - k, 1.4); + for (var i = 0; i < n; i++) { + if (i === cam.ci) { continue; } + var dy = k * CH * 0.9 * (0.6 + 0.04 * Math.abs(i - cam.ci)) * (i < cam.ci ? -1 : 1); + ctx.save(); + ctx.globalAlpha = a; + ctx.translate(0, dy); + ctx.fillStyle = INK3; + ctx.font = '11px ' + SANS; + ctx.textAlign = 'right'; + ctx.textBaseline = 'middle'; + ctx.fillText(short(M.chroms[i].n), gutG() - 10, rowY(i)); + ctx.textAlign = 'left'; + body(ctx, M.chroms[i], gx, rowY(i), 8, 1); + ctx.restore(); + } + } + function niceStep(span, px) { + var raw = span / Math.max(1, px / 90), p = Math.pow(10, Math.floor(Math.log10(raw))); + var m = raw / p; + return (m < 1.5 ? 1 : m < 3.5 ? 2 : m < 7.5 ? 5 : 10) * p; + } + function drawOpen() { + var k = cam.k, ch = chrom(), L = lanes(), v = view(), g = gut(), W = plotW(); + var X = function (bp) { return xAt(bp, true); }; + var yb = lerp(rowY(cam.ci), L.bands + L.bh / 2, k), hb = lerp(8, L.bh, k); + ctx.save(); + // The open row's label moves into the gutter. + ctx.fillStyle = INK3; + ctx.font = '11px ' + SANS; + ctx.textAlign = 'right'; + ctx.textBaseline = 'middle'; + ctx.fillText(k > 0.5 ? 'Bands' : short(ch.n), lerp(gutG() - 10, g - 12, k), yb); + ctx.restore(); + ctx.save(); + ctx.beginPath(); + ctx.rect(lerp(0, g - 2, k), 0, CW, CH); + ctx.clip(); + body(ctx, ch, X, yb, hb, 1); + ctx.restore(); + ctx.save(); + ctx.globalAlpha = k; + // Overview: the whole chromosome, the view as a brass window, a funnel to the ruler. + var ow = W, oX = function (bp) { return g + bp / ch.len * ow; }; + body(ctx, ch, oX, L.over + 6, 8, 0.8); + ctx.strokeStyle = BRASS; + ctx.lineWidth = 1.5; + var wa = oX(clamp(v[0], 0, ch.len)), wz = oX(clamp(v[1], 0, ch.len)); + ctx.strokeRect(wa, L.over, Math.max(3, wz - wa), 12); + ctx.globalAlpha = k * 0.35; + ctx.beginPath(); ctx.moveTo(wa, L.over + 12); ctx.lineTo(g, L.ruler); ctx.moveTo(wz, L.over + 12); ctx.lineTo(g + W, L.ruler); ctx.stroke(); + ctx.globalAlpha = k; + // The ruler, in Mb. + ctx.strokeStyle = 'rgba(255,255,255,.14)'; + ctx.lineWidth = 1; + ctx.beginPath(); ctx.moveTo(g, L.ruler); ctx.lineTo(g + W, L.ruler); ctx.stroke(); + var stp = niceStep(v[2], W); + ctx.fillStyle = INK3; + ctx.font = '10.5px ' + SANS; + ctx.textAlign = 'center'; + ctx.textBaseline = 'top'; + for (var at = Math.ceil(v[0] / stp) * stp; at <= v[1]; at += stp) { + var x = ox(at); + ctx.beginPath(); ctx.moveTo(x, L.ruler); ctx.lineTo(x, L.ruler + 5); ctx.stroke(); + ctx.fillText(mb(at, stp < 1e6 ? 1 : 0) + ' Mb', x, L.ruler + 8); + } + // Band names where they fit. + var b = M.bands[ch.n]; + if (b) { + ctx.font = '10px ' + SANS; + ctx.textBaseline = 'middle'; + for (var i = first(b, v[0]); i < b.n && b.s[i] < v[1]; i++) { + var a = ox(b.s[i]), z = ox(b.e[i]), nm = b.o.name[i]; + if (nm && z - a > measure('10px ' + SANS, nm) + 6) { + ctx.fillStyle = ['gpos75', 'gpos100'].indexOf(b.o.stain[i]) >= 0 ? INK : INK2; + ctx.fillText(nm, (Math.max(a, g) + Math.min(z, g + W)) / 2, L.bands + L.bh / 2); + } + hits.push({ kind: 'band', x0: a, x1: z, y0: L.bands, y1: L.bands + L.bh, name: nm, s: b.s[i], e: b.e[i] }); + } + } + (ch.masked || []).forEach(function (m) { + hits.push({ kind: 'masked', x0: ox(m[0]), x1: ox(m[1]), y0: L.bands, y1: L.bands + L.bh, s: m[0], e: m[1] }); + }); + ctx.textAlign = 'right'; + ctx.textBaseline = 'middle'; + ctx.fillStyle = INK3; + ctx.font = '11px ' + SANS; + ctx.fillText('Findings', g - 12, L.fy + 16); + ctx.fillText(phone ? 'Runs' : 'Homozygous runs', g - 12, L.ry + L.rh / 2); + ctx.textAlign = 'left'; + ctx.save(); + ctx.beginPath(); ctx.rect(g - 6, 0, W + 12, CH); ctx.clip(); + openFindings(L, v, ch); + openRoh(L, v, ch); + // The reading line: a dashed brass line through every lane at the centre, its place in a pill. + var cx = g + W / 2; + ctx.setLineDash([3, 3]); + ctx.strokeStyle = BRASS; + ctx.globalAlpha = k * 0.8; + ctx.beginPath(); ctx.moveTo(cx, L.ruler + 22); ctx.lineTo(cx, L.ry + L.rh + 6); ctx.stroke(); + ctx.setLineDash([]); + ctx.restore(); + var pill = mb(cam.c, 1) + ' Mb', pw = measure('600 10.5px ' + SANS, pill) + 12; + ctx.fillStyle = BRASS; + rounded(ctx, cx - pw / 2, L.ruler - 20, pw, 15, 7.5); + ctx.beginPath(); rounded(ctx, cx - pw / 2, L.ruler - 20, pw, 15, 7.5); ctx.fill(); + ctx.fillStyle = PAGE; + ctx.font = '600 10.5px ' + SANS; + ctx.textAlign = 'center'; + ctx.fillText(pill, cx, L.ruler - 12.5); + ctx.restore(); + } + function laneWords(c, text, x, y) { + c.fillStyle = INK3; + c.font = 'italic 11.5px ' + SANS; + c.textAlign = 'left'; + c.textBaseline = 'middle'; + c.fillText(text, x, y); + } + function openFindings(L, v, ch) { + var lay = M.layers.findings || {}, g = gut(); + if (!landed(lay)) { laneWords(ctx, layerWords(lay), g + 4, L.fy + 16); return; } + var fs = drawn().filter(function (r) { return r.chrom === ch.n; }); + var inView = fs.filter(function (r) { return r.e > v[0] && r.s < v[1]; }); + if (!inView.length) { + laneWords(ctx, fs.length ? 'No finding in this stretch.' : noFindingHere(), g + 4, L.fy + 16); + return; + } + var used = []; + inView.sort(function (a, b) { return PRI[a.level] - PRI[b.level] || a.s - b.s; }).forEach(function (r) { + var x = ox((r.s + r.e) / 2), y = L.fy + 14; + if (r.e - r.s > v[2] / 2000) { + ctx.fillStyle = LV[r.level] || LV.info; + ctx.fillRect(ox(r.s), y + 9, Math.max(2, ox(r.e) - ox(r.s)), 2); + } + ctx.strokeStyle = LV[r.level] || LV.info; + ctx.globalAlpha = 0.5; + ctx.lineWidth = 1.2; + ctx.beginPath(); ctx.moveTo(x, y + 8); ctx.lineTo(x, L.bands + L.bh); ctx.stroke(); + ctx.globalAlpha = cam.k; + glyph(ctx, r.level, x, y, 6); + hits.push({ kind: 'finding', r: r, x: x, y: y, rad: 9 }); + var name = r.gene || r.title, words = U.LEVEL_WORDS[r.level] || r.level; + var w = Math.max(measure('500 12px ' + SANS, name), measure('10.5px ' + SANS, words)); + var lx = x + 11; + if (lx + w > CW - 4 || used.some(function (u) { return !(lx + w < u[0] || lx > u[1]); })) { return; } + used.push([lx - 4, lx + w + 6]); + ctx.textBaseline = 'middle'; + label(ctx, name, lx, y - 1, '500 12px ' + SANS, INK); + label(ctx, words, lx, y + 13, '10.5px ' + SANS, INK3); + }); + } + function openRoh(L, v, ch) { + var lay = M.layers.roh || {}, g = gut(), y = L.ry + L.rh / 2; + if (!landed(lay)) { laneWords(ctx, layerWords(lay), g + 4, y); return; } + var r = M.roh[ch.n]; + if (!r || !r.n) { laneWords(ctx, 'No run of homozygosity on this chromosome.', g + 4, y); return; } + for (var i = first(r, v[0]); i < r.n && r.s[i] < v[1]; i++) { + var a = ox(r.s[i]), z = ox(r.e[i]), art = M.art[r.o.art[i]] || ''; + if (r.o.long[i]) { + ctx.fillStyle = INK2; + ctx.fillRect(a, y - 3, Math.max(1, z - a), 6); + var t = mb(r.e[i] - r.s[i]) + ' Mb run of homozygosity', font = '11px ' + SANS; + if (measure(font, t) + 8 > z - a) { t = mb(r.e[i] - r.s[i]) + ' Mb'; } + if (measure(font, t) + 8 < z - a) { ctx.textBaseline = 'middle'; label(ctx, t, a + 4, y - 11, font, INK2); } + } else if (art) { + ctx.fillStyle = hatch; + ctx.fillRect(a, y - 3, Math.max(1, z - a), 6); + } else { + ctx.fillStyle = 'rgba(195,194,183,.35)'; + ctx.fillRect(a, y - 0.5, Math.max(1, z - a), 1); + } + hits.push({ kind: 'roh', x0: a, x1: Math.max(a + 3, z), y0: y - 6, y1: y + 6, s: r.s[i], e: r.e[i], art: art, long: r.o.long[i] }); + } + } + function noFindingHere() { + return closedAny() ? 'No open finding with a place on this chromosome.' : 'No finding with a place on this chromosome.'; + } + function closedAny() { return M.veilIds.some(function (id) { return stateOf(id) === 'closed'; }); } + function layerWords(lay) { + if (lay.state === 'missing') { return 'Not on the map: ' + (lay.note || ('step ' + lay.step + ' wrote no ' + (lay.source || 'file') + '.')); } + if (lay.state === 'unreadable') { return 'Not on the map: ' + (lay.source || '') + ' could not be read' + (lay.note ? ' (' + lay.note + ')' : '') + '.'; } + if (lay.state === 'not_run') { return 'Not on the map: step ' + lay.step + ' was not part of this run.'; } + if (lay.state === 'not_in_this_run') { return 'Not on the map: not part of this run.'; } + if (lay.state === 'running' || lay.state === 'waiting') { return 'Not on the map yet: step ' + lay.step + ' has not ended.'; } + if (lay.state === 'failed') { return 'Not on the map: step ' + lay.step + ' failed.'; } + if (lay.state === 'skipped') { return 'Not on the map: step ' + lay.step + ' was skipped.'; } + return lay.note || ''; + } + + // --- the camera --- + function moving() { + return Math.abs(cam.k - tgt.k) > 0.002 || Math.abs(cam.ls - tgt.ls) > 0.002 || Math.abs(cam.c - tgt.c) > Math.exp(cam.ls) * 0.0005; + } + var last = 0; + function step() { + var now = performance.now(), dt = last ? Math.min(64, now - last) : 16; + last = now; + if (reduced || cam.ci !== tgt.ci && cam.k < 0.01) { cam.ci = tgt.ci; } + if (reduced) { cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; return; } + if (cam.ci !== tgt.ci) { + // Another chromosome: close the open one first, then open the new one. + cam.k += (0 - cam.k) * (1 - Math.exp(-dt / 60)); + if (cam.k < 0.01) { cam.k = 0; cam.ci = tgt.ci; cam.c = tgt.c; cam.ls = tgt.ls; } + return; + } + cam.c += (tgt.c - cam.c) * (1 - Math.exp(-dt / 75)); + cam.ls += (tgt.ls - cam.ls) * (1 - Math.exp(-dt / 75)); + cam.k += (tgt.k - cam.k) * (1 - Math.exp(-dt / 95)); + if (!moving()) { cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; } + } + function schedule() { if (!raf) { raf = window.requestAnimationFrame(function () { last = last || performance.now(); draw(); }); } } + function clampTgt() { + var ch = M.chroms[tgt.ci], span = clamp(Math.exp(tgt.ls), minSpan(ch.len), ch.len); + tgt.ls = Math.log(span); + tgt.c = clamp(tgt.c, span / 2, ch.len - span / 2); + } + function openChrom(i, c, span, push) { + if (!M || i < 0 || i >= M.chroms.length) { return; } + var ch = M.chroms[i]; + tgt.ci = i; + tgt.k = 1; + tgt.c = c === undefined ? ch.len / 2 : c; + tgt.ls = Math.log(span === undefined ? ch.len : span); + clampTgt(); + if (cam.k < 0.01) { cam.ci = i; cam.c = tgt.c; cam.ls = tgt.ls; } + changed(push); + } + function closeChrom(push) { + tgt.k = 0; + changed(push); + } + function changed(push) { + touched = true; + last = 0; + schedule(); + writeHash(push); + } + + // --- the address: #genome, #chr7, #chr7:117.4-137.4M; never a closed result's id, gene or place --- + function locusWords() { + if (tgt.k < 0.5) { return 'genome'; } + var ch = M.chroms[tgt.ci], span = Math.exp(tgt.ls); + if (span >= ch.len * 0.999) { return ch.n; } + return ch.n + ':' + mb(tgt.c - span / 2) + '-' + mb(tgt.c + span / 2) + 'M'; + } + function writeHash(push) { + if (!M || !window.history || !window.history.replaceState) { return; } + window.clearTimeout(hashT); + var go = function () { + var h = '#' + locusWords(); + if (window.location.hash === h) { return; } + if (push) { window.history.pushState(null, '', h); } else { window.history.replaceState(null, '', h); } + }; + if (push) { go(); } else { hashT = window.setTimeout(go, 250); } + } + // A place in the address: 'genome', 'chr7', 'chr7:117.4-117.7M', 'chr7:120k-140k', 'chr7:117480026-117668665' + // (1-based, inclusive), or the aliases view=genome, view=chromosome&chr=7, chr=7. Returns null when the + // address is not a map place, false when it is one that cannot be read. + function parsePlace(h) { + var m, kv = {}; + if (/^genome$/i.test(h)) { return { k: 0 }; } + if (/^(view|chr)=/.test(h)) { + h.split('&').forEach(function (p) { var x = p.split('='); kv[x[0]] = decodeURIComponent(x[1] || ''); }); + if (kv.view === 'genome') { return { k: 0 }; } + if (!kv.chr) { return kv.view ? false : null; } + h = 'chr' + String(kv.chr).replace(/^chr/i, ''); + } + if (!/^chr/i.test(h)) { return null; } + var name = 'chr' + h.slice(3).split(':')[0].toUpperCase().replace(/^0+/, ''); + if (!(name in M.by)) { return false; } + var ch = M.chroms[M.by[name]], rest = h.indexOf(':') > 0 ? h.slice(h.indexOf(':') + 1) : ''; + if (!rest) { return { k: 1, ci: M.by[name] }; } + if ((m = /^([0-9.]+)-([0-9.]+)([MmKk])$/.exec(rest))) { + var u = /m/i.test(m[3]) ? 1e6 : 1e3, a = Number(m[1]) * u, z = Number(m[2]) * u; + if (!(z > a)) { return false; } + return { k: 1, ci: M.by[name], s: a, e: z }; + } + if ((m = /^([0-9,]+)-([0-9,]+)$/.exec(rest))) { + var s = Number(m[1].replace(/,/g, '')) - 1, e = Number(m[2].replace(/,/g, '')); + if (!(e > s)) { return false; } + return { k: 1, ci: M.by[name], s: s, e: e }; + } + void ch; + return false; + } + function readHash(scroll) { + if (!M) { return; } + var h; + try { h = decodeURIComponent((window.location.hash || '').slice(1)); } catch (e) { return; } + if (!h) { return; } + var p = parsePlace(h); + if (p === null) { return; } + if (p === false) { say('Could not read the place “' + h + '”'); return; } + if (p.k === 0) { tgt.k = 0; } else { + var ch = M.chroms[p.ci]; + openChrom(p.ci, p.s === undefined ? ch.len / 2 : (p.s + p.e) / 2, p.s === undefined ? ch.len : p.e - p.s, false); + window.clearTimeout(hashT); + } + if (scroll) { + cam.ci = tgt.ci; cam.k = tgt.k; cam.c = tgt.c; cam.ls = tgt.ls; + fig.scrollIntoView({ block: 'start' }); + } + schedule(); + } + function say(words) { + toast.textContent = words; + toast.hidden = false; + window.clearTimeout(say.t); + say.t = window.setTimeout(function () { toast.hidden = true; }, 4000); + } + + // --- the margin: settled, not per frame --- + function settle() { + window.clearTimeout(settleT); + settleT = window.setTimeout(describe, 140); + } + function describe() { + if (!M) { return; } + var lv = level(), fs = drawn(), ch = chrom(); + fig.setAttribute('data-level', lv); + fig.setAttribute('data-chrom', lv === 'genome' ? '' : ch.n); + fig.setAttribute('data-view', locusWords()); + stopG.setAttribute('aria-pressed', String(lv === 'genome')); + stopC.setAttribute('aria-pressed', String(lv !== 'genome')); + crumb.textContent = ''; + crumb.hidden = lv === 'genome'; // the gauge already says Genome + var gl = el('button', { type: 'button', 'class': 'gm-crumb-g' }, 'Genome'); + gl.addEventListener('click', function () { closeChrom(true); }); + add(crumb, gl); + var nY = M.chroms.length; + if (lv === 'genome') { + title.textContent = M.label + '’s genome, all ' + nY + ' chromosomes'; + kicker.textContent = 'Figure 1 · Genome map · ' + M.label + '’s genome'; + mkick.textContent = 'The whole genome'; + mline.textContent = ''; + } else { + title.textContent = 'Chromosome ' + short(ch.n); + kicker.textContent = 'Figure 1 · Genome map · ' + M.label; + add(crumb, el('span', { 'class': 'sep', 'aria-hidden': 'true' }, ' › '), el('span', { 'aria-current': 'location' }, ch.n)); + mkick.textContent = 'Under the line'; + mline.textContent = ch.n + ' · ' + mb(cam.c, 2) + ' Mb' + bandAt(ch, cam.c); + } + canvas.setAttribute('aria-label', lv === 'genome' ? 'The genome map: ' + nY + ' chromosomes with ' + fs.length + ' findings drawn' + : 'Chromosome ' + short(ch.n) + ' open on the genome map'); + narr.textContent = ''; + var v = view(); + var shown = lv === 'genome' ? fs : fs.filter(function (r) { return r.chrom === ch.n && r.e > v[0] && r.s < v[1]; }); + var forms = (M.text.findings || {}); + if (lv === 'genome') { + var form = fs.length === 0 ? forms.none : fs.length === 1 ? forms.one : forms.many; + add(narr, runs((form || []).map(function (x) { return { t: x.t.replace('{n}', String(fs.length)), b: x.b }; }))); + } else if (M.text.chrom && M.text.chrom[ch.n]) { + add(narr, runs(M.text.chrom[ch.n])); + } + var lay = M.layers.findings || {}; + if (!landed(lay)) { + add(narr, el('p', {}, layerWords(lay))); + } else if (shown.length && !(phone && lv === 'genome')) { + // On a phone the genome level leaves the list to the sections under it: the page is long already. + var list = shown.slice().sort(function (a, b) { return PRI[a.level] - PRI[b.level] || M.by[a.chrom] - M.by[b.chrom] || a.s - b.s; }); + var p = el('p', { 'class': 'gm-phrases' }); + var cap = phone ? 3 : 6; + list.slice(0, cap).forEach(function (r, i) { if (i) { add(p, ' '); } add(p, r.phrase); }); + if (list.length > cap) { add(p, ' And ' + (list.length - cap) + ' more on the map.'); } + add(narr, p); + } else if (lv !== 'genome') { + add(narr, el('p', {}, fs.some(function (r) { return r.chrom === ch.n; }) ? 'No finding in this stretch.' : noFindingHere())); + } + if (lv === 'genome') { + add(narr, runs(M.text.genome)); + if (!phone) { add(narr, el('p', { 'class': 'dim' }, 'Click a chromosome to open it.')); } + } else { + var r = M.roh[ch.n], n = 0, longest = 0; + if (r) { for (var i = first(r, v[0]); i < r.n && r.s[i] < v[1]; i++) { n++; longest = Math.max(longest, r.e[i] - r.s[i]); } } + if (n && landed(M.layers.roh || {})) { + add(narr, el('p', {}, 'Runs of homozygosity in view: ' + n + ', the longest ' + mb(longest) + ' Mb.')); + } + if (!phone) { add(narr, el('p', { 'class': 'dim' }, 'Drag to move along it, scroll to zoom, Esc for the whole genome.')); } + } + drawLayers(); + drawMarks(); + } + function bandAt(ch, bp) { + var b = M.bands[ch.n]; + if (!b) { return ''; } + for (var i = first(b, bp); i < b.n && b.s[i] <= bp; i++) { + if (b.e[i] > bp) { return ' · band ' + short(ch.n) + b.o.name[i]; } + } + return ''; + } + function drawLayers() { + layersBox.textContent = ''; + [['findings', 'Findings'], ['roh', 'Runs of homozygosity']].forEach(function (x) { + var lay = M.layers[x[0]]; + var row = el('div', { 'class': 'gm-layer', 'data-layer': x[0], 'data-state': lay ? lay.state : 'missing' }); + var dd = el('dd', {}); + if (!lay) { add(dd, 'Not on the map: this server’s map has no ' + x[1].toLowerCase() + ' layer.'); } + else if (!landed(lay)) { add(dd, layerWords(lay)); } + else { + var n = x[0] === 'findings' ? drawn().length : lay.count; + add(dd, n === 0 && lay.empty ? lay.empty : n + ' on the map. '); + if (lay.note && n !== 0) { add(dd, lay.note); } + if (lay.caveat) { + var d = el('details', { 'class': 'gm-caveat' }); + add(d, el('summary', {}, 'Caveat'), el('p', {}, lay.caveat)); + add(dd, d); + } + } + add(row, el('dt', {}, x[1] + (lay && lay.step ? ' · step ' + lay.step : '')), dd); + add(layersBox, row); + }); + } + function drawMarks() { + marks.textContent = ''; + var lv = level(), ch = chrom(), v = view(); + drawn().filter(function (r) { return lv === 'genome' || (r.chrom === ch.n && r.e > v[0] && r.s < v[1]); }) + .sort(function (a, b) { return M.by[a.chrom] - M.by[b.chrom] || a.s - b.s; }).forEach(function (r) { + var a = el('a', { href: r.href, 'data-id': r.id, 'data-level': r.level, 'data-chrom': r.chrom, + 'data-s': String(r.s), 'data-e': String(r.e) }, (U.LEVEL_WORDS[r.level] || r.level) + ': ' + r.title + ', ' + r.chrom); + add(marks, add(el('li', {}), a)); + }); + } + function drawKey() { + keyBox.textContent = ''; + add(keyBox, el('div', { 'class': 'fig-label' }, 'Key')); + ['act', 'medicine', 'review', 'carrier', 'info'].forEach(function (lv) { add(keyBox, U.level(lv)); }); + add(keyBox, add(el('span', { 'class': 'gm-k-roh' }), el('span', { 'class': 'sw', 'aria-hidden': 'true' }), 'Run of homozygosity, 5 Mb or more')); + add(keyBox, add(el('span', { 'class': 'gm-k-mask' }), el('span', { 'class': 'sw', 'aria-hidden': 'true' }), 'Hatched: centromere or heterochromatin, few reads map there')); + } + + // --- the closed results: one dashed chip per kind, and the opened ones --- + function drawChips() { + chips.textContent = ''; + if (!M) { return; } + var acts = [], gates = [], opened = []; + M.veilIds.forEach(function (id) { + var s = stateOf(id); + if (s === 'closed') { (M.gated[id] ? gates : acts).push(id); } else if (s === 'opened') { opened.push(id); } + }); + if (acts.length) { + var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'act' }); + add(c, el('span', {}, (acts.length === 1 ? '1 result closed' : acts.length + ' results closed') + ' · Not drawn until opened on purpose')); + add(c, confirming === 'act' ? confirmBox(acts[0], false) : button('Open', function () { confirming = 'act'; drawChips(); })); + add(chips, c); + } + gates.forEach(function (id) { + var g = M.gated[id], name = g.gene || 'This result'; + var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'gate' }); + add(c, el('span', {}, name + ' closed · Not drawn until opened on purpose')); + add(c, confirming === id ? confirmBox(id, true) : button('Open ' + name, function () { confirming = id; drawChips(); })); + add(chips, c); + }); + opened.forEach(function (id) { + var r = M.findings.filter(function (x) { return x.id === id; })[0]; + var c = el('div', { 'class': 'gm-chip opened', 'data-opened': '1' }); + add(c, el('span', {}, 'Opened: ' + (r ? r.title : 'a result to act on'))); + add(c, button('Close it again', function () { confirming = null; opts.close(id); })); + add(chips, c); + }); + } + function confirmBox(id, gated) { + var box = el('span', { 'class': 'gm-confirm' }); + add(box, el('span', {}, gated ? 'It is kept behind its warning on every page. Open it here and in its section?' + : 'A result to act on in adult life is usually not looked at before the child can choose. Open it here and in its section?')); + add(box, button('Show the result', function () { confirming = null; opts.open(id); }, 'gm-yes')); + add(box, button('Keep it closed', function () { confirming = null; drawChips(); }, 'gm-no')); + return box; + } + function button(text, fn, cls) { + var b = el('button', { type: 'button', 'class': cls || '' }, text); + b.addEventListener('click', fn); + return b; + } + + // --- input --- + function at(e) { var r = canvas.getBoundingClientRect(); return [e.clientX - r.left, e.clientY - r.top]; } + function hitAt(x, y) { + var best = null; + for (var i = hits.length - 1; i >= 0; i--) { + var h = hits[i]; + if (h.kind === 'finding') { if (Math.abs(h.x - x) <= h.rad && Math.abs(h.y - y) <= h.rad) { return h; } } + else if (x >= h.x0 && x <= h.x1 && y >= h.y0 && y <= h.y1) { + if (!best || (h.kind === 'masked' && best.kind === 'band') || (h.kind === 'roh')) { best = h; } + } + } + return best; + } + function tipFor(h) { + var ch = M.chroms[cam.ci]; + if (h.kind === 'finding') { return [h.r.phrase, 'Click for the finding’s page']; } + if (h.kind === 'band') { return ['Band ' + short(ch.n) + (h.name || ''), ch.n + ':' + comma(h.s + 1) + '-' + comma(h.e)]; } + if (h.kind === 'masked') { + return ['Not readable here · ' + ch.n + ':' + comma(h.s + 1) + '-' + comma(h.e), + 'Centromere, heterochromatin or acrocentric arm: few reads map, so layers are cut here. Not missing data.']; + } + if (h.kind === 'roh') { + return [mb(h.e - h.s) + ' Mb run of homozygosity' + (h.art ? ' (' + h.art + ': not a real run)' : ''), ch.n + ':' + comma(h.s + 1) + '-' + comma(h.e)]; + } + if (h.kind === 'row') { return null; } + return null; + } + // A hidden tooltip keeps no words: it may have named a result that is closed now. + function hideTip() { tip.hidden = true; tip.textContent = ''; } + function showTip(h, x, y) { + var t = h && tipFor(h); + if (!t) { hideTip(); return; } + tip.textContent = ''; + add(tip, el('b', {}, t[0])); + if (t[1]) { add(tip, el('span', {}, t[1])); } + tip.hidden = false; + tip.style.setProperty('left', Math.min(x + 12, Math.max(0, CW - 260)) + 'px'); + tip.style.setProperty('top', (y + 14) + 'px'); + } + var drag = null; + canvas.addEventListener('pointermove', function (e) { + if (!M) { return; } + var p = at(e); + if (drag && level() !== 'genome') { + var v = view(); + tgt.c = drag.c - (p[0] - drag.x) / plotW() * v[2]; + clampTgt(); + cam.c = tgt.c; + drag.moved = drag.moved || Math.abs(p[0] - drag.x) > 3; + changed(false); + return; + } + var h = hitAt(p[0], p[1]); + var row = level() === 'genome' && h ? (h.kind === 'row' ? h.i : h.kind === 'finding' ? M.by[h.r.chrom] : -1) : -1; + if (row !== hover.row) { hover.row = row; schedule(); } + hover.hit = h; + canvas.style.setProperty('cursor', h && (h.kind === 'finding' || h.kind === 'row') ? 'pointer' : level() === 'genome' ? 'default' : 'grab'); + showTip(h, p[0], p[1]); + }); + canvas.addEventListener('pointerleave', function () { hover.row = -1; hideTip(); schedule(); }); + canvas.addEventListener('pointerdown', function (e) { + if (!M || level() === 'genome' || e.button !== 0) { return; } + drag = { x: at(e)[0], c: tgt.c, moved: false }; + if (canvas.setPointerCapture) { canvas.setPointerCapture(e.pointerId); } + }); + canvas.addEventListener('pointerup', function (e) { + var was = drag; + drag = null; + if (!M || (was && was.moved)) { if (was && was.moved) { writeHash(false); } return; } + var p = at(e), h = hitAt(p[0], p[1]); + if (h && h.kind === 'finding') { window.location.href = h.r.href; return; } + if (level() === 'genome' && h && h.kind === 'row') { openChrom(h.i, undefined, undefined, true); } + }); + canvas.addEventListener('wheel', function (e) { + if (!M || level() === 'genome') { return; } + e.preventDefault(); + var p = at(e), v = view(), ch = chrom(); + var bp = v[0] + (p[0] - gut()) / plotW() * v[2]; + var f = Math.exp(clamp(e.deltaY, -200, 200) * 0.0025); + if (f > 1 && Math.exp(tgt.ls) >= ch.len * 0.999) { closeChrom(true); return; } + var span = clamp(Math.exp(tgt.ls) * f, minSpan(ch.len), ch.len); + tgt.c = bp - (bp - tgt.c) * span / Math.exp(tgt.ls); + tgt.ls = Math.log(span); + clampTgt(); + changed(false); + }, { passive: false }); + canvas.addEventListener('keydown', function (e) { + if (!M || e.altKey || e.ctrlKey || e.metaKey) { return; } + var open = level() !== 'genome', span = Math.exp(tgt.ls); + var done = true; + if (e.key === 'Escape' || e.key === '0') { if (open) { closeChrom(true); } else { done = false; } } + else if (e.key === '[' || e.key === ']') { + var i = (open ? tgt.ci : (e.key === ']' ? -1 : M.chroms.length)) + (e.key === ']' ? 1 : -1); + openChrom((i + M.chroms.length) % M.chroms.length, undefined, undefined, true); + } else if (open && (e.key === '+' || e.key === '=' || e.key === '-' || e.key === '−')) { + tgt.ls = Math.log(span * (e.key === '-' || e.key === '−' ? 1.6 : 1 / 1.6)); + clampTgt(); + changed(false); + } else if (open && (e.key === 'ArrowLeft' || e.key === 'ArrowRight')) { + tgt.c += (e.key === 'ArrowLeft' ? -0.2 : 0.2) * span; + clampTgt(); + changed(false); + } else if (!open && e.key === 'Enter') { openChrom(tgt.ci, undefined, undefined, true); } + else { done = false; } + if (done) { e.preventDefault(); } + }); + stopG.addEventListener('click', function () { closeChrom(true); }); + stopC.addEventListener('click', function () { + if (level() !== 'genome') { openChrom(tgt.ci, undefined, undefined, true); return; } + // From the genome: the chromosome of the first finding by level, else chromosome 1. + var f = drawn().slice().sort(function (a, b) { return PRI[a.level] - PRI[b.level] || M.by[a.chrom] - M.by[b.chrom]; })[0]; + openChrom(f ? M.by[f.chrom] : 0, undefined, undefined, true); + }); + window.addEventListener('hashchange', function () { readHash(false); }); + window.addEventListener('popstate', function () { readHash(false); }); + if (window.ResizeObserver) { new window.ResizeObserver(function () { if (M) { size(); clampTgt(); schedule(); } }).observe(stage); } + + // --- loading --- + function fail(why, words) { + failure = why; + fig.setAttribute('data-pgp-map', why); + stage.hidden = true; + marks.hidden = true; + fallback.hidden = false; + fallback.textContent = ''; + add(fallback, el('p', { 'class': 'gm-fail' }, words)); + var rows = opts.fallback ? opts.fallback() : []; + if (rows.length) { + add(fallback, U.table({ columns: ['Level', 'Finding', 'Where'], rows: rows }, 12)); + } else { + add(fallback, el('p', { 'class': 'dim' }, 'No finding with a place on the map is open on this page; every finding is listed in its section.')); + } + narr.textContent = ''; + layersBox.textContent = ''; + } + function load(k) { + key = k; + if (typeof window.DecompressionStream === 'undefined' || !window.Response || !window.ReadableStream) { + fail('nodecompress', 'This browser cannot open the map file. The findings with a place are listed here instead.'); + return; + } + fig.setAttribute('data-pgp-map', 'loading'); + fetch(opts.url, { credentials: 'same-origin', cache: 'no-store' }).then(function (r) { + if (r.status === 401) { window.location.href = '/login?next=' + encodeURIComponent(window.location.pathname); return null; } + if (!r.ok) { + return r.text().then(function (t) { throw new Error((t || '').trim() || ('the server answered ' + r.status)); }, function () { throw new Error('the server answered ' + r.status); }); + } + var t1 = performance.now(); + return new Response(r.body.pipeThrough(new window.DecompressionStream('gzip'))).text().then(function (txt) { + var p0 = performance.now(); + var d = JSON.parse(txt); + var m = prepare(d); + var p1 = performance.now(); + return { m: m, parse: p1 - p0, arrived: t1 }; + }); + }).then(function (x) { + if (!x || k !== key) { return; } + var fresh = !M; + M = x.m; + failure = null; + stage.hidden = false; + marks.hidden = false; + fallback.hidden = true; + hatch = hatch || pattern(0.16); + fig.setAttribute('data-pgp-map-parse-ms', String(Math.round(x.parse))); + fig.removeAttribute('data-pgp-map-paint-ms'); + t0 = x.arrived; + size(); + if (fresh) { + tgt.ci = cam.ci = 0; + tgt.c = cam.c = M.chroms[0].len / 2; + tgt.ls = cam.ls = Math.log(M.chroms[0].len); + } + cacheKey = ''; + drawKey(); + drawChips(); + draw(); + describe(); + fig.setAttribute('data-pgp-map', 'drawn'); + if (fresh) { readHash(true); } + }).catch(function (err) { + if (k !== key) { return; } + fail('failed', 'The map could not be loaded: ' + String((err && err.message) || err).split('\n')[0] + + ' The findings with a place are listed here instead.'); + }); + } + + return { + node: fig, + // The page drew again: maybe new results (a new key fetches the document again), maybe a result + // opened or closed (the veil is asked again). + refresh: function (k) { + if (k !== key) { load(k); } + if (failure) { + fail(failure, fallback.firstChild ? fallback.firstChild.textContent : ''); + return; + } + if (!M) { return; } + cacheKey = ''; + hideTip(); // a tooltip of a result closed since leaves the page + drawChips(); + schedule(); + describe(); + } + }; + } + + window.PGPMap = { create: create, prepare: prepare }; +}()); diff --git a/assets/live/person.js b/assets/live/person.js index 6dcd7490f..83ad5a151 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -1,7 +1,7 @@ // person.js: a person's genome as a report, the Journal design (docs/design/journal/), on their page // (assets/live/index.html), from results.json (bin/live_results.py, served by // bin/owngenome_live_results.py) and the run's states (live.js). The page reads top to bottom: the lead -// paragraph built from the findings, the counts by level, the chromosome map of the findings, then ten +// paragraph built from the findings, the counts by level, the genome map of the findings (map.js), then ten // numbered sections: 1 To act on, 2 Medicines, 3 For your family, 4 Worth a look, 5 Can I trust this?, // 6 Origins, 7 Traits and scores, 8 Assessed, no finding, 9 What it cannot see, 10 Sources and files. // A run still going is the same report being written: each section carries its stamp (written, being @@ -21,6 +21,7 @@ // 'none': in no family; 'pending' or 'failed': in one whose roles are not known (yet); 'known'. var familyStatus = 'none'; var shown = {}; // veiled findings the reader opened, by id + var genomeMap = null, resultsKey = null; // Figure 1 (map.js), and the results it was built for var panel = document.getElementById('results'); var rail = document.getElementById('rail'); var NOT_ALL_CLEAR = 'No finding in what these steps assessed.'; @@ -428,8 +429,10 @@ add(text, lineList(lines)); if (c.hidden) { // APOE and the like: behind their warning, opened on purpose. - var d = el('details', { 'class': 'behind' }); + // Its fold is the map's APOE switch too (map.js): opening it here opens APOE on the map, and back. + var d = el('details', { 'class': 'behind', 'data-gate': 'apoe' }); add(d, el('summary', {}, c.hidden.summary), el('p', {}, c.hidden.text)); + d.addEventListener('toggle', function () { if (d.open !== !!shown.apoe) { setShown('apoe', d.open); } }); add(text, d); } if (c.explore && c.explore.length) { @@ -527,6 +530,37 @@ return !!(t && tagClosed(t)); } function openFinding(f) { shown[f.id] = true; var t = findingTag(f); if (t) { shown[t] = true; } } + // The genome map's veil (map.js): 'closed' while this page keeps the result closed, 'opened' once it was + // opened on purpose, 'open' when this page never closes it. A gated result (APOE, behind its warning) is + // closed on every page, an adult's too, until opened; a tag naming no finding of these results stays closed. + function mapVeil(id, gated) { + if (gated) { return shown[id] ? 'opened' : 'closed'; } + if (!closedPage()) { return 'open'; } + var f = findingById(id); + if (!f || veiled(f)) { return 'closed'; } + return f.level === 'act' || findingTag(f) ? 'opened' : 'open'; + } + // One switch for a result, from the map's chip or the card's "Show the result" (openFinding): opening it in + // one place opens it in both. Closing it closes the findings that were opened with it. + function setShown(id, on) { + var f = findingById(id); + if (on) { if (f) { openFinding(f); } else { shown[id] = true; } } else { + delete shown[id]; + ((results && results.findings) || []).forEach(function (x) { if (findingTag(x) === id) { delete shown[x.id]; } }); + } + redraw(true); + } + // Whether the page shows a finding of the map at this moment: its card's steps are done (a replay draws + // the run's end results from its first frame). A row with no finding of these results (the map's own APOE + // row) follows its card's fold instead. + function mapNow(id) { var f = findingById(id); var c = f && cardById(f.card); return !f || !!(c && gate(c) === 'show'); } + // Without the map (a browser with no DecompressionStream, or a map the server could not build): the findings + // with a place, as a table, each through this page's veil first. + function mapFallback() { + return findingsNow().filter(function (f) { return f.viewer && !veiled(f); }).map(function (f) { + return [U.LEVEL_WORDS[f.level] || f.level, f.title, f.viewer]; + }); + } function findingArticle(f, big) { var row = el('div', { 'class': 'row' }); var art = el('div', { 'class': 'finding', id: 'f-' + f.id, 'data-finding': f.id, 'data-level': f.level }); @@ -843,22 +877,16 @@ add(side, note('A research pipeline', el('span', {}, 'Not a clinical test: anything to act on is confirmed by a certified laboratory first.'))); add(row, main, side); add(slot, row); - // Figure 1: where the findings sit. - var marks = []; - fs.forEach(function (f) { - var l = U.locus(f.viewer); - if (l && !veiled(f)) { marks.push({ chr: l.chr, mb: l.mb, lv: f.level, label: f.gene || '', tip: f.title, href: findingHref(f) }); } - }); - var fig = el('figure', { 'class': 'row fig1' }); - var inner = el('div', {}); - add(inner, el('div', { 'class': 'fig-label' }, 'Figure 1 · Where the findings sit')); - var k = el('div', { 'class': 'kary', id: 'kary' }); - add(k, U.karyogram(marks)); - add(inner, k); - add(inner, el('figcaption', {}, marks.length ? 'Each mark is one finding with a position, on its chromosome. Shape and colour give the level, as in the row above; every finding is also listed in its section.' - : 'No finding with a position yet. The findings without one (medicine genes, repeat counts, copy numbers) are listed in their sections.')); - add(fig, inner, el('aside', {})); - add(slot, fig); + // Figure 1: the genome map (map.js), one figure kept across redraws so its camera stays where the + // reader left it. It asks this page's veil for each closed row, and opens or closes a result through it. + if (window.PGPMap) { + if (!genomeMap) { + genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, open: function (id) { setShown(id, true); }, + close: function (id) { setShown(id, false); }, fallback: mapFallback }); + } + add(slot, genomeMap.node); + genomeMap.refresh(resultsKey); + } } // --- the table of contents, the phone tab bar, the cover ------------------------------------------ @@ -1106,6 +1134,7 @@ if (key !== fetchedFor) { return; } if (!(r && (r.chapters || r.note))) { fetchedFor = null; return; } results = r; + resultsKey = key; redraw(true); route(); }).catch(function () { if (key === fetchedFor) { fetchedFor = null; } }); @@ -1145,6 +1174,8 @@ drawLead(); drawSections(); restoreFolds(); + // A gated result's fold follows its switch, whatever fold state the reader left (setShown). + Array.prototype.forEach.call(panel.querySelectorAll('details[data-gate]'), function (d) { d.open = !!shown[d.getAttribute('data-gate')]; }); window.scrollTo(0, y); document.getElementById('progress-q').textContent = running() ? 'Is it moving, when will it end, is anything wrong? One line per step.' : 'One line per step: when it ran, and what it found.'; From 39749a0b3072354407db70bb498836911785f970 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:28:26 +0100 Subject: [PATCH 010/255] ci(docs): build og so the demo site carries the map documents The demo build needs og to write map.json for each run. Docs now builds og, checks that each demo run has its map, and uploads screenshots of the map as the map-screens artifact. Bead: pgp-iqon.8 --- .github/workflows/docs.yml | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fe99ccddf..87f0a1229 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -31,6 +31,9 @@ on: - 'bin/**' - 'tests/demo/**' - 'tests/fixtures/live/**' + # The demo's Figure 1, the genome map, is built by og (og map build) from these. + - 'rust/**' + - 'tests/fixtures/map/**' # No paths filter on pull requests, on purpose: a check skipped by a path # filter stays Pending and blocks the merge once it is a required check # (repos.md step 5 makes green checks required). The build takes seconds. @@ -92,10 +95,41 @@ jobs: # (tests/demo/build_live_demo.py; standard library only). It is a static # folder beside the pages, not a page of the nav, so it goes in after the # site check. + # The demo's Figure 1 is the genome map, whose documents og writes (og map build). The map reads no + # BAM, so a native build without the bam feature is all it needs: no htslib, no container, and the + # cache keeps it to seconds. + - name: Install the toolchain rust/rust-toolchain.toml pins + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust/rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal + rustc --version + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: rust + + - name: Build og for the demo's map + working-directory: rust + run: cargo build --release --locked -p og --no-default-features + - name: Add the live-site demo run: | - python3 tests/demo/build_live_demo.py --out site/demo + python3 tests/demo/build_live_demo.py --out site/demo --og rust/target/release/og test -s site/demo/runs/DEMO-001/frames.json + for r in DEMO-001 DEMO-002 DEMO-003; do test -s "site/demo/runs/${r}/map.json"; done + + # The map at 1440 x 900 and 390 x 844, dark: genome, chromosome and a closed page, kept as the + # artifact map-screens (tests/test_live_demo.py --map-screens; Chrome is on the runner). + - name: Screenshots of the genome map + run: python3 tests/test_live_demo.py --map-screens "$RUNNER_TEMP/map-screens" site/demo + + - name: Upload the map screenshots + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: map-screens + path: ${{ runner.temp }}/map-screens/*.png + if-no-files-found: error + retention-days: 14 # Push to main only: workflow_dispatch can be run on any branch, and the # push filter above does not apply to it. Same line as docs-pages-helm.yml. From 77c4d0c0a11eafb43133e1ddddc0155314102e22 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:28:26 +0100 Subject: [PATCH 011/255] docs(live): describe Figure 1, the genome map Bead: pgp-iqon.8 --- docs/live-site.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/live-site.md b/docs/live-site.md index 11b784bc2..025b3f34c 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -10,7 +10,7 @@ https://geiserx.github.io/OwnGenome/demo/ is the live site on DEMO-001, the inve ![The person page of DEMO-001 at the end of the demo run: the name with its family and run state, the contents of the ten sections with a count per level, the lead paragraph (no finding for a doctor, two medicine results, six worth a look), the five-level ledger, the data-quality line and the start of the chromosome map of the findings](images/demo-live-site.png){ loading=lazy } -The pages and their plugins are the files of `assets/live/`, unchanged. Each moment of the replay is the server's own `state.json` at that moment: `tests/demo/build_live_demo.py` loads every plugin, stores the made-up messages of `tests/fixtures/live/demo-001.weblog.jsonl` one by one as the ingest port would, and reads the state after each. The cards come from the `results.json` the server would give at the end of the run; each shows once the replay reaches the end of its step. The home page, the charts, the search box, the run-health card, the database re-check and the family page work as they do on your own site. The genome viewer does not: it opens your own reads on your own computer, so its links lead to a page that says it is not in the demo. +The pages and their plugins are the files of `assets/live/`, unchanged. Each moment of the replay is the server's own `state.json` at that moment: `tests/demo/build_live_demo.py` loads every plugin, stores the made-up messages of `tests/fixtures/live/demo-001.weblog.jsonl` one by one as the ingest port would, and reads the state after each. The cards come from the `results.json` the server would give at the end of the run; each shows once the replay reaches the end of its step. The home page, the charts, the search box, the run-health card, the database re-check and the family page work as they do on your own site. The genome map of each person page is the document `og map build` writes for that run, built once when the demo is. The genome viewer does not work: it opens your own reads on your own computer, so its links lead to a page that says it is not in the demo. What differs from your own site: @@ -48,7 +48,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th **Home** (`/`). A date line, then one sentence on where everyone stands (how many reports are finished or being written, and whether any finding needs a doctor), and beside it the findings that need attention across everyone, one line per person and level, each opening its finding page. A run still going comes first, as "Being written", with its steps done and the sections already written. A run with no end that has written nothing for a long time reads "Stopped" instead, with the day it was last heard from, and comes after the runs still going: twice the longest step's usual hours (27 h, as DeepVariant's 13.5 h can pass with no message) and never under 6 h. It reads "Being written" again as soon as something new arrives (a message, a trace row, a status row). Then each family: a small pedigree in the members' colours and one tile per person, with the first lines of their report's lead and their counts by level. A family member with no run on this server is listed, dashed. Last, every run as a log: state, steps, hours, when it ended. Without scripts the page is a plain list of runs. -**A person** (`/runs//`). The name, the sex the reads show against the declared one, the person's place in the family and the run state. Down the left, the contents: the ten sections with how many findings each level has, then the step timeline and the family. The page opens on the lead: a paragraph built from the findings (data-quality problems first, then the findings for a doctor named one by one, the medicine genes, the carrier genes, what is worth a look, and the mean depth; with no finding for a doctor it says so only when a clinical screen ran, and otherwise that the clinical screens have no result yet), the ledger of the five levels with a count each, the data-quality line, and Figure 1, the chromosomes with each finding at its place. Then the sections: +**A person** (`/runs//`). The name, the sex the reads show against the declared one, the person's place in the family and the run state. Down the left, the contents: the ten sections with how many findings each level has, then the step timeline and the family. The page opens on the lead: a paragraph built from the findings (data-quality problems first, then the findings for a doctor named one by one, the medicine genes, the carrier genes, what is worth a look, and the mean depth; with no finding for a doctor it says so only when a clinical screen ran, and otherwise that the clinical screens have no result yet), the ledger of the five levels with a count each, the data-quality line, and Figure 1, the genome map. Then the sections: 1. **To act on**, 2. **Medicines**, 3. **For your family**, 4. **Worth a look**, 5. **Can I trust this?**: each finding of the level, with what it is, what to do, and in the margin its evidence, its caveat and its link to the report; then the cards of the report's sections that belong there. 6. **Origins** and 7. **Traits and scores**: the ancestry and trait cards. @@ -56,7 +56,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the chromosome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. @@ -70,6 +70,8 @@ While the run goes, the page still opens on the report being written. A short ba The header search finds a gene or rsID across the results and opens the card that names it; on a page that keeps results closed it leaves out every hit that names one (above). One row of the results that the page cannot read (a field of the wrong type) costs that row only: it shows as a finding on its card, "A row of this section could not be read", at the highest level its rule can give (to act on for a rule that can give one, so a page that keeps results closed keeps that card closed too), the card whose table cannot be drawn says so, and every other card and finding is on the page. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. +**Figure 1, the genome map.** The 24 chromosomes as rows, to scale (23 for a person with no Y), each with its bands, the centromere as a waist and the stretches few reads can map hatched (centromeres, heterochromatin, the acrocentric arms). Each finding with a place sits on its chromosome as its level's shape, with its gene beside it when the name fits; runs of homozygosity of 5 Mb or more are a line under the row, shorter ones a faint band. A finding that names a gene but no position (a medicine gene, Lp(a)) sits on that gene's span; a finding with neither stays in its section only. Click a chromosome (or, with the map focused, press `[` or `]`) and it opens in place: an overview with the part in view, a ruler in Mb, the bands with their names, a findings lane with each gene and its level's words, and a lane for the runs of homozygosity, each run with its length. Drag to move along it, scroll to zoom in (as far as about 10 Mb across a wide screen; closer views come later), and press Esc (or click Genome) to go back. A dashed line marks the centre, and the margin says what is under it: the chromosome, the position and the band, the findings in view and the runs. Every sentence in the margin is one the map document carries; the page only fills in the counts. The address follows the map: `#genome`, `#chr7`, or `#chr7:117.4-137.4M`, and a link to one of them opens that view; a place the page cannot read shows a short note and leaves the map as it was. The map comes from `/runs//map.json`, which the map plugin (`bin/owngenome_live_map.py`) builds with `og map build` from the run folder and the results the page gets (`rust/crates/og-map/README.md` describes the document). It needs og on the host (`scripts/setup.sh --og`); without it, or in a browser that cannot unpack the file (no `DecompressionStream`), the figure says so and lists the findings with a place in a table instead. A layer whose file is missing or cannot be read says so in words, never as a blank lane. + **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. **A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. On a phone the grid becomes a list of pairs. From f29223760904af9c9d691edd47edeadbafe5df52 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:25:11 +0100 Subject: [PATCH 012/255] fix(live): close every place of a child on the family page while any result of theirs is closed What the family page showed of a child depended on which result was closed and where it lay: a missing side-by-side row named the gene, the painting closed only when a grandparent stretch lay over the result, one 'Show the rows' opened every table, and the couple screen had no veil at all. Now, while a child has a result still closed (or the page cannot read the child's results), the painting's grandparents, the map image, every de novo and compound-het row, the child's medicines and HLA cells and the screen of a couple the child is part of close in the same way, one box each. Each click opens one result; under an unreadable state each box opens its own place. core.js veilId()/veilIds() lose their only caller and are removed. --- CHANGELOG.md | 2 +- assets/live/core.js | 49 +-- assets/live/family-page.js | 207 +++++---- bin/live_results.py | 7 +- docs/live-site.md | 4 +- tests/demo/make_demo_sample.py | 2 +- tests/fixtures/live/demo/README.md | 2 +- tests/test_live_demo.py | 636 +++++++++++++++++----------- tests/test_live_results.py | 64 +-- tests/test_owngenome_live_family.py | 15 + 10 files changed, 520 insertions(+), 468 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69de5401e..d309c0213 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The family page now keeps a child's results to act on closed, as the child's own page does. A de novo candidate or a compound-het pair that names a closed result, and a side-by-side row whose value in the child's column names one (a CPIC gene such as RYR1), stay behind "Show the result" until that result is opened there; a reload closes them again. While a stretch painted from a grandparent lies over a closed result, the child's painting shows every stretch from its parent alone. While the page cannot read the child's results, every such row of that child stays closed (in the side by side, every medicines and HLA row). On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: the files behind the pages hold every row. +- The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. - Step 49 (clonal haematopoiesis) now prepares the CHIP gene models and writes its two tables with the Rust `og` binary: `og chip prepare`, `og chip report` and `og chip check-table` replace the commands of `bin/chip_report.py`, which keeps only the detection limit and the column names the reports import. On Ensembl 116's full GFF3, prepare takes 1.7 s and 6 MB instead of 15 s and 20 MB. The two tables, `chip.gff3` and `chip.bed` are the same, byte for byte. What changes: a missing input file or a position that is not a whole number stops with an `ERROR:` line and exit 1, not a Python traceback, and each output is written whole or not at all. Step 49 now needs `og` (`scripts/setup.sh --og`). - VEP consequence ranking (steps 23 and 31) and the gnomAD constraint join now run in the Rust `og` binary, inside `BCFTOOLS_IMAGE`: `og vcf csq-rank` replaces `bin/csq_rank.awk` and `og table constraint-join` replaces `bin/constraint_join.awk`, which are gone. Neither awk program runs on the host anymore (step 23 still uses the host's `awk` for small table edits), and the ranker is no longer the slow stage of step 23's main pipe. Steps 23 and 31 now need `og`: `scripts/setup.sh --og` builds it. Their output is the same, byte for byte, except for these changes. diff --git a/assets/live/core.js b/assets/live/core.js index fff06e955..723309e3c 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -219,55 +219,8 @@ add(v, b); return v; } - // The id of the closed result a row belongs to, or null: bin/live_results.py veil_id() word for word, for the - // rows only the family page holds (a child's de novo and compound-het candidates). spans: results.json's - // veil.spans, [[id, gene, chrom, start, end]]. text: what the row shows (its gene names; its loci are read - // too); chrom, pos and end its own position. tests/test_live_results.py runs it under node against veil_id(). - var LOCUS_RE = /(?= s[3]; })) { return s[0]; } - } - return null; - } - // Every closed result a row belongs to, by the same rule taken one result at a time: a compound-het pair can - // name two, and a painted stretch can hold two, so the family page keeps such a row closed until each is opened. - function veilIds(spans, text, chrom, pos, end) { - return (spans || []).filter(function (s) { return veilId([s], text, chrom, pos, end); }).map(function (s) { return s[0]; }); - } - window.PGPUI = { el: el, svg: svg, add: add, getJSON: getJSON, avatar: avatar, day: day, caveat: caveat, facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, glyph: glyph, karyogram: karyogram, locus: locus, LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, - hm: hm, dm: dm, hours: hours, veil: veil, veilId: veilId, veilIds: veilIds }; + hm: hm, dm: dm, hours: hours, veil: veil }; }()); diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 174352935..584ed17d3 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -7,9 +7,8 @@ // members side by side. It draws what the steps wrote and computes nothing about genetics. Text through textContent, // the drawing through createElementNS; images from this server only. // A child's results to act on stay closed here as on the child's own page (person.js): each child's results.json -// gives the closed results (veil.spans), and a row of the child that names one (a de novo or compound-het -// candidate, a side-by-side row) stays closed until that result is opened on purpose here; while a stretch -// painted from a grandparent lies over one, the child's painting shows each stretch's parent only. +// gives the closed results (veil.spans), and while one is closed every place of this page that could carry one +// closes for that child, the same whichever result it is (the veil below). (function () { 'use strict'; var U = window.PGPUI; @@ -55,6 +54,7 @@ out.push([seq + ' genome' + (seq === 1 ? '' : 's') + ' of this family are read together here. ', false]); (st.couples || []).forEach(function (c) { if (!c.screened) { out.push([c.a + ' and ' + c.b + ' are not screened as a couple yet. ', false]); return; } + if (coupleShut(f, c).length) { out.push([c.a + ' and ' + c.b + '’s carrier screen stays closed in chapter III. ', false]); return; } if (c.shared) { out.push([c.a + ' and ' + c.b + ' both carry a change in ' + c.shared + ' gene' + (c.shared > 1 ? 's' : '') + ': chapter III gives the chance per pregnancy for each. ', true]); } else { out.push([c.a + ' and ' + c.b + ' share no carrier gene in what the screen sees; ', false], ['it does not see everything. ', false]); } if (c.x_linked) { out.push([c.x_linked + ' X-linked finding' + (c.x_linked > 1 ? 's' : '') + ' bear on sons. ', true]); } @@ -76,14 +76,21 @@ // --- the veil --------------------------------------------------------------------------------------- // What this page knows of each child's results: { status: 'pending', 'known' or 'failed', spans, acts }. A child - // with no run here, or whose results.json has not answered or could not be read, cannot say which rows name a - // result to act on, so every row of theirs that could stays closed (UNKNOWN) until opened on purpose. - // Opened results are kept for the life of this page, as on the child's page: a reload closes them again. They - // are kept per child and per run (two children can carry the same result, under the same id). + // with no run here, or whose results.json has not answered or could not be read, cannot say whether a result to + // act on is closed (UNKNOWN). + // What the page shows of a child never depends on which result is closed or where it lies: while a child has any + // result to act on still closed here, or while the page cannot tell, every place that could carry one closes for + // that child in the same way, with one box each and the same words whatever the result. The places: the + // painting's grandparents, the map as an image, the de novo and compound-het rows, the child's medicines and HLA + // cells of the side by side, and the carrier screen of a couple the child is part of. A known result opens with + // its own click, from any of those boxes; the places open once the last one is. While the page cannot tell, each + // box opens only its own place. Opened results are kept for the life of this page, as on the child's page: a + // reload closes them again. They are kept per child and per run (two children can carry the same result id). var UNKNOWN = '?'; var GENE_GROUPS = { Medicines: true, HLA: true }; // the side-by-side groups whose rows name a gene - var kids = {}, opened = {}, openedAll = {}, mapShown = {}, last = null; + var kids = {}, opened = {}, last = null; function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; } + function isChild(f, s) { return (f.state.children || []).some(function (c) { return c.child === s; }); } function loadKids(f) { (f.state.children || []).forEach(function (c) { var url = memberUrl(f, c.child); @@ -100,7 +107,8 @@ // Results the server could not build come back without their findings: as unreadable as no answer. var usable = !!(r && Array.isArray(r.findings)); var acts = usable ? r.findings.filter(function (x) { return x.level === 'act'; }).map(function (x) { return x.id; }) : []; - var next = usable ? { status: 'known', url: url, spans: v.spans || null, acts: acts } : { status: 'failed', url: url }; + var next = usable ? { status: 'known', url: url, spans: Array.isArray(v.spans) ? v.spans : null, acts: acts } + : { status: 'failed', url: url }; // An answer that cannot be read keeps what was known: the page never opens on a failed request. if (!usable && cur.status === 'known') { return; } var key = JSON.stringify([next.status, next.spans, next.acts]); @@ -116,47 +124,38 @@ }); }); } - // The closed results a row of CHILD belongs to: their ids, [UNKNOWN] when this page cannot tell, or []. - // COORDS: the row has a position and no gene name (a de novo candidate, a stretch). A closed result known by its - // gene alone (two changes in one gene, with no position) could sit at that position, and only the gene would - // tell: such a row stays closed with every result of that kind too. - function tagOf(child, text, chrom, pos, end, coords) { + // The results of CHILD still closed, as the place BOX sees them: their ids, [UNKNOWN] while this page cannot read + // them (until BOX's own button opens BOX), or [] once every one is opened. Results with act findings but no + // veil.spans cannot say which they are: UNKNOWN too. + function shutOf(child, box) { var k = kids[child]; - if (!k || k.status !== 'known') { return [UNKNOWN]; } - if (!k.spans) { return k.acts.length ? [UNKNOWN] : []; } // results without the tags: cannot tell - var ids = U.veilIds(k.spans, text, chrom, pos, end); - if (coords) { k.spans.forEach(function (s) { if (!s[2] && ids.indexOf(s[0]) < 0) { ids.push(s[0]); } }); } - return ids; - } - // The results of TAGS still closed for CHILD: a row stays closed while one of them is. - function stillClosed(child, tags) { - return tags.filter(function (t) { return t === UNKNOWN ? !openedAll[child] : !opened[openKey(child, t)]; }); - } - // The same for a whole drawing of CHILD (its painting, its map as an image): while this page cannot read the - // child's results, only the drawing's own button (KEY) opens it, never "Show the rows" of a table. - function drawingClosed(child, tags, key) { - return tags.filter(function (t) { return t === UNKNOWN ? !mapShown[openKey(child, key)] : !opened[openKey(child, t)]; }); - } - // The rows of CHILD split by the veil: { open: rows, closed: [the still closed results of each closed row] }. - function veiled(child, rows, tagFn) { - var out = { open: [], closed: [] }; - rows.forEach(function (r) { - var t = stillClosed(child, tagFn(r)); - if (t.length) { out.closed.push(t); } else { out.open.push(r); } - }); + if (!k || k.status !== 'known' || (!k.spans && k.acts.length)) { + return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN]; + } + var out = []; + (k.spans || []).forEach(function (s) { if (!opened[openKey(child, s[0])] && out.indexOf(s[0]) < 0) { out.push(s[0]); } }); return out; } - // The closed box under a table of CHILD: how many rows stay closed, and one button that opens one result. - function veilBox(child, closed, what, whats) { - if (!closed.length) { return null; } - var n = closed.length === 1 ? 'One ' + what + ' stays' : closed.length + ' ' + whats + ' stay'; - var first = closed[0][0]; - if (first === UNKNOWN) { - return U.veil(n + ' closed: this page cannot read ' + child + '’s results yet, so it cannot tell which ' + whats + - ' name a result to act on in adult life.', 'Show the ' + whats, function () { openedAll[child] = true; redraw(); }); - } - return U.veil(n + ' closed with the result to act on in adult life ' + (closed.length === 1 ? 'it belongs' : 'they belong') + - ' to, until that result is opened.', null, function () { opened[openKey(child, first)] = true; redraw(); }); + // The closed box of the place BOX of CHILD: WHAT says what stays closed; one button opens one result (BUTTON + // while the page cannot read the child's results, and then it opens this place only). + function veilBox(child, shut, box, what, button) { + if (!shut.length) { return null; } + var v = shut[0] === UNKNOWN + ? U.veil(what + ': closed, as this page cannot read ' + child + '’s results yet, so it cannot tell whether a ' + + 'result to act on in adult life is among them.', button, function () { opened[openKey(child, 'box:' + box)] = true; redraw(); }) + : U.veil(what + ': closed while ' + child + ' has a result to act on in adult life that is closed here.', null, + function () { opened[openKey(child, shut[0])] = true; redraw(); }); + v.setAttribute('data-veil-child', child); + v.setAttribute('data-veil-box', box); + return v; + } + + // A couple's carrier screen names the partners' carrier genes and their positions, so it closes while a partner + // who is a child of this family has a result still closed: [[that partner, their closed results]]. + function coupleShut(f, c) { + return [c.a, c.b].filter(function (m) { return isChild(f, m); }) + .map(function (m) { return [m, shutOf(m, 'couple:' + c.a + ' ' + c.b)]; }) + .filter(function (x) { return x[1].length; }); } // The pedigree drawn from the layout the server gives: a sequenced person with a run here is a link. @@ -261,6 +260,16 @@ add(s, p); return; } + var cs = coupleShut(f, c); + if (cs.length) { + // The whole screen closes: its counts, its boxes, its table and its report. + p.textContent = c.a + ' and ' + c.b + ': the carrier screen stays closed here.'; + add(s, p); + cs.forEach(function (x) { + add(s, veilBox(x[0], x[1], 'couple:' + c.a + ' ' + c.b, 'The carrier screen of ' + c.a + ' and ' + c.b, 'Show the screen')); + }); + return; + } var rows = (f.couples || {})[c.a + ' ' + c.b] || []; var one = rows.filter(function (r) { return r.result === 'one_partner'; }).length; // A count of zero is not a clean result: the screen sees only what it lists as assessed. @@ -377,15 +386,12 @@ var SIDE = { paternal: 'father', maternal: 'mother' }; function painting(f, child, all) { // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a - // closed result says whether the child got it. A stretch from a parent alone says nothing. Closing only that - // stretch would point at the closed result (a hatch, a gap in the table), so while one is closed every stretch - // of this child is drawn from its parent alone, and no grandparent, basis or closed mark is shown. - var shut = [], pk = openKey(child, 'paint'); - all.forEach(function (r) { - if (!r.ancestor || r.ancestor === '.') { return; } - drawingClosed(child, tagOf(child, r.chrom + ':' + r.start + '-' + r.end, r.chrom, r.start, r.end, true), 'paint') - .forEach(function (x) { if (shut.indexOf(x) < 0) { shut.push(x); } }); - }); + // closed result says whether the child got it. A stretch from a parent alone says nothing. Closing only the + // stretches over a result, or the painting only when one lies over it, would point at where the result lies, so + // while the child has any result still closed every stretch is drawn and listed from its parent alone, with no + // grandparent, basis or closed mark, behind one box. + var anc = all.some(function (r) { return r.ancestor && r.ancestor !== '.'; }); + var shut = anc ? shutOf(child, 'paint') : []; var rows = !shut.length ? all : all.map(function (r) { return { side: r.side, parent: r.parent, chrom: r.chrom, start: r.start, end: r.end, sites: r.sites, discordant: r.discordant, ancestor: '.', basis: 'parent' }; @@ -434,16 +440,8 @@ }); add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).')); add(fig, leg); - if (shut.length) { - // One box for the whole painting: a count of closed stretches would say how many sit over the result. - add(fig, shut[0] === UNKNOWN - ? U.veil('The grandparents of ' + child + '’s stretches stay hidden: this page cannot read ' + child + '’s results yet, ' + - 'so it cannot tell which stretch lies over a result to act on in adult life. Each stretch shows its parent only.', - 'Show the grandparents', function () { mapShown[pk] = true; redraw(); }) - : U.veil('The grandparents of ' + child + '’s stretches stay hidden while a result to act on in adult life is ' + - 'closed: they could show which copy the child got there. Each stretch shows its parent only.', null, - function () { opened[openKey(child, shut[0])] = true; redraw(); })); - } + // One box for the whole painting: each stretch shows its parent only. + add(fig, veilBox(child, shut, 'paint', 'The grandparents of ' + child + '’s stretches', 'Show the grandparents')); var d = el('details', { 'class': 'w-fallback' }); add(d, el('summary', {}, 'Every stretch (' + rows.length + ')')); add(d, U.table({ columns: ['Copy', 'Chromosome', 'Start', 'End', 'From', 'Sites', 'Discordant', 'Basis'], @@ -474,15 +472,11 @@ if (rows && rows.length) { add(d, painting(f, c.child, rows)); } else if (c.segments) { - // The drawing as an image cannot close one stretch: while this child has a result to act on that is - // closed (or this page cannot tell), it stays closed whole. - var k = kids[c.child] || {}; - var all = k.status !== 'known' || (!k.spans && k.acts.length) ? [UNKNOWN] : (k.spans || []).map(function (s) { return s[0]; }); - var shut = drawingClosed(c.child, all, 'map'), mk = openKey(c.child, 'map'); - // "Show the map" opens the map only: each result keeps its own closed rows until it is opened. - if (shut.length && !mapShown[mk]) { - add(d, U.veil('The chromosome map of ' + c.child + ' stays closed while a result to act on in adult life is: ' + - 'it could show which copy the child got there.', 'Show the map', function () { mapShown[mk] = true; redraw(); })); + // The drawing as an image is coloured by grandparent too, and cannot close one stretch: it stays closed + // whole, as the painting does. + var shut = shutOf(c.child, 'map'); + if (shut.length) { + add(d, veilBox(c.child, shut, 'map', 'The chromosome map of ' + c.child + ' and its colours', 'Show the map')); } else { add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); } @@ -492,24 +486,31 @@ var dn = (f.de_novo || []).filter(function (r) { return r.child === c.child; }); add(d, el('h3', {}, 'De novo candidates')); if (dn.length) { - var dv = veiled(c.child, dn, function (r) { return tagOf(c.child, r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.chrom, r.pos, null, true); }); add(d, el('p', { 'class': 'dim' }, 'In the child and in neither parent. Most candidates in a genome are genotype errors: ' + 'check the reads before reading anything into one.')); - add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], - rows: dv.open.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, - r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); - add(d, veilBox(c.child, dv.closed, 'row', 'rows')); + // Every row closes while a result is: a row closed on its own would say where the result lies. + var dshut = shutOf(c.child, 'denovo'); + if (dshut.length) { + add(d, veilBox(c.child, dshut, 'denovo', 'The de novo candidates of ' + c.child, 'Show the rows')); + } else { + add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], + rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, + r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); + } } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'de_novo' }, empty(c.child, 'de_novo', 'de_novo.tsv'))); } var ch = (f.compound_het || []).filter(function (r) { return r.child === c.child; }); add(d, el('h3', {}, 'Compound-het pairs')); if (ch.length) { - var cv = veiled(c.child, ch, function (r) { return tagOf(c.child, [r.gene, r.variant_a, r.variant_b].join(' ')); }); add(d, el('p', { 'class': 'dim' }, 'Two candidate variants in one gene, in trans first: one from each parent.')); - add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], - rows: cv.open.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); - add(d, veilBox(c.child, cv.closed, 'row', 'rows')); + var hshut = shutOf(c.child, 'het'); + if (hshut.length) { + add(d, veilBox(c.child, hshut, 'het', 'The compound-het pairs of ' + c.child, 'Show the rows')); + } else { + add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], + rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); + } } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'compound_het' }, empty(c.child, 'compound_het', 'compound_het.tsv'))); } @@ -525,26 +526,18 @@ if (!sb || !sb.rows.length) { add(s, el('p', { 'class': 'dim' }, 'No member has a summary yet, or no summary lists these results.')); return s; } add(s, el('p', { 'class': 'dim' }, 'Each member’s own report, one column per person: CYP2D6, the CPIC genes ' + '(phenotypes in CPIC’s words), HLA, the haplogroups and the blood groups.')); - // A child's value that names a closed result (a CPIC gene that is also the gene of a result to act on, RYR1 or - // CACNA1S) closes its whole row, as the child's own page closes that row of its medicines card: a closed cell - // under the gene's name would say which gene it is. While this page cannot read a child's results, every row - // that names a gene (the medicines and HLA rows) stays closed; the haplogroups and blood groups name none. - var kidsHere = (f.state.children || []).map(function (c) { return c.child; }); - var closedBy = {}, shown = []; - sb.rows.forEach(function (r) { - var open = true; - sb.members.forEach(function (m, i) { - if (kidsHere.indexOf(m) < 0) { return; } - var t = stillClosed(m, tagOf(m, r.label + ' ' + (r.values[i] || ''))); - if (t[0] === UNKNOWN && !GENE_GROUPS[r.group]) { return; } - if (t.length) { (closedBy[m] = closedBy[m] || []).push(t); open = false; } - }); - if (open) { shown.push(r); } - }); + // A child's medicines and HLA cells name genes (a CPIC gene can be the gene of a result to act on, RYR1 or + // CACNA1S). Closing only the row of a closed result's gene would leave a gap in a list everyone knows, so + // while a child has any result still closed every medicines and HLA cell of the child's column reads "closed": + // every row stays, with the parents' values. The haplogroups and blood groups name no gene. + var shutBy = {}; + sb.members.forEach(function (m) { if (isChild(f, m)) { shutBy[m] = shutOf(m, 'side'); } }); // A value from a section older than its input (state "stale") says so in its own cell. - var group = null, rows = [], old = 0; - shown.forEach(function (r) { + var group = null, rows = [], old = 0, closedCells = {}; + sb.rows.forEach(function (r) { rows.push([r.group === group ? '' : r.group, r.label].concat(r.values.map(function (v, i) { + var m = sb.members[i]; + if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length) { closedCells[m] = true; return 'closed'; } if (!(r.stale || [])[i]) { return v; } old++; return v + STALE; @@ -553,8 +546,7 @@ }); add(s, U.table({ columns: ['', 'Gene or system'].concat(sb.members), rows: rows }, 14)); sb.members.forEach(function (m) { - var box = closedBy[m] && veilBox(m, closedBy[m], 'row', 'rows'); - if (box) { box.setAttribute('data-veil-child', m); add(s, box); } + if (closedCells[m]) { add(s, veilBox(m, shutBy[m], 'side', 'The medicines and HLA cells of ' + m, 'Show the cells')); } }); if (old) { add(s, el('p', { 'class': 'dim', 'data-stale': String(old) }, 'Marked' + STALE + ': the member’s result file is from an ' + @@ -572,7 +564,12 @@ } if (f.state.reports.length) { var ul = el('ul', { 'class': 'lines' }); - f.state.reports.forEach(function (r) { add(ul, add(el('li', {}), el('a', { href: r.url }, r.name))); }); + // A couple report of a screen that stays closed is left out with it. + var shutReports = (f.state.couples || []).filter(function (c) { return c.report && coupleShut(f, c).length; }) + .map(function (c) { return c.report; }); + f.state.reports.forEach(function (r) { + if (shutReports.indexOf(r.url) < 0) { add(ul, add(el('li', {}), el('a', { href: r.url }, r.name))); } + }); add(s, ul); } return s; diff --git a/bin/live_results.py b/bin/live_results.py index a2202011f..3f0a3a603 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -1211,10 +1211,9 @@ def veil(summary, found, chapters, lead_parts=()): id}), every other finding that names a closed result (findings: finding id -> id; a symptom ranking can rank the gene) and the lead's segments that name one (lead: one per segment of LEAD_PARTS, the "act" names left out: the page closes those itself), each card's facts (facts: card -> one per fact) and the items - of its drawing (widget: card -> one per CHIP call or copy-number dot). spans is closable() itself, so the - family page (assets/live/family-page.js) can tag its own rows of a child, the de novo and compound-het - candidates, by this rule (core.js veilId(), checked against veil_id() under node). Empty when nothing can - be closed.""" + of its drawing (widget: card -> one per CHIP call or copy-number dot). spans is closable() itself: the family + page (assets/live/family-page.js) reads which results are closed from it, and closes every place of the child + that could carry one while one is. Empty when nothing can be closed.""" spans = closable(found) if not spans: return {} diff --git a/docs/live-site.md b/docs/live-site.md index d2f6ecf85..89d07f916 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -72,7 +72,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page: a de novo candidate or a compound-het pair that names one stays closed behind "Show the result" until that result is opened on this page, and a count says how many rows stay closed. A stretch painted from a grandparent over one says which grandparent's copy the child got there: while such a stretch exists, every stretch of that child is drawn and listed from its parent alone, with no grandparent, so the painting does not point at where the result lies, and "Show the result" brings the grandparents back. Each click opens one result, a row that names two closed results stays closed until both are opened, and a reload closes them again. Opening a result opens it for that child only. A result known by its gene alone, with no position, could sit at any de novo candidate or painted stretch, which name no gene: those rows stay closed with it. When the stretches come only as the drawn image, the image stays closed while a result is, and "Show the map" opens the image and no result. The page reads each child's `results.json` to know which results those are; while it cannot (no run here, no answer yet, a request that failed, or results the server could not build), every such row of that child stays closed, and "Show the rows" opens them on purpose. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. A row whose value in a child's column names a closed result of that child (a CPIC gene such as RYR1 that is also the gene of a result to act on) is left out whole, for every member, until that result is opened here: a closed cell under the gene's name would say which gene it is. While the page cannot read a child's results, every medicines and HLA row stays closed; the haplogroups and blood groups name no gene. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page, and what the page shows of a child never depends on which result is closed or where it lies. While a child has any result to act on still closed, every place of the page that could carry one closes for that child in the same way, each behind one box with the same words whatever the result: the grandparents of the painted stretches (every stretch is then drawn and listed from its parent alone), the chromosome map when it comes only as an image, every de novo candidate and every compound-het pair, the child's cell in every medicines and HLA row of the side by side, and the carrier screen of a couple the child is part of (its counts, its boxes, its table, the lead's words on it and its report link). Each "Show the result" opens one result, from whichever box it is clicked; the places come back once the child's last closed result is opened, and a reload closes them again. Opening a result opens it for that child only. The page reads each child's `results.json` to know whether a result is closed; while it cannot (no run here, no answer yet, a request that failed, results the server could not build, or results with findings to act on but no `veil.spans`), every place of that child stays closed, and each box opens only its own place on purpose: "Show the rows" its table, "Show the cells" the side by side, "Show the grandparents" the painting, "Show the map" the image and "Show the screen" the couple screen. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. Every row stays, with the parents' values: a child's medicines and HLA cells read "closed" while chapter IV's rule closes them (a row left out would leave a gap in a list everyone knows), and the haplogroups and blood groups, which name no gene, stay open. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. @@ -88,7 +88,7 @@ The reads cross the network to the browser that logged in: only the byte ranges `GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. -The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the rows outside a result's own card that close with it, and `spans`, the closed results with their gene and position, which the family page tags a child's rows with by the same rule). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. +The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the rows outside a result's own card that close with it, and `spans`, the closed results with their gene and position, from which the family page knows whether a child has a result still closed). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. ## Privacy and what is stored diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 352eb0b93..ab1b0be1e 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -108,7 +108,7 @@ def own_chroms(): # The son's results to act on ("act"): a pathogenic ClinVar hit in KCNH2, an ACMG secondary-findings gene that # is on no carrier list, and an AnnotSV class 5 deletion over it. Public gene name, invented positions and # alleles. tests/test_live_demo.py checks that his page names neither until each is opened on purpose, and that the -# family page keeps closed the compound-het pair in KCNH2 of tests/fixtures/live/demo/DEMO-FAMILY too. +# family page keeps his places closed with them, the compound-het pairs of tests/fixtures/live/demo/DEMO-FAMILY too. PLANTED = {"gene": "KCNH2", "hit": ("chr7", 150960000, "C", "T"), "sv": ("chr7", 150955000, 150965000)} # Stranger's thresholds per locus (normal up to, pathologic from), invented for the demo's catalog. STR_THRESHOLDS = {"HTT": (26, 40), "FMR1": (54, 200), "C9ORF72": (20, 30), "ATXN1": (35, 39), "DMPK": (34, 50), diff --git a/tests/fixtures/live/demo/README.md b/tests/fixtures/live/demo/README.md index d08c2eade..a78ec4866 100644 --- a/tests/fixtures/live/demo/README.md +++ b/tests/fixtures/live/demo/README.md @@ -15,6 +15,6 @@ sample DEMO-001 that `tests/demo/make_demo_sample.py` writes. No real sample, pe `de_novo.tsv`, `compound_het.tsv`: step 38's files for the trio, invented, in the columns step 38 writes, so the family page draws its grid, the son's painted stretches and his tables. `compound_het.tsv` also holds a pair in KCNH2 whose first variant is the son's planted result to act on - (`make_demo_sample.py` PLANTED): the family page keeps that row closed until the result is opened there. + (`make_demo_sample.py` PLANTED): the family page keeps the son's pairs closed until his results are opened there. The run itself is `../demo-001.weblog.jsonl`, written by `tests/demo/make_demo_events.py`. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 968eeaaa9..ca4f4fb45 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -53,24 +53,26 @@ marks and ranking only, and a reload closes them again; the home page and every finding page of the son (an open finding's rail and related list too) name them nowhere, and the symptom ranking's own page shows the gene - once opened on purpose; on the family page the son's rows that name a planted - result (the compound-het pair in KCNH2 of the fixture, and a de novo - candidate and a stretch painted from a grandparent planted in family.json) - name its gene, positions and that grandparent in no row, link, title or - count until "Show the result" is clicked there, a row that names both - planted results stays closed until both are opened (one per click), each - row then comes back, and a reload closes them again; while a stretch painted - from a grandparent lies over one, every stretch of his is drawn and listed from - its parent alone, none marked; with a closed result of his in RYR1 the side by - side's RYR1 row stays closed until opened there; with the son's results.json - unavailable, or answered as unreadable, every row of his that could name one - stays closed (in the side by side every medicines and HLA row), and "Show the - rows" opens neither his painting's grandparents nor his map as an image, which - open only by their own button; with a closed result known by its gene alone his de novo rows - (a position, no gene) stay closed with it; his map as an image stays closed, - and "Show the map" opens the map and no result; on his page - a CHIP call and a copy-number dot in the gene of a closed result stay out - of their drawings until it is opened; DEMO-001's page + once opened on purpose; on the family page, while the son has a planted + result closed (the fixture's compound-het pair in KCNH2, a de novo + candidate and a stretch painted from a grandparent planted in family.json), + his painting's grandparents, his de novo and compound-het rows and his + medicines and HLA cells of the side by side close, one box each, and no + text or attribute names its gene, positions or that grandparent; each click + opens one result, every place comes back once both are opened, and a reload + closes them again; the page is the same, byte for byte, whichever result is + closed (KCNH2, RYR1, HLA-B, a gene alone, a position) and wherever a + grandparent's stretch lies; while the page cannot read his results (no + results.json, an unreadable one, act findings without veil.spans) every + place stays closed and each box opens only its own place; with a second + child holding the same results, opening his leaves hers closed; a couple + screen whose partner is a child with a closed result closes whole (counts, + table, boxes, report link, the lead's words); his map as an image stays + closed until each result is opened from its box, and with his results + unreadable "Show the rows" leaves it closed and "Show the map" opens it; on + his page a CHIP call, a copy-number dot and a card's fact in the gene of a + closed result stay out of sight (on a phone the card's line too) until it + is opened; DEMO-001's page mid-replay and at the end leads with the report (a short run band above the lead while it runs, none once finished), ends with the step timeline folded, keeps the wire to at most eight one-sentence lines with the plain @@ -81,10 +83,13 @@ the veil, person.js and finding.js that read a missing state as a standalone adult, and search.js without its filter; for the planted results, results.json without its veil tags, person.js whose table rows ignore them, person.js whose findings ignore them, charts.js that keeps every mark, search.js that ignores a hit's tag -and finding.js whose findings ignore their tags; for the family page, family-page.js whose rows ignore the veil -core.js whose veilId() tags nothing, family-page.js without the gene-only fallback, a "Show the map" that opens -the results too, a closed painting that keeps each stretch's grandparent, a side by side that ignores the veil and a -drawing that "Show the rows" opens; for the drawings, person.js that draws every item), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +and finding.js whose findings ignore their tags; for the family page, de novo rows, compound-het rows, a painting, +a side by side's cells, a couple screen and a map image that ignore the veil, a closed painting that keeps each +stretch's grandparent, a box that opens every result with one click, de novo rows closed only when a result lies +under one, a painting closed only when a grandparent's stretch lies over a closed result, a side by side that closes +only the closed gene's cells, one opening shared by every box while the results cannot be read, results without +veil.spans read as nothing closed and an opened result kept per run only; for the son's page, person.js that draws +every item, shows every fact or builds a card's line from every fact), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -430,8 +435,13 @@ def family_depth(b, base): b.go("about:blank") b.wait("location.href === 'about:blank'") b.go(f"{base}/runs/{FAM}/") - return b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " - "(document.documentElement.dataset.pgpKids || '').indexOf('DEMO-003:known') >= 0 && JSON.stringify([" + b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " + "(document.documentElement.dataset.pgpKids || '').indexOf('DEMO-003:known') >= 0") + # The son's results to act on close his tables here: open each on purpose (one per click) to count them. + for _ in range(4): + b.eval("var v = document.querySelector('.veil[data-veil-child=\"DEMO-003\"] button'); v && v.click(); true") + time.sleep(0.2) + return b.wait("JSON.stringify([" "document.querySelectorAll('table.rel-grid td:not(.self):not(.none)').length, " "document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg').length, " "document.querySelectorAll('details[data-child=\"DEMO-003\"] table.rows').length, " @@ -739,12 +749,16 @@ def write(p, text): write(p, text) -# The family page: the son's rows that name his planted result to act on. -FAMILY_SHOWN_JS = ("[document.body.textContent].concat(Array.prototype.map.call(document.querySelectorAll('[href],[title],[aria-label]," - "[alt]'), function (e) { return ['href', 'title', 'aria-label', 'alt'].map(function (k) { " - "return e.getAttribute(k) || ''; }).join(' '); })).join(' ')") +# The family page: what it shows of the son while a result of his to act on is closed. +# Every text of the page and every attribute of every element: a gene, a position or a grandparent hidden in a +# title, a link, a label, a data attribute or a class is as shown as one in a cell. +FAMILY_SHOWN_JS = ("[document.body.textContent].concat(Array.prototype.map.call(document.querySelectorAll('*'), function (e) { " + "return Array.prototype.map.call(e.attributes, function (a) { return a.value; }).join(' '); })).join(' ')") GRANDPARENT = "DEMO-GF" PAIR_B = "150962400" # the second variant of the fixture's compound-het pair in KCNH2 +SON = OTHERS[1] +# The places that close for the son while a result of his is closed, one box each, in page order. +SON_BOXES = ["paint", "denovo", "het", "side"] def no_cache(b): @@ -764,302 +778,405 @@ def family_page(b, base): time.sleep(0.3) +def boxes_of(b, child=SON): + """The places closed for CHILD (data-veil-box of each of its closed boxes), in page order.""" + got = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('.veil[data-veil-child=\"" + child + + "\"]'), function (x) { return x.getAttribute('data-veil-box'); }))") + return json.loads(got) if got else None + + +def open_box(b, box, child=SON): + """Click the button of CHILD's closed box BOX; whether there was one.""" + ok = b.eval("var v = document.querySelector('.veil[data-veil-child=\"" + child + "\"][data-veil-box=\"" + box + "\"] button'); " + "if (v) { v.click(); } !!v") + time.sleep(0.4) + return ok + + +def son_tables(b, child=SON): + """[rows of each table] in CHILD's fold of the family page.""" + rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"" + child + "\"] " + "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") + return json.loads(rows) if rows else None + + +def side_rows(b): + """The side by side as [[group, label, value of each member]], the group carried down.""" + got = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows tbody tr'), " + "function (r) { return Array.prototype.map.call(r.cells, function (c) { return c.textContent; }); }))") + rows, group = [], "" + for r in json.loads(got) if got else []: + group = r[0] or group + rows.append([group] + r[1:]) + return rows + + +def side_problems_of(b, closed): + """[problems] of the side by side: the son's medicines and HLA cells all "closed" when CLOSED, none otherwise; a + parent's cell never closed, the haplogroups and blood groups never closed.""" + rows = side_rows(b) + members = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows thead th'), " + "function (x) { return x.textContent; }).slice(2))") + members = json.loads(members) if members else [] + if SON not in members or not rows: + return [f"no side by side with the son: {members}"] + out = [] + for r in rows: + for m, v in zip(members, r[2:]): + want = closed and m == SON and r[0] in ("Medicines", "HLA") + if (v == "closed") != want: + out.append(f"{m}'s {r[1]} reads {v!r}") + return out[:6] + + def family_closed(b, base): """[problems] of the family page before any click: the son's planted gene, its positions or the grandparent of - the stretch over it named anywhere; the rows that name nothing closed gone too.""" + the stretch over it named anywhere, or a place of his open: the painting's grandparents, his de novo and + compound-het rows and his medicines and HLA cells close, one box each.""" gene, hit, sv = planted_words() family_page(b, base) shown = b.eval(FAMILY_SHOWN_JS) or "" out = [f"{w} shown on the family page while closed" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] - rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " - "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") - closed = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") + boxes, tables = boxes_of(b), son_tables(b) segs = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg').length") marked = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg.closed, " "[data-painting=\"DEMO-003\"] .paint-seg.other').length") - # The 2 de novo rows of the fixture and its 2 other pairs stay, one closed box each; the painting closes whole - # (one box): every stretch drawn and listed from its parent, none marked, so none points at the closed result. - if not rows or json.loads(rows)[1:] != [2, 2] or closed != 3 or json.loads(rows)[0] != segs or marked: - out.append(f"the open rows and the closed boxes: tables {rows}, {closed} closed boxes, {segs} stretches drawn, " - f"{marked} marked") + # Only the painting's table of stretches stays, every stretch drawn and listed from its parent, none marked. + if boxes != SON_BOXES or not tables or len(tables) != 1 or tables[0] != segs or marked: + out.append(f"closed places {boxes}, tables {tables}, {segs} stretches drawn, {marked} marked") + out += side_problems_of(b, True) return out def family_problems(b, base): - """[problems] of the family page: closed, then opened one result per click, then reloaded. Each planted row - names both of the son's planted results (the secondary finding's position, and the deletion's span around it, - both in KCNH2), so the first click opens one result and every row stays closed for the other.""" + """[problems] of the family page: closed; one click on the compound-het box opens one of the son's two planted + results and leaves every place closed; a click on the de novo box opens the other and every place; a reload + closes them again.""" gene, hit, _ = planted_words() out = family_closed(b, base) if out: return out - click = "document.querySelector('details[data-child=\"DEMO-003\"] .veil button').click(); true" - b.eval(click) - time.sleep(0.5) + if not open_box(b, "het"): + return ["no compound-het box"] shown = b.eval(FAMILY_SHOWN_JS) or "" - left = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") - if any(w in shown for w in [gene, GRANDPARENT, PAIR_B] + hit) or left != 3: - out.append(f"one click opened a row that also names the other closed result ({left} closed boxes left)") - b.eval(click) - time.sleep(0.5) + if any(w in shown for w in [gene, GRANDPARENT, PAIR_B] + hit) or boxes_of(b) != SON_BOXES: + out.append(f"one click opened a place while the other result is still closed: {boxes_of(b)}") + open_box(b, "denovo") shown = b.eval(FAMILY_SHOWN_JS) or "" - rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " - "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") - left = b.eval("document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil').length") - if not all(w in shown for w in [gene, GRANDPARENT, hit[0]]) or json.loads(rows or "[0,0,0]")[1:] != [3, 3] or left: - out.append(f"two clicks on 'Show the result' did not bring back each row: tables {rows}, {left} closed boxes left") + tables = son_tables(b) + if not all(w in shown for w in [gene, GRANDPARENT, hit[0], PAIR_B]) or not tables or tables[1:] != [3, 3] or boxes_of(b): + out.append(f"opening both results did not bring back every place: tables {tables}, closed {boxes_of(b)}") + out += side_problems_of(b, False) out += [f"after a reload: {x}" for x in family_closed(b, base)] return out -def family_unknown(b, base): - """[problems] of the family page with the son's results.json unavailable: a row of his named, or any of his de - novo or compound-het rows open.""" +def family_view(b, base): + """The family page's HTML before any click.""" + family_page(b, base) + return b.eval("document.getElementById('family').innerHTML") or "" + + +def invariance_problems(b, base, rj, res, fj, fams): + """[problems]: the family page before any click differs between closed results of different genes and places + (FAMS: name -> family.json text; the results' spans set in RJ from RES), so it would tell which is closed.""" + views = {} + for name, spans in (("the planted pair in KCNH2", None), ("RYR1, a medicines gene", [["sf-ryr1", "RYR1", "chr19", 38440000, 38440000]]), + ("HLA-B", [["sf-hlab", "HLA-B", "chr6", 31353000, 31357000]]), + ("a gene alone", [["two-hits-g0999", "G0999", None, None, None]]), + ("a position under a de novo row", [["sf-pos", None, "chr3", 48211904, 48211904]])): + r = copy.deepcopy(res) + if spans is not None: + r["veil"]["spans"] = spans + with open(rj, "w") as f: + json.dump(r, f) + for fname, text in fams.items(): + with open(fj, "w") as f: + f.write(text) + views[f"{name}, {fname}"] = family_view(b, base) + with open(rj, "w") as f: + json.dump(res, f) + first = next(iter(views)) + return [f"the page differs between {first!r} and {k!r}" for k, v in views.items() if v != views[first]] + + +def family_unknown(b, base, opens=True): + """[problems] of the family page while it cannot read the son's results: a place of his open or a word of his + planted results shown; then (OPENS) the de novo box's own button opens his de novo rows and no other place.""" gene, hit, sv = planted_words() family_page(b, base) shown = b.eval(FAMILY_SHOWN_JS) or "" - out = [f"{w} shown with the son's results unavailable" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] - rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " - "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") - if rows is None or len(json.loads(rows)) != 1: - out.append(f"the son's de novo or compound-het rows open with his results unavailable: tables {rows}") - side = side_labels(b) - if any(g in side for g in SIDE_GENES) or "Mitochondrial" not in side: - out.append(f"side by side with the son's results unavailable: rows {sorted(side)[:8]}") + out = [f"{w} shown with the son's results unreadable" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] + if boxes_of(b) != SON_BOXES or son_tables(b) is None or len(son_tables(b)) != 1: + out.append(f"closed places {boxes_of(b)}, tables {son_tables(b)}") + words = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('.veil[data-veil-child=\"DEMO-003\"] button'), " + "function (x) { return x.textContent; }))") + if json.loads(words or "[]") != ["Show the grandparents", "Show the rows", "Show the rows", "Show the cells"]: + out.append(f"the buttons read {words}") + out += side_problems_of(b, True) + if not opens or out: + return out + open_box(b, "denovo") + shown = b.eval(FAMILY_SHOWN_JS) or "" + if boxes_of(b) != ["paint", "het", "side"] or (son_tables(b) or [])[1:] != [3] or GRANDPARENT in shown or PAIR_B in shown: + out.append(f"the de novo box opened another place: closed {boxes_of(b)}, tables {son_tables(b)}") + out += side_problems_of(b, True) return out -SIDE_GENES = ("RYR1", "CACNA1S", "CYP2D6", "HLA-A") - - -def click_veil(b, words): - """Click the son's closed box whose button says WORDS; whether there was one.""" - return b.eval("var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil " - "button'), function (x) { return x.textContent === " + json.dumps(words) + "; })[0]; " - "if (v) { v.click(); } !!v") - - -def unknown_drawing_problems(b, base, image): - """[problems] of the family page with the son's results unavailable: "Show the rows" of a table opening his - drawing (the map as an image, or the painting's grandparents), or the drawing's own button not opening it.""" +def unknown_map_problems(b, base): + """[problems] of the family page whose son has his map as an image, with his results unreadable: 'Show the + rows' of a table opening the image, or 'Show the map' not opening it.""" family_page(b, base) - out = [] - if not click_veil(b, "Show the rows"): - out.append("no 'Show the rows' box") - time.sleep(0.4) img = "!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')" - seen = (lambda: b.eval(img)) if image else (lambda: GRANDPARENT in (b.eval(FAMILY_SHOWN_JS) or "")) - if seen(): - out.append("'Show the rows' opened the drawing too") - if not click_veil(b, "Show the map" if image else "Show the grandparents"): - out.append("no closed box of the drawing") - time.sleep(0.4) - if not seen(): - out.append("the drawing's own button did not open it") + out = [] + if not open_box(b, "denovo"): + out.append("no de novo box") + if b.eval(img): + out.append("'Show the rows' opened the map too") + if not open_box(b, "map") or not b.eval(img): + out.append("'Show the map' did not open the map") return out -def side_labels(b): - """The 'Gene or system' column of the family page's side by side.""" - got = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows tbody tr'), " - "function (r) { return r.children[1] ? r.children[1].textContent : ''; }))") - return json.loads(got) if got else [] - - -def side_problems(b, base): - """[problems] of the family page's side by side with a closed result of the son's in RYR1: the RYR1 row (his - diplotype and phenotype under the gene's name) shown before it is opened there, the other rows closed with it, - or the row not back once opened.""" +def map_problems(b, base): + """[problems] of the family page whose son has his map as an image and two closed results: the image shown + before both are opened from its box (one per click), or not shown after.""" family_page(b, base) + img = "!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')" out = [] - labels = side_labels(b) - shown = b.eval("document.getElementById('ch-side-by-side').textContent") or "" - boxes = b.eval("document.querySelectorAll('#ch-side-by-side .veil[data-veil-child=\"DEMO-003\"]').length") - if "RYR1" in shown or "CACNA1S" not in labels or boxes != 1: - out.append(f"before any click: RYR1 {'shown' if 'RYR1' in shown else 'closed'}, {len(labels)} rows, {boxes} closed boxes") - b.eval("var x = document.querySelector('#ch-side-by-side .veil button'); x && x.click(); true") - time.sleep(0.4) - labels2 = side_labels(b) - if "RYR1" not in labels2 or len(labels2) != len(labels) + 1: - out.append(f"after 'Show the result': {len(labels2)} rows, RYR1 {'back' if 'RYR1' in labels2 else 'still closed'}") + if b.eval(img) or boxes_of(b) != ["map", "denovo", "het", "side"]: + out.append(f"the map shown while a result is closed, or a place open: {boxes_of(b)}") + open_box(b, "map") + if b.eval(img) or boxes_of(b) != ["map", "denovo", "het", "side"]: + out.append(f"one click opened a place while the other result is closed: {boxes_of(b)}") + open_box(b, "map") + if not b.eval(img) or boxes_of(b): + out.append(f"the map not back once both results are opened: closed {boxes_of(b)}") return out -def son_tables(b): - """[rows of each table] in the son's fold of the family page.""" - rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] " - "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") - return json.loads(rows) if rows else None - - -def gene_only_problems(b, base): - """[problems] of the family page when the son also has a closed result known by its gene alone: a de novo row - shown before it is opened, a pair in another gene closed, or the rows not back once each result is opened.""" +def two_children_problems(b, base): + """[problems] of a family with a second child, DEMO-004, whose run has the son's results under the same ids: + opening every result of the son leaves each of DEMO-004's places closed.""" family_page(b, base) out = [] - t = son_tables(b) - shown = b.eval(FAMILY_SHOWN_JS) or "" - # Open: the stretches' table and the two pairs in other genes; closed: both de novo rows. - if t != [80, 2] or "48211904" in shown: - out.append(f"before any click: tables {t}") - for _ in range(3): # the gene-only result, then the two planted ones (the pair in KCNH2) - b.eval("var x = document.querySelector('details[data-child=\"DEMO-003\"] .veil button'); x && x.click(); true") - time.sleep(0.4) - t = son_tables(b) - if t != [80, 2, 3]: - out.append(f"after opening each result: tables {t}") + if boxes_of(b, "DEMO-004") != SON_BOXES: + out.append(f"DEMO-004's closed places before: {boxes_of(b, 'DEMO-004')}") + for _ in range(2): + open_box(b, "denovo") + if boxes_of(b): + out.append(f"the son's places not opened: {boxes_of(b)}") + other = b.eval("document.querySelector('details[data-child=\"DEMO-004\"]').textContent") or "" + if boxes_of(b, "DEMO-004") != SON_BOXES or GRANDPARENT in other or planted_words()[1][0] in other: + out.append(f"opening the son's results opened DEMO-004's: {boxes_of(b, 'DEMO-004')}") return out -def map_problems(b, base): - """[problems] of the family page whose son has his map as an image: the image shown while a result is closed, - or 'Show the map' opening a result's rows.""" +def couple_problems(b, base): + """[problems] of a family where DEMO-001, a partner of the couple screen, is also a child (of an unsequenced + father) with a closed result in PRF1, a gene of the screen: the screen names it (its line's counts, its + table, its boxes, its report link, the lead) before the result is opened from its box, or not after.""" family_page(b, base) out = [] - img = b.eval("!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')") - if img: - out.append("the map shown while a result is closed") - b.eval("var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=\"DEMO-003\"] .veil'), " - "function (x) { return x.textContent.indexOf('Show the map') >= 0; })[0]; v && v.querySelector('button').click(); true") - time.sleep(0.4) - img = b.eval("!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')") - shown = b.eval(FAMILY_SHOWN_JS) or "" - if not img: - out.append("'Show the map' did not show the map") - if "KCNH2" in shown or PAIR_B in shown: - out.append("'Show the map' opened the closed pair too") + sect = b.eval("var s = document.getElementById('ch-couples'); s ? s.textContent + ' ' + s.innerHTML : ''") or "" + lead = b.eval("(document.getElementById('family-lead') || {}).textContent || ''") or "" + links = b.eval("Array.prototype.map.call(document.querySelectorAll('a[href]'), function (a) { return a.getAttribute('href'); }).join(' ')") or "" + bx = boxes_of(b, "DEMO-001") or [] + if ("PRF1" in sect or "70601223" in sect or "CPT2" in sect or "couple_DEMO" in links + or "carrier screen stays closed in chapter III" not in lead): + out.append("the screen of a couple whose partner has a closed result named before it is opened") + if bx != ["couple:DEMO-001 DEMO-002", "side"]: + out.append(f"DEMO-001's closed places: {bx}") + if not open_box(b, "couple:DEMO-001 DEMO-002", "DEMO-001"): + return out + ["no closed box under the screen"] + sect = b.eval("document.getElementById('ch-couples').textContent") or "" + links = b.eval("Array.prototype.map.call(document.querySelectorAll('a[href]'), function (a) { return a.getAttribute('href'); }).join(' ')") or "" + if "PRF1" not in sect or "couple_DEMO" not in links or boxes_of(b, "DEMO-001"): + out.append(f"the screen not back once the result is opened: closed {boxes_of(b, 'DEMO-001')}") return out def family_veil_checks(b, base, out): - """On the family page a child's rows that name a closed result stay closed as on the child's page, until that - result is opened there; a page that cannot read the child's results keeps every row of his that could closed.""" + """On the family page, while a child has a result to act on still closed, or the page cannot read the child's + results, every place that could carry one closes for that child, the same whichever result it is; each known + result opens with its own click, and each place opens on its own while the results cannot be read.""" no_cache(b) import make_demo_sample fj = os.path.join(out, "runs", FAM, "family.json") - rj = os.path.join(out, "runs", OTHERS[1], "results.json") - fp, cj = (os.path.join(out, "assets", x) for x in ("family-page.js", "core.js")) + rj = os.path.join(out, "runs", SON, "results.json") + rj1 = os.path.join(out, "runs", S, "results.json") + fp = os.path.join(out, "assets", "family-page.js") keep = {} - for p in (fj, fp, cj): + for p in (fj, fp, rj, rj1): with open(p) as f: keep[p] = f.read() def write(p, text): with open(p, "w") as f: f.write(text) + + def planted_control(desc, a, z, problems): + """Run PROBLEMS against family-page.js with A replaced by Z: it must find a problem.""" + if a not in keep[fp]: + check(f"control: {desc} (the planted line is in family-page.js)", False, a) + return + write(fp, keep[fp].replace(a, z)) + try: + control(desc, not problems()) + finally: + write(fp, keep[fp]) try: # A de novo candidate at the planted hit, and the father's copy over it painted from a grandparent. fam = json.loads(keep[fj]) c, pos, ref, alt = make_demo_sample.PLANTED["hit"] - row = dict(fam["de_novo"][0], chrom=c, pos=str(pos), ref=ref, alt=alt) - fam["de_novo"].append(row) - segs = fam["segments"]["DEMO-003"] + fam["de_novo"].append(dict(fam["de_novo"][0], chrom=c, pos=str(pos), ref=ref, alt=alt)) + segs = fam["segments"][SON] over = [s_ for s_ in segs if s_["chrom"] == c and s_["side"] == "paternal" and int(s_["start"]) <= pos <= int(s_["end"])] - if len(over) != 1: - check("browser: the son's stretches hold one paternal stretch over the planted hit", False, over) + away = [s_ for s_ in segs if s_["chrom"] != c and s_["side"] == "paternal"] + if len(over) != 1 or not away: + check("browser: the son's stretches hold one paternal stretch over the planted hit, and others", False, over) return over[0]["ancestor"] = GRANDPARENT - write(fj, json.dumps(fam)) - keep["planted"] = json.dumps(fam) + planted = json.dumps(fam) + over[0]["ancestor"] = "." + away[0]["ancestor"] = GRANDPARENT + elsewhere = json.dumps(fam) + write(fj, planted) bad = family_problems(b, base) - check("browser: on the family page the son's rows that name his planted result (a compound-het pair, a de novo " - "candidate, a stretch from a grandparent) name its gene, positions and grandparent nowhere until 'Show the " - "result' is clicked there, a row naming two closed results stays closed until both are opened, then each " - "row comes back, and a reload closes them", not bad, bad) - for p, a, z, desc in ( - (fp, "return tags.filter(function (t) {", "return [].filter(function (t) {", - "family-page.js whose rows ignore the veil"), - (cj, " if (!spans || !spans.length) { return null; }", " return null;", "core.js whose veilId() tags nothing"), - (fp, "ancestor: '.', basis: 'parent' };", "ancestor: r.ancestor, basis: r.basis };", - "a closed painting that keeps each stretch's grandparent")): - if a not in keep[p]: - check(f"control: {desc} (the planted line is in {os.path.basename(p)})", False, a) - continue - write(p, keep[p].replace(a, z)) - control(desc, not family_closed(b, base)) - write(p, keep[p]) + check("browser: on the family page, while the son has a result to act on closed, his painting's grandparents, " + "his de novo and compound-het rows and his medicines and HLA cells close, one box each, naming no gene, " + "position or grandparent; each click opens one result, every place comes back once both are opened, and " + "a reload closes them", not bad, bad) + for desc, a, z in ( + ("de novo rows that ignore the veil", "var dshut = shutOf(c.child, 'denovo');", "var dshut = [];"), + ("compound-het rows that ignore the veil", "var hshut = shutOf(c.child, 'het');", "var hshut = [];"), + ("a painting that ignores the veil", "var shut = anc ? shutOf(child, 'paint') : [];", "var shut = [];"), + ("a closed painting that keeps each stretch's grandparent", "ancestor: '.', basis: 'parent' };", + "ancestor: r.ancestor, basis: r.basis };"), + ("a side by side whose cells ignore the veil", "closedCells[m] = true; return 'closed';", "closedCells[m] = true; return v;")): + planted_control(desc, a, z, lambda: family_closed(b, base)) + planted_control("a box that opens every result with one click", + "function () { opened[openKey(child, shut[0])] = true; redraw(); });", + "function () { shut.forEach(function (t) { opened[openKey(child, t)] = true; }); redraw(); });", + lambda: family_problems(b, base)) + + # The same page whichever result is closed and wherever it lies. + res = json.loads(keep[rj]) + fams = {"a grandparent over the hit": planted, "a grandparent elsewhere": elsewhere} + bad = invariance_problems(b, base, rj, res, fj, fams) + check("browser: the family page is the same, byte for byte, whichever of the son's results is closed (a pair " + "in KCNH2, RYR1, HLA-B, a gene alone, a position) and wherever a grandparent's stretch lies", not bad, bad) + for desc, a, z in ( + ("de novo rows closed only when a result lies under one", "var dshut = shutOf(c.child, 'denovo');", + "var dshut = shutOf(c.child, 'denovo').filter(function (id) { return dn.some(function (r) { return " + "kids[c.child].spans.some(function (s) { return s[0] === id && s[2] === r.chrom && +r.pos >= s[3] && " + "+r.pos <= s[4]; }); }); });"), + ("a painting closed only when a grandparent's stretch lies over a closed result", + "var anc = all.some(function (r) { return r.ancestor && r.ancestor !== '.'; });", + "var anc = all.some(function (r) { return r.ancestor && r.ancestor !== '.' && ((kids[child] || {}).spans || [])" + ".some(function (s) { return !s[2] || (s[2] === r.chrom && s[3] <= +r.end && s[4] >= +r.start); }); });"), + ("a side by side that closes only the closed gene's cells", + "if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length) {", + "if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length && (kids[m].spans || []).some(function (s) { " + "return s[1] === r.label; })) {")): + planted_control(desc, a, z, lambda: invariance_problems(b, base, rj, res, fj, fams)) + write(fj, planted) + + # The page cannot read the son's results: no answer, an unreadable answer, act findings without veil.spans. os.replace(rj, rj + ".away") try: bad = family_unknown(b, base) - bad += unknown_drawing_problems(b, base, False) + check("browser: with the son's results.json unavailable every place of his stays closed, and the de novo " + "box's own button opens his de novo rows and no other place", not bad, bad) + planted_control("one opening shared by every box while the results cannot be read", + "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", + "return opened[openKey(child, 'box:denovo')] ? [] : [UNKNOWN];", + lambda: family_unknown(b, base)) finally: os.replace(rj + ".away", rj) - check("browser: with the son's results.json unavailable the family page keeps every row of his that could name " - "a result to act on closed", not bad, bad) - # The results route answers a summary it cannot build with a note and no findings (owngenome_live_results.py). - with open(rj) as f: - keep[rj] = f.read() - write(rj, json.dumps({"schema_version": 1, "sample": OTHERS[1], "chapters": [], + write(rj, json.dumps({"schema_version": 1, "sample": SON, "chapters": [], "note": "The results could not be read (KeyError)."})) - bad = family_unknown(b, base) - check("browser: with the son's results answered as unreadable (a note, no findings) the family page keeps every " - "row of his that could name a result to act on closed", not bad, bad) - - # A closed result known by its gene alone: a row with only a position (a de novo candidate) could be it. - res = json.loads(keep[rj]) - res["veil"]["spans"].append(["two-hits-g0999", "G0999", None, None, None]) - write(rj, json.dumps(res)) - write(fj, keep[fj]) # the fixture's family: two de novo rows, three pairs (one of them in KCNH2) - bad = gene_only_problems(b, base) - check("browser: with a closed result known by its gene alone, the son's de novo rows (a position, no gene) stay " - "closed until it is opened, and his pairs in other genes stay open", not bad, bad) - a = "if (coords) {" - if a not in keep[fp]: - check("control: family-page.js without the gene-only fallback (the planted line is in family-page.js)", False, a) - else: - write(fp, keep[fp].replace(a, "if (false) {")) - control("family-page.js without the gene-only fallback", not gene_only_problems(b, base)) - write(fp, keep[fp]) + bad = family_unknown(b, base, opens=False) + check("browser: with the son's results answered as unreadable (a note, no findings) every place of his stays " + "closed", not bad, bad) + nospans = json.loads(keep[rj]) + nospans.pop("veil", None) + write(rj, json.dumps(nospans)) + bad = family_unknown(b, base, opens=False) + check("browser: with the son's results holding findings to act on but no veil.spans every place of his stays " + "closed", not bad, bad) + planted_control("results without veil.spans read as nothing closed", + "if (!k || k.status !== 'known' || (!k.spans && k.acts.length)) {", + "if (!k || k.status !== 'known') {", lambda: family_unknown(b, base, opens=False)) write(rj, keep[rj]) - # A closed result in a CPIC gene: the side by side lists that gene with the son's diplotype in his column. - res = json.loads(keep[rj]) - res["veil"]["spans"].append(["sf-ryr1-demo", "RYR1", "chr19", 38440000, 38440000]) - write(rj, json.dumps(res)) - bad = side_problems(b, base) - check("browser: with a closed result of the son's in RYR1, the side by side's RYR1 row (his diplotype and " - "phenotype) stays closed until it is opened there, and the other rows stay", not bad, bad) - a = "var t = stillClosed(m, tagOf(m, r.label + ' ' + (r.values[i] || '')));" - if a not in keep[fp]: - check("control: a side by side that ignores the veil (the planted line is in family-page.js)", False, a) - else: - write(fp, keep[fp].replace(a, "var t = [];")) - control("a side by side that ignores the veil", not side_problems(b, base)) - write(fp, keep[fp]) - write(rj, keep[rj]) - - # The drawing as an image (no stretches in family.json): "Show the map" opens the map and no result. + # A second child whose run holds the same results under the same ids: opening is per child. + fam2 = json.loads(planted) + fam2["state"]["children"].append(dict(fam2["state"]["children"][0], child="DEMO-004")) + fam2["runs"]["DEMO-004"] = fam2["runs"][SON] + fam2["segments"]["DEMO-004"] = fam2["segments"][SON] + fam2["de_novo"] += [dict(r, child="DEMO-004") for r in fam2["de_novo"] if r["child"] == SON] + fam2["compound_het"] += [dict(r, child="DEMO-004") for r in fam2["compound_het"] if r["child"] == SON] + fam2["side_by_side"]["members"].append("DEMO-004") + for r in fam2["side_by_side"]["rows"]: + r["values"].append(r["values"][fam2["side_by_side"]["members"].index(SON)]) + if r.get("stale"): + r["stale"].append(False) + write(fj, json.dumps(fam2)) + bad = two_children_problems(b, base) + check("browser: with two children holding the same results, opening each of the son's leaves the other " + "child's places closed", not bad, bad) + planted_control("an opened result kept per run only, not per child", + "function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; }", + "function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + id; }", + lambda: two_children_problems(b, base)) + write(fj, planted) + + # A partner of the couple screen who is also a child, with a closed result in a gene of the screen. + famc = json.loads(keep[fj]) + famc["state"]["members"].insert(0, {"sample": GRANDPARENT, "sex": "male", "father": None, "mother": None, + "partner": None, "sequenced": False}) + for m in famc["state"]["members"]: + if m["sample"] == S: + m["father"] = GRANDPARENT + famc["state"]["children"].insert(0, {"child": S, "father": GRANDPARENT, "mother": None, "segments": None}) + rep = f"couple_{S}_{OTHERS[0]}.md" + famc["state"]["couples"][0]["report"] = rep + famc["state"]["reports"] = famc["state"].get("reports", []) + [{"name": rep, "url": rep}] + write(fj, json.dumps(famc)) + r1 = json.loads(keep[rj1]) + r1.setdefault("veil", {})["spans"] = [["two-hits-prf1", "PRF1", None, None, None]] + write(rj1, json.dumps(r1)) + bad = couple_problems(b, base) + check("browser: a couple screen whose partner is a child with a closed result stays closed whole (counts, " + "table, boxes, report link, the lead's words) until that result is opened from its box", not bad, bad) + planted_control("a couple screen that ignores the veil", "var cs = coupleShut(f, c);", "var cs = [];", + lambda: couple_problems(b, base)) + write(rj1, keep[rj1]) + + # The drawing as an image (no stretches in family.json). fam = json.loads(keep[fj]) fam["segments"] = {} fam["state"]["children"][0]["segments"] = fam["state"]["pedigree"] write(fj, json.dumps(fam)) bad = map_problems(b, base) - check("browser: the son's map as an image stays closed while a result is, and 'Show the map' opens the map " - "and leaves each result's rows closed", not bad, bad) + check("browser: the son's map as an image stays closed while a result is, and its box opens one result per " + "click, the map once both are opened", not bad, bad) + planted_control("a map image that ignores the veil", "var shut = shutOf(c.child, 'map');", "var shut = [];", + lambda: map_problems(b, base)) os.replace(rj, rj + ".away") try: - bad = unknown_drawing_problems(b, base, True) + bad = unknown_map_problems(b, base) check("browser: with the son's results unavailable, 'Show the rows' leaves his map as an image closed, and " "'Show the map' opens it", not bad, bad) - a = "return t === UNKNOWN ? !mapShown[openKey(child, key)] :" - if a not in keep[fp]: - check("control: a drawing that 'Show the rows' opens (the planted line is in family-page.js)", False, a) - else: - write(fp, keep[fp].replace(a, "return t === UNKNOWN ? !mapShown[openKey(child, key)] && !openedAll[child] :")) - control("a drawing that 'Show the rows' opens", not unknown_drawing_problems(b, base, True)) - write(fp, keep[fp]) + planted_control("a map image that 'Show the rows' opens", + "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", + "return opened[openKey(child, 'box:denovo')] ? [] : [UNKNOWN];", + lambda: unknown_map_problems(b, base)) finally: os.replace(rj + ".away", rj) - a = "function () { mapShown[mk] = true; redraw(); }" - if a not in keep[fp]: - check("control: a 'Show the map' that opens the results too (the planted line is in family-page.js)", False, a) - else: - write(fp, keep[fp].replace(a, "function () { mapShown[mk] = true; shut.forEach(function (t) { " - "opened[openKey(c.child, t)] = true; }); redraw(); }")) - control("a 'Show the map' that opens the results too", not map_problems(b, base)) - write(fp, keep[fp]) finally: - keep.pop("planted", None) for p, text in keep.items(): write(p, text) @@ -1084,6 +1201,24 @@ def drawn(): "&& document.querySelector('[data-widget=copies]') && document.querySelector('.veil') && 'drawn'", 15) return shown() + def phone_line(): + """The copy-number card's one line on a phone (its fold's summary, after the card's own title, the same for + everyone), drawn from its first open fact.""" + b.call("Emulation.setDeviceMetricsOverride", {"width": 390, "height": 844, "deviceScaleFactor": 2, "mobile": False}, + b.session) + try: + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{OTHERS[1]}/") + return b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.documentElement.dataset.pgpResults " + "&& (document.querySelector('[data-card=fold-paralogs] summary .what') || {}).textContent", 15) or "" + finally: + b.call("Emulation.clearDeviceMetricsOverride", {}, b.session) + + def phone_problems(): + line = phone_line() + return [] if line and "SMN1" not in line else [f"the copy-number card's line on a phone: {line!r}"] + def shown(): return b.eval("JSON.stringify([" + SHOWN_JS + ", (document.querySelector('[data-widget=copies]') || {}).textContent || ''])") @@ -1095,6 +1230,9 @@ def problems(text, opened): if ("p.R175H" in page) != opened or ("SMN1" in cops) != opened: out_.append(f"the CHIP call or the SMN1 dot of the closed results {'not ' if opened else ''}shown: " f"{'p.R175H' in page} | {cops[:200]}") + # The copy-number card's facts name SMN1: closed in its margin and in its line of section 8. + if ("SMN1 copies" in page) != opened: + out_.append(f"the card's fact on SMN1 {'not ' if opened else ''}shown") return out_ try: with open(os.path.join(os.path.dirname(out), "root", OTHERS[1], "summary.json")) as f: @@ -1112,21 +1250,29 @@ def problems(text, opened): res = live_results.build(OTHERS[1], summary) with open(rj, "w") as f: json.dump(res, f) - bad = problems(drawn(), False) + bad = problems(drawn(), False) + phone_problems() if not bad: for _ in range(2): # the CPSR card holds both results: one click opens one b.eval("document.querySelector('#card-cpsr .veil button').click(); true") time.sleep(0.5) bad = problems(shown(), True) - check("browser: on the son's page a CHIP call and a copy-number dot in the gene of a closed result stay out of " - "sight until it is opened", not bad, bad) - a = "add(text, widget(of[1]));" - if a not in keep_js: - check("control: person.js that draws every item (the planted line is in person.js)", False, a) - else: + check("browser: on the son's page a CHIP call, a copy-number dot and a card's fact in the gene of a closed " + "result stay out of sight (the fact in the card's line on a phone too) until it is opened", not bad, bad) + for desc, a, z in ( + ("person.js that draws every item", "add(text, widget(of[1]));", "add(text, widget(c.widget));"), + ("person.js that shows every fact", "facts = (c.facts || []).filter(function (f, i) { return !(ft[i] && tagClosed(ft[i])); });", + "facts = c.facts || [];"), + ("person.js whose card line reads every fact", + "(openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)", + "(c.facts && c.facts[0] ? c.facts[0][0] + ': ' + c.facts[0][1] : NOT_ALL_CLEAR)")): + if a not in keep_js: + check(f"control: {desc} (the planted line is in person.js)", False, a) + continue + with open(js, "w") as f: + f.write(keep_js.replace(a, z)) + control(desc, not (problems(drawn(), False) + phone_problems())) with open(js, "w") as f: - f.write(keep_js.replace(a, "add(text, widget(c.widget));")) - control("person.js that draws every item", not problems(drawn(), False)) + f.write(keep_js) finally: with open(rj, "w") as f: f.write(keep) diff --git a/tests/test_live_results.py b/tests/test_live_results.py index becc5e182..25fd54a9d 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -44,15 +44,13 @@ home page's view of the run (overview) lists no such finding among its first ones and its closed lead names none; a CHIP call, a copy-number dot and a fact that name a closed result's gene carry its id (the CHIP chart's - panel of genes never does), results.json carries the closed results as - veil.spans, and core.js veilId(), which tags the family page's rows of a - child, answers each row as veil_id() does, under node. + panel of genes never does), and results.json carries the closed results as + veil.spans (the family page closes a child's places while one is closed). Each check runs again on a planted mistake and must fail there: 1 with VNTYPER's entry dropped, 2 with one card's caveat blanked, 3 with "ok" planted in a card and "All clear" in a script, 4 with the hits emptied, 5 with the results route registered without the login, 6 with a row printed as its Python dict, 7 with the Slivar adapter returning nothing, -8 with a veil rule that tags nothing (rows, lines, facts and drawings) and one that tags every row, and -veilId() with its gene rule or its overlap rule dropped and veilIds() keeping only the first result. A check +8 with a veil rule that tags nothing (rows, lines, facts and drawings) and one that tags every row. A check that cannot fail is no check. Run: python3 tests/test_live_results.py @@ -437,48 +435,6 @@ def widget_problems(): return out -# Rows as the family page tags them (family-page.js): [text, chrom, pos, end], against two closed results. -VEIL_SPANS = [["sf-kcnh2", "KCNH2", "chr7", 150960000, 150960000], ["sv-del", "KCNH2", "chr7", 150955000, 150965000], - ["two-hits-hla", "HLA-A", None, None, None]] -VEIL_ROWS = [["chr7:150960000 C>T", "chr7", "150960000", None], ["chr7:150961000 G>A", "chr7", "150961000", None], - ["chr7:150970000 G>A", "chr7", "150970000", None], ["chr3:48211904 C>T", "chr3", "48211904", None], - ["KCNH2 chr7:94371020:G>A chr7:94398114:T>C", None, None, None], - ["G0187 chr7:150,962,000:G>A chr7:1:T>C", None, None, None], ["G0412 chr2:1:A>G", None, None, None], - ["chr7:61600000-159285973", "chr7", "61600000", "159285973"], ["in KCNH2.", None, None, None], - ["HLA-A*02:01", None, None, None], ["HLA-AB", None, None, None], ["kcnh2 lower case", None, None, None], - ["7:150960000", "7", 150960000, None], ["chrX:1-2", "chrX", "x", None], ["", "chr7", "150955000", "150965000"]] -CORE_JS = os.path.join(ASSETS, "core.js") -NODE_VEIL = ("const vm=require('vm'),fs=require('fs');const w={};" - "const doc={documentElement:{setAttribute(){}},addEventListener(){}};" - "vm.runInNewContext(fs.readFileSync(process.argv[1],'utf8'),{window:w,document:doc});" - "const sp=JSON.parse(process.argv[2]);" - "console.log(JSON.stringify(JSON.parse(process.argv[3]).map(r=>[w.PGPUI.veilId(sp,r[0],r[1],r[2],r[3])," - "w.PGPUI.veilIds(sp,r[0],r[1],r[2],r[3])])))") - - -def node_veil(js_path): - """[veilId(), veilIds()] of core.js at JS_PATH for each of VEIL_ROWS, under node, or the error.""" - node = shutil.which("node") - if not node: - return "no node" - p = subprocess.run([node, "-e", NODE_VEIL, js_path, json.dumps(VEIL_SPANS), json.dumps(VEIL_ROWS)], - capture_output=True, text=True) - return json.loads(p.stdout) if p.returncode == 0 else p.stderr - - -def veil_js_problems(js_path): - """What veilId() of core.js answers differently from veil_id() on VEIL_ROWS, and veilIds() from veil_id() taken - one closed result at a time (every result the row belongs to).""" - spans = [tuple(x) for x in VEIL_SPANS] - want = [[live_results.veil_id(spans, *r), [s_[0] for s_ in spans if live_results.veil_id([s_], *r)]] for r in VEIL_ROWS] - got = node_veil(js_path) - if got != want: - return [f"veilId, veilIds {got} != veil_id {want}"] - if len({w[0] for w in want}) < 4 or not any(len(w[1]) > 1 for w in want): - return [f"the rows exercise too few answers: {want}"] - return [] - - def overview_problems(): """What is wrong with the home page's view of a child's run (overview): a finding that names a closed result (a symptom ranking of its gene) is listed among the first findings, or the closed lead still names it.""" @@ -599,20 +555,6 @@ def tagged(sample_dir): bad = widget_problems() check("8. a CHIP call, a copy-number dot and a fact that name a closed result's gene carry its id, the CHIP " "chart's gene panel none, and results.json carries the closed results as veil.spans", not bad, bad) - check("8. node is on the PATH (ubuntu-latest has it)", shutil.which("node") is not None) - bad = veil_js_problems(CORE_JS) - check("8. core.js veilId() tags each family-page row as veil_id() does, and veilIds() with every result " - "veil_id() finds one result at a time (under node)", not bad, bad) - with open(CORE_JS) as f: - core = f.read() - for a, z, desc in (("if (s[1] && words[s[1]]) { return s[0]; }", "", "veilId() without its gene rule"), - ("l[1] <= s[4] && l[2] >= s[3]", "false", "veilId() without its overlap rule"), - (".map(function (s) { return s[0]; });\n }", ".map(function (s) { return s[0]; }).slice(0, 1);\n }", - "veilIds() keeping only the first result")): - planted = os.path.join(work, "core-planted.js") - with open(planted, "w") as f: - f.write(core.replace(a, z)) - control(desc, a in core and not veil_js_problems(planted)) bad = veil_problems(*tagged(d3)) check("8. the planted ClinVar hit's row, viewer link, tick and star bar carry its secondary finding's id, the " "deletion's span its own, every other card's row of the gene and the symptom ranking's finding one of them, " diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index 5965c1045..27c67ed60 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -344,6 +344,10 @@ def browser_checks(port, work): loaded = b.wait("var i = document.querySelector('img.pedigree'); i && i.complete && i.naturalWidth > 0 && " "i.naturalWidth", 10) check("browser: the pedigree image loads", loaded, loaded) + # father_a and mother_a are also children (of gf_a and gm_a) whose results this server cannot read, so the + # couple screen stays closed whole until the button of each opens it. + shut = open_screen(b) + check("browser: the couple screen of a partner who is a child stays closed until 'Show the screen'", shut, shut) couple = b.eval("var p = document.querySelector('p[data-couple=\\'father_a mother_a\\']'); p && p.textContent") check("browser: the couple line counts 1 shared and 1 X-linked and links the report", couple and "1 gene(s) where both carry a finding, 1 X-linked" in couple @@ -368,6 +372,16 @@ def browser_checks(port, work): b.close() +def open_screen(b): + """Click the closed boxes of father_a's and mother_a's couple screen, one per partner, once they show with no + table or link: whether they did and the screen then opened.""" + shut = b.wait("document.querySelectorAll('#ch-couples .veil').length === 2 && !document.querySelector('#ch-couples table') " + "&& !document.querySelector('#ch-couples p[data-couple] a') && 'shut'", 10) + for who in ("father_a", "mother_a"): + b.eval("var v = document.querySelector('#ch-couples .veil[data-veil-child=%s] button'); v && v.click(); true" % who) + return bool(shut) and bool(b.wait("!!document.querySelector('#ch-couples table') && !document.querySelector('#ch-couples .veil')", 5)) + + # A stale blood-group section with two systems past ABO and RhD, for child_a's summary.json. STALE_SUMMARY = {"sections": {"bloodtype": {"state": "stale", "data": { "abo": "A1", "rhd": "D+", "systems": [{"system": "ABO", "phenotype": "A1"}, {"system": "RhD", "phenotype": "D+"}, @@ -396,6 +410,7 @@ def depth_checks(b, base, root): clin = os.path.join(root, "child_a", "clinical") try: b.go(base + "/runs/family_a/") + open_screen(b) ready = b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " "!!document.querySelector('div[data-shared=GENE4]')") four = b.eval("var k = document.querySelector('div[data-shared=GENE1] .kids4 .grid4'); k ? k.children.length : -1") From c31290e0952b5662f91165d8dce10b52b65a4aa6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:40:27 +0100 Subject: [PATCH 013/255] fix(live): keep a phone's person page short with the genome map On Linux fonts DEMO-002's page at 390 px came to 8062 px, over the 8000 px bound. A phone now gets 10 px rows, one paragraph for the genome sentences and a chip without its reason. The map is also no longer sized while it has no width, which gave a phone the desktop's rows. Bead: pgp-iqon.8 --- assets/live/map.css | 11 +++++++++++ assets/live/map.js | 29 +++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/assets/live/map.css b/assets/live/map.css index cc3e42e00..8dc7df978 100644 --- a/assets/live/map.css +++ b/assets/live/map.css @@ -55,3 +55,14 @@ .gm-margin { margin-top: 12px; } .gm-narration { font-size: 14px; } } +@media (max-width: 520px) { + .gmap { margin-top: 22px; } + .gm-why { display: none; } + .gm-margin { margin-top: 4px; } + .gm-head { gap: 4px; margin-bottom: 6px; } + .gm-chip { padding: 4px 8px; } + .gm-chip button { padding: 3px 9px; } + .gm-narration { margin-top: 4px; } + .gm-titlerow { row-gap: 6px; } + .gm-title { font-size: 20px; } +} diff --git a/assets/live/map.js b/assets/live/map.js index 94ffbaee0..c8b2e3c35 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -30,8 +30,8 @@ var MONO = 'ui-monospace, "SF Mono", Menlo, monospace'; // The chromosome level ends at 16,667 bp per CSS pixel: closer views (region, gene, reads) come later. var CHROM_BPP = 16667; - // A genome row's least height: 22 CSS px, 14 on a phone (the page there is long enough already). - var ROW_MIN = 22, ROW_PHONE = 14; + // A genome row's least height: 22 CSS px, 10 on a phone (the page there is long enough already). + var ROW_MIN = 22, ROW_PHONE = 10; var reduced = window.matchMedia && window.matchMedia('(prefers-reduced-motion: reduce)').matches; function lerp(a, b, t) { return a + (b - a) * t; } @@ -180,8 +180,11 @@ } // --- the canvas size, the hatch pattern --- + // False while the stage has no width (not in the page yet, or hidden): sized then, a phone would get the + // desktop's rows. The ResizeObserver draws once it has one. function size() { - var w = Math.max(280, Math.floor(stage.clientWidth || 680)); + if (!stage.clientWidth) { return false; } + var w = Math.max(280, Math.floor(stage.clientWidth)); phone = w < 520; var n = M ? M.chroms.length : 24; var h = 18 + n * (phone ? ROW_PHONE : ROW_MIN); @@ -194,6 +197,7 @@ canvas.style.setProperty('height', h + 'px'); cacheKey = ''; } + return true; } function pattern(alpha) { var p = document.createElement('canvas'); @@ -295,7 +299,7 @@ M.chroms.forEach(function (ch, i) { var yc = rowY(i); c.fillStyle = INK3; - c.font = '11px ' + SANS; + c.font = (phone ? '9px ' : '11px ') + SANS; c.textAlign = 'right'; c.textBaseline = 'middle'; c.fillText(short(ch.n), gutG() - 10, yc); @@ -354,7 +358,7 @@ raf = 0; if (!M) { return; } var t = performance.now(); - size(); // the page may have changed width since the last frame (a phone turned, a fold opened) + if (!size()) { return; } // the page may have changed width since the last frame (a phone turned, a fold opened) step(); ctx.setTransform(1, 0, 0, 1, 0, 0); ctx.clearRect(0, 0, canvas.width, canvas.height); @@ -758,7 +762,14 @@ add(narr, el('p', {}, fs.some(function (r) { return r.chrom === ch.n; }) ? 'No finding in this stretch.' : noFindingHere())); } if (lv === 'genome') { - add(narr, runs(M.text.genome)); + var tail = runs(M.text.genome), lead = narr.children.length === 1 ? narr.firstChild : null; + if (phone && lead) { + // On a phone the count and the runs share one paragraph. + add(lead, ' '); + while (tail.firstChild) { lead.appendChild(tail.firstChild); } + } else { + add(narr, tail); + } if (!phone) { add(narr, el('p', { 'class': 'dim' }, 'Click a chromosome to open it.')); } } else { var r = M.roh[ch.n], n = 0, longest = 0; @@ -828,16 +839,18 @@ var s = stateOf(id); if (s === 'closed') { (M.gated[id] ? gates : acts).push(id); } else if (s === 'opened') { opened.push(id); } }); + // Why it is closed: said beside the count, left out on a phone (the card says it too). + function why() { return el('span', { 'class': 'gm-why' }, ' · Not drawn until opened on purpose'); } if (acts.length) { var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'act' }); - add(c, el('span', {}, (acts.length === 1 ? '1 result closed' : acts.length + ' results closed') + ' · Not drawn until opened on purpose')); + add(c, add(el('span', {}, acts.length === 1 ? '1 result closed' : acts.length + ' results closed'), why())); add(c, confirming === 'act' ? confirmBox(acts[0], false) : button('Open', function () { confirming = 'act'; drawChips(); })); add(chips, c); } gates.forEach(function (id) { var g = M.gated[id], name = g.gene || 'This result'; var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'gate' }); - add(c, el('span', {}, name + ' closed · Not drawn until opened on purpose')); + add(c, add(el('span', {}, name + ' closed'), why())); add(c, confirming === id ? confirmBox(id, true) : button('Open ' + name, function () { confirming = id; drawChips(); })); add(chips, c); }); From 1ff5652230905916288f108c91d06cdbfea43cb4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:47:45 +0100 Subject: [PATCH 014/255] test(live): check a closed family page only once the child's results have settled A page still waiting for a child's results keeps every place closed anyway, so the checks of the unreadable state could pass on a request that never ended. They now first require the page to report the child's results as known or failed. --- tests/test_live_demo.py | 6 +++++- tests/test_owngenome_live_family.py | 9 ++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index ca4f4fb45..6dd6b8df9 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -100,6 +100,7 @@ import copy import json import os +import re import shutil import sys import tempfile @@ -905,8 +906,11 @@ def family_unknown(b, base, opens=True): planted results shown; then (OPENS) the de novo box's own button opens his de novo rows and no other place.""" gene, hit, sv = planted_words() family_page(b, base) + # A page still waiting for his results keeps everything closed anyway: the check is on the state it settles in. + kids = b.eval("document.documentElement.dataset.pgpKids || ''") or "" shown = b.eval(FAMILY_SHOWN_JS) or "" - out = [f"{w} shown with the son's results unreadable" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] + out = [] if re.search(r"\bDEMO-003:(known|failed)\b", kids) else [f"the son's results never settled: {kids!r}"] + out += [f"{w} shown with the son's results unreadable" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] if boxes_of(b) != SON_BOXES or son_tables(b) is None or len(son_tables(b)) != 1: out.append(f"closed places {boxes_of(b)}, tables {son_tables(b)}") words = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('.veil[data-veil-child=\"DEMO-003\"] button'), " diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index 27c67ed60..42d4253fa 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -355,9 +355,11 @@ def browser_checks(port, work): b.eval("document.querySelector('details[data-child=child_a]').open = true; true") # child_a's results cannot be built here (its summary is cut short), so the page cannot tell whether a # result to act on is closed: the map stays closed until "Show the map" is clicked. - shut = b.wait("var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=child_a] .veil'), " + # Only once the page has given up on child_a's results: a page still waiting keeps the map closed anyway. + shut = b.wait("/\\bchild_a:failed\\b/.test(document.documentElement.dataset.pgpKids || '') && " + "(function () { var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=child_a] .veil'), " "function (x) { return x.textContent.indexOf('Show the map') >= 0; })[0]; " - "v && !document.querySelector('details[data-child=child_a] img') && (v.querySelector('button').click(), true)", 10) + "return v && !document.querySelector('details[data-child=child_a] img') && (v.querySelector('button').click(), true); }())", 10) check("browser: child_a's chromosome map stays closed while its results cannot be read", shut, shut) seg = b.wait("var i = document.querySelector('details[data-child=child_a] img'); " "i && i.complete && i.naturalWidth > 0 && i.getAttribute('src')", 10) @@ -375,7 +377,8 @@ def browser_checks(port, work): def open_screen(b): """Click the closed boxes of father_a's and mother_a's couple screen, one per partner, once they show with no table or link: whether they did and the screen then opened.""" - shut = b.wait("document.querySelectorAll('#ch-couples .veil').length === 2 && !document.querySelector('#ch-couples table') " + shut = b.wait("!/:pending\\b/.test(document.documentElement.dataset.pgpKids || ':pending') && " + "document.querySelectorAll('#ch-couples .veil').length === 2 && !document.querySelector('#ch-couples table') " "&& !document.querySelector('#ch-couples p[data-couple] a') && 'shut'", 10) for who in ("father_a", "mother_a"): b.eval("var v = document.querySelector('#ch-couples .veil[data-veil-child=%s] button'); v && v.click(); true" % who) From 4409a9680e7345eeb3f24131b9da00ada3f6b039 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:44:08 +0100 Subject: [PATCH 015/255] fix(live): keep an overlapping closed result off the map and answer the map review A row that carries another result's tag now also asks its own result, so opening one closed result never draws a second one under its tag. The runs of homozygosity wait for their step in a replay, Back to an address with no place returns to the whole genome, a DPR change resizes the bitmap, a failed reload clears the chips, and an open chromosome gets zoom buttons for touch. The tests use OG when set and take no map screenshot of a map that did not draw. --- assets/live/map.css | 3 ++ assets/live/map.js | 72 +++++++++++++++++++++++++++----------- assets/live/person.js | 7 ++-- docs/live-site.md | 2 +- tests/test_live_demo.py | 76 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 133 insertions(+), 27 deletions(-) diff --git a/assets/live/map.css b/assets/live/map.css index 8dc7df978..99d59a657 100644 --- a/assets/live/map.css +++ b/assets/live/map.css @@ -66,3 +66,6 @@ .gm-titlerow { row-gap: 6px; } .gm-title { font-size: 20px; } } +.gm-zoom { display: inline-flex; border: 1px solid var(--hair-2); border-radius: 999px; overflow: hidden; } +.gm-zoom[hidden] { display: none; } +.gm-zoom button { border: 0; border-radius: 0; background: transparent; color: var(--ink-2); padding: 6px 14px; font-size: 14px; line-height: 1; } diff --git a/assets/live/map.js b/assets/live/map.js index c8b2e3c35..e7032c7c9 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -3,8 +3,9 @@ // the 24 chromosomes as rows with their bands, the masked spans hatched, each finding with a place as its // level's glyph, and the runs of homozygosity; a click opens a chromosome in place, where the bands, the // findings and the runs get lanes of their own under a ruler. Every mark comes from the document, never from -// results.json, and every sentence the margin shows is one the document wrote (text, phrase, empty, note), -// with the counts filled in here. +// results.json. The margin's findings count, chromosome sentences, runs sentence and finding phrases are the +// document's (text, phrase, empty, note); this file adds only short lines of its own: an empty lane or +// stretch, the runs in view, how to move, and why a layer is not on the map. // // The veil. The document is the same for every reader: each row a closed result could reveal carries the id // of that result (veil). Whether it is closed is the page's question (person.js veil()): a closed row is not @@ -93,14 +94,18 @@ }); var gated = {}; rows.forEach(function (r) { if (r.gate) { gated[r.id] = r; } }); + // What can be closed: each veil tag, and each result to act on that carries another result's tag (a row + // over two of them carries only the first one's tag, and one over APOE carries APOE's): its own id too. + var closable = veilIds.slice(); + rows.forEach(function (r) { if (r.veil && r.level === 'act' && !r.gate && closable.indexOf(r.id) < 0) { closable.push(r.id); } }); return { doc: d, id: id, label: person.label || id, chroms: chroms, by: by, bands: bands, roh: roh, bins: bins, - binSize: (per.roh_bins && per.roh_bins.size) || 1e6, findings: rows, gated: gated, veilIds: veilIds, + binSize: (per.roh_bins && per.roh_bins.size) || 1e6, findings: rows, gated: gated, veilIds: veilIds, closable: closable, layers: d.layers || {}, text: d.text || {}, art: (d.dict && d.dict.art) || [] }; } // --- the figure --------------------------------------------------------------------------------------- // create({url, veil(id, gated) -> 'open'|'closed'|'opened', now(id) -> shown at this moment, open(id), - // close(id), fallback() -> rows}): one figure per page. The page calls refresh(key) on each of its redraws; a new key + // close(id), stepDone(step) -> ended at this moment, fallback() -> rows}): one figure per page. The page calls refresh(key) on each of its redraws; a new key // fetches the document again and keeps the camera. function create(opts) { var M = null, key = null, failure = null, t0 = 0; @@ -113,9 +118,14 @@ var stopG = el('button', { type: 'button', 'data-stop': 'genome' }, 'Genome'); var stopC = el('button', { type: 'button', 'data-stop': 'chromosome' }, 'Chromosome'); add(gauge, stopG, stopC); + // Zoom buttons for an open chromosome: the touch path to what the wheel and the + and - keys do. + var zoom = el('div', { 'class': 'gm-zoom', role: 'group', 'aria-label': 'Zoom', hidden: 'hidden' }); + var zoomOut = el('button', { type: 'button', 'aria-label': 'Zoom out', 'data-zoom': 'out' }, '−'); + var zoomIn = el('button', { type: 'button', 'aria-label': 'Zoom in', 'data-zoom': 'in' }, '+'); + add(zoom, zoomOut, zoomIn); var crumb = el('nav', { 'class': 'gm-crumb mono', 'aria-label': 'Where the map is' }); var chips = el('div', { 'class': 'gm-chips' }); - add(head, kicker, add(el('div', { 'class': 'gm-titlerow' }), title, gauge), crumb, chips); + add(head, kicker, add(el('div', { 'class': 'gm-titlerow' }), title, gauge, zoom), crumb, chips); var stage = el('div', { 'class': 'gm-stage' }); var canvas = el('canvas', { 'class': 'gm-canvas', tabindex: '0', role: 'img', 'aria-describedby': 'gm-narration' }); var tip = el('div', { 'class': 'gm-tip', role: 'tooltip', hidden: 'hidden' }); @@ -149,7 +159,19 @@ function stateOf(id) { return opts.veil(id, !!(M && M.gated[id])); } // A row shows when its result is not closed and the page shows its finding at this moment (a replay // before the finding's step ended does not). - function shownRow(r) { return (!r.veil || stateOf(r.veil) !== 'closed') && (!opts.now || opts.now(r.id)); } + // A row under another result's tag also stays hidden while its own result is closed. + function shownRow(r) { + return (!r.veil || stateOf(r.veil) !== 'closed') && (r.veil === r.id || M.closable.indexOf(r.id) < 0 || + stateOf(r.id) !== 'closed') && (!opts.now || opts.now(r.id)); + } + // The runs of homozygosity, as this page shows them now: a replay before their step ended has none yet. + function rohLayer() { + var lay = M.layers.roh; + if (lay && landed(lay) && lay.step && opts.stepDone && !opts.stepDone(lay.step)) { + return { state: 'waiting', step: lay.step, step_name: lay.step_name }; + } + return lay; + } function drawn() { return M ? M.findings.filter(shownRow) : []; } // --- geometry --- @@ -189,7 +211,7 @@ var n = M ? M.chroms.length : 24; var h = 18 + n * (phone ? ROW_PHONE : ROW_MIN); DPR = Math.min(2, window.devicePixelRatio || 1); - if (w !== CW || h !== CH) { + if (w !== CW || h !== CH || canvas.width !== Math.round(w * DPR)) { CW = w; CH = h; canvas.width = Math.round(w * DPR); canvas.height = Math.round(h * DPR); @@ -306,7 +328,7 @@ c.textAlign = 'left'; body(c, ch, gx, yc, 8, 1); // Runs of homozygosity under the body: the long ones solid, the rest as 1 Mb coverage bins. - var bins = M.bins[ch.n], ry = yc + 6.5; + var rohOn = landed(rohLayer()), bins = rohOn && M.bins[ch.n], ry = yc + 6.5; if (bins) { c.fillStyle = INK2; for (var b = 0; b < bins.length; b++) { @@ -316,7 +338,7 @@ } c.globalAlpha = 1; } - var rr = M.roh[ch.n]; + var rr = rohOn && M.roh[ch.n]; if (rr) { c.strokeStyle = INK; c.lineWidth = 2.5; @@ -545,7 +567,7 @@ }); } function openRoh(L, v, ch) { - var lay = M.layers.roh || {}, g = gut(), y = L.ry + L.rh / 2; + var lay = rohLayer() || {}, g = gut(), y = L.ry + L.rh / 2; if (!landed(lay)) { laneWords(ctx, layerWords(lay), g + 4, y); return; } var r = M.roh[ch.n]; if (!r || !r.n) { laneWords(ctx, 'No run of homozygosity on this chromosome.', g + 4, y); return; } @@ -570,7 +592,7 @@ function noFindingHere() { return closedAny() ? 'No open finding with a place on this chromosome.' : 'No finding with a place on this chromosome.'; } - function closedAny() { return M.veilIds.some(function (id) { return stateOf(id) === 'closed'; }); } + function closedAny() { return M.closable.some(function (id) { return stateOf(id) === 'closed'; }); } function layerWords(lay) { if (lay.state === 'missing') { return 'Not on the map: ' + (lay.note || ('step ' + lay.step + ' wrote no ' + (lay.source || 'file') + '.')); } if (lay.state === 'unreadable') { return 'Not on the map: ' + (lay.source || '') + ' could not be read' + (lay.note ? ' (' + lay.note + ')' : '') + '.'; } @@ -682,7 +704,8 @@ if (!M) { return; } var h; try { h = decodeURIComponent((window.location.hash || '').slice(1)); } catch (e) { return; } - if (!h) { return; } + // Back to the address with no place (the page as first loaded): the whole genome again. + if (!h) { if (!scroll && tgt.k) { tgt.k = 0; last = 0; schedule(); } return; } var p = parsePlace(h); if (p === null) { return; } if (p === false) { say('Could not read the place “' + h + '”'); return; } @@ -717,6 +740,7 @@ fig.setAttribute('data-view', locusWords()); stopG.setAttribute('aria-pressed', String(lv === 'genome')); stopC.setAttribute('aria-pressed', String(lv !== 'genome')); + zoom.hidden = lv === 'genome'; crumb.textContent = ''; crumb.hidden = lv === 'genome'; // the gauge already says Genome var gl = el('button', { type: 'button', 'class': 'gm-crumb-g' }, 'Genome'); @@ -762,7 +786,8 @@ add(narr, el('p', {}, fs.some(function (r) { return r.chrom === ch.n; }) ? 'No finding in this stretch.' : noFindingHere())); } if (lv === 'genome') { - var tail = runs(M.text.genome), lead = narr.children.length === 1 ? narr.firstChild : null; + var rl = rohLayer(); + var tail = rl === M.layers.roh ? runs(M.text.genome) : runs([{ t: layerWords(rl), b: false }]), lead = narr.children.length === 1 ? narr.firstChild : null; if (phone && lead) { // On a phone the count and the runs share one paragraph. add(lead, ' '); @@ -774,10 +799,10 @@ } else { var r = M.roh[ch.n], n = 0, longest = 0; if (r) { for (var i = first(r, v[0]); i < r.n && r.s[i] < v[1]; i++) { n++; longest = Math.max(longest, r.e[i] - r.s[i]); } } - if (n && landed(M.layers.roh || {})) { + if (n && landed(rohLayer())) { add(narr, el('p', {}, 'Runs of homozygosity in view: ' + n + ', the longest ' + mb(longest) + ' Mb.')); } - if (!phone) { add(narr, el('p', { 'class': 'dim' }, 'Drag to move along it, scroll to zoom, Esc for the whole genome.')); } + if (!phone) { add(narr, el('p', { 'class': 'dim' }, 'Drag to move along it, scroll or use + and − to zoom, Esc for the whole genome.')); } } drawLayers(); drawMarks(); @@ -793,7 +818,7 @@ function drawLayers() { layersBox.textContent = ''; [['findings', 'Findings'], ['roh', 'Runs of homozygosity']].forEach(function (x) { - var lay = M.layers[x[0]]; + var lay = x[0] === 'roh' ? rohLayer() : M.layers[x[0]]; var row = el('div', { 'class': 'gm-layer', 'data-layer': x[0], 'data-state': lay ? lay.state : 'missing' }); var dd = el('dd', {}); if (!lay) { add(dd, 'Not on the map: this server’s map has no ' + x[1].toLowerCase() + ' layer.'); } @@ -835,7 +860,7 @@ chips.textContent = ''; if (!M) { return; } var acts = [], gates = [], opened = []; - M.veilIds.forEach(function (id) { + M.closable.forEach(function (id) { var s = stateOf(id); if (s === 'closed') { (M.gated[id] ? gates : acts).push(id); } else if (s === 'opened') { opened.push(id); } }); @@ -971,9 +996,7 @@ var i = (open ? tgt.ci : (e.key === ']' ? -1 : M.chroms.length)) + (e.key === ']' ? 1 : -1); openChrom((i + M.chroms.length) % M.chroms.length, undefined, undefined, true); } else if (open && (e.key === '+' || e.key === '=' || e.key === '-' || e.key === '−')) { - tgt.ls = Math.log(span * (e.key === '-' || e.key === '−' ? 1.6 : 1 / 1.6)); - clampTgt(); - changed(false); + zoomBy(e.key === '-' || e.key === '−' ? 1.6 : 1 / 1.6); } else if (open && (e.key === 'ArrowLeft' || e.key === 'ArrowRight')) { tgt.c += (e.key === 'ArrowLeft' ? -0.2 : 0.2) * span; clampTgt(); @@ -982,6 +1005,14 @@ else { done = false; } if (done) { e.preventDefault(); } }); + function zoomBy(f) { + if (!M || level() === 'genome') { return; } + tgt.ls = Math.log(Math.exp(tgt.ls) * f); + clampTgt(); + changed(false); + } + zoomOut.addEventListener('click', function () { zoomBy(1.6); }); + zoomIn.addEventListener('click', function () { zoomBy(1 / 1.6); }); stopG.addEventListener('click', function () { closeChrom(true); }); stopC.addEventListener('click', function () { if (level() !== 'genome') { openChrom(tgt.ci, undefined, undefined, true); return; } @@ -1010,6 +1041,7 @@ } narr.textContent = ''; layersBox.textContent = ''; + chips.textContent = ''; // a chip drawn before the failure may name a result closed since } function load(k) { key = k; diff --git a/assets/live/person.js b/assets/live/person.js index 83ad5a151..021cff5e1 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -553,7 +553,10 @@ // Whether the page shows a finding of the map at this moment: its card's steps are done (a replay draws // the run's end results from its first frame). A row with no finding of these results (the map's own APOE // row) follows its card's fold instead. - function mapNow(id) { var f = findingById(id); var c = f && cardById(f.card); return !f || !!(c && gate(c) === 'show'); } + // A finding this page keeps closed is not shown either, whatever row tag it shares with another result. + function mapNow(id) { var f = findingById(id); var c = f && cardById(f.card); return !f || (!!(c && gate(c) === 'show') && !veiled(f)); } + // Whether a step of the run has ended at this moment (a replay before it has not): the map's layers follow it. + function stepDone(step) { var s = stepsBy()[step]; return !s || s.state === 'done'; } // Without the map (a browser with no DecompressionStream, or a map the server could not build): the findings // with a place, as a table, each through this page's veil first. function mapFallback() { @@ -881,7 +884,7 @@ // reader left it. It asks this page's veil for each closed row, and opens or closes a result through it. if (window.PGPMap) { if (!genomeMap) { - genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, open: function (id) { setShown(id, true); }, + genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, stepDone: stepDone, open: function (id) { setShown(id, true); }, close: function (id) { setShown(id, false); }, fallback: mapFallback }); } add(slot, genomeMap.node); diff --git a/docs/live-site.md b/docs/live-site.md index 025b3f34c..d5c57e616 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -70,7 +70,7 @@ While the run goes, the page still opens on the report being written. A short ba The header search finds a gene or rsID across the results and opens the card that names it; on a page that keeps results closed it leaves out every hit that names one (above). One row of the results that the page cannot read (a field of the wrong type) costs that row only: it shows as a finding on its card, "A row of this section could not be read", at the highest level its rule can give (to act on for a rule that can give one, so a page that keeps results closed keeps that card closed too), the card whose table cannot be drawn says so, and every other card and finding is on the page. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. -**Figure 1, the genome map.** The 24 chromosomes as rows, to scale (23 for a person with no Y), each with its bands, the centromere as a waist and the stretches few reads can map hatched (centromeres, heterochromatin, the acrocentric arms). Each finding with a place sits on its chromosome as its level's shape, with its gene beside it when the name fits; runs of homozygosity of 5 Mb or more are a line under the row, shorter ones a faint band. A finding that names a gene but no position (a medicine gene, Lp(a)) sits on that gene's span; a finding with neither stays in its section only. Click a chromosome (or, with the map focused, press `[` or `]`) and it opens in place: an overview with the part in view, a ruler in Mb, the bands with their names, a findings lane with each gene and its level's words, and a lane for the runs of homozygosity, each run with its length. Drag to move along it, scroll to zoom in (as far as about 10 Mb across a wide screen; closer views come later), and press Esc (or click Genome) to go back. A dashed line marks the centre, and the margin says what is under it: the chromosome, the position and the band, the findings in view and the runs. Every sentence in the margin is one the map document carries; the page only fills in the counts. The address follows the map: `#genome`, `#chr7`, or `#chr7:117.4-137.4M`, and a link to one of them opens that view; a place the page cannot read shows a short note and leaves the map as it was. The map comes from `/runs//map.json`, which the map plugin (`bin/owngenome_live_map.py`) builds with `og map build` from the run folder and the results the page gets (`rust/crates/og-map/README.md` describes the document). It needs og on the host (`scripts/setup.sh --og`); without it, or in a browser that cannot unpack the file (no `DecompressionStream`), the figure says so and lists the findings with a place in a table instead. A layer whose file is missing or cannot be read says so in words, never as a blank lane. +**Figure 1, the genome map.** The 24 chromosomes as rows, to scale (23 for a person with no Y), each with its bands, the centromere as a waist and the stretches few reads can map hatched (centromeres, heterochromatin, the acrocentric arms). Each finding with a place sits on its chromosome as its level's shape, with its gene beside it when the name fits; runs of homozygosity of 5 Mb or more are a line under the row, shorter ones a faint band. A finding that names a gene but no position (a medicine gene, Lp(a)) sits on that gene's span; a finding with neither stays in its section only. Click a chromosome (or, with the map focused, press `[` or `]`) and it opens in place: an overview with the part in view, a ruler in Mb, the bands with their names, a findings lane with each gene and its level's words, and a lane for the runs of homozygosity, each run with its length. Drag to move along it, scroll or press the + and − buttons beside the zoom stops to zoom in (as far as about 10 Mb across a wide screen; closer views come later), and press Esc (or click Genome) to go back. Back in the browser returns to the view before, the whole genome when the address had no place. A dashed line marks the centre, and the margin says what is under it: the chromosome, the position and the band, the findings in view and the runs. The findings count, the chromosome sentences, the runs sentence and each finding's phrase come from the map document; the page adds its own short lines: a lane or stretch with no finding, the runs in view, how to move, and why a layer is not on the map. A replay draws the runs of homozygosity only once their step has ended. The address follows the map: `#genome`, `#chr7`, or `#chr7:117.4-137.4M`, and a link to one of them opens that view; a place the page cannot read shows a short note and leaves the map as it was. The map comes from `/runs//map.json`, which the map plugin (`bin/owngenome_live_map.py`) builds with `og map build` from the run folder and the results the page gets (`rust/crates/og-map/README.md` describes the document). It needs og on the host (`scripts/setup.sh --og`); without it, or in a browser that cannot unpack the file (no `DecompressionStream`), the figure says so and lists the findings with a place in a table instead. A layer whose file is missing or cannot be read says so in words, never as a blank lane. **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 0fedd27d5..65a965532 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1185,9 +1185,16 @@ def fold_checks(b, base, out): def og_binary(): - """An og that runs here: bin/og when scripts/setup.sh --og built it, else one cargo builds now from rust/ - without the BAM feature (the map reads no BAM), kept in CARGO_TARGET_DIR or a folder of the temp dir. - None when neither works.""" + """An og that runs here: OG when set (Guard's unit-tests job and the docs builds set it to the og they + built, as tests/og_binary.py reads it), else bin/og when scripts/setup.sh --og built it, else one cargo + builds now from rust/ without the BAM feature (the map reads no BAM), kept in CARGO_TARGET_DIR or a folder + of the temp dir. None when none of them works.""" + named = os.environ.get("OG") + if named: + if os.access(named, os.X_OK): + return named + print(f"OG={named} is not an executable og binary", file=sys.stderr) + return None og = os.path.join(REPO, "bin", "og") if subprocess.run([og, "map", "--help"], capture_output=True, check=False).returncode == 0: return og @@ -1426,6 +1433,48 @@ def leaks(c): return bad +def overlap_problems(b, base, out): + """On the son's closed page, a second result to act on over the first carries only the first one's tag + (rust/crates/og-map/README.md). Opening the first leaves the second closed: in no mark, sentence, link, + title or address of the map.""" + son = OTHERS[1] + path = os.path.join(out, "runs", son, "map.json") + with open(path, "rb") as fh: + keep = fh.read() + try: + doc = read_map(path) + f = doc["per"][son]["findings"] + acts = [i for i, lv in enumerate(f["level"]) if lv == "act" and not f["gate"][i]] + if not acts: + return ["the son's map has no result to act on to close"] + i, n = acts[0], len(f["id"]) + fid = f["id"][i] + for k in f: + if isinstance(f[k], list) and len(f[k]) == n: + f[k].append(f[k][i]) + plant = {"id": "planted-overlap", "gene": "PLANTEDGENE", "title": "Planted overlapping result", + "href": "finding/planted-overlap/", "phrase": "Planted overlap phrase."} + for k, v in plant.items(): + f[k][-1] = v + write_map(path, doc) + c = map_page(b, base, son, "#" + f["chrom"][i]) + if c["state"] != "drawn": + return [f"the planted map: {c['state']}"] + b.eval("document.querySelector('#f-%s .veil button').click(); true" % fid) + c = map_now(b) + bad = [] + if fid not in [m["id"] for m in c["marks"]]: + bad.append(f"the first result did not open: marks {[m['id'] for m in c['marks']]}") + leaked = [w for w in plant.values() if w in c["text"] or w in c["tip"] or w in c["hash"]] + leaked += [m["id"] for m in c["marks"] if m["id"] == "planted-overlap"] + if leaked: + bad.append(f"opening the first shows the second: {leaked}") + return bad + finally: + with open(path, "wb") as fh: + fh.write(keep) + + def missing_layer_problems(b, base, out): """A map whose ROH layer is missing says so in words, in the margin, never a blank.""" path = os.path.join(out, "runs", S, "map.json") @@ -1478,7 +1527,12 @@ def map_screens(b, base, dest): b.call("Emulation.setDeviceMetricsOverride", {"width": w, "height": h, "deviceScaleFactor": 1, "mobile": False}, b.session) for run, link, name in ((S, "", "genome"), (S, "#chr16", "chromosome"), (OTHERS[1], "", "closed")): - map_page(b, base, run, link) + got = map_page(b, base, run, link) + want = "chromosome" if link else "genome" + if got["state"] != "drawn" or got["level"] != want: + print(f"ERROR: {run}{link}: the map is {got['state']} at {got['level'] or 'no'} level, " + f"want drawn at {want}: no screenshot") + continue b.eval("document.getElementById('genome-map').scrollIntoView({block: 'start'}); true") time.sleep(0.5) png = b.call("Page.captureScreenshot", {"format": "png"}, b.session)["data"] @@ -1545,6 +1599,20 @@ def map_checks(b, base, out): finally: with open(js, "w") as fh: fh.write(keep_js) + bad = overlap_problems(b, base, out) + check("map: a second closed result to act on over the first stays closed when the first is opened", not bad, bad) + mjs = os.path.join(out, "assets", "map.js") + with open(mjs) as fh: + keep_mjs = fh.read() + own = "(r.veil === r.id || M.closable.indexOf(r.id) < 0 ||\n stateOf(r.id) !== 'closed') && " + try: + with open(mjs, "w") as fh: + fh.write(keep_mjs.replace(own, "")) + control("map.js whose rows ask only their tag, never their own result", keep_mjs.count(own) != 1 or + not overlap_problems(b, base, out)) + finally: + with open(mjs, "w") as fh: + fh.write(keep_mjs) bad = missing_layer_problems(b, base, out) check("map: a missing layer says why in words, never a blank", not bad, bad) bad = fallback_problems(b, base, out) From c3c5abc5351a008481ebdb7cfe675561a2afc02b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:14:00 +0100 Subject: [PATCH 016/255] fix(live): read malformed veil.spans as unreadable and test each box, failed request, nothing closed and a moved run The family page took any array as veil.spans. An entry that is not [id, ...] blanked the page, and an empty list beside findings to act on opened every place. Such spans now read as unreadable, so every place stays closed. The tests now click each box in turn while the results cannot be read (the son's four boxes, the map image against the tables, the couple screen against the partner's cells), cover a results.json cut short, a child with nothing to act on and a run that moves while the page is open, and compare whole pages for a copy-number span, a CHIP call, the couple fixture and the map image, each view settled on knowing the results. Each new check has a planted control that turns it red. --- assets/live/family-page.js | 6 +- docs/live-site.md | 2 +- tests/test_live_demo.py | 275 ++++++++++++++++++++++++++++++++----- 3 files changed, 248 insertions(+), 35 deletions(-) diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 584ed17d3..575d1ee72 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -107,7 +107,11 @@ // Results the server could not build come back without their findings: as unreadable as no answer. var usable = !!(r && Array.isArray(r.findings)); var acts = usable ? r.findings.filter(function (x) { return x.level === 'act'; }).map(function (x) { return x.id; }) : []; - var next = usable ? { status: 'known', url: url, spans: Array.isArray(v.spans) ? v.spans : null, acts: acts } + // Spans that are not [id, ...] entries naming every result to act on cannot say which are closed: read + // as none given (UNKNOWN while there are results to act on). + var spansOk = Array.isArray(v.spans) && v.spans.every(function (s) { return Array.isArray(s) && typeof s[0] === 'string'; }) && + acts.every(function (id) { return v.spans.some(function (s) { return s[0] === id; }); }); + var next = usable ? { status: 'known', url: url, spans: spansOk ? v.spans : null, acts: acts } : { status: 'failed', url: url }; // An answer that cannot be read keeps what was known: the page never opens on a failed request. if (!usable && cur.status === 'known') { return; } diff --git a/docs/live-site.md b/docs/live-site.md index 89d07f916..245d2297a 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -72,7 +72,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page, and what the page shows of a child never depends on which result is closed or where it lies. While a child has any result to act on still closed, every place of the page that could carry one closes for that child in the same way, each behind one box with the same words whatever the result: the grandparents of the painted stretches (every stretch is then drawn and listed from its parent alone), the chromosome map when it comes only as an image, every de novo candidate and every compound-het pair, the child's cell in every medicines and HLA row of the side by side, and the carrier screen of a couple the child is part of (its counts, its boxes, its table, the lead's words on it and its report link). Each "Show the result" opens one result, from whichever box it is clicked; the places come back once the child's last closed result is opened, and a reload closes them again. Opening a result opens it for that child only. The page reads each child's `results.json` to know whether a result is closed; while it cannot (no run here, no answer yet, a request that failed, results the server could not build, or results with findings to act on but no `veil.spans`), every place of that child stays closed, and each box opens only its own place on purpose: "Show the rows" its table, "Show the cells" the side by side, "Show the grandparents" the painting, "Show the map" the image and "Show the screen" the couple screen. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. Every row stays, with the parents' values: a child's medicines and HLA cells read "closed" while chapter IV's rule closes them (a row left out would leave a gap in a list everyone knows), and the haplogroups and blood groups, which name no gene, stay open. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page, and what the page shows of a child never depends on which result is closed or where it lies. While a child has any result to act on still closed, every place of the page that could carry one closes for that child in the same way, each behind one box with the same words whatever the result: the grandparents of the painted stretches (every stretch is then drawn and listed from its parent alone), the chromosome map when it comes only as an image, every de novo candidate and every compound-het pair, the child's cell in every medicines and HLA row of the side by side, and the carrier screen of a couple the child is part of (its counts, its boxes, its table, the lead's words on it and its report link). Each "Show the result" opens one result, from whichever box it is clicked; the places come back once the child's last closed result is opened, and a reload closes them again. Opening a result opens it for that child only. The page reads each child's `results.json` to know whether a result is closed; while it cannot (no run here, no answer yet, a request that failed, results the server could not build, or results with findings to act on whose `veil.spans` is missing or does not list each of them), every place of that child stays closed, and each box opens only its own place on purpose: "Show the rows" its table, "Show the cells" the side by side, "Show the grandparents" the painting, "Show the map" the image and "Show the screen" the couple screen. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. Every row stays, with the parents' values: a child's medicines and HLA cells read "closed" while chapter IV's rule closes them (a row left out would leave a gap in a list everyone knows), and the haplogroups and blood groups, which name no gene, stay open. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 6dd6b8df9..4c38fcf84 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -61,15 +61,22 @@ text or attribute names its gene, positions or that grandparent; each click opens one result, every place comes back once both are opened, and a reload closes them again; the page is the same, byte for byte, whichever result is - closed (KCNH2, RYR1, HLA-B, a gene alone, a position) and wherever a - grandparent's stretch lies; while the page cannot read his results (no - results.json, an unreadable one, act findings without veil.spans) every - place stays closed and each box opens only its own place; with a second - child holding the same results, opening his leaves hers closed; a couple - screen whose partner is a child with a closed result closes whole (counts, - table, boxes, report link, the lead's words); his map as an image stays - closed until each result is opened from its box, and with his results - unreadable "Show the rows" leaves it closed and "Show the map" opens it; on + closed (KCNH2, RYR1, HLA-B, a gene alone, a position, a copy-number span, + a CHIP call) and wherever a grandparent's stretch lies, each view settled + on knowing his results; while the page cannot read his results (no + results.json, an unreadable one, one cut short, act findings without + veil.spans or with malformed ones) every place stays closed and each box + opens only its own place; with nothing of his to act on every place is + open with no box; with a second child holding the same results, opening + his leaves hers closed, and results opened close again when his run moves; + a couple screen whose partner is a child with a closed result closes whole + (counts, table, boxes, report link, the lead's words), the same page + whether that result is in a gene of the screen or not, and with the + partner's results unreadable its box and the cells' box each open only + their own place; his map as an image stays closed until each result is + opened from its box, the same page whichever result is closed, and with + his results unreadable "Show the rows" leaves it closed and "Show the map" + opens it and no table; on his page a CHIP call, a copy-number dot and a card's fact in the gene of a closed result stay out of sight (on a phone the card's line too) until it is opened; DEMO-001's page @@ -87,8 +94,12 @@ a side by side's cells, a couple screen and a map image that ignore the veil, a closed painting that keeps each stretch's grandparent, a box that opens every result with one click, de novo rows closed only when a result lies under one, a painting closed only when a grandparent's stretch lies over a closed result, a side by side that closes -only the closed gene's cells, one opening shared by every box while the results cannot be read, results without -veil.spans read as nothing closed and an opened result kept per run only; for the son's page, person.js that draws +only the closed gene's cells or stays open for a wide span, a page that never reads the son's results, one opening +shared by every box or 'Show the grandparents' opening the side by side while the results cannot be read, a failed +request read as nothing closed, malformed veil.spans read as given, results without veil.spans read as nothing +closed or as closed with nothing to act on, an opened result kept per run only or per child only, a couple screen +closed only for a result in its genes or opened by 'Show the cells', a map image closed only for a result with a +position, and de novo rows that 'Show the map' opens; for the son's page, person.js that draws every item, shows every fact or builds a card's line from every fact), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing @@ -872,38 +883,53 @@ def family_problems(b, base): return out -def family_view(b, base): - """The family page's HTML before any click.""" +def family_view(b, base, child=SON): + """The family page's HTML before any click, or None when the page did not settle on knowing CHILD's results.""" family_page(b, base) + kids = b.eval("document.documentElement.dataset.pgpKids || ''") or "" + if not re.search(r"\b" + re.escape(child) + r":known\b", kids): + return None return b.eval("document.getElementById('family').innerHTML") or "" -def invariance_problems(b, base, rj, res, fj, fams): +# Closed results of different kinds and places, as veil.spans holds them. +CLOSED_CASES = (("RYR1, a medicines gene", [["sf-ryr1", "RYR1", "chr19", 38440000, 38440000]]), + ("HLA-B", [["sf-hlab", "HLA-B", "chr6", 31353000, 31357000]]), + ("a gene alone", [["two-hits-g0999", "G0999", None, None, None]]), + ("a position under a de novo row", [["sf-pos", None, "chr3", 48211904, 48211904]]), + ("a copy-number span in SMN1", [["sv-chr5-70925000-70953000", "SMN1", "chr5", 70925000, 70953000]]), + ("a CHIP call in TP53", [["chip-tp53", "TP53", "chr17", 7675088, 7675088]])) + + +def invariance_problems(b, base, rj, res, fj, fams, cases=None, child=SON): """[problems]: the family page before any click differs between closed results of different genes and places - (FAMS: name -> family.json text; the results' spans set in RJ from RES), so it would tell which is closed.""" + (FAMS: name -> family.json text; CASES: name -> spans set in RJ, CHILD's results, from RES, its findings to act + on renamed to match, None for RES as it is), so it would tell which is closed; or it never knew CHILD's results.""" views = {} - for name, spans in (("the planted pair in KCNH2", None), ("RYR1, a medicines gene", [["sf-ryr1", "RYR1", "chr19", 38440000, 38440000]]), - ("HLA-B", [["sf-hlab", "HLA-B", "chr6", 31353000, 31357000]]), - ("a gene alone", [["two-hits-g0999", "G0999", None, None, None]]), - ("a position under a de novo row", [["sf-pos", None, "chr3", 48211904, 48211904]])): + for name, spans in cases or (("the planted pair in KCNH2", None),) + CLOSED_CASES: r = copy.deepcopy(res) if spans is not None: - r["veil"]["spans"] = spans + act = [x for x in r.get("findings", []) if x.get("level") == "act"] + r["veil"] = dict(r.get("veil") or {}, spans=spans) + if act: + r["findings"] = [x for x in r["findings"] if x.get("level") != "act"] + [dict(act[0], id=s_[0]) for s_ in spans] with open(rj, "w") as f: json.dump(r, f) for fname, text in fams.items(): with open(fj, "w") as f: f.write(text) - views[f"{name}, {fname}"] = family_view(b, base) + views[f"{name}, {fname}"] = family_view(b, base, child) with open(rj, "w") as f: json.dump(res, f) + out = [f"the page never knew {child}'s results with {k!r}" for k, v in views.items() if v is None] first = next(iter(views)) - return [f"the page differs between {first!r} and {k!r}" for k, v in views.items() if v != views[first]] + return out + [f"the page differs between {first!r} and {k!r}" for k, v in views.items() if v != views[first]] def family_unknown(b, base, opens=True): """[problems] of the family page while it cannot read the son's results: a place of his open or a word of his - planted results shown; then (OPENS) the de novo box's own button opens his de novo rows and no other place.""" + planted results shown; then (OPENS) each box's own button, from a fresh page each, opens its own place and no + other.""" gene, hit, sv = planted_words() family_page(b, base) # A page still waiting for his results keeps everything closed anyway: the check is on the state it settles in. @@ -920,17 +946,29 @@ def family_unknown(b, base, opens=True): out += side_problems_of(b, True) if not opens or out: return out - open_box(b, "denovo") - shown = b.eval(FAMILY_SHOWN_JS) or "" - if boxes_of(b) != ["paint", "het", "side"] or (son_tables(b) or [])[1:] != [3] or GRANDPARENT in shown or PAIR_B in shown: - out.append(f"the de novo box opened another place: closed {boxes_of(b)}, tables {son_tables(b)}") - out += side_problems_of(b, True) + # What each place shows once open: the painting its grandparent, the de novo rows the planted hit (not the + # pair's second variant), the compound-het rows the pair's second variant, the side by side his cells. + for box in SON_BOXES: + family_page(b, base) + if not open_box(b, box): + out.append(f"no {box} box") + continue + shown = b.eval(FAMILY_SHOWN_JS) or "" + rest, tables = [x for x in SON_BOXES if x != box], son_tables(b) or [] + want = {"paint": GRANDPARENT in shown and PAIR_B not in shown and len(tables) == 1, + "denovo": GRANDPARENT not in shown and PAIR_B not in shown and tables[1:] == [3], + "het": GRANDPARENT not in shown and PAIR_B in shown and tables[1:] == [3], + "side": GRANDPARENT not in shown and PAIR_B not in shown and len(tables) == 1}[box] + if boxes_of(b) != rest or not want: + out.append(f"the {box} box opened another place, or not its own: closed {boxes_of(b)}, tables {tables}") + out += [f"after the {box} box: {x}" for x in side_problems_of(b, box != "side")] return out def unknown_map_problems(b, base): """[problems] of the family page whose son has his map as an image, with his results unreadable: 'Show the - rows' of a table opening the image, or 'Show the map' not opening it.""" + rows' of a table opening the image, 'Show the map' not opening it, or (from a fresh page) 'Show the map' opening + a table.""" family_page(b, base) img = "!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')" out = [] @@ -940,6 +978,9 @@ def unknown_map_problems(b, base): out.append("'Show the rows' opened the map too") if not open_box(b, "map") or not b.eval(img): out.append("'Show the map' did not open the map") + family_page(b, base) + if not open_box(b, "map") or not b.eval(img) or boxes_of(b) != ["denovo", "het", "side"] or son_tables(b): + out.append(f"'Show the map' opened another place, or not the map: closed {boxes_of(b)}, tables {son_tables(b)}") return out @@ -1001,6 +1042,85 @@ def couple_problems(b, base): return out +def nothing_closed_problems(b, base): + """[problems] of the family page whose son has nothing to act on (no veil): a box of his, or a place of his + closed: the painting's grandparent, his de novo and compound-het rows, his cells.""" + family_page(b, base) + kids = b.eval("document.documentElement.dataset.pgpKids || ''") or "" + shown = b.eval(FAMILY_SHOWN_JS) or "" + tables = son_tables(b) or [] + out = [] if re.search(r"\bDEMO-003:known\b", kids) else [f"the son's results never known: {kids!r}"] + if boxes_of(b) or GRANDPARENT not in shown or len(tables) != 3 or tables[1:] != [3, 3]: + out.append(f"a place closed with nothing to act on: closed {boxes_of(b)}, tables {tables}") + return out + side_problems_of(b, False) + + +def new_run_problems(b, base, fj, before, after): + """[problems]: with family.json BEFORE, the son's results opened from the page; then family.json AFTER (his run + in another folder) is fetched by the page itself, and his places must close again. The page asks every 30 s: + here every 0.5 s.""" + b.call("Page.enable", {}, b.session) # the script below runs only with the Page domain on + fast = b.call("Page.addScriptToEvaluateOnNewDocument", {"source": "(function () { var s = window.setInterval; " + "window.setInterval = function (f, t) { return s(f, t === 30000 ? 500 : t); }; }());"}, b.session) + try: + with open(fj, "w") as f: + f.write(before) + family_page(b, base) + out = [] + for _ in range(2): + open_box(b, "denovo") + if boxes_of(b): + out.append(f"the son's places not opened: {boxes_of(b)}") + with open(fj, "w") as f: + f.write(after) + moved = b.wait("var r = Array.prototype.some.call(document.querySelectorAll('a[href]'), function (a) { " + "return a.getAttribute('href').indexOf('-again/') >= 0; }); r && " + "/DEMO-003:known/.test(document.documentElement.dataset.pgpKids || '') && 'moved'", 10) + time.sleep(1.5) + if moved != "moved": + out.append("the page never fetched the moved run") + elif boxes_of(b) != SON_BOXES: + out.append(f"opened results stayed open for the son's new run: closed {boxes_of(b)}") + return out + finally: + b.call("Page.removeScriptToEvaluateOnNewDocument", {"identifier": (fast or {}).get("identifier", "")}, b.session) + b.call("Page.disable", {}, b.session) + + +def gene_cells(b, child): + """The values of CHILD's medicines and HLA cells in the side by side.""" + rows = side_rows(b) + members = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows thead th'), " + "function (x) { return x.textContent; }).slice(2))") + members = json.loads(members) if members else [] + if child not in members: + return [] + i = members.index(child) + return [r[2 + i] for r in rows if r[0] in ("Medicines", "HLA")] + + +def unknown_couple_problems(b, base): + """[problems] of the couple fixture while the page cannot read DEMO-001's results: from a fresh page each, 'Show + the screen' opens the screen and leaves DEMO-001's cells closed, and 'Show the cells' opens the cells and leaves + the screen closed.""" + out, box = [], "couple:DEMO-001 DEMO-002" + screen = "!!document.querySelector('#ch-couples table.rows')" + for which in (box, "side"): + family_page(b, base) + if boxes_of(b, "DEMO-001") != [box, "side"] or b.eval(screen) or set(gene_cells(b, "DEMO-001")) != {"closed"}: + out.append(f"DEMO-001's places not all closed: {boxes_of(b, 'DEMO-001')}") + continue + if not open_box(b, which, "DEMO-001"): + out.append(f"no {which} box") + continue + cells = set(gene_cells(b, "DEMO-001")) + rest = [x for x in (box, "side") if x != which] + ok = (b.eval(screen) and cells == {"closed"}) if which == box else (not b.eval(screen) and cells and "closed" not in cells) + if boxes_of(b, "DEMO-001") != rest or not ok: + out.append(f"the {which} box opened another place, or not its own: closed {boxes_of(b, 'DEMO-001')}, cells {sorted(cells)[:3]}") + return out + + def family_veil_checks(b, base, out): """On the family page, while a child has a result to act on still closed, or the page cannot read the child's results, every place that could carry one closes for that child, the same whichever result it is; each known @@ -1070,7 +1190,8 @@ def planted_control(desc, a, z, problems): fams = {"a grandparent over the hit": planted, "a grandparent elsewhere": elsewhere} bad = invariance_problems(b, base, rj, res, fj, fams) check("browser: the family page is the same, byte for byte, whichever of the son's results is closed (a pair " - "in KCNH2, RYR1, HLA-B, a gene alone, a position) and wherever a grandparent's stretch lies", not bad, bad) + "in KCNH2, RYR1, HLA-B, a gene alone, a position, a copy-number span in SMN1, a CHIP call in TP53) and " + "wherever a grandparent's stretch lies, each view settled on knowing his results", not bad, bad) for desc, a, z in ( ("de novo rows closed only when a result lies under one", "var dshut = shutOf(c.child, 'denovo');", "var dshut = shutOf(c.child, 'denovo').filter(function (id) { return dn.some(function (r) { return " @@ -1083,7 +1204,13 @@ def planted_control(desc, a, z, problems): ("a side by side that closes only the closed gene's cells", "if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length) {", "if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length && (kids[m].spans || []).some(function (s) { " - "return s[1] === r.label; })) {")): + "return s[1] === r.label; })) {"), + ("a side by side whose cells stay open for a span wider than 10 kb (a copy-number region)", + "if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length) {", + "if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length && (kids[m].spans || []).every(function (s) { " + "return !(s[4] - s[3] > 10000); })) {"), + ("a page that never reads the son's results (every view the same, all closed as unreadable)", + "U.getJSON(url + 'results.json')", "U.getJSON(url + 'missing.json')")): planted_control(desc, a, z, lambda: invariance_problems(b, base, rj, res, fj, fams)) write(fj, planted) @@ -1091,12 +1218,17 @@ def planted_control(desc, a, z, problems): os.replace(rj, rj + ".away") try: bad = family_unknown(b, base) - check("browser: with the son's results.json unavailable every place of his stays closed, and the de novo " - "box's own button opens his de novo rows and no other place", not bad, bad) + check("browser: with the son's results.json unavailable every place of his stays closed, and each box's " + "own button (the painting's, the de novo, the compound-het, the side by side's) opens its own place " + "and no other", not bad, bad) planted_control("one opening shared by every box while the results cannot be read", "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", "return opened[openKey(child, 'box:denovo')] ? [] : [UNKNOWN];", lambda: family_unknown(b, base)) + planted_control("'Show the grandparents' that opens the side by side while the results cannot be read", + "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", + "return opened[openKey(child, box === 'side' ? 'box:paint' : 'box:' + box)] ? [] : [UNKNOWN];", + lambda: family_unknown(b, base)) finally: os.replace(rj + ".away", rj) write(rj, json.dumps({"schema_version": 1, "sample": SON, "chapters": [], @@ -1104,6 +1236,29 @@ def planted_control(desc, a, z, problems): bad = family_unknown(b, base, opens=False) check("browser: with the son's results answered as unreadable (a note, no findings) every place of his stays " "closed", not bad, bad) + write(rj, keep[rj][:5000]) + bad = family_unknown(b, base, opens=False) + check("browser: with the son's results.json cut short (a request that fails) every place of his stays closed", + not bad, bad) + planted_control("a failed request read as nothing closed", "cur.status = 'failed';", + "cur.status = 'known'; cur.spans = null; cur.acts = [];", lambda: family_unknown(b, base, opens=False)) + for desc, spans in (("[null]", [None]), ("[] while he has findings to act on", [])): + bad_spans = json.loads(keep[rj]) + bad_spans["veil"]["spans"] = spans + write(rj, json.dumps(bad_spans)) + bad = family_unknown(b, base, opens=False) + check(f"browser: with the son's veil.spans malformed ({desc}) every place of his stays closed", not bad, bad) + planted_control(f"malformed veil.spans ({desc}) read as given", "spans: spansOk ? v.spans : null", + "spans: Array.isArray(v.spans) ? v.spans : null", lambda: family_unknown(b, base, opens=False)) + clean = json.loads(keep[rj]) + clean["findings"] = [x for x in clean["findings"] if x.get("level") != "act"] + clean.pop("veil", None) + write(rj, json.dumps(clean)) + bad = nothing_closed_problems(b, base) + check("browser: with nothing of the son's to act on (no veil) every place of his is open, with no box", not bad, bad) + planted_control("results without veil.spans read as closed even with nothing to act on", + "if (!k || k.status !== 'known' || (!k.spans && k.acts.length)) {", + "if (!k || k.status !== 'known' || !k.spans) {", lambda: nothing_closed_problems(b, base)) nospans = json.loads(keep[rj]) nospans.pop("veil", None) write(rj, json.dumps(nospans)) @@ -1137,6 +1292,23 @@ def planted_control(desc, a, z, problems): lambda: two_children_problems(b, base)) write(fj, planted) + # The son's run moves to another folder (same results) while the page is open: what was opened closes again. + again = os.path.join(out, "runs", SON + "-again") + shutil.copytree(os.path.join(out, "runs", SON), again) + moved = json.loads(planted) + moved["runs"][SON]["url"] = moved["runs"][SON]["url"].replace(f"/{SON}/", f"/{SON}-again/") + try: + bad = new_run_problems(b, base, fj, planted, json.dumps(moved)) + check("browser: results the reader opened close again when the son's run changes while the page is open", + not bad, bad) + planted_control("an opened result kept per child only, not per run", + "function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; }", + "function openKey(child, id) { return child + '|' + id; }", + lambda: new_run_problems(b, base, fj, planted, json.dumps(moved))) + finally: + shutil.rmtree(again, ignore_errors=True) + write(fj, planted) + # A partner of the couple screen who is also a child, with a closed result in a gene of the screen. famc = json.loads(keep[fj]) famc["state"]["members"].insert(0, {"sample": GRANDPARENT, "sex": "male", "father": None, "mother": None, @@ -1157,6 +1329,29 @@ def planted_control(desc, a, z, problems): "table, boxes, report link, the lead's words) until that result is opened from its box", not bad, bad) planted_control("a couple screen that ignores the veil", "var cs = coupleShut(f, c);", "var cs = [];", lambda: couple_problems(b, base)) + couple_cases = (("PRF1, a gene of the screen", [["two-hits-prf1", "PRF1", None, None, None]]), + ("RYR1, a gene the screen does not name", [["sf-ryr1", "RYR1", "chr19", 38440000, 38440000]])) + bad = invariance_problems(b, base, rj1, json.loads(keep[rj1]), fj, {"the couple": json.dumps(famc)}, + couple_cases, S) + check("browser: the family page is the same, byte for byte, whether the partner's closed result is in a gene " + "of the screen or not", not bad, bad) + planted_control("a couple screen closed only when the closed result is in one of its genes", + "var cs = coupleShut(f, c);", + "var cs = coupleShut(f, c).filter(function (x) { return (kids[x[0]].spans || []).some(function (s) { " + "return ((f.couples || {})[c.a + ' ' + c.b] || []).some(function (r) { return r.gene === s[1]; }); }); });", + lambda: invariance_problems(b, base, rj1, json.loads(keep[rj1]), fj, {"the couple": json.dumps(famc)}, + couple_cases, S)) + os.replace(rj1, rj1 + ".away") + try: + bad = unknown_couple_problems(b, base) + check("browser: with the partner's results unavailable, 'Show the screen' opens the screen and leaves the " + "partner's cells closed, and 'Show the cells' opens the cells and leaves the screen closed", not bad, bad) + planted_control("a couple screen that 'Show the cells' opens while the results cannot be read", + "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", + "return opened[openKey(child, box.indexOf('couple:') === 0 ? 'box:side' : 'box:' + box)] ? [] : [UNKNOWN];", + lambda: unknown_couple_problems(b, base)) + finally: + os.replace(rj1 + ".away", rj1) write(rj1, keep[rj1]) # The drawing as an image (no stretches in family.json). @@ -1169,6 +1364,16 @@ def planted_control(desc, a, z, problems): "click, the map once both are opened", not bad, bad) planted_control("a map image that ignores the veil", "var shut = shutOf(c.child, 'map');", "var shut = [];", lambda: map_problems(b, base)) + mapfam = {"the map as an image": json.dumps(fam)} + bad = invariance_problems(b, base, rj, json.loads(keep[rj]), fj, mapfam) + check("browser: with the son's map as an image the family page is the same, byte for byte, whichever of his " + "results is closed", not bad, bad) + planted_control("a map image closed only when the closed result has a position", + "var shut = shutOf(c.child, 'map');", + "var shut = shutOf(c.child, 'map').filter(function (id) { return (kids[c.child].spans || []).some(" + "function (s) { return s[0] === id && s[2]; }); });", + lambda: invariance_problems(b, base, rj, json.loads(keep[rj]), fj, mapfam)) + write(fj, json.dumps(fam)) os.replace(rj, rj + ".away") try: bad = unknown_map_problems(b, base) @@ -1178,6 +1383,10 @@ def planted_control(desc, a, z, problems): "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", "return opened[openKey(child, 'box:denovo')] ? [] : [UNKNOWN];", lambda: unknown_map_problems(b, base)) + planted_control("de novo rows that 'Show the map' opens", + "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", + "return opened[openKey(child, box === 'denovo' ? 'box:map' : 'box:' + box)] ? [] : [UNKNOWN];", + lambda: unknown_map_problems(b, base)) finally: os.replace(rj + ".away", rj) finally: From b170ad774ce341fd0b475fb4c0a7b1e47c919e8f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:48:25 +0100 Subject: [PATCH 017/255] fix(live): keep a shown finding on the map when a closed result lies in its gene An open finding whose card the page shows was dropped from the map while a closed result sat inside its gene span, and came back when that result was opened: that placed the closed result inside the gene. A page finding now follows the page's own veil only; a row with no page finding (APOE) still follows its tag. A result opened by its own card now shows even when it overlaps another closed result. Also: APOE's chip and switch wait for the traits card in a replay, the chromosome's runs sentence waits for its step, and a failed reload empties the mark links and hides the tooltip. --- assets/live/map.js | 22 +++-- assets/live/person.js | 13 ++- docs/live-site.md | 4 +- tests/test_live_demo.py | 190 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 216 insertions(+), 13 deletions(-) diff --git a/assets/live/map.js b/assets/live/map.js index e7032c7c9..054471c96 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -105,7 +105,8 @@ // --- the figure --------------------------------------------------------------------------------------- // create({url, veil(id, gated) -> 'open'|'closed'|'opened', now(id) -> shown at this moment, open(id), - // close(id), stepDone(step) -> ended at this moment, fallback() -> rows}): one figure per page. The page calls refresh(key) on each of its redraws; a new key + // close(id), known(id) -> a finding of the page's results, stepDone(step) -> ended at this moment, + // fallback() -> rows}): one figure per page. The page calls refresh(key) on each of its redraws; a new key // fetches the document again and keeps the camera. function create(opts) { var M = null, key = null, failure = null, t0 = 0; @@ -157,12 +158,16 @@ // --- the veil: what this page keeps closed at this moment --- function stateOf(id) { return opts.veil(id, !!(M && M.gated[id])); } - // A row shows when its result is not closed and the page shows its finding at this moment (a replay + // A row shows when its own result is not closed and the page shows its finding at this moment (a replay // before the finding's step ended does not). - // A row under another result's tag also stays hidden while its own result is closed. + // A row that is a finding of the page (opts.known) follows the page's own veil only: the page already + // hides every finding it keeps closed (opts.now), and its card shows the rest. Hiding such a row for + // another result's tag would drop a finding whose card the page shows, and so place that closed result + // inside it. A row the page has no finding for (the map's APOE row) follows its tag too. function shownRow(r) { - return (!r.veil || stateOf(r.veil) !== 'closed') && (r.veil === r.id || M.closable.indexOf(r.id) < 0 || - stateOf(r.id) !== 'closed') && (!opts.now || opts.now(r.id)); + var own = M.closable.indexOf(r.id) < 0 || stateOf(r.id) !== 'closed'; + var tag = (opts.known && opts.known(r.id)) || !r.veil || stateOf(r.veil) !== 'closed'; + return own && tag && (!opts.now || opts.now(r.id)); } // The runs of homozygosity, as this page shows them now: a replay before their step ended has none yet. function rohLayer() { @@ -768,7 +773,8 @@ if (lv === 'genome') { var form = fs.length === 0 ? forms.none : fs.length === 1 ? forms.one : forms.many; add(narr, runs((form || []).map(function (x) { return { t: x.t.replace('{n}', String(fs.length)), b: x.b }; }))); - } else if (M.text.chrom && M.text.chrom[ch.n]) { + } else if (M.text.chrom && M.text.chrom[ch.n] && rohLayer() === M.layers.roh) { + // The chromosome's sentence is about its runs of homozygosity: a replay before their step ended has none. add(narr, runs(M.text.chrom[ch.n])); } var lay = M.layers.findings || {}; @@ -861,6 +867,8 @@ if (!M) { return; } var acts = [], gates = [], opened = []; M.closable.forEach(function (id) { + // A gated result (APOE) has a chip only once the page shows its card (a replay before its step has none). + if (M.gated[id] && opts.now && !opts.now(id)) { return; } var s = stateOf(id); if (s === 'closed') { (M.gated[id] ? gates : acts).push(id); } else if (s === 'opened') { opened.push(id); } }); @@ -1042,6 +1050,8 @@ narr.textContent = ''; layersBox.textContent = ''; chips.textContent = ''; // a chip drawn before the failure may name a result closed since + marks.textContent = ''; // and so may a mark link, hidden or not + hideTip(); } function load(k) { key = k; diff --git a/assets/live/person.js b/assets/live/person.js index 021cff5e1..04685da90 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -552,9 +552,16 @@ } // Whether the page shows a finding of the map at this moment: its card's steps are done (a replay draws // the run's end results from its first frame). A row with no finding of these results (the map's own APOE - // row) follows its card's fold instead. + // row) follows the card that holds its fold (the traits card): a replay before that card shows has no APOE. // A finding this page keeps closed is not shown either, whatever row tag it shares with another result. - function mapNow(id) { var f = findingById(id); var c = f && cardById(f.card); return !f || (!!(c && gate(c) === 'show') && !veiled(f)); } + function mapNow(id) { + var f = findingById(id); + if (!f) { var t = cards().filter(function (c) { return c.hidden; })[0]; return !!(t && gate(t) === 'show'); } + var c = cardById(f.card); + return !!(c && gate(c) === 'show') && !veiled(f); + } + // Whether a row of the map is a finding of these results: then the page's own veil alone decides it (map.js). + function mapKnown(id) { return !!findingById(id); } // Whether a step of the run has ended at this moment (a replay before it has not): the map's layers follow it. function stepDone(step) { var s = stepsBy()[step]; return !s || s.state === 'done'; } // Without the map (a browser with no DecompressionStream, or a map the server could not build): the findings @@ -884,7 +891,7 @@ // reader left it. It asks this page's veil for each closed row, and opens or closes a result through it. if (window.PGPMap) { if (!genomeMap) { - genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, stepDone: stepDone, open: function (id) { setShown(id, true); }, + genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, stepDone: stepDone, open: function (id) { setShown(id, true); }, close: function (id) { setShown(id, false); }, fallback: mapFallback }); } add(slot, genomeMap.node); diff --git a/docs/live-site.md b/docs/live-site.md index d5c57e616..70b462198 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,7 +56,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. @@ -70,7 +70,7 @@ While the run goes, the page still opens on the report being written. A short ba The header search finds a gene or rsID across the results and opens the card that names it; on a page that keeps results closed it leaves out every hit that names one (above). One row of the results that the page cannot read (a field of the wrong type) costs that row only: it shows as a finding on its card, "A row of this section could not be read", at the highest level its rule can give (to act on for a rule that can give one, so a page that keeps results closed keeps that card closed too), the card whose table cannot be drawn says so, and every other card and finding is on the page. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. -**Figure 1, the genome map.** The 24 chromosomes as rows, to scale (23 for a person with no Y), each with its bands, the centromere as a waist and the stretches few reads can map hatched (centromeres, heterochromatin, the acrocentric arms). Each finding with a place sits on its chromosome as its level's shape, with its gene beside it when the name fits; runs of homozygosity of 5 Mb or more are a line under the row, shorter ones a faint band. A finding that names a gene but no position (a medicine gene, Lp(a)) sits on that gene's span; a finding with neither stays in its section only. Click a chromosome (or, with the map focused, press `[` or `]`) and it opens in place: an overview with the part in view, a ruler in Mb, the bands with their names, a findings lane with each gene and its level's words, and a lane for the runs of homozygosity, each run with its length. Drag to move along it, scroll or press the + and − buttons beside the zoom stops to zoom in (as far as about 10 Mb across a wide screen; closer views come later), and press Esc (or click Genome) to go back. Back in the browser returns to the view before, the whole genome when the address had no place. A dashed line marks the centre, and the margin says what is under it: the chromosome, the position and the band, the findings in view and the runs. The findings count, the chromosome sentences, the runs sentence and each finding's phrase come from the map document; the page adds its own short lines: a lane or stretch with no finding, the runs in view, how to move, and why a layer is not on the map. A replay draws the runs of homozygosity only once their step has ended. The address follows the map: `#genome`, `#chr7`, or `#chr7:117.4-137.4M`, and a link to one of them opens that view; a place the page cannot read shows a short note and leaves the map as it was. The map comes from `/runs//map.json`, which the map plugin (`bin/owngenome_live_map.py`) builds with `og map build` from the run folder and the results the page gets (`rust/crates/og-map/README.md` describes the document). It needs og on the host (`scripts/setup.sh --og`); without it, or in a browser that cannot unpack the file (no `DecompressionStream`), the figure says so and lists the findings with a place in a table instead. A layer whose file is missing or cannot be read says so in words, never as a blank lane. +**Figure 1, the genome map.** The 24 chromosomes as rows, to scale (23 for a person with no Y), each with its bands, the centromere as a waist and the stretches few reads can map hatched (centromeres, heterochromatin, the acrocentric arms). Each finding with a place sits on its chromosome as its level's shape, with its gene beside it when the name fits; runs of homozygosity of 5 Mb or more are a line under the row, shorter ones a faint band. A finding that names a gene but no position (a medicine gene, Lp(a)) sits on that gene's span; a finding with neither stays in its section only. Click a chromosome (or, with the map focused, press `[` or `]`) and it opens in place: an overview with the part in view, a ruler in Mb, the bands with their names, a findings lane with each gene and its level's words, and a lane for the runs of homozygosity, each run with its length. Drag to move along it, scroll or press the + and − buttons beside the zoom stops to zoom in (as far as about 10 Mb across a wide screen; closer views come later), and press Esc (or click Genome) to go back. Back in the browser returns to the view before, the whole genome when the address had no place. A dashed line marks the centre, and the margin says what is under it: the chromosome, the position and the band, the findings in view and the runs. The findings count, the chromosome sentences, the runs sentence and each finding's phrase come from the map document; the page adds its own short lines: a lane or stretch with no finding, the runs in view, how to move, and why a layer is not on the map. A replay draws the runs of homozygosity, and says the sentences about them, only once their step has ended. The address follows the map: `#genome`, `#chr7`, or `#chr7:117.4-137.4M`, and a link to one of them opens that view; a place the page cannot read shows a short note and leaves the map as it was. The map comes from `/runs//map.json`, which the map plugin (`bin/owngenome_live_map.py`) builds with `og map build` from the run folder and the results the page gets (`rust/crates/og-map/README.md` describes the document). It needs og on the host (`scripts/setup.sh --og`); without it, or in a browser that cannot unpack the file (no `DecompressionStream`), the figure says so and lists the findings with a place in a table instead. A layer whose file is missing or cannot be read says so in words, never as a blank lane. **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 65a965532..171c1c6f0 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -75,6 +75,7 @@ import base64 import copy import gzip +import hashlib import json import os import shutil @@ -1475,6 +1476,158 @@ def overlap_problems(b, base, out): fh.write(keep) +# The figure as a reader gets it before any click: its HTML (the chips aside, which count the closed results +# on purpose, and the timing attributes), its marks, margin and layers, and a hash of the canvas's pixels. +FIGURE_JS = r"""JSON.stringify((function () { + var f = document.getElementById('genome-map'); + if (!f) { return null; } + var c = f.cloneNode(true); + Array.prototype.forEach.call(c.querySelectorAll('.gm-chips'), function (x) { x.textContent = ''; }); + c.removeAttribute('data-pgp-map-parse-ms'); + c.removeAttribute('data-pgp-map-paint-ms'); + return { html: c.outerHTML, canvas: f.querySelector('.gm-canvas').toDataURL() }; +}()))""" + + +def plant_son(out, extra, og): + """The son's results.json with the findings EXTRA added and his map.json built again from it by og map + build, as the map plugin builds it: og tags each row itself. Returns what to write back afterwards.""" + son = OTHERS[1] + rj, mj = (os.path.join(out, "runs", son, n) for n in ("results.json", "map.json")) + keep = {} + for p in (rj, mj): + with open(p, "rb") as fh: + keep[p] = fh.read() + res = json.loads(keep[rj]) + res["findings"] = res["findings"] + [dict(f) for f in extra] + sec = {"act": 1, "medicine": 2, "carrier": 3, "review": 4} + for f in extra: # each in its level's section, as live_results lists them: its card shows on the page + for j in res["journal"]: + if j["n"] == sec[f["level"]]: + j["findings"] = j["findings"] + [f["id"]] + with open(rj, "w") as fh: + json.dump(res, fh) + root = os.path.join(os.path.dirname(out), "root") + doc = build_live_demo.og_map(og, ["build", "--root", root, "--run", son, "--assets", os.path.join(REPO, "assets"), + "--cache-dir", os.path.join(root, ".owngenome-live", "map")], + json.dumps({"results": res}).encode()) + with open(mj, "wb") as fh: + fh.write(doc) + return keep + + +def unplant(keep): + for p, data in keep.items(): + with open(p, "wb") as fh: + fh.write(data) + + +def row_tag(out, fid): + """The id whose tag og put on FID's row of the son's map, or None.""" + son = OTHERS[1] + doc = read_map(os.path.join(out, "runs", son, "map.json")) + f = doc["per"][son]["findings"] + if fid not in f["id"]: + return None + v = f["veil"][f["id"].index(fid)] + return doc["dict"]["veil"][v] if v >= 0 else None + + +def son_figure(b, base, hash_=""): + """The son's map before any click: (the map's facts, the figure with its canvas hashed).""" + c = map_page(b, base, OTHERS[1], hash_) + fig = json.loads(b.eval(FIGURE_JS) or "null") or {} + fig["canvas"] = hashlib.sha256((fig.get("canvas") or "").encode()).hexdigest()[:12] + return c, fig + + +def act(fid, gene, viewer, title): + return {"id": fid, "level": "act", "card": "clinical-filter", "gene": gene, "viewer": viewer, "title": title, + "detail": f"{gene} {viewer}", "text": f"{title}, planted."} + + +# An open finding over a closed result whose card names another gene: og tags the open finding's row (the +# whole gene span) with the closed result, while the page shows its card. A map that hides that row only +# while the result is closed places the closed result inside that gene. Each planting: (what, the closed +# results, the open findings in both pages, the open finding to watch, chromosome). +UNDER = (("a gene-only review finding (CFTR) over a closed deletion naming another gene", + [act("pl-iota", "PLIOTA", "chr7:117400000-117700000", "Planted iota deletion")], + [{"id": "pl-kappa", "level": "review", "card": "exomiser", "gene": "CFTR", "title": "Planted kappa review", + "detail": "CFTR", "text": "A planted review finding that names CFTR only."}], "pl-kappa", "chr7"), + ("the page's own CYP2D6 medicine finding over a closed variant naming another gene", + [act("pl-theta", "PLTHETA", "chr22:42125000", "Planted theta result")], [], "pgx-cyp2d6", "chr22")) + + +def under_problems(b, base, out, og): + """For each planting of UNDER: the son's map before any click is the same, figure HTML, canvas, marks, + margin and layers, with the closed result and without it, at the genome level and on its chromosome; and + the open finding is drawn in both.""" + bad = [] + for what, closed, opened, watch, chrom in UNDER: + got = {} + for name, extra in (("without", opened), ("with", opened + closed)): + keep = plant_son(out, extra, og) + try: + if name == "with" and row_tag(out, watch) != closed[0]["id"]: + bad.append(f"{what}: og did not tag {watch}'s row with {closed[0]['id']} ({row_tag(out, watch)}): " + "the planting tests nothing") + got[name] = [son_figure(b, base, h) for h in ("", "#" + chrom)] + with open(os.path.join(out, "runs", OTHERS[1], "results.json")) as fh: + title = [f["title"] for f in json.load(fh)["findings"] if f["id"] == watch][0] + if not b.eval("document.body.innerText.indexOf(%s) >= 0" % json.dumps(title)): + bad.append(f"{what}: the page does not show {watch}'s card ({title}): the planting tests nothing") + finally: + unplant(keep) + for (c0, f0), (c1, f1), lv in zip(got["without"], got["with"], ("genome", chrom)): + if watch not in [m["id"] for m in c0["marks"]]: + bad.append(f"{what}, {lv}: {watch} is not on the map even with no closed result") + if f0 != f1: + diff = [k for k in ("html", "canvas") if f0.get(k) != f1.get(k)] + bad.append(f"{what}, {lv}: the figure differs with the closed result ({diff}): marks " + f"{[m['id'] for m in c0['marks']]} vs {[m['id'] for m in c1['marks']]}") + return bad + + +def second_open_problems(b, base, out, og): + """Two overlapping closed results on the son's page (og tags the second with the first): opening the + second by its own card draws exactly it on the map, never the first.""" + two = [act("pl-alpha", "PLALPHA", "chr17:43050000-43060000", "Planted alpha result"), + act("pl-beta", "PLBETA", "chr17:43055000", "Planted beta result")] + keep = plant_son(out, two, og) + try: + if row_tag(out, "pl-beta") != "pl-alpha": + return [f"og did not tag pl-beta's row with pl-alpha ({row_tag(out, 'pl-beta')}): the planting tests nothing"] + c = map_page(b, base, OTHERS[1], "#chr17") + ids = [m["id"] for m in c["marks"]] + if "pl-alpha" in ids or "pl-beta" in ids: + return [f"closed: marks {ids}"] + b.eval("document.querySelector('#f-pl-beta .veil button').click(); true") + c = map_now(b) + ids = [m["id"] for m in c["marks"]] + bad = [] + if "pl-beta" not in ids: + bad.append(f"opened by its card, pl-beta is not on the map: marks {ids}, chips {c['chips']}") + if "pl-alpha" in ids or "PLALPHA" in c["text"]: + bad.append(f"opening pl-beta shows pl-alpha: marks {ids}") + return bad + finally: + unplant(keep) + + +def replay_apoe_problems(b, base): + """DEMO-001 at the first frame of its replay, before the traits step: the map has no APOE chip, no + switch to open it and no APOE mark, as its card has no fold yet.""" + map_page(b, base, S) + b.eval("var s = document.getElementById('replay-slider'); s.value = 0; s.dispatchEvent(new Event('input')); true") + c = map_now(b) + bad = [] + if b.eval("!!document.querySelector('details[data-gate=apoe]')"): + bad.append("the traits card has its APOE fold at the first frame: the check tests nothing") + if any("APOE" in t for t in c["chips"]) or any(m["id"] == "apoe" for m in c["marks"]): + bad.append(f"before the traits step: chips {c['chips']}, marks {[m['id'] for m in c['marks']]}") + return bad + + def missing_layer_problems(b, base, out): """A map whose ROH layer is missing says so in words, in the margin, never a blank.""" path = os.path.join(out, "runs", S, "map.json") @@ -1604,15 +1757,48 @@ def map_checks(b, base, out): mjs = os.path.join(out, "assets", "map.js") with open(mjs) as fh: keep_mjs = fh.read() - own = "(r.veil === r.id || M.closable.indexOf(r.id) < 0 ||\n stateOf(r.id) !== 'closed') && " + own = "var own = M.closable.indexOf(r.id) < 0 || stateOf(r.id) !== 'closed';" try: with open(mjs, "w") as fh: - fh.write(keep_mjs.replace(own, "")) + fh.write(keep_mjs.replace(own, "var own = true;")) control("map.js whose rows ask only their tag, never their own result", keep_mjs.count(own) != 1 or not overlap_problems(b, base, out)) finally: with open(mjs, "w") as fh: fh.write(keep_mjs) + og = og_binary() or build_live_demo.OG + bad = under_problems(b, base, out, og) + check("map: an open finding over a closed result naming another gene stays on the map; before any click the " + "figure is the same with the closed result and without it", not bad, bad) + bad = second_open_problems(b, base, out, og) + check("map: the second of two overlapping closed results, opened by its own card, is drawn, and the first " + "is not", not bad, bad) + known = "(opts.known && opts.known(r.id)) || " + try: + with open(mjs, "w") as fh: + fh.write(keep_mjs.replace(known, "")) + control("map.js whose page findings follow another result's tag", keep_mjs.count(known) != 1 or + not under_problems(b, base, out, og)) + control("map.js whose page findings follow another result's tag (the second result opened)", + keep_mjs.count(known) != 1 or not second_open_problems(b, base, out, og)) + finally: + with open(mjs, "w") as fh: + fh.write(keep_mjs) + bad = replay_apoe_problems(b, base) + check("map: before the traits step a replay has no APOE chip, switch or mark", not bad, bad) + gated = "if (M.gated[id] && opts.now && !opts.now(id)) { return; }" + try: + with open(js, "w") as fh: + fh.write(keep_js.replace("if (!f) { var t = cards()", "if (!f) { return true; var t = cards()")) + with open(mjs, "w") as fh: + fh.write(keep_mjs.replace(gated, "")) + control("person.js and map.js that show APOE's chip and row before its card", keep_mjs.count(gated) != 1 or + not replay_apoe_problems(b, base)) + finally: + with open(js, "w") as fh: + fh.write(keep_js) + with open(mjs, "w") as fh: + fh.write(keep_mjs) bad = missing_layer_problems(b, base, out) check("map: a missing layer says why in words, never a blank", not bad, bad) bad = fallback_problems(b, base, out) From ae3716fac9de6594007eaee757593bd37b1c6980 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:44:01 +0100 Subject: [PATCH 018/255] fix(live): count every closed result on the map chip and stop drawing after a failed reload The dashed chip now counts the results to act on the page keeps closed, so it no longer tells whether a closed result has a place on the map. Open opens the placed ones first, then the next closed result in its section. A failed reload drops the frame and margin the last refresh scheduled, so the old document is not drawn again. New planted checks, each with a control watched red: a finding the server closes by name and a tagged map row the page has no finding for stay hidden; the chip count is the same with a closed result that has a place and one that has none; the runs sentence waits for its step; nothing is drawn after a failed reload. plant_son now computes results.veil.findings with live_results.veil, as the server does. --- assets/live/map.js | 31 +++-- assets/live/person.js | 7 +- docs/live-site.md | 4 +- rust/crates/og-map/README.md | 2 +- tests/test_live_demo.py | 219 ++++++++++++++++++++++++++++++++++- 5 files changed, 247 insertions(+), 16 deletions(-) diff --git a/assets/live/map.js b/assets/live/map.js index 054471c96..a9448a17e 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -105,8 +105,9 @@ // --- the figure --------------------------------------------------------------------------------------- // create({url, veil(id, gated) -> 'open'|'closed'|'opened', now(id) -> shown at this moment, open(id), - // close(id), known(id) -> a finding of the page's results, stepDone(step) -> ended at this moment, - // fallback() -> rows}): one figure per page. The page calls refresh(key) on each of its redraws; a new key + // close(id), known(id) -> a finding of the page's results, closed() -> the ids of the results to act on the + // page keeps closed at this moment, stepDone(step) -> ended at this moment, fallback() -> rows}): one figure + // per page. The page calls refresh(key) on each of its redraws; a new key // fetches the document again and keeps the camera. function create(opts) { var M = null, key = null, failure = null, t0 = 0; @@ -383,7 +384,7 @@ function draw() { raf = 0; - if (!M) { return; } + if (!M || failure) { return; } // a failed reload leaves the old document undrawn var t = performance.now(); if (!size()) { return; } // the page may have changed width since the last frame (a phone turned, a fold opened) step(); @@ -597,7 +598,17 @@ function noFindingHere() { return closedAny() ? 'No open finding with a place on this chromosome.' : 'No finding with a place on this chromosome.'; } - function closedAny() { return M.closable.some(function (id) { return stateOf(id) === 'closed'; }); } + function closedAny() { return closedActs().length > 0 || M.closable.some(function (id) { return stateOf(id) === 'closed'; }); } + // The results to act on this page keeps closed, in the order Open opens them: the ones with a place on the + // map first, in map order, then the rest of the page's closed results, which have no place to draw. Its + // length is what the chip counts, so the count never tells whether a closed result has a place. + function closedActs() { + var page = opts.closed ? opts.closed() : null; + var placed = M.closable.filter(function (id) { + return !M.gated[id] && stateOf(id) === 'closed' && (!page || page.indexOf(id) >= 0); + }); + return page ? placed.concat(page.filter(function (id) { return placed.indexOf(id) < 0; })) : placed; + } function layerWords(lay) { if (lay.state === 'missing') { return 'Not on the map: ' + (lay.note || ('step ' + lay.step + ' wrote no ' + (lay.source || 'file') + '.')); } if (lay.state === 'unreadable') { return 'Not on the map: ' + (lay.source || '') + ' could not be read' + (lay.note ? ' (' + lay.note + ')' : '') + '.'; } @@ -630,7 +641,7 @@ cam.k += (tgt.k - cam.k) * (1 - Math.exp(-dt / 95)); if (!moving()) { cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; } } - function schedule() { if (!raf) { raf = window.requestAnimationFrame(function () { last = last || performance.now(); draw(); }); } } + function schedule() { if (!raf && !failure) { raf = window.requestAnimationFrame(function () { last = last || performance.now(); draw(); }); } } function clampTgt() { var ch = M.chroms[tgt.ci], span = clamp(Math.exp(tgt.ls), minSpan(ch.len), ch.len); tgt.ls = Math.log(span); @@ -738,7 +749,7 @@ settleT = window.setTimeout(describe, 140); } function describe() { - if (!M) { return; } + if (!M || failure) { return; } var lv = level(), fs = drawn(), ch = chrom(); fig.setAttribute('data-level', lv); fig.setAttribute('data-chrom', lv === 'genome' ? '' : ch.n); @@ -865,12 +876,12 @@ function drawChips() { chips.textContent = ''; if (!M) { return; } - var acts = [], gates = [], opened = []; + var acts = closedActs(), gates = [], opened = []; M.closable.forEach(function (id) { // A gated result (APOE) has a chip only once the page shows its card (a replay before its step has none). if (M.gated[id] && opts.now && !opts.now(id)) { return; } var s = stateOf(id); - if (s === 'closed') { (M.gated[id] ? gates : acts).push(id); } else if (s === 'opened') { opened.push(id); } + if (s === 'closed') { if (M.gated[id]) { gates.push(id); } } else if (s === 'opened') { opened.push(id); } }); // Why it is closed: said beside the count, left out on a phone (the card says it too). function why() { return el('span', { 'class': 'gm-why' }, ' · Not drawn until opened on purpose'); } @@ -1035,6 +1046,10 @@ // --- loading --- function fail(why, words) { failure = why; + // The frame and the margin the last refresh scheduled would draw the old document again: drop them. + window.cancelAnimationFrame(raf); + raf = 0; + window.clearTimeout(settleT); fig.setAttribute('data-pgp-map', why); stage.hidden = true; marks.hidden = true; diff --git a/assets/live/person.js b/assets/live/person.js index 04685da90..f00936988 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -560,6 +560,11 @@ var c = cardById(f.card); return !!(c && gate(c) === 'show') && !veiled(f); } + // The results to act on this page keeps closed at this moment, in the page's order: the map's chip counts + // them, whether or not they have a place on the map. + function mapClosed() { + return findingsNow().filter(function (f) { return f.level === 'act' && veiled(f); }).map(function (f) { return f.id; }); + } // Whether a row of the map is a finding of these results: then the page's own veil alone decides it (map.js). function mapKnown(id) { return !!findingById(id); } // Whether a step of the run has ended at this moment (a replay before it has not): the map's layers follow it. @@ -891,7 +896,7 @@ // reader left it. It asks this page's veil for each closed row, and opens or closes a result through it. if (window.PGPMap) { if (!genomeMap) { - genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, stepDone: stepDone, open: function (id) { setShown(id, true); }, + genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, closed: mapClosed, stepDone: stepDone, open: function (id) { setShown(id, true); }, close: function (id) { setShown(id, false); }, fallback: mapFallback }); } add(slot, genomeMap.node); diff --git a/docs/live-site.md b/docs/live-site.md index 70b462198..1fd6d8c42 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,7 +56,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens the closed results with a place first, in map order; after them it opens the next closed result in its section, and the map then draws nothing new, as that result has no place on it. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. @@ -70,7 +70,7 @@ While the run goes, the page still opens on the report being written. A short ba The header search finds a gene or rsID across the results and opens the card that names it; on a page that keeps results closed it leaves out every hit that names one (above). One row of the results that the page cannot read (a field of the wrong type) costs that row only: it shows as a finding on its card, "A row of this section could not be read", at the highest level its rule can give (to act on for a rule that can give one, so a page that keeps results closed keeps that card closed too), the card whose table cannot be drawn says so, and every other card and finding is on the page. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. -**Figure 1, the genome map.** The 24 chromosomes as rows, to scale (23 for a person with no Y), each with its bands, the centromere as a waist and the stretches few reads can map hatched (centromeres, heterochromatin, the acrocentric arms). Each finding with a place sits on its chromosome as its level's shape, with its gene beside it when the name fits; runs of homozygosity of 5 Mb or more are a line under the row, shorter ones a faint band. A finding that names a gene but no position (a medicine gene, Lp(a)) sits on that gene's span; a finding with neither stays in its section only. Click a chromosome (or, with the map focused, press `[` or `]`) and it opens in place: an overview with the part in view, a ruler in Mb, the bands with their names, a findings lane with each gene and its level's words, and a lane for the runs of homozygosity, each run with its length. Drag to move along it, scroll or press the + and − buttons beside the zoom stops to zoom in (as far as about 10 Mb across a wide screen; closer views come later), and press Esc (or click Genome) to go back. Back in the browser returns to the view before, the whole genome when the address had no place. A dashed line marks the centre, and the margin says what is under it: the chromosome, the position and the band, the findings in view and the runs. The findings count, the chromosome sentences, the runs sentence and each finding's phrase come from the map document; the page adds its own short lines: a lane or stretch with no finding, the runs in view, how to move, and why a layer is not on the map. A replay draws the runs of homozygosity, and says the sentences about them, only once their step has ended. The address follows the map: `#genome`, `#chr7`, or `#chr7:117.4-137.4M`, and a link to one of them opens that view; a place the page cannot read shows a short note and leaves the map as it was. The map comes from `/runs//map.json`, which the map plugin (`bin/owngenome_live_map.py`) builds with `og map build` from the run folder and the results the page gets (`rust/crates/og-map/README.md` describes the document). It needs og on the host (`scripts/setup.sh --og`); without it, or in a browser that cannot unpack the file (no `DecompressionStream`), the figure says so and lists the findings with a place in a table instead. A layer whose file is missing or cannot be read says so in words, never as a blank lane. +**Figure 1, the genome map.** The 24 chromosomes as rows, to scale (23 for a person with no Y), each with its bands, the centromere as a waist and the stretches few reads can map hatched (centromeres, heterochromatin, the acrocentric arms). Each finding with a place sits on its chromosome as its level's shape, with its gene beside it when the name fits; runs of homozygosity of 5 Mb or more are a line under the row, shorter ones a faint band. A finding that names a gene but no position (a medicine gene, Lp(a)) sits on that gene's span; a finding with neither stays in its section only. Click a chromosome (or, with the map focused, press `[` or `]`) and it opens in place: an overview with the part in view, a ruler in Mb, the bands with their names, a findings lane with each gene and its level's words, and a lane for the runs of homozygosity, each run with its length. Drag to move along it, scroll or press the + and − buttons beside the zoom stops to zoom in (as far as about 10 Mb across a wide screen; closer views come later), and press Esc (or click Genome) to go back. Back in the browser returns to the view before, the whole genome when the address had no place. A dashed line marks the centre, and the margin says what is under it: the chromosome, the position and the band, the findings in view and the runs. The findings count, the chromosome sentences, the runs sentence and each finding's phrase come from the map document; the page adds its own short lines: a lane or stretch with no finding, the runs in view, how to move, and why a layer is not on the map. A replay draws the runs of homozygosity, and says the sentences about them, only once their step has ended. The address follows the map: `#genome`, `#chr7`, or `#chr7:117.4-137.4M`, and a link to one of them opens that view; a place the page cannot read shows a short note and leaves the map as it was. The map comes from `/runs//map.json`, which the map plugin (`bin/owngenome_live_map.py`) builds with `og map build` from the run folder and the results the page gets (`rust/crates/og-map/README.md` describes the document). It needs og on the host (`scripts/setup.sh --og`); without it, or in a browser that cannot unpack the file (no `DecompressionStream`), the figure says so and lists the findings with a place in a table instead. A map that fails to load again after it was drawn says so the same way and draws nothing more from the map it had. A layer whose file is missing or cannot be read says so in words, never as a blank lane. **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. diff --git a/rust/crates/og-map/README.md b/rust/crates/og-map/README.md index 8d84d50dc..eee8c3f5d 100644 --- a/rust/crates/og-map/README.md +++ b/rust/crates/og-map/README.md @@ -74,7 +74,7 @@ One gzip member with mtime 0 and no file name, holding JSON written as Python's - `ref.chroms`: chr1-22, X and Y. `len` is the chromosome's last band end, `cen` the span of its `acen` bands, `masked` its `acen`, `gvar` and `stalk` bands with touching neighbours merged. - `L`, a layer entry: `state` (`ok`, `missing` or `unreadable`), `step`, `step_name`, `source` (the file read or looked for, relative to GENOME_DIR), `note` (why it is missing, or `file:line: reason` when unreadable), `count` (rows on the map), `totals` (`all` rows read and `kept` rows on the map, when the file was read), `caveat`, and `empty` (the sentence for a read file with no rows). A missing input is `missing`, an input that cannot be read is `unreadable`, and a read input with no rows is `ok` with `count` 0: three answers, never a blank lane. - `findings`: one flat column per field, rows sorted by chromosome then start. A finding's `viewer` (`chrom:pos` or `chrom:start-end`, 1-based, `chr` optional) places it: a position gives `pos` and `[pos - 1, pos)`, a span gives `[start - 1, end)` and `pos` null. Without a usable `viewer`, a finding that names a gene sits on that gene's span from the GENCODE genes file (the first in map order when a name is on several chromosomes). A finding with neither is not a row. `gate` is set for a finding the page keeps behind its own warning: the traits card holds APOE that way, so the map adds the finding `apoe` on the APOE gene, with `gate` `apoe`; the hidden text, which names the genotype, never enters the document. `zyg` (`het`, `hom`, `hemi` or `unknown`) comes from the finding's own `Genotype` evidence; no VCF is read. `hgvs` is its `Variant` evidence. `copy` is null until the family map. `href` is the finding's page relative to the person's page, `reads` the place as the reads viewer takes it (1-based), `phrase` the sentence the map shows. `veil` is the index in `dict.veil` of the first closable finding (in `dict.veil` order) whose span the row overlaps: its own for a closable finding with no earlier overlap, else -1. So an open finding over a closable one's span is tagged too. The layer's `count` and `totals.kept` are the rows whose `veil` is -1; `totals.all` adds the findings that are not closable and have no place, and the `note` counts those ("N findings with no place on the map stay on the page"). -- **The veil.** The document is the same for every reader. Whether a page is closed is the page's own rule; the document says only what a closed result could reveal. Closable findings are those of level `act` and every gated one (APOE closes on every page, an adult's too). Each closable finding with a place veils its span, and `dict.veil` lists their ids: gated ones first, then the act ones in the page's order. A row of a layer that can be a finding's own source (structural variants, mosaic calls and ClinVar hits, as their layers arrive), and every findings row, that overlaps a veiled span carries the index of the first such finding in `dict.veil` in a `veil` column, else -1. A row holds one tag only. Gated findings come first, so a row over both APOE and an act finding carries APOE, which is closed on every page; a row over two act findings carries the first, and both close under the same rule. The page hides a tagged row while its result is closed. Counts, totals and density bins cover untagged rows only, so no number counts a closed row. Runs of homozygosity and bands are never a finding's source and carry no tag. +- **The veil.** The document is the same for every reader. Whether a page is closed is the page's own rule; the document says only what a closed result could reveal. Closable findings are those of level `act` and every gated one (APOE closes on every page, an adult's too). Each closable finding with a place veils its span, and `dict.veil` lists their ids: gated ones first, then the act ones in the page's order. A row of a layer that can be a finding's own source (structural variants, mosaic calls and ClinVar hits, as their layers arrive), and every findings row, that overlaps a veiled span carries the index of the first such finding in `dict.veil` in a `veil` column, else -1. A row holds one tag only. Gated findings come first, so a row over both APOE and an act finding carries APOE, which is closed on every page; a row over two act findings carries the first, and both close under the same rule. The page decides its own findings with its own veil, and the tag binds only the rows that are not one of the page's findings (APOE, or a row the page's results do not have yet): the page hides such a row while its result is closed. Counts, totals and density bins cover untagged rows only, so no number counts a closed row. Runs of homozygosity and bands are never a finding's source and carry no tag. - `roh`: the `RG` rows of step 11's `bcftools roh` text (`{run}/vcf/{run}_roh.txt`, else `{run}/roh/{run}_roh.txt`). bcftools writes 1-based inclusive positions, so `s = start - 1` and `e = end`. `ST` lines are skipped; rows on chrM and other contigs count in `totals.all` only. `markers` and `q` are bcftools' columns 7 and 8. `art` indexes `dict.art`: `one X` on a male's chrX, `one Y` on a male's chrY, else `centromere` when more than half the run lies in masked spans. An unknown sex sets neither flag. `long` is 1 for a run of 5 Mb or more with no `art`. - `roh_bins`: bases covered by any run per 1 Mb bin, for each chromosome with a run. - `text`: the sentences the page shows, in pieces; `b` marks bold words. `text.findings` is the count of findings on the map in three forms (`none`, `one`, `many`): the page counts the findings rows whose `veil` is -1 plus the tagged rows whose result is open, picks the form and puts the number in place of `{n}`. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 171c1c6f0..11811cf33 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1378,8 +1378,12 @@ def leaks(c): return [f"the son's map: {c['state']}"] if leaks(c): bad.append(f"closed: the map shows {leaks(c)}") - if not any("1 result closed" in t for t in c["chips"]): - bad.append(f"closed: no chip counting it: {c['chips']}") + # The chip counts every result to act on the page keeps closed, with a place on the map or not. + want = b.eval("new Set(Array.prototype.filter.call(document.querySelectorAll('.finding[data-level=act]'), " + "function (a) { return a.querySelector('.veil'); }).map(function (a) { return a.dataset.finding; })).size") + label = "1 result closed" if want == 1 else f"{want} results closed" + if not want or not any(t.startswith(label + " ") for t in c["chips"]): + bad.append(f"closed: no chip counting the page's {want} closed results: {c['chips']}") b.eval("document.querySelector('.gm-chip[data-closed=act] button').click(); true") c = map_now(b) if not b.eval("!!document.querySelector('.gm-chip[data-closed=act] .gm-confirm')") or leaks(c): @@ -1490,8 +1494,9 @@ def overlap_problems(b, base, out): def plant_son(out, extra, og): - """The son's results.json with the findings EXTRA added and his map.json built again from it by og map - build, as the map plugin builds it: og tags each row itself. Returns what to write back afterwards.""" + """The son's results.json with the findings EXTRA added, its findings' veil tags (results.veil.findings) + computed again by live_results.veil as the server computes them, and his map.json built again from it by og + map build, as the map plugin builds it: og tags each row itself. Returns what to write back afterwards.""" son = OTHERS[1] rj, mj = (os.path.join(out, "runs", son, n) for n in ("results.json", "map.json")) keep = {} @@ -1505,6 +1510,8 @@ def plant_son(out, extra, og): for j in res["journal"]: if j["n"] == sec[f["level"]]: j["findings"] = j["findings"] + [f["id"]] + tags = live_results.veil({}, res["findings"], []).get("findings") or {} + res["veil"] = dict(res.get("veil") or {}, findings=dict((res.get("veil") or {}).get("findings") or {}, **tags)) with open(rj, "w") as fh: json.dump(res, fh) root = os.path.join(os.path.dirname(out), "root") @@ -1614,6 +1621,157 @@ def second_open_problems(b, base, out, og): unplant(keep) +# A finding the server tags with a closed result because it names its gene (results.veil.findings: a symptom +# ranking), once with og's tag on its row too (CFTR, inside the deletion's span) and once with none (BRCA1). +NAMED = (act("pl-x", "PLXGENE", "chr7:117400000-117700000", "Planted x deletion"), + {"id": "pl-q", "level": "review", "card": "exomiser", "gene": "CFTR", "title": "Symptom ranking: PLXGENE ranks 1", + "detail": "CFTR", "text": "A planted ranking that names the deletion's gene."}, + {"id": "pl-r", "level": "review", "card": "exomiser", "gene": "BRCA1", "title": "Symptom ranking: PLXGENE ranks 2", + "detail": "BRCA1", "text": "A planted ranking that names the deletion's gene."}) +# A row of the map with no finding in results.json (a newer finding the map holds before the page has it), +# on the closed deletion's place and with its tag. +GHOST = {"id": "pl-ghost", "level": "review", "gene": "GHOSTGENE", "title": "Ghost review", "href": "finding/pl-ghost/", + "phrase": "Ghost review phrase."} + + +def named_problems(b, base, out, og): + """On the son's closed page, with the planted deletion pl-x closed: the findings the server closes with it + (pl-q, pl-r) and a tagged map row the page has no finding for (pl-ghost) are in no mark, sentence, link, + title or address, and the figure is the one of the page without them; opening pl-x by its card draws pl-x, + pl-q and pl-r.""" + son = OTHERS[1] + _, before = son_figure(b, base) + keep = plant_son(out, NAMED, og) + try: + with open(os.path.join(out, "runs", son, "results.json")) as fh: + tags = (json.load(fh).get("veil") or {}).get("findings") or {} + if tags.get("pl-q") != "pl-x" or tags.get("pl-r") != "pl-x" or row_tag(out, "pl-r"): + return [f"the server's tags {tags}, og's tag on pl-r {row_tag(out, 'pl-r')}: the planting tests nothing"] + path = os.path.join(out, "runs", son, "map.json") + doc = read_map(path) + f = doc["per"][son]["findings"] + if "pl-x" not in f["id"] or "pl-x" not in doc["dict"]["veil"]: + return ["og did not place or tag pl-x: the planting tests nothing"] + i, n = f["id"].index("pl-x"), len(f["id"]) + for k in f: + if isinstance(f[k], list) and len(f[k]) == n: + f[k].append(f[k][i]) + for k, v in GHOST.items(): + f[k][-1] = v + f["veil"][-1] = doc["dict"]["veil"].index("pl-x") + write_map(path, doc) + c, fig = son_figure(b, base) + bad = [] + ids = [m["id"] for m in c["marks"]] + shown = [x for x in ("pl-x", "pl-q", "pl-r", "pl-ghost") if x in ids] + shown += [w for w in ("PLXGENE", "Ghost", "GHOSTGENE") if w in c["text"] or w in c["tip"] or w in c["hash"]] + if shown: + bad.append(f"closed: the map shows {shown}") + if fig != before: + bad.append(f"closed: the figure differs from the page without them " + f"({[k for k in ('html', 'canvas') if fig.get(k) != before.get(k)]})") + b.eval("document.querySelector('#f-pl-x .veil button').click(); true") + c = map_now(b) + ids = [m["id"] for m in c["marks"]] + if not {"pl-x", "pl-q", "pl-r"} <= set(ids): + bad.append(f"opened by its card: marks {ids}") + return bad + finally: + unplant(keep) + + +def chip_problems(b, base, out, og): + """The son's map before any click is the same, its chips included, whether his planted closed result has a + place on the map (chr13) or none (no viewer, a gene not in GENCODE): the chip counts the results the page + keeps closed, never only the placed ones.""" + got, bad = {}, [] + for name, fid, viewer in (("no place", "pl-nowhere", ""), ("a place", "pl-somewhere", "chr13:32340000")): + keep = plant_son(out, [act(fid, "PL" + fid[3:].upper(), viewer, "Planted closed result")], og) + try: + placed = fid in read_map(os.path.join(out, "runs", OTHERS[1], "map.json"))["per"][OTHERS[1]]["findings"]["id"] + if placed != bool(viewer): + bad.append(f"{name}: og placed {fid} {placed}: the planting tests nothing") + c, fig = son_figure(b, base) + got[name] = (c["chips"], fig) + finally: + unplant(keep) + if got["no place"] != got["a place"]: + bad.append(f"the chips {got['no place'][0]} with no place, {got['a place'][0]} with one" + + ("" if got["no place"][1] == got["a place"][1] else ", and the figure differs")) + return bad + + +def runs_sentence(b, base): + """Whether DEMO-001's planted runs sentence on chr1 is in the margin at the first frame of the replay and + at the last: (first, last, the level and chromosome at the first frame).""" + map_page(b, base, S, "#chr1") + seen = [] + for to in ("0", "s.max"): + b.eval("var s = document.getElementById('replay-slider'); s.value = %s; s.dispatchEvent(new Event('input')); true" % to) + c = map_now(b) + seen.append(("PLANTEDRUNS" in c["text"], c["level"] + " " + c["chrom"])) + return seen[0][0], seen[1][0], seen[0][1] + + +def runs_sentence_problems(b, base, out): + """A replay before the runs-of-homozygosity step ended has no sentence about them on an open chromosome; + at its end it has it.""" + path = os.path.join(out, "runs", S, "map.json") + with open(path, "rb") as fh: + keep = fh.read() + try: + doc = read_map(path) + doc.setdefault("text", {}).setdefault("chrom", {})["chr1"] = [{"t": "PLANTEDRUNS sentence.", "b": False}] + write_map(path, doc) + first, last, where = runs_sentence(b, base) + if where != "chromosome chr1": + return [f"the first frame is at {where}: the check tests nothing"] + return (["the first frame says the runs sentence"] if first else []) + ([] if last else ["the last frame lacks it"]) + finally: + with open(path, "wb") as fh: + fh.write(keep) + + +# A second figure on DEMO-001's page with a veil that keeps nothing closed: drawn from map.json, then reloaded +# from a file that is not there. +SECOND_JS = r"""(function () { + var o = { url: 'map.json', veil: function () { return 'open'; }, now: function () { return true; }, + known: function () { return true; }, stepDone: function () { return true; }, open: function () {}, + close: function () {}, fallback: function () { return []; } }; + var g = window.PGPMap.create(o); + g.node.id = 'gm-second'; + document.body.appendChild(g.node); + window.pgpSecond = { g: g, o: o }; + g.refresh('k1'); + return true; +}())""" +SECOND_FACTS = r"""JSON.stringify((function () { + var f = document.getElementById('gm-second'); + return { state: f.dataset.pgpMap, marks: f.querySelectorAll('.gm-marks a').length, + narr: f.querySelector('.gm-narration').textContent, layers: f.querySelector('.gm-layers').textContent, + tip: f.querySelector('.gm-tip').hidden ? '' : f.querySelector('.gm-tip').textContent }; +}()))""" + + +def failed_reload_problems(b, base): + """A figure whose reload fails draws nothing more of the document it had: no mark link, no margin sentence, + no layer line, no tooltip.""" + map_page(b, base, S) + b.eval(SECOND_JS) + if not b.wait("document.getElementById('gm-second').dataset.pgpMap === 'drawn'"): + return ["the second figure never drew"] + time.sleep(0.6) + drawn = json.loads(b.eval(SECOND_FACTS)) + if not drawn["marks"] or not drawn["narr"]: + return [f"the second figure drew no mark or sentence: the check tests nothing ({drawn})"] + b.eval("pgpSecond.o.url = 'no-such-map.json'; pgpSecond.g.refresh('k2'); true") + if not b.wait("document.getElementById('gm-second').dataset.pgpMap === 'failed'"): + return ["the second figure's reload did not fail"] + time.sleep(1.0) + c = json.loads(b.eval(SECOND_FACTS)) + return [f"after the failed reload: {c}"] if c["marks"] or c["narr"] or c["layers"] or c["tip"] else [] + + def replay_apoe_problems(b, base): """DEMO-001 at the first frame of its replay, before the traits step: the map has no APOE chip, no switch to open it and no APOE mark, as its card has no fold yet.""" @@ -1784,6 +1942,33 @@ def map_checks(b, base, out): finally: with open(mjs, "w") as fh: fh.write(keep_mjs) + bad = named_problems(b, base, out, og) + check("map: a finding the server closes with a closed result, and a tagged map row the page has no finding " + "for, stay off the map until the result is opened", not bad, bad) + now = "&& (!opts.now || opts.now(r.id));" + try: + with open(mjs, "w") as fh: + fh.write(keep_mjs.replace(now, "&& true;")) + control("map.js whose rows ignore what the page shows now", keep_mjs.count(now) != 1 or + not named_problems(b, base, out, og)) + with open(mjs, "w") as fh: + fh.write(keep_mjs.replace(known, "true || ")) + control("map.js that takes every row for a finding of the page", keep_mjs.count(known) != 1 or + not named_problems(b, base, out, og)) + finally: + with open(mjs, "w") as fh: + fh.write(keep_mjs) + bad = chip_problems(b, base, out, og) + check("map: the closed-results chip counts the results the page keeps closed, with a place on the map or " + "without", not bad, bad) + try: + with open(js, "w") as fh: + fh.write(keep_js.replace("closed: mapClosed, ", "")) + control("person.js that does not tell the map which results it keeps closed", + keep_js.count("closed: mapClosed, ") != 1 or not chip_problems(b, base, out, og)) + finally: + with open(js, "w") as fh: + fh.write(keep_js) bad = replay_apoe_problems(b, base) check("map: before the traits step a replay has no APOE chip, switch or mark", not bad, bad) gated = "if (M.gated[id] && opts.now && !opts.now(id)) { return; }" @@ -1801,6 +1986,32 @@ def map_checks(b, base, out): fh.write(keep_mjs) bad = missing_layer_problems(b, base, out) check("map: a missing layer says why in words, never a blank", not bad, bad) + bad = runs_sentence_problems(b, base, out) + check("map: a replay says an open chromosome's runs sentence only once the runs' step ended", not bad, bad) + waits = "&& rohLayer() === M.layers.roh" + try: + with open(mjs, "w") as fh: + fh.write(keep_mjs.replace(waits, "")) + control("map.js whose runs sentence does not wait for its step", keep_mjs.count(waits) != 1 or + not runs_sentence_problems(b, base, out)) + finally: + with open(mjs, "w") as fh: + fh.write(keep_mjs) + bad = failed_reload_problems(b, base) + check("map: after a failed reload the figure draws nothing more of its old document", not bad, bad) + undone = [("if (!M || failure) { return; }", "if (!M) { return; }", 2), ("if (!raf && !failure) {", "if (!raf) {", 1), + ("window.cancelAnimationFrame(raf);\n raf = 0;\n window.clearTimeout(settleT);\n", "", 1)] + try: + planted_mjs = keep_mjs + for a, z, _ in undone: + planted_mjs = planted_mjs.replace(a, z) + with open(mjs, "w") as fh: + fh.write(planted_mjs) + control("map.js that keeps drawing after a failed reload", + any(keep_mjs.count(a) != n for a, _, n in undone) or not failed_reload_problems(b, base)) + finally: + with open(mjs, "w") as fh: + fh.write(keep_mjs) bad = fallback_problems(b, base, out) check("map: a browser with no DecompressionStream gets the sentence and a findings table through the veil", not bad, bad) b.call("Emulation.setDeviceMetricsOverride", {"width": 390, "height": 844, "deviceScaleFactor": 2, "mobile": False}, b.session) From a6e3f1527910b6d6e6f51f588b8a6502f1c2427e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 05:22:18 +0100 Subject: [PATCH 019/255] fix(live): open the map chip's closed results in the page's order, one per click The chip's Open opened the closed results with a place on the map first, so after one click the order showed whether a result still closed has a place. It now opens the next closed result in the page's own order. A result with no place gets an "Opened in its section" chip with a link to its card and a button to close it again. The keyboard follows the chip's buttons. The map checks now assert that each Open opens exactly one result and that the count falls by one, that the order is the same whichever result has a place, and where focus goes, each with its planted control. The map's ready wait no longer runs to its timeout on a browser without DecompressionStream, and the state check reuses its first page load. --- assets/live/map.js | 45 +++++-- assets/live/person.js | 4 +- docs/live-site.md | 2 +- rust/crates/og-map/README.md | 2 +- tests/test_live_demo.py | 220 ++++++++++++++++++++++++++++++----- 5 files changed, 230 insertions(+), 43 deletions(-) diff --git a/assets/live/map.js b/assets/live/map.js index a9448a17e..f19b2d39d 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -154,6 +154,8 @@ var CW = 0, CH = 0, DPR = 1, phone = false; var cam = { ci: 0, c: 0, ls: 0, k: 0 }, tgt = { ci: 0, c: 0, ls: 0, k: 0 }; var hover = { row: -1, hit: null }, hits = [], raf = 0, settleT = 0, hashT = 0, confirming = null, touched = false; + // The results with no place on the map that its chip opened: each gets an "Opened in its section" chip while open. + var inSection = []; var cache = null, cacheKey = ''; var hatch = null; @@ -599,15 +601,14 @@ return closedAny() ? 'No open finding with a place on this chromosome.' : 'No finding with a place on this chromosome.'; } function closedAny() { return closedActs().length > 0 || M.closable.some(function (id) { return stateOf(id) === 'closed'; }); } - // The results to act on this page keeps closed, in the order Open opens them: the ones with a place on the - // map first, in map order, then the rest of the page's closed results, which have no place to draw. Its - // length is what the chip counts, so the count never tells whether a closed result has a place. + // The results to act on this page keeps closed, in the order Open opens them: the page's own order (the + // order of its closed cards), with a place on the map or without. Its length is what the chip counts, so + // neither the count nor the order tells whether a closed result has a place. A page that passes no list + // closes only the map's own rows, in map order. function closedActs() { var page = opts.closed ? opts.closed() : null; - var placed = M.closable.filter(function (id) { - return !M.gated[id] && stateOf(id) === 'closed' && (!page || page.indexOf(id) >= 0); - }); - return page ? placed.concat(page.filter(function (id) { return placed.indexOf(id) < 0; })) : placed; + if (page) { return page.filter(function (id) { return !M.gated[id]; }); } + return M.closable.filter(function (id) { return !M.gated[id] && stateOf(id) === 'closed'; }); } function layerWords(lay) { if (lay.state === 'missing') { return 'Not on the map: ' + (lay.note || ('step ' + lay.step + ' wrote no ' + (lay.source || 'file') + '.')); } @@ -888,30 +889,48 @@ if (acts.length) { var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'act' }); add(c, add(el('span', {}, acts.length === 1 ? '1 result closed' : acts.length + ' results closed'), why())); - add(c, confirming === 'act' ? confirmBox(acts[0], false) : button('Open', function () { confirming = 'act'; drawChips(); })); + add(c, confirming === 'act' ? confirmBox(acts[0], false) : button('Open', function () { confirming = 'act'; drawChips(); keyTo('.gm-yes'); })); add(chips, c); } gates.forEach(function (id) { var g = M.gated[id], name = g.gene || 'This result'; var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'gate' }); add(c, add(el('span', {}, name + ' closed'), why())); - add(c, confirming === id ? confirmBox(id, true) : button('Open ' + name, function () { confirming = id; drawChips(); })); + add(c, confirming === id ? confirmBox(id, true) : button('Open ' + name, function () { confirming = id; drawChips(); keyTo('.gm-yes'); })); add(chips, c); }); opened.forEach(function (id) { var r = M.findings.filter(function (x) { return x.id === id; })[0]; var c = el('div', { 'class': 'gm-chip opened', 'data-opened': '1' }); add(c, el('span', {}, 'Opened: ' + (r ? r.title : 'a result to act on'))); - add(c, button('Close it again', function () { confirming = null; opts.close(id); })); + add(c, button('Close it again', function () { confirming = null; opts.close(id); keyTo(); })); + add(chips, c); + }); + // A result the chip opened that has no place: named only once it is open, with a link to its card. + inSection = inSection.filter(function (id) { return stateOf(id) !== 'closed' && opts.title && opts.title(id); }); + inSection.forEach(function (id) { + var c = el('div', { 'class': 'gm-chip opened', 'data-opened': 'section' }); + add(c, add(el('span', {}, 'Opened in its section: '), el('a', { href: '#f/' + id }, opts.title(id)))); + add(c, button('Close it again', function () { confirming = null; opts.close(id); keyTo(); })); add(chips, c); }); } + // Where the keyboard goes once a chip's button rebuilt the chips: the chips' first button, else the map. + function keyTo(sel) { + var b = chips.querySelector(sel || 'button'); + (b || canvas).focus({ preventScroll: true }); + } function confirmBox(id, gated) { var box = el('span', { 'class': 'gm-confirm' }); add(box, el('span', {}, gated ? 'It is kept behind its warning on every page. Open it here and in its section?' : 'A result to act on in adult life is usually not looked at before the child can choose. Open it here and in its section?')); - add(box, button('Show the result', function () { confirming = null; opts.open(id); }, 'gm-yes')); - add(box, button('Keep it closed', function () { confirming = null; drawChips(); }, 'gm-no')); + add(box, button('Show the result', function () { + confirming = null; + if (!gated && M.closable.indexOf(id) < 0 && inSection.indexOf(id) < 0) { inSection.push(id); } + opts.open(id); // the page draws again, and the map's chips with it (refresh) + keyTo(); + }, 'gm-yes')); + add(box, button('Keep it closed', function () { confirming = null; drawChips(); keyTo(); }, 'gm-no')); return box; } function button(text, fn, cls) { @@ -1047,6 +1066,8 @@ function fail(why, words) { failure = why; // The frame and the margin the last refresh scheduled would draw the old document again: drop them. + // These lines and the failure guards of draw(), describe() and schedule() cover for each other, so no one + // of them can go red alone: the failed-reload check undoes them together. window.cancelAnimationFrame(raf); raf = 0; window.clearTimeout(settleT); diff --git a/assets/live/person.js b/assets/live/person.js index f00936988..82ad6b21a 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -565,6 +565,8 @@ function mapClosed() { return findingsNow().filter(function (f) { return f.level === 'act' && veiled(f); }).map(function (f) { return f.id; }); } + // The title of a result once the page shows it, for the map's "Opened in its section" chip; never a closed one's. + function mapTitle(id) { var f = findingById(id); return f && !veiled(f) ? f.title : ''; } // Whether a row of the map is a finding of these results: then the page's own veil alone decides it (map.js). function mapKnown(id) { return !!findingById(id); } // Whether a step of the run has ended at this moment (a replay before it has not): the map's layers follow it. @@ -896,7 +898,7 @@ // reader left it. It asks this page's veil for each closed row, and opens or closes a result through it. if (window.PGPMap) { if (!genomeMap) { - genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, closed: mapClosed, stepDone: stepDone, open: function (id) { setShown(id, true); }, + genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, closed: mapClosed, title: mapTitle, stepDone: stepDone, open: function (id) { setShown(id, true); }, close: function (id) { setShown(id, false); }, fallback: mapFallback }); } add(slot, genomeMap.node); diff --git a/docs/live-site.md b/docs/live-site.md index 1fd6d8c42..80f70e55c 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,7 +56,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens the closed results with a place first, in map order; after them it opens the next closed result in its section, and the map then draws nothing new, as that result has no place on it. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows and lines, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. diff --git a/rust/crates/og-map/README.md b/rust/crates/og-map/README.md index eee8c3f5d..b2906b391 100644 --- a/rust/crates/og-map/README.md +++ b/rust/crates/og-map/README.md @@ -77,7 +77,7 @@ One gzip member with mtime 0 and no file name, holding JSON written as Python's - **The veil.** The document is the same for every reader. Whether a page is closed is the page's own rule; the document says only what a closed result could reveal. Closable findings are those of level `act` and every gated one (APOE closes on every page, an adult's too). Each closable finding with a place veils its span, and `dict.veil` lists their ids: gated ones first, then the act ones in the page's order. A row of a layer that can be a finding's own source (structural variants, mosaic calls and ClinVar hits, as their layers arrive), and every findings row, that overlaps a veiled span carries the index of the first such finding in `dict.veil` in a `veil` column, else -1. A row holds one tag only. Gated findings come first, so a row over both APOE and an act finding carries APOE, which is closed on every page; a row over two act findings carries the first, and both close under the same rule. The page decides its own findings with its own veil, and the tag binds only the rows that are not one of the page's findings (APOE, or a row the page's results do not have yet): the page hides such a row while its result is closed. Counts, totals and density bins cover untagged rows only, so no number counts a closed row. Runs of homozygosity and bands are never a finding's source and carry no tag. - `roh`: the `RG` rows of step 11's `bcftools roh` text (`{run}/vcf/{run}_roh.txt`, else `{run}/roh/{run}_roh.txt`). bcftools writes 1-based inclusive positions, so `s = start - 1` and `e = end`. `ST` lines are skipped; rows on chrM and other contigs count in `totals.all` only. `markers` and `q` are bcftools' columns 7 and 8. `art` indexes `dict.art`: `one X` on a male's chrX, `one Y` on a male's chrY, else `centromere` when more than half the run lies in masked spans. An unknown sex sets neither flag. `long` is 1 for a run of 5 Mb or more with no `art`. - `roh_bins`: bases covered by any run per 1 Mb bin, for each chromosome with a run. -- `text`: the sentences the page shows, in pieces; `b` marks bold words. `text.findings` is the count of findings on the map in three forms (`none`, `one`, `many`): the page counts the findings rows whose `veil` is -1 plus the tagged rows whose result is open, picks the form and puts the number in place of `{n}`. +- `text`: the sentences the page shows, in pieces; `b` marks bold words. `text.findings` is the count of findings on the map in three forms (`none`, `one`, `many`): the page counts the rows it draws: each of its own findings that its veil shows at this moment, whatever the row's tag, and each other row whose tag is -1 or names a result that is open; it picks the form and puts the number in place of `{n}`. ## Tests diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 11811cf33..4978049fb 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1255,7 +1255,7 @@ def write_map(path, doc): hash: location.hash }; }()))""" MAP_READY = ("document.documentElement.dataset.pgpResults && (function () { var f = document.getElementById('genome-map'); " - "return f && /^(drawn|failed|nodecompress)$/.test(f.dataset.pgpMap || '') && f.dataset.level !== undefined " + "return f && /^(drawn|failed|nodecompress)$/.test(f.dataset.pgpMap || '') " "&& (f.dataset.pgpMap !== 'drawn' || !!f.dataset.level); }())") @@ -1275,6 +1275,61 @@ def map_now(b): return json.loads(b.eval(MAP_JS)) +# The page's closed results to act on and the map's chips, read at once (a chip's button rebuilds the chips at +# once: nothing to wait for): the closed act cards in page order, the act chip's count, whether its confirm step +# shows, where the keyboard is (first: the chips' first button; canvas: the map), and the opened chips. +CHIP_JS = r"""JSON.stringify((function () { + var ids = []; + Array.prototype.forEach.call(document.querySelectorAll('.finding[data-level=act]'), function (a) { + if (a.querySelector('.veil') && ids.indexOf(a.dataset.finding) < 0) { ids.push(a.dataset.finding); } }); + var c = document.querySelector('.gm-chip[data-closed=act]'), f = document.activeElement; + var first = document.querySelector('.gm-chips button'); + return { closed: ids, buttons: !!first, chip: c ? c.firstChild.textContent : '', confirm: !!(c && c.querySelector('.gm-confirm')), + focus: f && f.classList.contains('gm-yes') ? 'gm-yes' : f === first ? 'first' : f === document.querySelector('.gm-canvas') ? 'canvas' + : (f ? f.tagName + '.' + f.className : ''), + opened: Array.prototype.map.call(document.querySelectorAll('.gm-chip.opened'), function (o) { var a = o.querySelector('a'); + return { kind: o.dataset.opened, text: o.textContent, href: a ? a.getAttribute('href') : '' }; }) }; +}()))""" + + +def chips_now(b): + time.sleep(0.05) # a chip's button draws the page and the chips again before its click returns + return json.loads(b.eval(CHIP_JS)) + + +def closed_words(n): + return "" if n == 0 else "1 result closed" if n == 1 else f"{n} results closed" + + +def chip_open(b): + """The act chip's Open, then its Show the result, as a keyboard reader does it. Returns (the id it opened, + problems): one Open opens exactly one of the page's closed results, the count falls by one, the keyboard + goes to Show the result and then to the chips' first button (the map when no chip is left), and an + "Opened in its section" chip never names a result that is still closed.""" + before = chips_now(b) + b.eval("var o = document.querySelector('.gm-chip[data-closed=act] button'); o.focus(); o.click(); true") + mid = chips_now(b) + bad = [] + if not mid["confirm"] or mid["closed"] != before["closed"]: + bad.append(f"Open: confirm step {mid['confirm']}, closed {before['closed']} -> {mid['closed']}") + if mid["focus"] != "gm-yes": + bad.append(f"Open: the keyboard is on {mid['focus']!r}, not Show the result") + b.eval("document.querySelector('.gm-chip[data-closed=act] .gm-yes').click(); true") + after = chips_now(b) + gone = [x for x in before["closed"] if x not in after["closed"]] + n = len(before["closed"]) - 1 + if len(gone) != 1 or len(after["closed"]) != n: + bad.append(f"Show the result opened {gone}, not one: closed {before['closed']} -> {after['closed']}") + if not after["chip"].startswith(closed_words(n)) or bool(after["chip"]) != (n > 0): + bad.append(f"Show the result: the chip reads {after['chip']!r}, want {closed_words(n)!r}") + if after["focus"] != ("first" if after["buttons"] else "canvas"): + bad.append(f"Show the result: the keyboard is on {after['focus']!r}") + named = [o["href"] for o in after["opened"] if o["kind"] == "section" and o["href"][3:] in after["closed"]] + if named: + bad.append(f"an Opened in its section chip names a closed result: {named}") + return (gone[0] if len(gone) == 1 else None), bad + + def parity_problems(b, base, rows): """The karyogram's marks the map does not place with the same level and chromosome within 0.01 Mb.""" bad = [] @@ -1323,6 +1378,14 @@ def state_problems(b, base): g = map_page(b, base, S) if not (g["level"] == "genome" and "all 24 chromosomes" in g["title"] and g["view"] == "genome"): bad.append(f"genome: {g['level']} {g['title']!r} {g['view']}") + click_canvas(b, 200, 9 + 2 * 22 + 11) + c = map_now(b) + if not (c["level"] == "chromosome" and c["chrom"] == "chr3" and c["hash"] == "#chr3"): + bad.append(f"a click on chromosome 3's row: {c['level']} {c['chrom']} {c['hash']}") + b.eval("document.querySelector('.gm-canvas').dispatchEvent(new KeyboardEvent('keydown', {key: 'Escape', bubbles: true})); true") + c = map_now(b) + if c["level"] != "genome" or c["hash"] != "#genome": + bad.append(f"Esc: {c['level']} {c['hash']}") for link, chrom, view in (("#chr16", "chr16", "chr16"), ("#chr16:10.0-40.0M", "chr16", "chr16:10.0-40.0M"), ("#view=chromosome&chr=11", "chr11", "chr11"), ("#chr2:20000001-60000000", "chr2", "chr2:20.0-60.0M")): c = map_page(b, base, S, link) @@ -1330,32 +1393,24 @@ def state_problems(b, base): bad.append(f"{link}: {c['level']} {c['chrom']} {c['view']}") elif any(m["chrom"] != chrom for m in c["marks"]): bad.append(f"{link}: marks of another chromosome listed: {c['marks']}") - c = map_page(b, base, S, "#chr16") - b.eval("location.hash = '#genome'; true") + b.eval("location.hash = '#genome'; true") # after the last link's place c = map_now(b) if c["level"] != "genome": - bad.append(f"#genome after #chr16: {c['level']}") + bad.append(f"#genome after a place: {c['level']}") b.eval("location.hash = '#chrQ'; true") c = map_now(b) if "Could not read the place" not in c["toast"] or c["level"] != "genome": bad.append(f"#chrQ: toast {c['toast']!r}, level {c['level']}") - g = map_page(b, base, S) - click_canvas(b, 200, 9 + 2 * 22 + 11) - c = map_now(b) - if not (c["level"] == "chromosome" and c["chrom"] == "chr3" and c["hash"] == "#chr3"): - bad.append(f"a click on chromosome 3's row: {c['level']} {c['chrom']} {c['hash']}") - b.eval("document.querySelector('.gm-canvas').dispatchEvent(new KeyboardEvent('keydown', {key: 'Escape', bubbles: true})); true") - c = map_now(b) - if c["level"] != "genome" or c["hash"] != "#genome": - bad.append(f"Esc: {c['level']} {c['hash']}") return bad def map_veil_problems(b, base, out): """On the son's closed page: his closed result to act on (the planted deletion) is in no mark, tooltip, - chip, sentence, link or address of the map until opened; the chip's Open (after its confirm) and the card's - "Show the result" each bring back exactly its mark, on the map and in the card; Close it again and a reload - close it. On DEMO-001, an adult's page, APOE stays closed until opened, from the chip or its card's fold.""" + chip, sentence, link or address of the map until opened; the chip's Open (after its confirm) opens one + result per click in the page's order (chip_open), so his result with no place opens first, in its section; + then Open and the card's "Show the result" each bring back exactly the deletion's mark, on the map and in + the card; Close it again and a reload close it. On DEMO-001, an adult's page, APOE stays closed until + opened, from the chip or its card's fold.""" bad = [] son = OTHERS[1] doc = read_map(os.path.join(out, "runs", son, "map.json")) @@ -1368,8 +1423,8 @@ def map_veil_problems(b, base, out): words = [gene, f"{f['chrom'][i]}:{f['s'][i] + 1}", f"{f['s'][i] + 1:,}", fid] x, y = genome_xy(b, doc, f["chrom"][i], (f["s"][i] + f["e"][i]) / 2) - def leaks(c): - return [w for w in words if w in c["text"] or w in c["tip"] or w in c["hash"]] + \ + def leaks(c, but=""): + return [w for w in words if w not in but and (w in c["text"] or w in c["tip"] or w in c["hash"])] + \ [m["id"] for m in c["marks"] if m["id"] == fid] c = map_page(b, base, son) click_canvas(b, x, y, move_only=True) @@ -1384,11 +1439,23 @@ def leaks(c): label = "1 result closed" if want == 1 else f"{want} results closed" if not want or not any(t.startswith(label + " ") for t in c["chips"]): bad.append(f"closed: no chip counting the page's {want} closed results: {c['chips']}") - b.eval("document.querySelector('.gm-chip[data-closed=act] button').click(); true") - c = map_now(b) - if not b.eval("!!document.querySelector('.gm-chip[data-closed=act] .gm-confirm')") or leaks(c): - bad.append("Open: no confirm step, or it opened at once") - b.eval("document.querySelector('.gm-chip[data-closed=act] .gm-yes').click(); true") + # Open opens the page's closed results one per click, in the page's order, so the son's own result with no + # place (his first closed card) opens before the deletion: the map then names it only in its section chip. + order = chips_now(b)["closed"] + for k in range(want): + got, more = chip_open(b) + if more: + return bad + [f"Open {k + 1}: {x}" for x in more] + if got != order[k]: + return bad + [f"Open {k + 1} opened {got}, not the page's next closed result {order[k]} ({order})"] + if got == fid: + break + section = [o for o in chips_now(b)["opened"] if o["kind"] == "section"] + if [o["href"] for o in section] != [f"#f/{got}"] or "Opened in its section: " not in section[0]["text"]: + bad.append(f"Open {k + 1} opened {got}, which has no place: section chips {section}") + c = map_now(b) + if leaks(c, " ".join(o["text"] for o in section)): # the opened result names its own gene + bad.append(f"Open {k + 1} ({got}): the map shows {leaks(c)}") c = map_now(b) card = b.eval("!!document.querySelector('#f-%s h3') && !document.querySelector('#f-%s .veil')" % (fid, fid)) if [m["id"] for m in c["marks"] if m["id"] == fid] != [fid] or not card: @@ -1401,7 +1468,12 @@ def leaks(c): c = map_now(b) if gene not in c["tip"]: bad.append(f"Open: its tooltip at its place does not name it: {c['tip']!r}") - b.eval("document.querySelector('.gm-chip.opened button').click(); true") + # The results opened before it in their section close again from their chips, and those chips go. + b.eval("Array.prototype.forEach.call(document.querySelectorAll('.gm-chip.opened[data-opened=section]'), " + "function () { document.querySelector('.gm-chip.opened[data-opened=section] button').click(); }); true") + if [o for o in chips_now(b)["opened"] if o["kind"] == "section"] or chips_now(b)["closed"] != order[:k]: + bad.append(f"Close it again on the section chips: chips {chips_now(b)['opened']}, closed {chips_now(b)['closed']}") + b.eval("document.querySelector('.gm-chip.opened[data-opened=\\'1\\'] button').click(); true") c = map_now(b) card = b.eval("!!document.querySelector('#f-%s .veil')" % fid) if leaks(c) or not card: @@ -1701,6 +1773,61 @@ def chip_problems(b, base, out, og): return bad +# Two planted closed results, one with a place on the map and one without, in both orders of the page. +OPEN_ORDER = (("unplaced pl-a, placed pl-b", [act("pl-a", "PLAGENE", "", "Planted a result"), + act("pl-b", "PLBGENE", "chr13:32340000", "Planted b result")]), + ("placed pl-a, unplaced pl-b", [act("pl-a", "PLAGENE", "chr13:32340000", "Planted a result"), + act("pl-b", "PLBGENE", "", "Planted b result")])) + + +def open_order_problems(b, base, out, og): + """For each planting of OPEN_ORDER, the chip's Open clicked until nothing is closed (chip_open checks each + click): the ids it opens, click by click, are the same in both, so the order never tells which closed result + has a place. A result with no place opens with an "Opened in its section" chip that links its card; its + Close it again closes it on the page and takes the chip away. Before that, Open then Keep it closed opens + nothing and gives the keyboard back to the chips.""" + seqs, bad = {}, [] + for k, (what, extra) in enumerate(OPEN_ORDER): + keep = plant_son(out, extra, og) + try: + placed = read_map(os.path.join(out, "runs", OTHERS[1], "map.json"))["per"][OTHERS[1]]["findings"]["id"] + if [x["id"] in placed for x in extra] != [bool(x["viewer"]) for x in extra]: + return [f"{what}: og placed {[x['id'] for x in extra if x['id'] in placed]}: the planting tests nothing"] + map_page(b, base, OTHERS[1]) + first = chips_now(b) + if k == 0: + b.eval("var o = document.querySelector('.gm-chip[data-closed=act] button'); o.focus(); o.click(); true") + b.eval("document.querySelector('.gm-chip[data-closed=act] .gm-no').click(); true") + kept = chips_now(b) + if kept["closed"] != first["closed"] or kept["confirm"] or kept["focus"] != "first": + return [f"{what}: Keep it closed: closed {kept['closed']}, confirm {kept['confirm']}, " + f"keyboard on {kept['focus']!r}"] + seq = [] + for _ in first["closed"]: + got, more = chip_open(b) + if more: + return bad + [f"{what}, Open {len(seq) + 1}: {x}" for x in more] + seq.append(got) + opened = chips_now(b)["opened"] + section = [o["href"] for o in opened if o["kind"] == "section"] + if (got in placed) == (f"#f/{got}" in section): + return [f"{what}: {got} (placed {got in placed}) opened with section chips {section}"] + seqs[what] = seq + if k == 0: + gone = [x["id"] for x in extra if not x["viewer"]][0] + b.eval("var o = document.querySelector('.gm-chip.opened[data-opened=section] a[href=\"#f/%s\"]'); " + "o.parentNode.parentNode.querySelector('button').click(); true" % gone) + after = chips_now(b) + if after["closed"] != [gone] or any(o["kind"] == "section" and gone in o["href"] for o in after["opened"]): + bad.append(f"{what}: Close it again on {gone}'s section chip: closed {after['closed']}, " + f"opened chips {after['opened']}") + finally: + unplant(keep) + if len(set(map(tuple, seqs.values()))) != 1: + bad.append(f"the order Open opens them tells which has a place: {seqs}") + return bad + + def runs_sentence(b, base): """Whether DEMO-001's planted runs sentence on chr1 is in the margin at the first frame of the replay and at the last: (first, last, the level and chromosome at the first frame).""" @@ -1900,21 +2027,36 @@ def map_checks(b, base, out): keep_js = fh.read() bad = map_veil_problems(b, base, out) check("map: the son's closed result is in no mark, tooltip, chip, sentence, link or address until opened; Open " - "(after its confirm) and the card's Show the result each bring back exactly its mark there and in the card; " + "(after its confirm) opens one result per click, in the page's order, and the count falls by one; Open and " + "the card's Show the result each bring back exactly its mark there and in the card; " "Close it again and a reload close it; APOE stays closed on an adult's page until opened, from the chip " "or its card's fold", not bad, bad) + mjs = os.path.join(out, "assets", "map.js") + with open(mjs) as fh: + keep_mjs = fh.read() + one = "opts.open(id); // " + every = "f.level === 'act' && veiled(f); })" try: with open(js, "w") as fh: fh.write(keep_js.replace("function mapVeil(id, gated) {", "function mapVeil(id, gated) { return 'open';")) control("person.js whose map veil keeps nothing closed", not map_veil_problems(b, base, out)) + with open(js, "w") as fh: + fh.write(keep_js.replace(every, "f.level === 'act'; })")) + control("person.js whose map chip counts every result to act on, closed or open", keep_js.count(every) != 1 or + not map_veil_problems(b, base, out)) + with open(js, "w") as fh: + fh.write(keep_js) + with open(mjs, "w") as fh: + fh.write(keep_mjs.replace(one, "(gated ? [id] : closedActs()).forEach(function (x) { opts.open(x); }); // ")) + control("map.js whose Open opens every closed result at once", keep_mjs.count(one) != 1 or + not map_veil_problems(b, base, out)) finally: with open(js, "w") as fh: fh.write(keep_js) + with open(mjs, "w") as fh: + fh.write(keep_mjs) bad = overlap_problems(b, base, out) check("map: a second closed result to act on over the first stays closed when the first is opened", not bad, bad) - mjs = os.path.join(out, "assets", "map.js") - with open(mjs) as fh: - keep_mjs = fh.read() own = "var own = M.closable.indexOf(r.id) < 0 || stateOf(r.id) !== 'closed';" try: with open(mjs, "w") as fh: @@ -1969,6 +2111,28 @@ def map_checks(b, base, out): finally: with open(js, "w") as fh: fh.write(keep_js) + bad = open_order_problems(b, base, out, og) + check("map: the chip's Open opens one closed result per click in the page's order, whether it has a place or " + "not; one with no place gets an Opened in its section chip until closed again; the keyboard follows", + not bad, bad) + concat = "if (page) { return page.filter(function (id) { return !M.gated[id]; }); }" + placed_first = ("if (page) { var placed = M.closable.filter(function (id) { return !M.gated[id] && stateOf(id) === " + "'closed' && page.indexOf(id) >= 0; }); return placed.concat(page.filter(function (id) { return " + "placed.indexOf(id) < 0; })); }") + alive = "stateOf(id) !== 'closed' && opts.title && opts.title(id);" + pushed = "inSection.push(id);" + focus = "(b || canvas).focus({ preventScroll: true });" + try: + for desc, a, z in (("map.js whose Open opens the closed results with a place first", concat, placed_first), + ("map.js whose section chip stays for a closed result", alive, "true;"), + ("map.js with no section chip for a result with no place", pushed, ""), + ("map.js that leaves the keyboard where the chips were rebuilt", focus, "void b;")): + with open(mjs, "w") as fh: + fh.write(keep_mjs.replace(a, z)) + control(desc, keep_mjs.count(a) != 1 or not open_order_problems(b, base, out, og)) + finally: + with open(mjs, "w") as fh: + fh.write(keep_mjs) bad = replay_apoe_problems(b, base) check("map: before the traits step a replay has no APOE chip, switch or mark", not bad, bad) gated = "if (M.gated[id] && opts.now && !opts.now(id)) { return; }" From e5306541a8c1cf81bcc1efa6975a78901e152736 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:28:58 +0100 Subject: [PATCH 020/255] ci(guard): give the unit-test shards 15 minutes so the browser tests stop racing the limit --- .github/workflows/guard.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 3c9248867..1d59f9971 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -156,11 +156,12 @@ jobs: # Three shards: one job ran every test file in about 9.5 of its 10 minutes, # so each new browser test pushed it over. File i runs in shard i % 3, so - # every file runs in exactly one shard. + # every file runs in exactly one shard. The limit is 15 minutes: the shard + # that holds the browser tests of the live page already takes about 9. unit-tests: needs: og runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 strategy: fail-fast: false matrix: From 1a23510c3caff68a3051481dfc38402dda695b8b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:16:27 +0100 Subject: [PATCH 021/255] fix(live): close every gene of a closed deletion and list the fixed-panel cards A closed structural variant tagged only its first gene, so a row or a finding that named its second gene stayed open on a child's page. closable() now gives one span per gene the finding names, and each structural-variant finding lists its genes. veil.panels lists the cards whose rows are the same genes for everyone (FIXED_PANELS, one table), so the page can close them whole. Bead: pgp-iqon.28 Bead: pgp-iqon.29 --- bin/live_results.py | 41 +++++++++++++++++++++++++++---- tests/test_live_results.py | 50 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/bin/live_results.py b/bin/live_results.py index 3f0a3a603..629abda5c 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -888,6 +888,9 @@ def gene(x): gene=(x.get("genes") or [None])[0], detail=x.get("sv"), evidence=[["Class", x.get("acmg_class")], ["Callers", rr.annotsv_support_text(x)], ["Genotype", x.get("gt")]], viewer=str(x.get("sv") or "").replace(" ", "") or None)) + # Every gene it covers, not only the first: a closed one closes the rows that name any of them (closable()). + gl = x.get("genes") + out[-1]["genes"] = [str(g) for g in gl if g] if isinstance(gl, list) else [] # 6. Repeat expansions outside Stranger's "normal", or in an intermediate zone. for l in _items(_data(S, "expansions").get("loci")): with _Row(out, "expansions", "act"): @@ -1144,6 +1147,24 @@ def lead(found, levels, summary): # belongs to a closed result when it names the result's gene, or its position falls inside the result's # position or span; a row that is the result's own position or span belongs to that result first, else # the first such result in finding order. The veil lives in the browser: the files still hold every row. +# The cards whose rows are a fixed panel: the same genes or loci for everyone, known before the run. A row or +# dot of one that closes with a result would tell which gene the result is in by what is missing, so a child's +# page closes each of these cards whole while any result to act on is closed, behind one box with the same words +# whatever the result (results.json veil.panels; assets/live/person.js panelShut()). The one table: a card of a +# fixed panel is listed here and nowhere else. +FIXED_PANELS = { + "expansions": "the repeat loci of the catalog", + "vntyper": "MUC1", + "lpa": "LPA", + "chip": "the CHIP genes and their detection limits", + "paralogs": "SMN1, SMN2, HBA1, HBA2, the RCCX module and the other paralog genes", + "cpic": "the CPIC genes", + "cyp2d6": "CYP2D6", + "hla": "the HLA genes", + "locityper": "the Locityper loci", + "traits-panel": "the trait panel's genes", + "bloodtype": "the blood group systems", +} LOCUS_RE = re.compile(r"(? id; a symptom ranking can rank the gene) and the lead's segments that name one (lead: one per segment of LEAD_PARTS, the "act" names left out: the page closes those itself), each card's facts (facts: card -> one per fact) and the items - of its drawing (widget: card -> one per CHIP call or copy-number dot). spans is closable() itself: the family + of its drawing (widget: card -> one per CHIP call or copy-number dot), and the cards of a fixed panel + (panels: FIXED_PANELS, which close whole while any result is closed). spans is closable() itself: the family page (assets/live/family-page.js) reads which results are closed from it, and closes every place of the child that could carry one while one is. Empty when nothing can be closed.""" spans = closable(found) if not spans: return {} out = {"rows": {}, "viewer": {}, "lines": {}, "facts": {}, "widget": {}, "marks": {}, "findings": {}, "lead": [], - "spans": [list(s) for s in spans]} + "spans": [list(s) for s in spans], + "panels": [c["id"] for ch in chapters for c in ch["sections"] if c["id"] in FIXED_PANELS and c.get("state") != "missing"]} S = summary.get("sections") or {} hits = [h for h in _items(_data(S, "clinvar").get("hits"))[:50]] tags = [veil_id(spans, str(h.get("gene") or ""), h.get("chrom"), h.get("pos")) if isinstance(h, dict) else None diff --git a/tests/test_live_results.py b/tests/test_live_results.py index f5bec4523..8faefe11f 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -48,12 +48,16 @@ first ones and its closed lead names none; a CHIP call, a copy-number dot and a fact that name a closed result's gene carry its id (the CHIP chart's panel of genes never does), and results.json carries the closed results as - veil.spans (the family page closes a child's places while one is closed). + veil.spans (the family page closes a child's places while one is closed); + a closed structural variant over two genes closes the rows and the finding + that name only its second gene, and veil.panels lists the cards of a fixed + panel (live_results.FIXED_PANELS). Each check runs again on a planted mistake and must fail there: 1 with VNTYPER's entry dropped, 2 with one card's caveat blanked, 3 with "ok" planted in a card and "All clear" in a script, 4 with the hits emptied, 5 with the results route registered without the login, 6 with a row printed as its Python dict, 7 with the Slivar adapter returning nothing, -8 with a veil rule that tags nothing (rows, lines, facts and drawings) and one that tags every row. A check +8 with a veil rule that tags nothing (rows, lines, facts and drawings), one that tags every row and one that tags +only a structural variant's first gene. A check that cannot fail is no check. Run: python3 tests/test_live_results.py @@ -464,6 +468,39 @@ def widget_problems(): return out +def sv_genes_problems(): + """What is wrong with the veil's tags of a closed structural variant over two genes: a ClinVar row and a carrier + row that name only its second gene (the hit sits elsewhere, so only the gene can tie it) carry its id, and so + does the carrier finding; veil.spans lists it once per gene; veil.panels lists the cards of a fixed panel.""" + S = {"annotsv": {"state": "ok", "data": {"rows": [ + {"sv": "chr17:43044295-43170245", "type": "deletion", "length": -125950, "acmg_class": "5", + "genes": ["BRCA1", "NBR2"], "gt": "0/1", "support": 2, "support_callers": ["Manta", "Delly"]}]}}, + "clinvar": {"state": "ok", "data": {"hits": [ + {"gene": "NBR2", "chrom": "chr1", "pos": 5, "ref": "A", "alt": "G", "significance": "Pathogenic"}, + {"gene": "GENE5", "chrom": "chr1", "pos": 9, "ref": "A", "alt": "G", "significance": "Pathogenic"}]}}, + "carrier": {"state": "ok", "data": {"genes_on_list": 2, "carriers": [ + {"gene": "NBR2", "variant": "c.1A>G", "genotype": "het", "significance": "Pathogenic"}]}}, + "paralogs": {"state": "ok", "data": {"genes": [{"gene": "SMN1", "copy_number": 2, "status": "called"}]}}} + res = live_results.build("T", {"sections": S}) + v = res.get("veil") or {} + sv = [f["id"] for f in res["findings"] if f["card"] == "structural-variants"] + car = [f["id"] for f in res["findings"] if f["card"] == "carrier-list"] + out = [] + if len(sv) != 1 or len(car) != 1: + return [f"the planted findings: {[f['id'] for f in res['findings']]}"] + if (v.get("rows") or {}).get("clinvar") != [sv[0], None]: + out.append(f"ClinVar rows tagged {(v.get('rows') or {}).get('clinvar')}, want the deletion's id on the NBR2 row") + if (v.get("rows") or {}).get("carrier-list") != [sv[0]]: + out.append(f"carrier rows tagged {(v.get('rows') or {}).get('carrier-list')}") + if (v.get("findings") or {}).get(car[0]) != sv[0]: + out.append(f"the NBR2 carrier finding tagged {(v.get('findings') or {}).get(car[0])}") + if sorted(s[1] for s in v.get("spans") or [] if s[0] == sv[0]) != ["BRCA1", "NBR2"]: + out.append(f"spans {v.get('spans')}") + if v.get("panels") != ["paralogs"]: + out.append(f"panels {v.get('panels')}, want the copy-number card, the only fixed panel here") + return out + + def overview_problems(): """What is wrong with the home page's view of a child's run (overview): a finding that names a closed result (a symptom ranking of its gene) is listed among the first findings, or the closed lead still names it.""" @@ -584,6 +621,15 @@ def tagged(sample_dir): bad = widget_problems() check("8. a CHIP call, a copy-number dot and a fact that name a closed result's gene carry its id, the CHIP " "chart's gene panel none, and results.json carries the closed results as veil.spans", not bad, bad) + bad = sv_genes_problems() + check("8. a closed structural variant over two genes closes the rows and the finding that name only its second " + "gene, veil.spans lists it per gene, and veil.panels lists the fixed-panel cards", not bad, bad) + first = live_results.closable + try: + live_results.closable = lambda found: first([{k: v for k, v in f.items() if k != "genes"} for f in found]) + control("a veil that tags only a structural variant's first gene", not sv_genes_problems()) + finally: + live_results.closable = first bad = veil_problems(*tagged(d3)) check("8. the planted ClinVar hit's row, viewer link, tick and star bar carry its secondary finding's id, the " "deletion's span its own, every other card's row of the gene and the symptom ranking's finding one of them, " From 9ef50594e0ac18a3c75bb810de6b9c5d4e216cb3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:16:28 +0100 Subject: [PATCH 022/255] fix(live): make a child's page the same whichever result is closed A closed finding's element and the address carried its id, which names the gene and variant. They now carry a handle, its place among the page's closed findings, and a link to #f/ lands on the closed card with the handle in the address. A fixed-panel card closes whole while any result is closed, and the note on a card's closed rows no longer counts them. The contents mark the section the address names on every redraw, so that mark no longer depends on timing. tests/test_live_demo.py compares two pages of the son that differ only in which result is closed (HTML, attributes, address, map canvas), with controls for the id, the whole-card rule and the count, and its map check no longer dies on an unbound k when nothing is closed. Bead: pgp-iqon.27 Bead: pgp-iqon.29 --- assets/live/person.js | 88 +++++++++-- tests/test_live_demo.py | 329 ++++++++++++++++++++++++++++++++-------- 2 files changed, 340 insertions(+), 77 deletions(-) diff --git a/assets/live/person.js b/assets/live/person.js index b4363b225..6e7cbb039 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -362,6 +362,32 @@ return [facts, w, shut]; } function veiledOf(c) { return (c.findings || []).map(findingById).filter(function (f) { return f && veiled(f); }); } + // The results to act on this page keeps closed, drawn yet or not. + function closedActs() { return ((results && results.findings) || []).filter(function (f) { return veiled(f) && f.level === 'act'; }); } + // A card of a fixed panel (results.veil.panels, bin/live_results.py FIXED_PANELS: the same genes or loci for + // everyone, known in advance): a row or dot of it closed with a result would tell the result's gene by what is + // missing. So it closes whole while this page keeps any result to act on closed, behind one box with the same + // words whatever the result, and comes back once the last one is opened. + function panelShut(c) { + var p = (results && results.veil && results.veil.panels) || []; + return closedPage() && p.indexOf(c.id) >= 0 && closedActs().length > 0; + } + var PANEL_WORDS = 'This section lists the same genes for everyone, so it stays closed while a result to act on is closed ' + + 'on this page: what it left out would tell which gene. It opens once every such result is opened.'; + var PANEL_LINE = 'The same genes for everyone, closed while a result about a child is closed.'; + // The next closed result in the page's order (the map's Open opens the same one). + function nextClosed() { var ids = mapClosed(); return ids.length ? findingById(ids[0]) : closedActs()[0]; } + // A closed finding's handle in the page: its place in the page's list of closed findings, never its id, which + // names its gene and variant. The id comes back once the finding is opened on purpose (#f/ too). + function closedOrder() { + var out = []; + sectionsList().forEach(function (sec) { + (sec.findings || []).forEach(function (id) { var f = findingById(id); if (f && veiled(f) && out.indexOf(id) < 0) { out.push(id); } }); + }); + ((results && results.findings) || []).forEach(function (f) { if (veiled(f) && out.indexOf(f.id) < 0) { out.push(f.id); } }); + return out; + } + function handle(f) { return veiled(f) ? 'closed-' + (closedOrder().indexOf(f.id) + 1) : f.id; } // The search box (plugins/search.js) asks the same question for each hit, by the summary key and the anchor // of the section it comes from: a hit from a card this page keeps closed is left out. A closed page with // no results.json yet cannot tell which cards those are, so it leaves every hit out until it can. @@ -372,7 +398,7 @@ if (!results || key === null) { return true; } if (tag && tagClosed(tag)) { return true; } return cards().some(function (c) { - return (c.id === anchor || (c.keys || []).indexOf(key) >= 0) && veiledOf(c).length > 0; + return (c.id === anchor || (c.keys || []).indexOf(key) >= 0) && (veiledOf(c).length > 0 || panelShut(c)); }); }; @@ -419,6 +445,9 @@ add(text, U.veil((closedFs.length === 1 ? 'This section holds a result' : 'This section holds ' + closedFs.length + ' results') + ' to act on in adult life. Its rows stay closed until ' + (closedFs.length === 1 ? 'it is' : 'each is') + ' opened.', null, function () { openFinding(closedFs[0]); redraw(true); var x = document.getElementById('f-' + closedFs[0].id) || document.getElementById('card-' + c.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); + } else if (panelShut(c)) { + // One button opens one result, the next closed one in the page's order, as the map's Open does. + add(text, U.veil(PANEL_WORDS, null, function () { var f = nextClosed(); if (f) { openFinding(f); } redraw(true); })); } else { (c.notes || []).forEach(function (n) { add(text, el('p', { 'class': 'note-stale' }, n)); }); if (c.headline) { add(text, el('p', { 'class': 'headline' }, c.headline)); } @@ -435,12 +464,9 @@ var shut = ot[1] + (c.lines || []).length - lines.length, items = of[2]; if (tb) { add(text, tb); } else if (c.empty_text && !ot[1]) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } if (shut || items) { - // Rows of its table and lines, and items of its facts and drawing, counted apart. - var what = [shut ? (shut === 1 ? 'One row' : shut + ' rows') : '', items ? (items === 1 ? (shut ? 'one item' : 'One item') : items + ' items') + - ' of its numbers or drawing' : ''].filter(Boolean).join(' and '); - var many = shut + items > 1; - add(text, el('p', { 'class': 'dim veil-rows' }, what + (many ? ' stay' : ' stays') + - ' closed with the result to act on ' + (many ? 'they belong' : 'it belongs') + ' to, until that result is opened.')); + // The same words however many rows or items close, and whichever result they belong to: a count would + // tell one closed result from another. + add(text, el('p', { 'class': 'dim veil-rows' }, 'Some of its rows or numbers stay closed with a result to act on, until that result is opened.')); } add(text, lineList(lines)); if (c.hidden) { @@ -465,7 +491,7 @@ add(foot, el('a', { href: 'report.html#' + encodeURIComponent(c.anchor), 'class': 'report' }, 'Open in the report')); var vtags = veilTags('viewer', c.id); var reads = (c.viewer || []).filter(function (x, i) { return !(vtags[i] && tagClosed(vtags[i])); }); - if (g === 'show' && reads.length) { + if (g === 'show' && reads.length && !panelShut(c)) { add(foot, el('a', { href: 'viewer/#' + reads[0], 'class': 'viewer' }, 'See the reads at ' + reads[0])); } (g === 'show' ? c.links || [] : []).forEach(function (f) { @@ -545,7 +571,29 @@ var t = findingTag(f); return !!(t && tagClosed(t)); } - function openFinding(f) { shown[f.id] = true; var t = findingTag(f); if (t) { shown[t] = true; } } + function openFinding(f) { keepHash(function () { shown[f.id] = true; var t = findingTag(f); if (t) { shown[t] = true; } }); } + // The address never keeps a closed finding's id. #f/ of a finding this page keeps closed (a link from another + // page, or a finding closed again since) becomes #f/closed-, its handle, and #f/ again once it is opened. + // Opening or closing a result moves the handles of the others: the address follows the finding it named. The + // finding behind a handle in the address is kept here only (hashId); a new address from the reader forgets it. + var hashId = null; + window.addEventListener('hashchange', function () { hashId = null; }); + function hashFinding() { + var h; + try { h = decodeURIComponent((window.location.hash || '').slice(1)); } catch (e) { return null; } + if (h.indexOf('f/') !== 0) { return null; } + var x = h.slice(2); + if (/^closed-\d+$/.test(x)) { return (hashId && findingById(hashId)) || findingById(closedOrder()[Number(x.slice(7)) - 1]) || null; } + return findingById(x) || null; + } + function syncHash() { + var f = hashFinding(); + if (!f || !window.history || !window.history.replaceState) { return; } + hashId = f.id; + var to = '#f/' + handle(f); + if (window.location.hash !== to) { window.history.replaceState(null, '', to); } + } + function keepHash(change) { var at = hashFinding(); if (at) { hashId = at.id; } change(); syncHash(); } // The genome map's veil (map.js): 'closed' while this page keeps the result closed, 'opened' once it was // opened on purpose, 'open' when this page never closes it. A gated result (APOE, behind its warning) is // closed on every page, an adult's too, until opened; a tag naming no finding of these results stays closed. @@ -561,8 +609,10 @@ function setShown(id, on) { var f = findingById(id); if (on) { if (f) { openFinding(f); } else { shown[id] = true; } } else { - delete shown[id]; - ((results && results.findings) || []).forEach(function (x) { if (findingTag(x) === id) { delete shown[x.id]; } }); + keepHash(function () { + delete shown[id]; + ((results && results.findings) || []).forEach(function (x) { if (findingTag(x) === id) { delete shown[x.id]; } }); + }); } redraw(true); } @@ -596,7 +646,7 @@ } function findingArticle(f, big) { var row = el('div', { 'class': 'row' }); - var art = el('div', { 'class': 'finding', id: 'f-' + f.id, 'data-finding': f.id, 'data-level': f.level }); + var art = el('div', { 'class': 'finding', id: 'f-' + handle(f), 'data-finding': handle(f), 'data-level': f.level }); var at = newAt(f); if (at) { art.classList.add('new'); add(art, el('span', { 'class': 'newband' }, 'New at ' + U.hm(at))); add(art, el('br', {})); } add(art, U.findingLevel(f.level, f.level_words)); @@ -632,7 +682,7 @@ return row; } function findingLine(f) { - var li = el('li', { id: 'f-' + f.id, 'data-finding': f.id, 'data-level': f.level }); + var li = el('li', { id: 'f-' + handle(f), 'data-finding': handle(f), 'data-level': f.level }); if (veiled(f)) { // A finding that names a closed result (a symptom ranking of its gene): closed with it, opened with it. var b = el('button', { type: 'button', 'class': 'btn' }, 'Show the result'); @@ -766,6 +816,7 @@ function cardLine(c) { var g = gate(c); return g === 'show' && veiledOf(c).length ? 'A result about a child, closed until opened.' + : g === 'show' && panelShut(c) ? PANEL_LINE : g === 'show' ? (c.headline || (openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)) : pending(g) ? 'Waiting for ' + stepWords(c) + '.' : g === 'failed' ? 'Not assessed: its step failed.' : 'Not assessed in this run.'; } @@ -1104,7 +1155,7 @@ if (h.indexOf('s/') === 0 || (h && !/^(s\d+|f\/.*|q\/.*|progress)$/.test(h) && cardById(h))) { target = document.getElementById('card-' + (h.indexOf('s/') === 0 ? h.slice(2) : h)); isCard = true; - } else if (h.indexOf('f/') === 0) { target = document.getElementById('f-' + h.slice(2)); } + } else if (h.indexOf('f/') === 0) { var hf = hashFinding(); target = document.getElementById('f-' + (hf ? handle(hf) : h.slice(2))); } else if (h.indexOf('q/') === 0) { target = document.getElementById('s' + (CHAPTER_SECTION[h.slice(2)] || 1)); } else { target = document.getElementById(h); } return target ? { target: target, isCard: isCard } : null; @@ -1124,12 +1175,18 @@ } } function route() { + syncHash(); var t = routeTarget(); if (!t) { return; } var target = t.target; openAround(t); target.scrollIntoView({ block: 'start' }); if (target.focus) { target.focus({ preventScroll: true }); } + markToc(target); + } + // The contents mark the section the address names; a redraw marks it again, so the mark never depends on + // whether the page drew again after the address was read. + function markToc(target) { Array.prototype.forEach.call(rail.querySelectorAll('[data-toc]'), function (a) { var sec = target.closest ? target.closest('section.sec') : null; a.setAttribute('aria-current', sec && sec.id === 's' + a.getAttribute('data-toc') ? 'true' : 'false'); @@ -1205,6 +1262,9 @@ // A gated result's fold follows its switch, whatever fold state the reader left (setShown). Array.prototype.forEach.call(panel.querySelectorAll('details[data-gate]'), function (d) { d.open = !!shown[d.getAttribute('data-gate')]; }); window.scrollTo(0, y); + syncHash(); + var rt = routeTarget(); + if (rt) { markToc(rt.target); } document.getElementById('progress-q').textContent = running() ? 'Is it moving, when will it end, is anything wrong? One line per step.' : 'One line per step: when it ran, and what it found.'; // The step timeline closes the report, folded once the page has a report to show, while the run goes diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 02fbf1b47..487fa3d2e 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -77,9 +77,14 @@ opened from its box, the same page whichever result is closed, and with his results unreadable "Show the rows" leaves it closed and "Show the map" opens it and no table; on - his page a CHIP call, a copy-number dot and a card's fact in the gene of a - closed result stay out of sight (on a phone the card's line too) until it - is opened; DEMO-001's page + his page the copy-number and CHIP cards, a fixed panel, close whole while + a result of his is closed (on a phone their line too), their box opens one + result per click and every dot, call and fact is back once all are open; + two of his pages that differ only in which result is closed (two SNV + genes, two deletions of two genes each, SMN1 against HBA1, a result with a + place on the map against one without, one ClinVar row against two, a link + to each) are the same before any click, HTML, attributes, address and map + canvas, and a closed deletion's second gene is named nowhere; DEMO-001's page mid-replay and at the end leads with the report (a short run band above the lead while it runs, none once finished), ends with the step timeline folded, keeps the wire to at most eight one-sentence lines with the plain @@ -99,8 +104,9 @@ request read as nothing closed, malformed veil.spans read as given, results without veil.spans read as nothing closed or as closed with nothing to act on, an opened result kept per run only or per child only, a couple screen closed only for a result in its genes or opened by 'Show the cells', a map image closed only for a result with a -position, and de novo rows that 'Show the map' opens; for the son's page, person.js that draws -every item, shows every fact or builds a card's line from every fact), the phone page (person.js that never folds, and a route that opens only the nearest fold), the +position, and de novo rows that 'Show the map' opens; for the son's page, person.js that draws a fixed-panel +card's open rows or builds its line from its facts, that puts a closed card's id back or counts the closed rows, +and a veil that tags only a deletion's first gene), the phone page (person.js that never folds, and a route that opens only the nearest fold), the folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and the person page's layout (a wire showing every line or every plain finish, a card showing every line, a step timeline left open) also run against a planted mistake and must @@ -110,6 +116,7 @@ """ import base64 import copy +import difflib import gzip import hashlib import json @@ -358,6 +365,7 @@ def browser_checks(out): map_checks(b, base, out) family_veil_checks(b, base, out) widget_veil_checks(b, base, out) + infer_checks(b, base, out) finally: b.close() srv.shutdown() @@ -402,8 +410,9 @@ def person_checks(b, base): WIDGETS = [("DEMO-002", "expansions", "str", "[data-widget=str] .w-row", 8), ("DEMO-002", "lpa", "ci", "[data-widget=ci] .w-tick", 1), ("DEMO-002", "chip", "vaf", "[data-widget=vaf] .w-tick.w-call", 1), - # five copy-number genes from Paraphase, then Parascopy's SMN1 and SMN2 - ("DEMO-003", "paralogs", "copies", "[data-widget=copies] li", 7), + # five copy-number genes from Paraphase, then Parascopy's SMN1 and SMN2, on the partner's page: the son's keeps + # the copy-number card closed while a result of his is closed (a fixed panel) + ("DEMO-002", "paralogs", "copies", "[data-widget=copies] li", 7), ("DEMO-003", "y-chromosome", "ypath", "[data-widget=ypath] li", 5), ("DEMO-002", "clinvar", "MoChA bar", "figure[data-chart=genome] rect.mca", 1), ("DEMO-003", "clinvar", "AnnotSV span", "figure[data-chart=genome] rect.sv", 1)] @@ -636,19 +645,32 @@ def planted_closed(b, base): return out -def planted_problems(b, base): +def open_closed(b, fid): + """Opens the closed result FID as a reader who holds a link to it does: #f/ lands on its closed card, whose + address and element carry its handle (#f/closed-), never its id; then its "Show the result". False when the + link lands on no closed card.""" + b.eval("location.hash = %s; true" % json.dumps("#f/" + fid)) + if not b.wait("/^#f\\/closed-\\d+$/.test(location.hash) && document.getElementById('f-' + location.hash.slice(3)) " + "&& location.hash", 5): + return False + b.eval("document.getElementById('f-' + location.hash.slice(3)).querySelector('.veil button').click(); true") + return True + + +def planted_problems(b, base, out_dir): """[problems] of the planted results on the son's page: closed, then opened one at a time, then reloaded.""" gene, hit, sv = planted_words() out = planted_closed(b, base) - ids = b.eval("Array.prototype.map.call(document.querySelectorAll('[data-finding][data-level=act]'), " - "function (e) { return e.getAttribute('data-finding'); }).join(' ')") or "" + with open(os.path.join(out_dir, "runs", OTHERS[1], "results.json")) as f: + ids = " ".join(x["id"] for x in json.load(f)["findings"] if x["level"] == "act") sf = next((i for i in ids.split() if i.startswith("sf-")), None) import make_demo_sample start = str(make_demo_sample.PLANTED["sv"][1]) del_ = next((i for i in ids.split() if i.startswith("sv-") and start in i), None) if not sf or not del_: return out + [f"the planted results are not among the son's results to act on: {ids}"] - b.eval("document.querySelector('#f-%s .veil button').click(); true" % sf) + if not open_closed(b, sf): + return out + [f"a link to {sf} lands on no closed card"] time.sleep(0.8) shown = b.eval(SHOWN_JS) or "" rows = b.eval("Array.prototype.filter.call(document.querySelectorAll('#card-clinvar tbody tr'), function (r) { " @@ -665,7 +687,8 @@ def planted_problems(b, base): hits, _ = son_search(b, gene) if hits in (None, "0"): out.append(f"the search for {gene} finds nothing once its result is opened") - b.eval("document.querySelector('#f-%s .veil button').click(); true" % del_) + if not open_closed(b, del_): + return out + [f"a link to {del_} lands on no closed card"] time.sleep(0.8) shown = b.eval(SHOWN_JS) or "" spans = b.eval("document.querySelectorAll('figure[data-chart=genome] a[data-mark^=sv-]').length") @@ -730,7 +753,7 @@ def write(p, text): with open(p, "w") as f: f.write(text) try: - bad = planted_problems(b, base) + bad = planted_problems(b, base, out) check("browser: the son's planted secondary finding and class 5 deletion name their gene and positions nowhere " "(cards, charts, titles, links, search) until each is opened, each opens only its own rows and marks, " "and a reload closes them", not bad, bad) @@ -1399,10 +1422,174 @@ def planted_control(desc, a, z, problems): write(p, text) +# What a reader of the son's page gets before any click: the whole document (every text and every attribute, the +# map's timing attributes aside), the address, and a hash of the map's canvas. +PAGE_JS = r"""JSON.stringify((function () { + var c = document.documentElement.cloneNode(true); + Array.prototype.forEach.call(c.querySelectorAll('[data-pgp-map-parse-ms],[data-pgp-map-paint-ms]'), function (e) { + e.removeAttribute('data-pgp-map-parse-ms'); e.removeAttribute('data-pgp-map-paint-ms'); }); + var cv = document.querySelector('.gm-canvas'); + return { html: c.outerHTML, href: location.href, canvas: cv ? cv.toDataURL() : '' }; +}()))""" + + +def page_now(b, base, hash_=""): + """The son's page before any click, once its results, its charts and its map have drawn: (lines of its HTML, + its address, its canvas hash).""" + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{OTHERS[1]}/{hash_}") + b.wait("var d = document.documentElement.dataset; d.pgpResults && d.pgpCharts && d.pgpStatus === 'ready' && (" + MAP_READY + + ") && 'ready'", 15) + time.sleep(0.8) # the margin follows the camera 140 ms after it settles + got = json.loads(b.eval(PAGE_JS) or "{}") + return (got.get("html") or "").replace("><", ">\n<").split("\n"), got.get("href") or "", \ + hashlib.sha256((got.get("canvas") or "").encode()).hexdigest()[:12] + + +def plant_summary(out, og, change): + """The son's results.json built again by live_results from his summary.json with CHANGE(sections) applied, and + his map.json built again from it by og map build. Returns what to write back afterwards.""" + son = OTHERS[1] + rj, mj = (os.path.join(out, "runs", son, n) for n in ("results.json", "map.json")) + keep = {} + for p in (rj, mj): + with open(p, "rb") as fh: + keep[p] = fh.read() + root = os.path.join(os.path.dirname(out), "root") + with open(os.path.join(root, son, "summary.json")) as fh: + summary = json.load(fh) + change(summary.setdefault("sections", {})) + res = live_results.build(son, summary) + with open(rj, "w") as fh: + json.dump(res, fh) + doc = build_live_demo.og_map(og, ["build", "--root", root, "--run", son, "--assets", os.path.join(REPO, "assets"), + "--cache-dir", os.path.join(root, ".owngenome-live", "map")], + json.dumps({"results": res}).encode()) + with open(mj, "wb") as fh: + fh.write(doc) + return keep + + +def cpsr(gene): + """A CPSR result to act on in GENE, the same variant whatever the gene: a closed result known by its gene.""" + def change(S): + S["cpsr"] = {"state": "ok", "data": {"call_floor_dp": 10, "low_confidence": 0, "plp": [ + {"gene": gene, "variant": "c.68_69del", "final": "Pathogenic", "clinvar": "Pathogenic"}]}} + return change + + +def sv_over(locus, genes): + """A class 5 deletion over GENES, and a carrier row that names only its second gene.""" + def change(S): + rows = S["annotsv"]["data"]["rows"] + S["annotsv"]["data"]["rows"] = rows + [{"sv": locus, "type": "DEL", "gt": "0/1", "length": "-100000", + "acmg_class": "5", "genes": genes, "caller": "Manta", "support": 1, + "support_callers": ["Manta"]}] + S["carrier"]["data"]["carriers"] = [{"gene": genes[1], "variant": "c.100C>T", "genotype": "het", + "significance": "Pathogenic", "stars": 2, "inheritance": "AR"}] + return change + + +def second_hit(S): + """A second ClinVar row in the gene of the son's planted secondary finding, the card's counts kept.""" + S["clinvar"]["data"]["hits"] = S["clinvar"]["data"]["hits"] + [dict( + [h for h in S["clinvar"]["data"]["hits"] if h.get("gene") == "KCNH2"][0], pos=150970000)] + + +# Pairs of son's pages that differ only in which result is closed (or in what only a closed result names): the +# page before any click must be the same, byte for byte. +INFER = (("two SNV results in different genes (BRCA1, BRCA2)", cpsr("BRCA1"), cpsr("BRCA2")), + ("a deletion over BRCA1 and NBR2 against one over CFTR and CTTNBP2", + sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"]), sv_over("chr7:117480025-117580025", ["CFTR", "CTTNBP2"])), + ("SMN1 against HBA1, the same copy numbers", cpsr("SMN1"), cpsr("HBA1")), + ("a result with a place on the map (BRCA2) against one without (MLH1)", cpsr("BRCA2"), cpsr("MLH1")), + ("a closed result with one ClinVar row against two", lambda S: None, second_hit), + ("a link to each of two closed results (#f/)", cpsr("BRCA1"), cpsr("BRCA2"), + ("#f/cpsr-brca1-c-68-69del", "#f/cpsr-brca2-c-68-69del"))) + + +def infer_problems(b, base, out, og, cases=INFER): + """[problems]: for each pair of CASES, the son's page before any click differs (its HTML, an attribute, the + address or the map's canvas), with the number of differing lines and the first ones.""" + bad = [] + for what, one, two, *hashes in cases: + got = [] + for change, h in zip((one, two), (hashes or [("", "")])[0]): + keep = plant_summary(out, og, change) + try: + got.append(page_now(b, base, h)) + finally: + unplant(keep) + (h1, a1, c1), (h2, a2, c2) = got + diff = [x for x in difflib.unified_diff(h1, h2, lineterm="", n=0) if x[:1] in "+-" and x[:3] not in ("+++", "---")] + if diff or a1 != a2 or c1 != c2: + bad.append(f"{what}: {len(diff)} lines differ, address {a1 == a2}, canvas {c1 == c2}: {diff[:6]}") + return bad + + +def sv_gene_problems(b, base, out, og): + """[problems]: with a closed deletion over BRCA1 and NBR2, the page names NBR2 nowhere (a carrier row and its + finding name only NBR2) before any click.""" + keep = plant_summary(out, og, sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"])) + try: + html = "\n".join(page_now(b, base)[0]) + finally: + unplant(keep) + return [f"NBR2 on the page: ...{html[max(0, html.find('NBR2') - 120):html.find('NBR2') + 40]}..."] if "NBR2" in html else [] + + +def infer_checks(b, base, out): + """The son's page is a function of his open results and the count of his closed ones: two pages that differ only + in which result is closed are the same before any click (INFER), and a closed deletion's second gene is named + nowhere. Controls: person.js that puts a closed card's id back, that draws a fixed-panel card's open rows, or + that counts the closed rows; a veil that tags only a deletion's first gene.""" + og = og_binary() or build_live_demo.OG + if not os.path.isfile(os.path.join(out, "runs", OTHERS[1], "map.json")): + print("[SKIP] infer checks: the demo has no map.json") + return + t0 = time.time() + bad = infer_problems(b, base, out, og) + check("browser: two pages of the son that differ only in which result is closed are the same before any click, " + "HTML, attributes, address and map canvas (" + "; ".join(x[0] for x in INFER) + ")", not bad, bad) + bad = sv_gene_problems(b, base, out, og) + check("browser: a closed deletion over two genes closes the rows that name only its second gene", not bad, bad) + js = os.path.join(out, "assets", "person.js") + with open(js) as fh: + keep_js = fh.read() + try: + for desc, a, z, case in ( + ("person.js that puts a closed card's id back", + "function handle(f) { return veiled(f) ? 'closed-' + (closedOrder().indexOf(f.id) + 1) : f.id; }", + "function handle(f) { return f.id; }", INFER[0]), + ("person.js that draws a fixed-panel card's open rows", "} else if (panelShut(c)) {", "} else if (false) {", + INFER[2]), + ("person.js that counts the closed rows", + "'Some of its rows or numbers stay closed with a result to act on, until that result is opened.'", + "(ot[1] + (c.lines || []).length - lines.length) + ' rows stay closed with a result to act on.'", INFER[4])): + if a not in keep_js: + check(f"control: {desc} (the planted line is in person.js)", False, a) + continue + with open(js, "w") as fh: + fh.write(keep_js.replace(a, z)) + control(desc, not infer_problems(b, base, out, og, [case])) + finally: + with open(js, "w") as fh: + fh.write(keep_js) + first = live_results.closable + try: + live_results.closable = lambda found: first([{k: v for k, v in f.items() if k != "genes"} for f in found]) + control("a veil that tags only a deletion's first gene", not sv_gene_problems(b, base, out, og)) + finally: + live_results.closable = first + print(f" infer checks: {time.time() - t0:.0f} s") + + def widget_veil_checks(b, base, out): - """On the son's page a CHIP call and a copy-number dot in the gene of a closed result stay out of sight until that - result is opened; the other dot stays drawn. A CHIP call is also a finding of its card that names its gene, so - the CHIP card closes whole with the result; SMN1 at two copies is no finding, so only its dot's tag closes it.""" + """On the son's page the copy-number card and the CHIP card, whose rows are a fixed panel (the same genes for + everyone), close whole while any result of his to act on is closed: no dot, no call, no fact of theirs shows, + and their line on a phone is the same words whatever the result. Each "Show the result" of their box opens + one result; once every result is opened every dot, call and fact is back.""" no_cache(b) rj = os.path.join(out, "runs", OTHERS[1], "results.json") js = os.path.join(out, "assets", "person.js") @@ -1416,12 +1603,11 @@ def drawn(): b.wait("location.href === 'about:blank'") b.go(f"{base}/runs/{OTHERS[1]}/") b.wait("document.documentElement.dataset.pgpResults && document.documentElement.dataset.pgpStatus === 'ready' " - "&& document.querySelector('[data-widget=copies]') && document.querySelector('.veil') && 'drawn'", 15) + "&& document.querySelector('#card-paralogs') && document.querySelector('.veil') && 'drawn'", 15) return shown() def phone_line(): - """The copy-number card's one line on a phone (its fold's summary, after the card's own title, the same for - everyone), drawn from its first open fact.""" + """The copy-number card's one line on a phone (its fold's summary, after the card's own title).""" b.call("Emulation.setDeviceMetricsOverride", {"width": 390, "height": 844, "deviceScaleFactor": 2, "mobile": False}, b.session) try: @@ -1431,26 +1617,25 @@ def phone_line(): return b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.documentElement.dataset.pgpResults " "&& (document.querySelector('[data-card=fold-paralogs] summary .what') || {}).textContent", 15) or "" finally: - b.call("Emulation.clearDeviceMetricsOverride", {}, b.session) + b.call("Emulation.setDeviceMetricsOverride", {"width": 1440, "height": 900, "deviceScaleFactor": 1, "mobile": False}, + b.session) def phone_problems(): line = phone_line() - return [] if line and "SMN1" not in line else [f"the copy-number card's line on a phone: {line!r}"] + want = "The same genes for everyone, closed while a result about a child is closed." + return [] if line == want else [f"the copy-number card's line on a phone: {line!r}, want {want!r}"] def shown(): - return b.eval("JSON.stringify([" + SHOWN_JS + ", (document.querySelector('[data-widget=copies]') || {}).textContent || ''])") + return b.eval("JSON.stringify([" + SHOWN_JS + ", (document.querySelector('[data-widget=copies]') || {}).textContent || '', " + "document.querySelectorAll('.finding[data-level=act] .veil').length])") def problems(text, opened): - page, cops = json.loads(text) if text else ["", ""] + page, cops, _ = json.loads(text) if text else ["", "", 0] out_ = [] - if "SMN2" not in cops: - out_.append(f"the open dot is not drawn: {cops[:120]}") - if ("p.R175H" in page) != opened or ("SMN1" in cops) != opened: - out_.append(f"the CHIP call or the SMN1 dot of the closed results {'not ' if opened else ''}shown: " - f"{'p.R175H' in page} | {cops[:200]}") - # The copy-number card's facts name SMN1: closed in its margin and in its line of section 8. - if ("SMN1 copies" in page) != opened: - out_.append(f"the card's fact on SMN1 {'not ' if opened else ''}shown") + if opened and not ("SMN1" in cops and "SMN2" in cops and "p.R175H" in page and "SMN1 copies" in page): + out_.append(f"every result opened, yet a dot, the call or the fact is missing: {cops[:200]}") + if not opened and (cops or "p.R175H" in page or "SMN1 copies" in page): + out_.append(f"a closed result, yet the copy-number or CHIP card shows its rows: {cops[:200]}") return out_ try: with open(os.path.join(os.path.dirname(out), "root", OTHERS[1], "summary.json")) as f: @@ -1470,19 +1655,23 @@ def problems(text, opened): json.dump(res, f) bad = problems(drawn(), False) + phone_problems() if not bad: - for _ in range(2): # the CPSR card holds both results: one click opens one - b.eval("document.querySelector('#card-cpsr .veil button').click(); true") - time.sleep(0.5) - bad = problems(shown(), True) - check("browser: on the son's page a CHIP call, a copy-number dot and a card's fact in the gene of a closed " - "result stay out of sight (the fact in the card's line on a phone too) until it is opened", not bad, bad) + # The copy-number card's box opens one result per click, until none is left closed. + n = json.loads(shown())[2] + for k in range(n): + b.eval("var v = document.querySelector('#card-paralogs .veil button'); v && v.click(); true") + time.sleep(0.3) + left = json.loads(shown())[2] + if left != n - k - 1: + bad.append(f"click {k + 1} on the copy-number card's box left {left} of {n} results closed") + break + bad += problems(shown(), True) + check("browser: on the son's page the copy-number and CHIP cards, a fixed panel, close whole while a result is " + "closed (their line on a phone too), their box opens one result per click, and every dot, call and fact is " + "back once every result is opened", not bad, bad) for desc, a, z in ( - ("person.js that draws every item", "add(text, widget(of[1]));", "add(text, widget(c.widget));"), - ("person.js that shows every fact", "facts = (c.facts || []).filter(function (f, i) { return !(ft[i] && tagClosed(ft[i])); });", - "facts = c.facts || [];"), - ("person.js whose card line reads every fact", - "(openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)", - "(c.facts && c.facts[0] ? c.facts[0][0] + ': ' + c.facts[0][1] : NOT_ALL_CLEAR)")): + ("person.js that draws a fixed-panel card's open rows", "} else if (panelShut(c)) {", "} else if (false) {"), + ("person.js whose card line reads a fixed-panel card's facts", ": g === 'show' && panelShut(c) ? PANEL_LINE", + ": false ? PANEL_LINE")): if a not in keep_js: check(f"control: {desc} (the planted line is in person.js)", False, a) continue @@ -2053,15 +2242,17 @@ def map_now(b): # The page's closed results to act on and the map's chips, read at once (a chip's button rebuilds the chips at -# once: nothing to wait for): the closed act cards in page order, the act chip's count, whether its confirm step +# once: nothing to wait for): the act cards in page order (ids: each one's id, its handle while it is closed) and +# the places of the closed ones among them (closed: a closed card names no id, so a result is known by its place), the act chip's count, whether its confirm step # shows, where the keyboard is (first: the chips' first button; canvas: the map), and the opened chips. CHIP_JS = r"""JSON.stringify((function () { - var ids = []; - Array.prototype.forEach.call(document.querySelectorAll('.finding[data-level=act]'), function (a) { - if (a.querySelector('.veil') && ids.indexOf(a.dataset.finding) < 0) { ids.push(a.dataset.finding); } }); + var ids = [], closed = []; + Array.prototype.forEach.call(document.querySelectorAll('.finding[data-level=act]'), function (a, i) { + ids.push(a.dataset.finding); + if (a.querySelector('.veil')) { closed.push(i); } }); var c = document.querySelector('.gm-chip[data-closed=act]'), f = document.activeElement; var first = document.querySelector('.gm-chips button'); - return { closed: ids, buttons: !!first, chip: c ? c.firstChild.textContent : '', confirm: !!(c && c.querySelector('.gm-confirm')), + return { closed: closed, ids: ids, buttons: !!first, chip: c ? c.firstChild.textContent : '', confirm: !!(c && c.querySelector('.gm-confirm')), focus: f && f.classList.contains('gm-yes') ? 'gm-yes' : f === first ? 'first' : f === document.querySelector('.gm-canvas') ? 'canvas' : (f ? f.tagName + '.' + f.className : ''), opened: Array.prototype.map.call(document.querySelectorAll('.gm-chip.opened'), function (o) { var a = o.querySelector('a'); @@ -2079,8 +2270,8 @@ def closed_words(n): def chip_open(b): - """The act chip's Open, then its Show the result, as a keyboard reader does it. Returns (the id it opened, - problems): one Open opens exactly one of the page's closed results, the count falls by one, the keyboard + """The act chip's Open, then its Show the result, as a keyboard reader does it. Returns (the id it opened, its + place among the act cards, problems): one Open opens exactly one of the page's closed results, the count falls by one, the keyboard goes to Show the result and then to the chips' first button (the map when no chip is left), and an "Opened in its section" chip never names a result that is still closed.""" before = chips_now(b) @@ -2101,10 +2292,15 @@ def chip_open(b): bad.append(f"Show the result: the chip reads {after['chip']!r}, want {closed_words(n)!r}") if after["focus"] != ("first" if after["buttons"] else "canvas"): bad.append(f"Show the result: the keyboard is on {after['focus']!r}") - named = [o["href"] for o in after["opened"] if o["kind"] == "section" and o["href"][3:] in after["closed"]] + named = [o["href"] for o in after["opened"] if o["kind"] == "section" and o["href"][3:] not in after["ids"]] if named: - bad.append(f"an Opened in its section chip names a closed result: {named}") - return (gone[0] if len(gone) == 1 else None), bad + bad.append(f"an Opened in its section chip names a result that is not open: {named}") + leaked = [after["ids"][i] for i in after["closed"] if not re.match(r"^closed-\d+$", after["ids"][i] or "")] + if leaked: + bad.append(f"a closed card carries an id, not a handle: {leaked}") + if len(gone) != 1: + return None, None, bad + return after["ids"][gone[0]], gone[0], bad def parity_problems(b, base, rows): @@ -2219,12 +2415,15 @@ def leaks(c, but=""): # Open opens the page's closed results one per click, in the page's order, so the son's own result with no # place (his first closed card) opens before the deletion: the map then names it only in its section chip. order = chips_now(b)["closed"] + if not want: + return bad + ["the son's page shows no closed result to open"] + k, at = 0, None for k in range(want): - got, more = chip_open(b) + got, at, more = chip_open(b) if more: return bad + [f"Open {k + 1}: {x}" for x in more] - if got != order[k]: - return bad + [f"Open {k + 1} opened {got}, not the page's next closed result {order[k]} ({order})"] + if at != order[k]: + return bad + [f"Open {k + 1} opened {got}, the act card at {at}, not the page's next closed one {order[k]} ({order})"] if got == fid: break section = [o for o in chips_now(b)["opened"] if o["kind"] == "section"] @@ -2252,10 +2451,10 @@ def leaks(c, but=""): bad.append(f"Close it again on the section chips: chips {chips_now(b)['opened']}, closed {chips_now(b)['closed']}") b.eval("document.querySelector('.gm-chip.opened[data-opened=\\'1\\'] button').click(); true") c = map_now(b) - card = b.eval("!!document.querySelector('#f-%s .veil')" % fid) + card = at in chips_now(b)["closed"] if leaks(c) or not card: bad.append(f"Close it again: map shows {leaks(c)}, card closed {card}") - b.eval("document.querySelector('#f-%s .veil button').click(); true" % fid) + b.eval("document.querySelectorAll('.finding[data-level=act]')[%d].querySelector('.veil button').click(); true" % at) c = map_now(b) if [m["id"] for m in c["marks"] if m["id"] == fid] != [fid]: bad.append("the card's Show the result does not open it on the map") @@ -2314,7 +2513,8 @@ def overlap_problems(b, base, out): c = map_page(b, base, son, "#" + f["chrom"][i]) if c["state"] != "drawn": return [f"the planted map: {c['state']}"] - b.eval("document.querySelector('#f-%s .veil button').click(); true" % fid) + if not open_closed(b, fid): + return [f"a link to {fid} lands on no closed card"] c = map_now(b) bad = [] if fid not in [m["id"] for m in c["marks"]]: @@ -2457,7 +2657,8 @@ def second_open_problems(b, base, out, og): ids = [m["id"] for m in c["marks"]] if "pl-alpha" in ids or "pl-beta" in ids: return [f"closed: marks {ids}"] - b.eval("document.querySelector('#f-pl-beta .veil button').click(); true") + if not open_closed(b, "pl-beta"): + return ["a link to pl-beta lands on no closed card"] c = map_now(b) ids = [m["id"] for m in c["marks"]] bad = [] @@ -2519,7 +2720,8 @@ def named_problems(b, base, out, og): if fig != before: bad.append(f"closed: the figure differs from the page without them " f"({[k for k in ('html', 'canvas') if fig.get(k) != before.get(k)]})") - b.eval("document.querySelector('#f-pl-x .veil button').click(); true") + if not open_closed(b, "pl-x"): + return bad + ["a link to pl-x lands on no closed card"] c = map_now(b) ids = [m["id"] for m in c["marks"]] if not {"pl-x", "pl-q", "pl-r"} <= set(ids): @@ -2581,7 +2783,7 @@ def open_order_problems(b, base, out, og): f"keyboard on {kept['focus']!r}"] seq = [] for _ in first["closed"]: - got, more = chip_open(b) + got, _, more = chip_open(b) if more: return bad + [f"{what}, Open {len(seq) + 1}: {x}" for x in more] seq.append(got) @@ -2592,10 +2794,11 @@ def open_order_problems(b, base, out, og): seqs[what] = seq if k == 0: gone = [x["id"] for x in extra if not x["viewer"]][0] + at = chips_now(b)["ids"].index(gone) if gone in chips_now(b)["ids"] else -1 b.eval("var o = document.querySelector('.gm-chip.opened[data-opened=section] a[href=\"#f/%s\"]'); " "o.parentNode.parentNode.querySelector('button').click(); true" % gone) after = chips_now(b) - if after["closed"] != [gone] or any(o["kind"] == "section" and gone in o["href"] for o in after["opened"]): + if after["closed"] != [at] or any(o["kind"] == "section" and gone in o["href"] for o in after["opened"]): bad.append(f"{what}: Close it again on {gone}'s section chip: closed {after['closed']}, " f"opened chips {after['opened']}") finally: From 283ee24573a19381e1ad868e0b4279a8f0ae2036 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 10:16:28 +0100 Subject: [PATCH 023/255] docs(live): say how a child's page keeps which result is closed to itself Bead: pgp-iqon.27 Bead: pgp-iqon.28 Bead: pgp-iqon.29 --- CHANGELOG.md | 1 + docs/live-site.md | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4dff4278..333b93b80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. +- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; a link to `#f/` lands on the closed card with the handle in the address, and the id comes back once the result is opened. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; nothing is renamed or removed. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. - Steps 21, 32 and 9b (and the CYRIUS, PYPGX and STRANGER_RFC1 modules) now run the CYP2D6 depth check and the RFC1 motif count with `og`, and `bin/cyp2d6_depth_check.py` and `bin/rfc1_motif.py` are gone. A run that reaches these steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull; without it the step stops with `og is not built`. The output files, their names and formats do not change; in CI, every e2e case that runs these steps in bash checks `og` against the deleted Python, byte for byte. diff --git a/docs/live-site.md b/docs/live-site.md index b2b16746b..0710b9792 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,7 +56,12 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a copy-number dot (a CHIP call is a finding of its own card, so the CHIP card closes whole), a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A count says how many rows stay closed, and opening the result opens them with it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. The pages differ only in what is open and in how many results are closed, never in which result is closed: + +- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call) still show in their sections. +- A closed finding carries no id in the page or the address, since its id names its gene and variant. It is known by its place among the page's closed findings (`#f/closed-2`). A link to `#f/` of a closed result lands on its card, and the address becomes that handle. The id comes back once the result is opened, and goes again when it is closed. + +Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. While the traits card is closed whole on a child's page, the chip is the switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. @@ -90,7 +95,7 @@ The reads cross the network to the browser that logged in: only the byte ranges `GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. -The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the rows outside a result's own card that close with it, and `spans`, the closed results with their gene and position, from which the family page knows whether a child has a result still closed). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. +The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the rows outside a result's own card that close with it; `spans`, the closed results with their position, one row per gene they name, from which the family page knows whether a child has a result still closed; and `panels`, the cards of a fixed panel). A structural variant's finding also lists every gene it covers in `genes`. It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. ## Privacy and what is stored From bd9fde0e4d9eac4fe721284bf3c22a01f6d55946 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:10:02 +0100 Subject: [PATCH 024/255] fix(live): close the known mobile element insertions card whole as well Its rows are the catalog of assets/mei_sites.tsv, the same for every reader, so a row closed with a result would name its gene. It joins FIXED_PANELS. Bead: pgp-iqon.29 --- CHANGELOG.md | 2 +- bin/live_results.py | 1 + docs/live-site.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 333b93b80..b842bb3d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. -- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; a link to `#f/` lands on the closed card with the handle in the address, and the id comes back once the result is opened. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; nothing is renamed or removed. +- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; a link to `#f/` lands on the closed card with the handle in the address, and the id comes back once the result is opened. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; nothing is renamed or removed. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. - Steps 21, 32 and 9b (and the CYRIUS, PYPGX and STRANGER_RFC1 modules) now run the CYP2D6 depth check and the RFC1 motif count with `og`, and `bin/cyp2d6_depth_check.py` and `bin/rfc1_motif.py` are gone. A run that reaches these steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull; without it the step stops with `og is not built`. The output files, their names and formats do not change; in CI, every e2e case that runs these steps in bash checks `og` against the deleted Python, byte for byte. diff --git a/bin/live_results.py b/bin/live_results.py index 629abda5c..80c3954fb 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -1153,6 +1153,7 @@ def lead(found, levels, summary): # whatever the result (results.json veil.panels; assets/live/person.js panelShut()). The one table: a card of a # fixed panel is listed here and nowhere else. FIXED_PANELS = { + "mei": "the known disease-causing insertions of assets/mei_sites.tsv", "expansions": "the repeat loci of the catalog", "vntyper": "MUC1", "lpa": "LPA", diff --git a/docs/live-site.md b/docs/live-site.md index 0710b9792..0800efa7d 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -58,7 +58,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. The pages differ only in what is open and in how many results are closed, never in which result is closed: -- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call) still show in their sections. +- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call) still show in their sections. - A closed finding carries no id in the page or the address, since its id names its gene and variant. It is known by its place among the page's closed findings (`#f/closed-2`). A link to `#f/` of a closed result lands on its card, and the address becomes that handle. The id comes back once the result is opened, and goes again when it is closed. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. While the traits card is closed whole on a child's page, the chip is the switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. From a80d92cd71c5bb20328cf2a8753d683b25607c33 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:53:27 +0100 Subject: [PATCH 025/255] fix(live): close the ClinVar card's deep intronic lines all together The ClinVar card prints one line per known deep intronic site, hit or not, a fixed panel. Each line was tagged by its gene, so a closed result in ATM or GLA left one line out of six and named the gene. veil() now tags every such line with one panel tag (FIXED_LINES), and person.js keeps those lines closed while any result to act on is closed and opens them once the last one is opened. The docs say only what the tests prove. --- CHANGELOG.md | 2 +- assets/live/person.js | 9 +++++++- bin/live_results.py | 17 +++++++++----- docs/live-site.md | 4 ++-- tests/test_live_demo.py | 45 ++++++++++++++++++++++++++++++-------- tests/test_live_results.py | 22 +++++++++++++------ 6 files changed, 74 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b842bb3d3..ee46ec17c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. -- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; a link to `#f/` lands on the closed card with the handle in the address, and the id comes back once the result is opened. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; nothing is renamed or removed. +- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; a link to `#f/` lands on the closed card with the handle in the address, and the id comes back once the result is opened. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and so do the ClinVar card's lines for the known deep intronic sites, all together; a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels`, the tag `*panel*` in `veil.lines`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; nothing is renamed or removed. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. - Steps 21, 32 and 9b (and the CYRIUS, PYPGX and STRANGER_RFC1 modules) now run the CYP2D6 depth check and the RFC1 motif count with `og`, and `bin/cyp2d6_depth_check.py` and `bin/rfc1_motif.py` are gone. A run that reaches these steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull; without it the step stops with `og is not built`. The output files, their names and formats do not change; in CI, every e2e case that runs these steps in bash checks `og` against the deleted Python, byte for byte. diff --git a/assets/live/person.js b/assets/live/person.js index 6e7cbb039..bbd82b4fb 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -325,7 +325,14 @@ // A row outside its result's card (a ClinVar row, a card's line, a viewer link, a chart's mark, a search hit) // carries the id of the result to act on it names (results.veil, bin/live_results.py veil()): it stays closed // while that result is, and "Show the result" opens it with it. A tag naming no finding of these results stays closed. - function tagClosed(id) { var f = findingById(id); return closedPage() && (!f || veiled(f)); } + // A line of a fixed panel (PANEL, bin/live_results.py FIXED_LINES: a known deep intronic site) names no result: + // it stays closed while any result to act on is, whichever, so what is missing never tells the gene. + var PANEL = '*panel*'; + function tagClosed(id) { + if (id === PANEL) { return closedPage() && closedActs().length > 0; } + var f = findingById(id); + return closedPage() && (!f || veiled(f)); + } function veilTags(kind, key) { return ((results && results.veil && results.veil[kind]) || {})[key] || []; } // The charts (plugins/charts.js) ask for each mark. A closed page with no results.json yet cannot tell which // marks those are, so it leaves every mark out until it can. diff --git a/bin/live_results.py b/bin/live_results.py index 80c3954fb..c6beb862f 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -179,7 +179,7 @@ def a_clinvar(S, s): lines = [x for x in (rr.cdna_scan_lines(d) or [])] for x in d.get("deep_intronic") or []: # Always reported by the report, hit or not: each site's call stays visible on the card. - lines.append(f"Known deep intronic site (always reported): {x.get('gene')} {x.get('hgvs')} " + lines.append(f"{FIXED_LINES['clinvar']} {x.get('gene')} {x.get('hgvs')} " f"{x.get('chrom')}:{x.get('pos')} {x.get('ref')}>{x.get('alt')}: {rr.deep_intronic_call(x)}") freq = rr.clinvar_freq_line(d) if freq: @@ -1166,6 +1166,12 @@ def lead(found, levels, summary): "traits-panel": "the trait panel's genes", "bloodtype": "the blood group systems", } +# The lines of a card that are a fixed panel inside it (card -> the words each such line starts with): the ClinVar +# card prints one line per known deep intronic site of assets/deep_intronic_sites.tsv, hit or not. veil() tags +# each of them PANEL, never by its gene, so they close all together while any result to act on is closed and +# open all together once the last one is opened (assets/live/person.js tagClosed()). +FIXED_LINES = {"clinvar": "Known deep intronic site (always reported):"} +PANEL = "*panel*" LOCUS_RE = re.compile(r"(? one id or None per row; the ClinVar card's by each hit's own position), each card's viewer - links (viewer: card -> one per link), each card's lines (lines: card -> one per line; a known deep - intronic site is printed whether hit or not), the marks of the report's charts (marks: chart -> {mark id: - id}), every other finding that names a closed result (findings: finding id -> id; a symptom ranking can + links (viewer: card -> one per link), each card's lines (lines: card -> one per line; a line of a fixed + panel, such as a known deep intronic site, printed whether hit or not, is tagged PANEL: FIXED_LINES), the + marks of the report's charts (marks: chart -> {mark id: id}), every other finding that names a closed result (findings: finding id -> id; a symptom ranking can rank the gene) and the lead's segments that name one (lead: one per segment of LEAD_PARTS, the "act" names left out: the page closes those itself), each card's facts (facts: card -> one per fact) and the items of its drawing (widget: card -> one per CHIP call or copy-number dot), and the cards of a fixed panel @@ -1271,7 +1277,8 @@ def veil(summary, found, chapters, lead_parts=()): tags = [(rt[i] if i < len(rt) else None) or t for i, t in enumerate(tags)] if any(tags): out["viewer"][c["id"]] = tags - tags = [veil_id(spans, ln) for ln in c.get("lines") or []] + fixed = FIXED_LINES.get(c["id"]) + tags = [PANEL if fixed and str(ln).startswith(fixed) else veil_id(spans, ln) for ln in c.get("lines") or []] if any(tags): out["lines"][c["id"]] = tags tags = [veil_id(spans, " ".join(str(x) for x in f)) for f in c.get("facts") or []] diff --git a/docs/live-site.md b/docs/live-site.md index 0800efa7d..1315b98d2 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,9 +56,9 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. The pages differ only in what is open and in how many results are closed, never in which result is closed: +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. Two pages whose open rows are the same differ only in how many results are closed, not in which; the section that holds a closed result still says it holds one: -- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call) still show in their sections. +- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call) still show in their sections. A card can hold a fixed panel among its lines: the ClinVar card prints a line for each known deep intronic site, hit or not. Those lines close all together while any result to act on is closed and come back once every result is opened (`FIXED_LINES` in `bin/live_results.py`; `veil.lines` tags each of them `*panel*`). - A closed finding carries no id in the page or the address, since its id names its gene and variant. It is known by its place among the page's closed findings (`#f/closed-2`). A link to `#f/` of a closed result lands on its card, and the address becomes that handle. The id comes back once the result is opened, and goes again when it is closed. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. While the traits card is closed whole on a child's page, the chip is the switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 487fa3d2e..5f2a8ffcb 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1479,6 +1479,23 @@ def change(S): return change +def deep_sites(S): + """The ClinVar card's line for each known deep intronic site of assets/deep_intronic_sites.tsv, reference calls: + a fixed panel the son's demo summary leaves out.""" + with open(os.path.join(REPO, "assets", "deep_intronic_sites.tsv")) as fh: + rows = [ln.rstrip("\n").split("\t") for ln in fh if ln.strip() and not ln.startswith("#")] + S["clinvar"]["data"]["deep_intronic"] = [dict(zip(rows[0], r), call="ref", detail="GQ 50") for r in rows[1:]] + return len(rows) - 1 + + +def with_sites(gene): + """cpsr(GENE) with the known deep intronic sites' lines (deep_sites), the same on both pages.""" + def change(S): + cpsr(gene)(S) + deep_sites(S) + return change + + def sv_over(locus, genes): """A class 5 deletion over GENES, and a carrier row that names only its second gene.""" def change(S): @@ -1505,6 +1522,8 @@ def second_hit(S): ("SMN1 against HBA1, the same copy numbers", cpsr("SMN1"), cpsr("HBA1")), ("a result with a place on the map (BRCA2) against one without (MLH1)", cpsr("BRCA2"), cpsr("MLH1")), ("a closed result with one ClinVar row against two", lambda S: None, second_hit), + ("a closed result in a gene of a known deep intronic site (ATM) against one in none (BRCA2)", with_sites("ATM"), + with_sites("BRCA2")), ("a link to each of two closed results (#f/)", cpsr("BRCA1"), cpsr("BRCA2"), ("#f/cpsr-brca1-c-68-69del", "#f/cpsr-brca2-c-68-69del"))) @@ -1588,8 +1607,9 @@ def infer_checks(b, base, out): def widget_veil_checks(b, base, out): """On the son's page the copy-number card and the CHIP card, whose rows are a fixed panel (the same genes for everyone), close whole while any result of his to act on is closed: no dot, no call, no fact of theirs shows, - and their line on a phone is the same words whatever the result. Each "Show the result" of their box opens - one result; once every result is opened every dot, call and fact is back.""" + and their line on a phone is the same words whatever the result; so do the ClinVar card's lines for the known + deep intronic sites, a fixed panel inside it. Each "Show the result" of their box opens one result; once every + result is opened every dot, call, fact and deep intronic line is back.""" no_cache(b) rj = os.path.join(out, "runs", OTHERS[1], "results.json") js = os.path.join(out, "assets", "person.js") @@ -1632,10 +1652,14 @@ def shown(): def problems(text, opened): page, cops, _ = json.loads(text) if text else ["", "", 0] out_ = [] - if opened and not ("SMN1" in cops and "SMN2" in cops and "p.R175H" in page and "SMN1 copies" in page): - out_.append(f"every result opened, yet a dot, the call or the fact is missing: {cops[:200]}") - if not opened and (cops or "p.R175H" in page or "SMN1 copies" in page): - out_.append(f"a closed result, yet the copy-number or CHIP card shows its rows: {cops[:200]}") + deep = page.count(live_results.FIXED_LINES["clinvar"]) + if opened and not ("SMN1" in cops and "SMN2" in cops and "p.R175H" in page and "SMN1 copies" in page + and deep == sites[0]): + out_.append(f"every result opened, yet a dot, the call, the fact or a deep intronic line ({deep} of {sites[0]}) is " + f"missing: {cops[:200]}") + if not opened and (cops or "p.R175H" in page or "SMN1 copies" in page or deep): + out_.append(f"a closed result, yet the copy-number or CHIP card shows its rows or the ClinVar card " + f"{deep} deep intronic lines: {cops[:200]}") return out_ try: with open(os.path.join(os.path.dirname(out), "root", OTHERS[1], "summary.json")) as f: @@ -1648,6 +1672,7 @@ def problems(text, opened): {"gene": g, "protein_change": ch, "vaf": v, "alt_reads": 6, "depth": 75, "status": "pass", "note": "a clone"} for g, ch, v in (("DNMT3A", "p.R882H", 0.12), ("TP53", "p.R175H", 0.08))], "genes": [{"gene": g, "detection_limit_vaf": 0.05, "mean_depth": 75, "status": "ok"} for g in ("DNMT3A", "TP53")]}} + sites = [deep_sites(S)] S["paralogs"] = {"state": "ok", "data": {"genes": [{"gene": g, "copy_number": n, "status": "called", "detail": "."} for g, n in (("SMN1", 2), ("SMN2", 2))]}} res = live_results.build(OTHERS[1], summary) @@ -1666,12 +1691,14 @@ def problems(text, opened): break bad += problems(shown(), True) check("browser: on the son's page the copy-number and CHIP cards, a fixed panel, close whole while a result is " - "closed (their line on a phone too), their box opens one result per click, and every dot, call and fact is " - "back once every result is opened", not bad, bad) + "closed (their line on a phone too), and so do the ClinVar card's deep intronic lines; the box opens one " + "result per click, and every dot, call, fact and line is back once every result is opened", not bad, bad) for desc, a, z in ( ("person.js that draws a fixed-panel card's open rows", "} else if (panelShut(c)) {", "} else if (false) {"), ("person.js whose card line reads a fixed-panel card's facts", ": g === 'show' && panelShut(c) ? PANEL_LINE", - ": false ? PANEL_LINE")): + ": false ? PANEL_LINE"), + ("person.js that draws a fixed panel's lines", "if (id === PANEL) { return closedPage() && closedActs().length > 0; }", + "if (id === PANEL) { return false; }")): if a not in keep_js: check(f"control: {desc} (the planted line is in person.js)", False, a) continue diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 8faefe11f..321525727 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -36,9 +36,10 @@ link of the planted hit, its genome-chart tick and its review-star bar carry the secondary finding's id, the deletion's span carries its own, the search index's items that name the gene carry one of them, and nothing else is - tagged; DEMO-001, with no result to act on, has no tag; and a known deep - intronic site's line in the gene of a closed secondary finding carries - that finding's id (the card prints each site whether hit or not), a gene + tagged; DEMO-001, with no result to act on, has no tag; and every known + deep intronic site's line carries the fixed-panel tag, never the id of a + closed result in its gene (the card prints each site whether hit or not, + so one missing line would name the gene), a gene named before a full stop is that gene, and a ClinVar row closed by its gene alone closes its "See the reads at" link with it; every other card's row that names the gene (the son's symptom ranking ranks it) carries one of @@ -404,7 +405,8 @@ def veil_problems(res, idx): def veil_lines_problems(): """What is wrong with the veil's tags of the ClinVar card's lines: a known deep intronic site is printed - whether hit or not, so its line names the gene of a closed secondary finding in that gene.""" + whether hit or not, a fixed panel, so every such line carries the panel tag (live_results.PANEL), the one in + a closed secondary finding's gene and the others alike: a line closed alone would name the gene.""" S = {"clinvar": {"state": "ok", "data": {"hits": [], "deep_intronic": [ {"gene": "GLA", "hgvs": "NM_000169.3:c.640-801G>A", "chrom": "chrX", "pos": 101399747, "ref": "C", "alt": "T", "call": "hom", "detail": "GQ 40"}, @@ -413,9 +415,9 @@ def veil_lines_problems(): lines = live_results.a_clinvar(S, {"sections": S}).get("lines") or [] found = [{"id": "sf-gla-test", "level": "act", "gene": "GLA", "detail": "GLA"}] v = live_results.veil({"sections": S}, found, [{"sections": [{"id": "clinvar", "lines": lines}]}]) - want = ["sf-gla-test" if "GLA " in ln else None for ln in lines] + want = [live_results.PANEL if ln.startswith(live_results.FIXED_LINES["clinvar"]) else None for ln in lines] got = (v.get("lines") or {}).get("clinvar") - out = [] if "sf-gla-test" in want and None in want and got == want else [f"lines {lines} tagged {got}, want {want}"] + out = [] if want.count(live_results.PANEL) == 2 and got == want else [f"lines {lines} tagged {got}, want {want}"] # A gene named at the end of a sentence ("in GLA.") is that gene. if live_results.veil_id(live_results.closable(found), "A change in GLA.") != "sf-gla-test": out.append("'A change in GLA.' carries no tag") @@ -612,9 +614,15 @@ def tagged(sample_dir): r3["_summary"] = s3 return r3, results_index.build(s3, sample_dir, rr.SECTION_OF) bad = veil_lines_problems() - check("8. a known deep intronic site's line in a closed secondary finding's gene carries its id, a gene " + check("8. every known deep intronic site's line carries the fixed-panel tag, not a closed result's id, a gene " "before a full stop is that gene, a ClinVar row closed by its gene closes its reads link, and a lead " "segment naming the gene carries its id", not bad, bad) + fixed = live_results.FIXED_LINES + try: # the card still prints its lines; veil() no longer knows they are a panel + live_results.FIXED_LINES = type("Unknown", (dict,), {"get": lambda self, k, d=None: None})(fixed) + control("a veil that tags a known deep intronic site's line by its gene", not veil_lines_problems()) + finally: + live_results.FIXED_LINES = fixed bad = overview_problems() check("8. a child's home tile lists no finding that names a closed result, and its closed lead names none", not bad, bad) From 1edf2e9049ac21efe6e040e1c2c1aff9bb6ce033 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:02:05 +0100 Subject: [PATCH 026/255] fix(live): close a child's fixed panels, guessed ids and history entries the same whatever result is closed A fixed-panel card's own findings (a medicine gene, a CHIP call) and the lead's words on them now carry the panel tag, so a closed result in CYP2D6 no longer turns the medicine card's box into "This section holds a result" or moves its finding among the open ones. The carrier list's two-hit line and the callability card's rows join the fixed lines (FIXED_LINES is now a card to list-of-prefixes table). A row closes with a result only when it names the gene as a gene symbol: prose words, ontology ids and units no longer match, and the score card is never tagged. The address never holds a closable finding's id: #f/opened- once opened, and any #f/ that is not an open finding lands on the first closed one, so a guessed id tells nothing. The home page links a child's closed result to the report, not to its own page, and the finding page treats panel findings as the person page does. closedActs() checks the level before the veil, which a panel-tagged finding would otherwise recurse through. --- CHANGELOG.md | 2 +- assets/live/finding.js | 13 ++- assets/live/home.js | 3 +- assets/live/map.js | 5 +- assets/live/person.js | 50 ++++++--- bin/live_results.py | 92 ++++++++++++---- docs/live-site.md | 4 +- tests/test_live_demo.py | 213 ++++++++++++++++++++++++++++++++++--- tests/test_live_results.py | 89 +++++++++++++++- 9 files changed, 416 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee46ec17c..cb03678de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. -- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; a link to `#f/` lands on the closed card with the handle in the address, and the id comes back once the result is opened. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and so do the ClinVar card's lines for the known deep intronic sites, all together; a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels`, the tag `*panel*` in `veil.lines`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; nothing is renamed or removed. +- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; the address never holds a closable finding's id, closed (`#f/closed-`) or opened (`#f/opened-`), so no history entry keeps it, and any other `#f/` on such a page, a closed result's id or nothing, lands on the first closed finding, so a guessed id tells nothing. The home page links a child's closed result to the child's report, not to the finding's own page. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and so do their own findings (a medicine gene, a CHIP call) and the lead's words on them, the ClinVar card's lines for the known deep intronic sites, the carrier list's two-hit line and the callability card's rows, all together; a row or line closes with a result only when it names its gene as a gene symbol (never a prose word, an ontology id or a unit such as `5 MB`), and the polygenic score card is never tagged; a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels`, the tag `*panel*` in `veil.lines`, `veil.rows`, `veil.findings` and `veil.lead`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; nothing is renamed or removed. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. - Steps 21, 32 and 9b (and the CYRIUS, PYPGX and STRANGER_RFC1 modules) now run the CYP2D6 depth check and the RFC1 motif count with `og`, and `bin/cyp2d6_depth_check.py` and `bin/rfc1_motif.py` are gone. A run that reaches these steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull; without it the step stops with `og is not built`. The output files, their names and formats do not change; in CI, every e2e case that runs these steps in bash checks `og` against the deleted Python, byte for byte. diff --git a/assets/live/finding.js b/assets/live/finding.js index 5f6fc3a31..fbcdee3c8 100644 --- a/assets/live/finding.js +++ b/assets/live/finding.js @@ -83,9 +83,18 @@ var tags = (r.veil && r.veil.findings) || {}; function tagOf(x) { var t = tags[x.id]; return t && t !== x.id ? t : null; } // Opening this finding opens it and the result it names, nothing else: another result keeps its own veil. + // A finding of a fixed-panel card (tag PANEL, bin/live_results.py FIXED_PANELS) names no result: it stays closed + // while any result to act on is, whichever, as on the person page. + var PANEL = '*panel*'; var opened = {}; - if (draw.open) { opened[f.id] = true; if (tagOf(f)) { opened[tagOf(f)] = true; } } - function shut(x) { return child && (x.level === 'act' ? !opened[x.id] : !!tagOf(x) && !opened[tagOf(x)]); } + if (draw.open) { opened[f.id] = true; if (tagOf(f) && tagOf(f) !== PANEL) { opened[tagOf(f)] = true; } } + function actShut() { return (r.findings || []).some(function (x) { return x.level === 'act' && !opened[x.id]; }); } + function shut(x) { + if (!child) { return false; } + if (x.level === 'act') { return !opened[x.id]; } + var t = tagOf(x); + return t === PANEL ? !opened[x.id] && actShut() : !!t && !opened[t]; + } // Closed: no title, gene or variant anywhere on the page (its title, crumb, cover, rail) until opened. var closed = shut(f); var title = !closed ? f.title : f.level === 'act' ? 'A result to act on' : 'A finding about a result to act on'; diff --git a/assets/live/home.js b/assets/live/home.js index 6f25e7bb6..2afb54fa9 100644 --- a/assets/live/home.js +++ b/assets/live/home.js @@ -192,7 +192,8 @@ }); items.sort(function (a, b) { return order[a.f.level] - order[b.f.level]; }); items.slice(0, 6).forEach(function (x) { - var a = el('a', { href: x.r.url + 'finding/' + encodeURIComponent(x.f.id) + '/' }); + // A child's closed result links to the run's page, never to its own page: its id names its gene and variant. + var a = el('a', { href: x.r.url + (closedRuns[x.r.name] && x.f.level === 'act' ? '' : 'finding/' + encodeURIComponent(x.f.id) + '/') }); var who = el('span', { 'class': 'who' }); var pc = el('span', { 'class': 'pc' }, x.r.name); if (colourOf[x.r.name]) { pc.style.setProperty('--pcol', colourOf[x.r.name]); } diff --git a/assets/live/map.js b/assets/live/map.js index f19b2d39d..860382ebe 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -106,7 +106,8 @@ // --- the figure --------------------------------------------------------------------------------------- // create({url, veil(id, gated) -> 'open'|'closed'|'opened', now(id) -> shown at this moment, open(id), // close(id), known(id) -> a finding of the page's results, closed() -> the ids of the results to act on the - // page keeps closed at this moment, stepDone(step) -> ended at this moment, fallback() -> rows}): one figure + // page keeps closed at this moment, stepDone(step) -> ended at this moment, href(id) -> the address of a finding + // the page opened (never its id on a child's page), fallback() -> rows}): one figure // per page. The page calls refresh(key) on each of its redraws; a new key // fetches the document again and keeps the camera. function create(opts) { @@ -910,7 +911,7 @@ inSection = inSection.filter(function (id) { return stateOf(id) !== 'closed' && opts.title && opts.title(id); }); inSection.forEach(function (id) { var c = el('div', { 'class': 'gm-chip opened', 'data-opened': 'section' }); - add(c, add(el('span', {}, 'Opened in its section: '), el('a', { href: '#f/' + id }, opts.title(id)))); + add(c, add(el('span', {}, 'Opened in its section: '), el('a', { href: '#f/' + (opts.href ? opts.href(id) : id) }, opts.title(id)))); add(c, button('Close it again', function () { confirming = null; opts.close(id); keyTo(); })); add(chips, c); }); diff --git a/assets/live/person.js b/assets/live/person.js index bbd82b4fb..ddfbdbbf8 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -370,7 +370,7 @@ } function veiledOf(c) { return (c.findings || []).map(findingById).filter(function (f) { return f && veiled(f); }); } // The results to act on this page keeps closed, drawn yet or not. - function closedActs() { return ((results && results.findings) || []).filter(function (f) { return veiled(f) && f.level === 'act'; }); } + function closedActs() { return ((results && results.findings) || []).filter(function (f) { return f.level === 'act' && veiled(f); }); } // A card of a fixed panel (results.veil.panels, bin/live_results.py FIXED_PANELS: the same genes or loci for // everyone, known in advance): a row or dot of it closed with a result would tell the result's gene by what is // missing. So it closes whole while this page keeps any result to act on closed, behind one box with the same @@ -447,14 +447,15 @@ 'Step ' + (c.steps || []).join(', ') + ' failed: the step timeline says why, and a rerun of the step fills this section.')); } else if (g === 'empty') { add(text, el('p', { 'class': 'notassessed' }, 'Not assessed: its step ended without a result for this section.')); + } else if (panelShut(c)) { + // A fixed-panel card says the same words whichever result is closed, its own findings' too: so it comes first. + // One button opens one result, the next closed one in the page's order, as the map's Open does. + add(text, U.veil(PANEL_WORDS, null, function () { var f = nextClosed(); if (f) { openFinding(f); } redraw(true); })); } else if (closedFs.length) { // One button opens one result: a card with two results to act on asks twice. add(text, U.veil((closedFs.length === 1 ? 'This section holds a result' : 'This section holds ' + closedFs.length + ' results') + ' to act on in adult life. Its rows stay closed until ' + (closedFs.length === 1 ? 'it is' : 'each is') + ' opened.', null, function () { openFinding(closedFs[0]); redraw(true); var x = document.getElementById('f-' + closedFs[0].id) || document.getElementById('card-' + c.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); - } else if (panelShut(c)) { - // One button opens one result, the next closed one in the page's order, as the map's Open does. - add(text, U.veil(PANEL_WORDS, null, function () { var f = nextClosed(); if (f) { openFinding(f); } redraw(true); })); } else { (c.notes || []).forEach(function (n) { add(text, el('p', { 'class': 'note-stale' }, n)); }); if (c.headline) { add(text, el('p', { 'class': 'headline' }, c.headline)); } @@ -578,26 +579,45 @@ var t = findingTag(f); return !!(t && tagClosed(t)); } - function openFinding(f) { keepHash(function () { shown[f.id] = true; var t = findingTag(f); if (t) { shown[t] = true; } }); } - // The address never keeps a closed finding's id. #f/ of a finding this page keeps closed (a link from another - // page, or a finding closed again since) becomes #f/closed-, its handle, and #f/ again once it is opened. + // A finding of a fixed-panel card (tag PANEL) stays closed while any result to act on is: opening it opens the next + // closed result in the page's order, as its card's box does. + function openFinding(f) { + var t = findingTag(f), n = t === PANEL ? nextClosed() : null; + keepHash(function () { shown[f.id] = true; if (n) { shown[n.id] = true; } else if (t && t !== PANEL) { shown[t] = true; } }); + } + // The address never holds the id of a finding this page can close, closed or opened: #f/closed- while it is + // closed (its handle) and #f/opened- once opened on purpose (its place among the opened ones), so no history + // entry keeps the id. Any other #f/ on a page that keeps a result closed, a closed result's id or no id at all, + // lands the same way, on #f/closed-1: the page never picks a closed card by its id, so a guessed id tells nothing. // Opening or closing a result moves the handles of the others: the address follows the finding it named. The // finding behind a handle in the address is kept here only (hashId); a new address from the reader forgets it. var hashId = null; window.addEventListener('hashchange', function () { hashId = null; }); + function closable(f) { return closedPage() && (f.level === 'act' || !!findingTag(f)); } + function openedOrder() { + var out = []; + sectionsList().forEach(function (sec) { + (sec.findings || []).forEach(function (id) { var f = findingById(id); if (f && closable(f) && !veiled(f) && out.indexOf(id) < 0) { out.push(id); } }); + }); + ((results && results.findings) || []).forEach(function (f) { if (closable(f) && !veiled(f) && out.indexOf(f.id) < 0) { out.push(f.id); } }); + return out; + } + function addr(f) { return !closable(f) ? f.id : veiled(f) ? handle(f) : 'opened-' + (openedOrder().indexOf(f.id) + 1); } function hashFinding() { var h; try { h = decodeURIComponent((window.location.hash || '').slice(1)); } catch (e) { return null; } if (h.indexOf('f/') !== 0) { return null; } - var x = h.slice(2); - if (/^closed-\d+$/.test(x)) { return (hashId && findingById(hashId)) || findingById(closedOrder()[Number(x.slice(7)) - 1]) || null; } - return findingById(x) || null; + var x = h.slice(2), m = /^(closed|opened)-(\d+)$/.exec(x); + var f = (m && hashId && findingById(hashId)) || + (m ? findingById((m[1] === 'closed' ? closedOrder() : openedOrder())[Number(m[2]) - 1]) : findingById(x)); + if (f && !(veiled(f) && !m)) { return f; } + return closedOrder().length ? findingById(closedOrder()[0]) : null; } function syncHash() { var f = hashFinding(); if (!f || !window.history || !window.history.replaceState) { return; } hashId = f.id; - var to = '#f/' + handle(f); + var to = '#f/' + addr(f); if (window.location.hash !== to) { window.history.replaceState(null, '', to); } } function keepHash(change) { var at = hashFinding(); if (at) { hashId = at.id; } change(); syncHash(); } @@ -639,6 +659,8 @@ return findingsNow().filter(function (f) { return f.level === 'act' && veiled(f); }).map(function (f) { return f.id; }); } // The title of a result once the page shows it, for the map's "Opened in its section" chip; never a closed one's. + // The address of a result the map opened, for its "Opened in its section" link: never its id (addr()). + function mapHref(id) { var f = findingById(id); return f ? addr(f) : id; } function mapTitle(id) { var f = findingById(id); return f && !veiled(f) ? f.title : ''; } // Whether a row of the map is a finding of these results: then the page's own veil alone decides it (map.js). function mapKnown(id) { return !!findingById(id); } @@ -682,7 +704,7 @@ var rel = (f.related || []).map(findingById).filter(function (r) { return r && !veiled(r); }); if (rel.length) { var box = el('div', {}); - rel.slice(0, 4).forEach(function (r, i) { if (i) { add(box, el('br', {})); } add(box, el('a', { href: '#f/' + r.id }, r.title)); }); + rel.slice(0, 4).forEach(function (r, i) { if (i) { add(box, el('br', {})); } add(box, el('a', { href: '#f/' + addr(r) }, r.title)); }); add(side, note('Linked', box, 'linked-note')); } add(row, art, side); @@ -955,7 +977,7 @@ add(trust, el('span', { title: f[0] + ': ' + f[1] }, String(f[0]).replace(/\s*\([^)]*\)/g, '') + ': ' + String(f[1]).split(/[.;(]\s/)[0].replace(/[.;]$/, '').trim())); }); } else { add(trust, el('span', {}, 'Waiting for the quality steps.')); } - openProblems.forEach(function (f) { add(trust, el('span', { 'class': 'sep' }, '·'), el('a', { href: '#f/' + f.id }, f.title)); }); + openProblems.forEach(function (f) { add(trust, el('span', { 'class': 'sep' }, '·'), el('a', { href: '#f/' + addr(f) }, f.title)); }); add(trust, el('span', { 'class': 'sep' }, '·'), el('a', { href: '#s5' }, 'Can I trust this?')); add(main, trust); var side = el('aside', {}); @@ -967,7 +989,7 @@ // reader left it. It asks this page's veil for each closed row, and opens or closes a result through it. if (window.PGPMap) { if (!genomeMap) { - genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, closed: mapClosed, title: mapTitle, stepDone: stepDone, open: function (id) { setShown(id, true); }, + genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, closed: mapClosed, title: mapTitle, href: mapHref, stepDone: stepDone, open: function (id) { setShown(id, true); }, close: function (id) { setShown(id, false); }, fallback: mapFallback }); } add(slot, genomeMap.node); diff --git a/bin/live_results.py b/bin/live_results.py index c6beb862f..65c542fe9 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -179,7 +179,7 @@ def a_clinvar(S, s): lines = [x for x in (rr.cdna_scan_lines(d) or [])] for x in d.get("deep_intronic") or []: # Always reported by the report, hit or not: each site's call stays visible on the card. - lines.append(f"{FIXED_LINES['clinvar']} {x.get('gene')} {x.get('hgvs')} " + lines.append(f"{FIXED_LINES['clinvar'][0]} {x.get('gene')} {x.get('hgvs')} " f"{x.get('chrom')}:{x.get('pos')} {x.get('ref')}>{x.get('alt')}: {rr.deep_intronic_call(x)}") freq = rr.clinvar_freq_line(d) if freq: @@ -1166,14 +1166,57 @@ def lead(found, levels, summary): "traits-panel": "the trait panel's genes", "bloodtype": "the blood group systems", } -# The lines of a card that are a fixed panel inside it (card -> the words each such line starts with): the ClinVar -# card prints one line per known deep intronic site of assets/deep_intronic_sites.tsv, hit or not. veil() tags -# each of them PANEL, never by its gene, so they close all together while any result to act on is closed and -# open all together once the last one is opened (assets/live/person.js tagClosed()). -FIXED_LINES = {"clinvar": "Known deep intronic site (always reported):"} +# The lines and table rows of a card that are a fixed panel inside it (card -> the words such a line, or a cell of +# such a row, starts with): the ClinVar card prints one line per known deep intronic site of +# assets/deep_intronic_sites.tsv, hit or not; the carrier list prints its two-hit line, which names SMN1, HBA1 and +# HBA2 as the callers it read, whatever was found; the callability card lists the catalog genes this reference does +# not hold, the same for every genome on it, and the hard genes the reads did not cover, which are nearly the same +# for every genome (segmental duplications, collapsed regions). veil() tags each of them PANEL, never by a gene, so they close all +# together while any result to act on is closed and open all together once the last one is opened +# (assets/live/person.js tagClosed()). Every other line or row says something about this genome alone. +FIXED_LINES = {"clinvar": ("Known deep intronic site (always reported):",), + "carrier-list": ("Two hits across callers",), + "callability": ("not on this reference", "not read")} PANEL = "*panel*" +# The cards whose rows, lines and facts name no gene, only things known before the run: veil() never tags them. The +# score card names its scores and their traits (assets/pgs_scores.tsv) and its words "OR 1.57", "HR 1.4", "(TG)" +# are odds and hazard ratios and a trait, though HR and TG are gene symbols too. +NAMES_NO_GENE = {"prs": "the polygenic scores of assets/pgs_scores.tsv"} + + +def panel_finding(f): + """Whether finding F is one of a fixed-panel card's own (a medicine gene, a CHIP call, a repeat), not a result + to act on: veil() tags it PANEL whatever it names.""" + return f.get("level") != "act" and f.get("card") in FIXED_PANELS + + +def fixed_line(card_id, text): + """Whether TEXT, a line of the card CARD_ID or a cell of one of its rows, is one of its fixed lines (FIXED_LINES).""" + return any(str(text).startswith(w) for w in FIXED_LINES.get(card_id) or ()) LOCUS_RE = re.compile(r"(? one per link), each card's lines (lines: card -> one per line; a line of a fixed panel, such as a known deep intronic site, printed whether hit or not, is tagged PANEL: FIXED_LINES), the marks of the report's charts (marks: chart -> {mark id: id}), every other finding that names a closed result (findings: finding id -> id; a symptom ranking can - rank the gene) and the lead's segments that name one (lead: one per segment of LEAD_PARTS, the "act" - names left out: the page closes those itself), each card's facts (facts: card -> one per fact) and the items + rank the gene; PANEL for every finding of a FIXED_PANELS card) and the lead's segments that name one or name a + fixed-panel card's finding (lead: one per segment of LEAD_PARTS, the "act" names left out: the page closes those + itself), each card's facts (facts: card -> one per fact) and the items of its drawing (widget: card -> one per CHIP call or copy-number dot), and the cards of a fixed panel (panels: FIXED_PANELS, which close whole while any result is closed). spans is closable() itself: the family page (assets/live/family-page.js) reads which results are closed from it, and closes every place of the child @@ -1265,9 +1308,12 @@ def veil(summary, found, chapters, lead_parts=()): out["rows"]["clinvar"] = tags for ch in chapters: for c in ch["sections"]: + if c["id"] in NAMES_NO_GENE: + continue rows = (c.get("table") or {}).get("rows") or [] if c["id"] != "clinvar" and rows: - tags = [veil_id(spans, " ".join(str(x) for x in r) if isinstance(r, (list, tuple)) else str(r)) + tags = [PANEL if any(fixed_line(c["id"], x) for x in (r if isinstance(r, (list, tuple)) else [r])) + else veil_id(spans, " ".join(str(x) for x in r) if isinstance(r, (list, tuple)) else str(r)) for r in rows] if any(tags): out["rows"][c["id"]] = tags @@ -1277,8 +1323,7 @@ def veil(summary, found, chapters, lead_parts=()): tags = [(rt[i] if i < len(rt) else None) or t for i, t in enumerate(tags)] if any(tags): out["viewer"][c["id"]] = tags - fixed = FIXED_LINES.get(c["id"]) - tags = [PANEL if fixed and str(ln).startswith(fixed) else veil_id(spans, ln) for ln in c.get("lines") or []] + tags = [PANEL if fixed_line(c["id"], ln) else veil_id(spans, ln) for ln in c.get("lines") or []] if any(tags): out["lines"][c["id"]] = tags tags = [veil_id(spans, " ".join(str(x) for x in f)) for f in c.get("facts") or []] @@ -1303,13 +1348,23 @@ def veil(summary, found, chapters, lead_parts=()): m.get("end")) if fid: out["marks"].setdefault(sp["id"], {})[m["id"]] = fid + # A finding of a fixed-panel card (a medicine gene, a CHIP call, a repeat) is tagged PANEL whatever it names: it + # closes with its card while any result to act on is closed, so the findings left open never tell which gene the + # closed one is in. So is each segment of the lead that names one (its gene or its title). + panel_found = [f for f in found if panel_finding(f)] + panel_spans = [(PANEL, str(g).upper(), None, None, None) for f in panel_found for g in [f.get("gene")] if g] for f in found: if f.get("level") != "act": - fid = veil_id(spans, f"{f.get('gene') or ''} {f.get('detail') or ''} {f.get('title') or ''} " - f"{f.get('viewer') or ''}") + fid = PANEL if f in panel_found else veil_id(spans, f"{f.get('gene') or ''} {f.get('detail') or ''} " + f"{f.get('title') or ''} {f.get('viewer') or ''}") if fid: out["findings"][f["id"]] = fid - tags = [None if len(x) > 2 else veil_id(spans, x[0]) for x in lead_parts or ()] + + def lead_tag(t): + if veil_id(panel_spans, t) or any(f.get("title") and f["title"] in t for f in panel_found): + return PANEL + return veil_id(spans, t) + tags = [None if len(x) > 2 else lead_tag(x[0]) for x in lead_parts or ()] out["lead"] = tags if any(tags) else [] return {k: v for k, v in out.items() if v} @@ -1354,7 +1409,8 @@ def overview(summary): order["trust"] = -1 # a problem with the data comes before any finding read from it # A finding that names a result to act on (veil: a symptom ranking of its gene) is not listed: a child's tile # closes only the results to act on themselves, and the result it names already comes first. - named = (r.get("veil") or {}).get("findings") or {} + # A finding tagged PANEL (a fixed-panel card's own, veil()) names no result: it stays, the same for everyone. + named = {k: v for k, v in ((r.get("veil") or {}).get("findings") or {}).items() if v != PANEL} top = sorted((f for f in r["findings"] if f["level"] != "info" and f["id"] not in named), key=lambda f: order[f["level"]])[:4] tags = (r.get("veil") or {}).get("lead") or [] diff --git a/docs/live-site.md b/docs/live-site.md index 1315b98d2..7d9ea19ac 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -58,8 +58,8 @@ Every result carries one of six levels, with a word, a shape and a colour, so th A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. Two pages whose open rows are the same differ only in how many results are closed, not in which; the section that holds a closed result still says it holds one: -- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call) still show in their sections. A card can hold a fixed panel among its lines: the ClinVar card prints a line for each known deep intronic site, hit or not. Those lines close all together while any result to act on is closed and come back once every result is opened (`FIXED_LINES` in `bin/live_results.py`; `veil.lines` tags each of them `*panel*`). -- A closed finding carries no id in the page or the address, since its id names its gene and variant. It is known by its place among the page's closed findings (`#f/closed-2`). A link to `#f/` of a closed result lands on its card, and the address becomes that handle. The id comes back once the result is opened, and goes again when it is closed. +- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call, a repeat) close with them, whatever gene they name, and so does a segment of the lead that names one (`veil.findings` and `veil.lead` tag them `*panel*`): a medicine finding that closed only when the closed result is in its gene would tell the gene. A card can hold a fixed panel among its lines or rows: the ClinVar card prints a line for each known deep intronic site, hit or not; the carrier list prints its two-hit line, which names the copy-number callers it read; and the callability card lists the genes this reference does not hold and the hard genes the reads did not cover, nearly the same for everyone. Those lines and rows close all together while any result to act on is closed and come back once every result is opened (`FIXED_LINES` in `bin/live_results.py`; `veil.lines` and `veil.rows` tag each of them `*panel*`). A row or line closes with a result only when it names the result's gene written as a gene symbol: a prose word ("set"), an ontology id (`HP:0001250`) or a unit after a number (`5 MB`, `48-52% GC`) names no gene, and the polygenic score card, which names scores and traits only, is never tagged (`NAMES_NO_GENE`). +- A closed finding carries no id in the page or the address, since its id names its gene and variant. It is known by its place among the page's closed findings (`#f/closed-2`). On a page that keeps a result closed, the address never holds the id of a finding the page can close, open or closed: once opened on purpose it is `#f/opened-`, its place among the opened ones, so no history entry keeps the id. Any other `#f/` on such a page, the id of a closed result or no id at all, lands the same way, on the first closed finding (`#f/closed-1`): the page never picks a closed card by its id, so a guessed id tells nothing. On the home page, a child's closed result links to the child's report, never to the finding's own page, whose address names it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. While the traits card is closed whole on a child's page, the chip is the switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 5f2a8ffcb..eda441dc1 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1423,28 +1423,39 @@ def planted_control(desc, a, z, problems): # What a reader of the son's page gets before any click: the whole document (every text and every attribute, the -# map's timing attributes aside), the address, and a hash of the map's canvas. +# map's timing attributes aside), the address, where the page is scrolled to and what has the focus, and a hash of +# the map's canvas. PAGE_JS = r"""JSON.stringify((function () { var c = document.documentElement.cloneNode(true); Array.prototype.forEach.call(c.querySelectorAll('[data-pgp-map-parse-ms],[data-pgp-map-paint-ms]'), function (e) { e.removeAttribute('data-pgp-map-parse-ms'); e.removeAttribute('data-pgp-map-paint-ms'); }); - var cv = document.querySelector('.gm-canvas'); - return { html: c.outerHTML, href: location.href, canvas: cv ? cv.toDataURL() : '' }; + // The root's data attributes come from scripts that answer in any order: sort them by name. + var at = Array.prototype.map.call(c.attributes, function (x) { return [x.name, x.value]; }).sort(); + at.forEach(function (x) { c.removeAttribute(x[0]); }); + at.forEach(function (x) { c.setAttribute(x[0], x[1]); }); + var cv = document.querySelector('.gm-canvas'), a = document.activeElement; + return { html: c.outerHTML, href: location.href + ' scroll ' + Math.round(window.scrollY) + ' focus ' + + (a ? a.tagName + '#' + a.id : ''), canvas: cv ? cv.toDataURL() : '' }; }()))""" -def page_now(b, base, hash_=""): +def page_now(b, base, hash_="", typed=False): """The son's page before any click, once its results, its charts and its map have drawn: (lines of its HTML, - its address, its canvas hash).""" + its address with its scroll position and focus, its canvas hash). TYPED: the page loads without HASH_, which is + then set as a reader types it into the address bar (a hashchange); its canvas is left out then (a hashchange + can move the map's camera).""" b.go("about:blank") b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/{hash_}") + b.go(f"{base}/runs/{OTHERS[1]}/{'' if typed else hash_}") + # The son's family answers on its own request: wait for its name in the kicker too, or a pair can differ by it. b.wait("var d = document.documentElement.dataset; d.pgpResults && d.pgpCharts && d.pgpStatus === 'ready' && (" + MAP_READY - + ") && 'ready'", 15) + + ") && document.getElementById('kicker').textContent.indexOf('Family') >= 0 && 'ready'", 15) + if typed: + b.eval(f"location.hash = {json.dumps(hash_)}; true") time.sleep(0.8) # the margin follows the camera 140 ms after it settles got = json.loads(b.eval(PAGE_JS) or "{}") return (got.get("html") or "").replace("><", ">\n<").split("\n"), got.get("href") or "", \ - hashlib.sha256((got.get("canvas") or "").encode()).hexdigest()[:12] + "" if typed else hashlib.sha256((got.get("canvas") or "").encode()).hexdigest()[:12] def plant_summary(out, og, change): @@ -1515,7 +1526,13 @@ def second_hit(S): # Pairs of son's pages that differ only in which result is closed (or in what only a closed result names): the -# page before any click must be the same, byte for byte. +# page before any click must be the same, byte for byte. The two sides of each pair differ in the thing it guards: +# a gene that a fixed-panel card's own finding names (the medicine card's CYP2D6 and NAT2) against one no card names, +# a gene of the carrier list's two-hit line (SMN1) against one it does not name, a gene of the callability card's +# list of genes not on this reference (GSTT1) or of hard genes not covered (NBPF1) against one not on it, a gene +# spelled like a word or unit the report prints (HR, MB) against one that is not, and one guessed address on two +# pages where it is a closed result's id on one side only, loaded with the page and typed after it. +GUESS = "#f/cpsr-brca1-c-68-69del" INFER = (("two SNV results in different genes (BRCA1, BRCA2)", cpsr("BRCA1"), cpsr("BRCA2")), ("a deletion over BRCA1 and NBR2 against one over CFTR and CTTNBP2", sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"]), sv_over("chr7:117480025-117580025", ["CFTR", "CTTNBP2"])), @@ -1525,19 +1542,33 @@ def second_hit(S): ("a closed result in a gene of a known deep intronic site (ATM) against one in none (BRCA2)", with_sites("ATM"), with_sites("BRCA2")), ("a link to each of two closed results (#f/)", cpsr("BRCA1"), cpsr("BRCA2"), - ("#f/cpsr-brca1-c-68-69del", "#f/cpsr-brca2-c-68-69del"))) + ("#f/cpsr-brca1-c-68-69del", "#f/cpsr-brca2-c-68-69del")), + ("a gene of the medicine card's own finding (CYP2D6) against the other one (NAT2)", cpsr("CYP2D6"), cpsr("NAT2")), + ("a gene of the medicine card's own finding (CYP2D6) against one no card names (BRCA2)", cpsr("CYP2D6"), + cpsr("BRCA2")), + ("a gene the carrier list's two-hit line names (SMN1) against one it does not (BRCA2)", cpsr("SMN1"), + cpsr("BRCA2")), + ("a gene not on this reference (GSTT1) against one on it (BRCA2)", cpsr("GSTT1"), cpsr("BRCA2")), + ("a hard gene the reads did not cover (NBPF1) against one they did (BRCA2)", cpsr("NBPF1"), cpsr("BRCA2")), + ("a gene spelled like a word of the report (HR) against one that is not (BRCA2)", cpsr("HR"), + cpsr("BRCA2")), + ("a gene spelled like a unit of the report (MB) against one that is not (BRCA2)", cpsr("MB"), cpsr("BRCA2")), + ("one guessed address, a closed result's id on one page only (BRCA1 against BRCA2 closed)", cpsr("BRCA1"), + cpsr("BRCA2"), (GUESS, GUESS)), + ("the same guessed address typed after the page drew (BRCA1 against BRCA2 closed)", cpsr("BRCA1"), + cpsr("BRCA2"), (GUESS, GUESS), True)) def infer_problems(b, base, out, og, cases=INFER): """[problems]: for each pair of CASES, the son's page before any click differs (its HTML, an attribute, the address or the map's canvas), with the number of differing lines and the first ones.""" bad = [] - for what, one, two, *hashes in cases: + for what, one, two, *more in cases: got = [] - for change, h in zip((one, two), (hashes or [("", "")])[0]): + for change, h in zip((one, two), (more or [("", "")])[0]): keep = plant_summary(out, og, change) try: - got.append(page_now(b, base, h)) + got.append(page_now(b, base, h, typed=more[1:] == [True])) finally: unplant(keep) (h1, a1, c1), (h2, a2, c2) = got @@ -1558,6 +1589,104 @@ def sv_gene_problems(b, base, out, og): return [f"NBR2 on the page: ...{html[max(0, html.find('NBR2') - 120):html.find('NBR2') + 40]}..."] if "NBR2" in html else [] +def panel_words_problems(b, base, out, og): + """[problems]: with a closed result in CYP2D6, a gene the medicine card's own finding names, the medicine card shows + the fixed-panel box's words, never 'This section holds a result'.""" + keep = plant_summary(out, og, cpsr("CYP2D6")) + try: + page_now(b, base) + text = b.eval("(document.querySelector('#card-cpic .card-text') || {}).textContent || ''") + finally: + unplant(keep) + ok = "This section lists the same genes for everyone" in text and "This section holds" not in text + return [] if ok else [f"the medicine card says {text[:160]!r}"] + + +ADDRESS_JS = ("window.__seen = window.__seen || []; if (!window.__heard) { window.__heard = 1; ['hashchange', 'popstate']" + ".forEach(function (e) { window.addEventListener(e, function () { window.__seen.push(e + ' ' + location.hash); }); " + "}); } true") + + +def address_problems(b, base, out): + """[problems]: on the son's page, a result opened from its card and the reader's moves after it never put the + id of a result his page can close into the address or a history entry: #f/closed-1, Show the result, a link to + section 3, Back.""" + with open(os.path.join(out, "runs", OTHERS[1], "results.json")) as fh: + res = json.load(fh) + tags = (res.get("veil") or {}).get("findings") or {} + ids = [f["id"] for f in res["findings"] if f["level"] == "act" or f["id"] in tags] + seen = [] + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{OTHERS[1]}/#f/closed-1") + b.wait("document.documentElement.dataset.pgpResults && document.documentElement.dataset.pgpStatus === 'ready' && " + "document.querySelector('.finding[data-level=act] .veil button') && 'ready'", 15) + b.eval(ADDRESS_JS) + seen.append(b.eval("location.hash")) + b.eval("document.querySelector('.finding[data-level=act] .veil button').click(); true") + time.sleep(0.3) + seen.append(b.eval("location.hash")) + b.eval("location.hash = '#s3'; true") + time.sleep(0.3) + b.eval("history.back(); true") + time.sleep(0.5) + seen += [b.eval("location.hash")] + json.loads(b.eval("JSON.stringify(window.__seen)") or "[]") + bad = [h for h in seen if any(i in (h or "") for i in ids)] + if not str(seen[1]).startswith("#f/opened-"): + bad.append(f"the opened result's address: {seen[1]!r}, want #f/opened-") + return [f"addresses {seen}: {bad}"] if bad else [] + + +def home_now(b, base, out, change): + """The home page with the son's view (runs.json, live_results.overview) built from his summary with CHANGE applied: + the whole document.""" + p = os.path.join(out, "runs.json") + with open(p, "rb") as fh: + keep = fh.read() + with open(os.path.join(os.path.dirname(out), "root", OTHERS[1], "summary.json")) as fh: + summary = json.load(fh) + change(summary.setdefault("sections", {})) + idx = json.loads(keep) + idx["runs"] = [dict(r, **live_results.overview(summary)) if r["name"] == OTHERS[1] else r for r in idx["runs"]] + try: + with open(p, "w") as fh: + json.dump(idx, fh) + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/") + b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.documentElement.dataset.pgpPeople " + "&& document.querySelector('#attention li') && 'ready'", 15) + time.sleep(0.3) + return b.eval("document.documentElement.outerHTML").replace("><", ">\n<").split("\n") + finally: + with open(p, "wb") as fh: + fh.write(keep) + + +def renamed(gene): + """The son's summary with his planted gene (KCNH2) named GENE everywhere: his first result to act on, the one the + home page lists, is then in GENE.""" + def change(S): + S.update(json.loads(json.dumps(S).replace("KCNH2", gene))) + return change + + +HOME = (("his first result in BRCA1 against BRCA2", renamed("BRCA1"), renamed("BRCA2")), + ("a gene of the medicine card's own finding (CYP2D6) against BRCA2", cpsr("CYP2D6"), cpsr("BRCA2"))) + + +def home_problems(b, base, out, cases=HOME): + """[problems]: the home page is the same whichever of the son's results is closed (his tile and the list of what + needs attention: no link to a closed result's own page, whose address names its gene).""" + bad = [] + for what, one, two in cases: + h1, h2 = home_now(b, base, out, one), home_now(b, base, out, two) + diff = [x for x in difflib.unified_diff(h1, h2, lineterm="", n=0) if x[:1] in "+-" and x[:3] not in ("+++", "---")] + if diff: + bad.append(f"home, {what}: {len(diff)} lines differ: {diff[:4]}") + return bad + + def infer_checks(b, base, out): """The son's page is a function of his open results and the count of his closed ones: two pages that differ only in which result is closed are the same before any click (INFER), and a closed deletion's second gene is named @@ -1573,11 +1702,55 @@ def infer_checks(b, base, out): "HTML, attributes, address and map canvas (" + "; ".join(x[0] for x in INFER) + ")", not bad, bad) bad = sv_gene_problems(b, base, out, og) check("browser: a closed deletion over two genes closes the rows that name only its second gene", not bad, bad) + bad = panel_words_problems(b, base, out, og) + check("browser: a fixed-panel card whose own finding names the closed result's gene shows the panel's words", not bad, + bad) + bad = address_problems(b, base, out) + check("browser: opening a closed result and moving on never puts its id in the address or a history entry", not bad, + bad) + bad = home_problems(b, base, out) + check("browser: the home page is the same whichever of the son's results is closed (" + "; ".join(x[0] for x in HOME) + + ")", not bad, bad) + home = os.path.join(out, "assets", "home.js") + with open(home) as fh: + keep_home = fh.read() + a = "x.r.url + (closedRuns[x.r.name] && x.f.level === 'act' ? '' : 'finding/' + encodeURIComponent(x.f.id) + '/')" + try: + if a not in keep_home: + check("control: home.js that links a closed result's own page (the planted line is in home.js)", False, a) + else: + with open(home, "w") as fh: + fh.write(keep_home.replace(a, "x.r.url + 'finding/' + encodeURIComponent(x.f.id) + '/'")) + control("home.js that links a closed result's own page", not home_problems(b, base, out, HOME[:1])) + finally: + with open(home, "w") as fh: + fh.write(keep_home) + first = live_results.panel_finding + try: + live_results.panel_finding = lambda f: False + control("a veil that tags a fixed-panel card's own finding by the closed result's gene", + not infer_problems(b, base, out, og, [x for x in INFER if "(NAT2)" in x[0]])) + finally: + live_results.panel_finding = first + fixed = live_results.FIXED_LINES + try: + live_results.FIXED_LINES = {k: v for k, v in fixed.items() if k != "carrier-list"} + control("a veil that tags the carrier list's two-hit line by its genes", + not infer_problems(b, base, out, og, [x for x in INFER if "two-hit" in x[0]])) + finally: + live_results.FIXED_LINES = fixed js = os.path.join(out, "assets", "person.js") with open(js) as fh: keep_js = fh.read() try: for desc, a, z, case in ( + # The guard is the pair of lines: either alone lands every guess the same way. + ("person.js that picks a closed card by a guessed id", + " if (f && !(veiled(f) && !m)) { return f; }\n return closedOrder().length ? findingById(closedOrder()[0]) : null;", + " return f || null;", [x for x in INFER if x[0].startswith("the same guessed")][0]), + ("person.js that picks a closed card by a guessed id, at load", + " if (f && !(veiled(f) && !m)) { return f; }\n return closedOrder().length ? findingById(closedOrder()[0]) : null;", + " return f || null;", [x for x in INFER if x[0].startswith("one guessed")][0]), ("person.js that puts a closed card's id back", "function handle(f) { return veiled(f) ? 'closed-' + (closedOrder().indexOf(f.id) + 1) : f.id; }", "function handle(f) { return f.id; }", INFER[0]), @@ -1592,6 +1765,18 @@ def infer_checks(b, base, out): with open(js, "w") as fh: fh.write(keep_js.replace(a, z)) control(desc, not infer_problems(b, base, out, og, [case])) + for desc, a, z, still in ( + ("person.js whose fixed-panel card says 'This section holds a result' first", "} else if (panelShut(c)) {", + "} else if (panelShut(c) && !closedFs.length) {", lambda: not panel_words_problems(b, base, out, og)), + ("person.js that writes an opened result's id into the address", + "function addr(f) { return !closable(f) ? f.id : veiled(f) ? handle(f) : 'opened-' + (openedOrder().indexOf(f.id) + 1); }", + "function addr(f) { return veiled(f) ? handle(f) : f.id; }", lambda: not address_problems(b, base, out))): + if a not in keep_js: + check(f"control: {desc} (the planted line is in person.js)", False, a) + continue + with open(js, "w") as fh: + fh.write(keep_js.replace(a, z)) + control(desc, still()) finally: with open(js, "w") as fh: fh.write(keep_js) @@ -1652,7 +1837,7 @@ def shown(): def problems(text, opened): page, cops, _ = json.loads(text) if text else ["", "", 0] out_ = [] - deep = page.count(live_results.FIXED_LINES["clinvar"]) + deep = page.count(live_results.FIXED_LINES["clinvar"][0]) if opened and not ("SMN1" in cops and "SMN2" in cops and "p.R175H" in page and "SMN1 copies" in page and deep == sites[0]): out_.append(f"every result opened, yet a dot, the call, the fact or a deep intronic line ({deep} of {sites[0]}) is " diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 321525727..2562d3abb 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -394,7 +394,9 @@ def veil_problems(res, idx): out.append(f"{cid} rows tagged wrong: {bad or got}") exo = (v.get("rows") or {}).get("exomiser") or [] ranked = [f["id"] for f in res["findings"] if f["level"] != "act" and gene in f["title"]] - if sf not in exo or not ranked or v.get("findings") != {i: sf for i in ranked}: + # A fixed-panel card's own findings (a medicine gene) carry the panel tag, whatever the closed result. + named = {k: t for k, t in (v.get("findings") or {}).items() if t != live_results.PANEL} + if sf not in exo or not ranked or named != {i: sf for i in ranked}: out.append(f"the symptom ranking of {gene}: rows tagged {exo}, findings tagged {v.get('findings')}, want {ranked} -> {sf}") tagged = {(t, it["section"]): it.get("veil") for t, items in idx["terms"].items() for it in items if it.get("veil")} named = {(gene, it["section"]) for it in idx["terms"].get(gene) or []} @@ -503,6 +505,87 @@ def sv_genes_problems(): return out +# The veil's tables, pinned here so that an entry dropped from one turns this file red (each is checked by what +# veil() does with it, not by reading the table back): the fixed-panel cards, the fixed lines of a card (a sample +# line or row naming a gene, its card, and whether it is a row), the cards that name no gene, and the words that +# name no gene though a gene has their name. +WANT_PANELS = ("mei", "expansions", "vntyper", "lpa", "chip", "paralogs", "cpic", "cyp2d6", "hla", "locityper", + "traits-panel", "bloodtype") +WANT_LINES = (("clinvar", "Known deep intronic site (always reported): GLA c.640-801G>A chrX:1 C>T: hom", False), + ("carrier-list", "Two hits across callers (a deletion plus a variant on the other copy): none found; " + "deletions read from Parascopy's SMN1 count (step 35)", False), + ("callability", ["GSTT1", "not on this reference", ".", "ALT-only"], True), + ("callability", ["NBPF1", "not read", "69%", "segdup, collapsed"], True)) +WANT_NO_GENE = (("prs", ["Asthma", "PGS002727", "68.3 (EUR)", "2605/2872", "HR 1.57 (EUR, PPM014751)"], "HR"),) +WANT_NOT_A_GENE = (("SET", "the same set of genes"), ("MB", "0.0 MB in runs of 5 MB or more"), + ("HP", "Seizure (HP:0001250)"), ("GC", "reads per million with 48-52% GC")) +WANT_A_GENE = (("KCNH2", "A change in KCNH2."), ("C9ORF72", "C9orf72 repeat"), ("HLA-A", "HLA-A*02:01")) + + +def gene_of(text): + """The first upper-case word of TEXT (a list is a table row) that looks like a gene symbol.""" + t = " ".join(map(str, text)) if isinstance(text, list) else text + return next(w for w in re.findall(r"[A-Z][A-Z0-9]{2,}", t.split(":", 1)[-1] if not isinstance(text, list) else t) + if w not in ("ALT", "SMN")) + + +def table_problems(): + """What is wrong with veil()'s tables (WANT_*): a fixed-panel card missing from veil.panels, a fixed line or row + tagged by the gene it names instead of the panel tag, a card that names no gene tagged, a word that names no gene + read as one, or a gene symbol not read as one.""" + out = [] + found = lambda g: [{"id": "sf-" + g.lower(), "level": "act", "gene": g, "detail": g}] + chapters = [{"sections": [{"id": c, "state": "ok"} for c in WANT_PANELS]}] + got = live_results.veil({"sections": {}}, found("BRCA2"), chapters).get("panels") or [] + out += [f"card {c} is not in veil.panels" for c in WANT_PANELS if c not in got] + for card, item, is_row in WANT_LINES: + g = gene_of(item) + sec = {"id": card, "table": {"head": ["a", "b", "c", "d"], "rows": [item]}} if is_row else {"id": card, "lines": [item]} + v = live_results.veil({"sections": {}}, found(g), [{"sections": [sec]}]) + tag = ((v.get("rows" if is_row else "lines") or {}).get(card) or [None])[0] + if tag != live_results.PANEL: + out.append(f"{card}: {str(item)[:60]} tagged {tag} with a closed result in {g}, want the panel tag") + for card, row, g in WANT_NO_GENE: + sec = {"id": card, "table": {"head": ["a"] * len(row), "rows": [row]}, "lines": [" ".join(row)], "facts": [row[:2]]} + v = live_results.veil({"sections": {}}, found(g), [{"sections": [sec]}]) + if any((v.get(k) or {}).get(card) for k in ("rows", "lines", "facts")): + out.append(f"{card} names no gene, yet a closed result in {g} tags it: {v}") + for g, text in WANT_NOT_A_GENE: + if live_results.veil_id(live_results.closable(found(g)), text): + out.append(f"{text!r} read as naming {g}") + for g, text in WANT_A_GENE: + if not live_results.veil_id(live_results.closable(found(g)), text): + out.append(f"{text!r} not read as naming {g}") + return out + + +def table_controls(): + """Each entry of the veil's tables dropped in turn: table_problems() must fail every time.""" + keep = (live_results.FIXED_PANELS, live_results.FIXED_LINES, live_results.NAMES_NO_GENE, live_results.UNIT_RE) + first_words = live_results.gene_words + try: + for c in WANT_PANELS: + live_results.FIXED_PANELS = {k: v for k, v in keep[0].items() if k != c} + control(f"FIXED_PANELS without {c}", not table_problems()) + live_results.FIXED_PANELS = keep[0] + for card, words in keep[1].items(): + for w in words: + live_results.FIXED_LINES = dict(keep[1], **{card: tuple(x for x in words if x != w)}) + control(f"FIXED_LINES[{card}] without {w!r}", not table_problems()) + live_results.FIXED_LINES = keep[1] + live_results.NAMES_NO_GENE = {} + control("NAMES_NO_GENE empty", not table_problems()) + live_results.NAMES_NO_GENE = keep[2] + live_results.UNIT_RE = re.compile(r"(?!)") + control("gene_words with no units", not table_problems()) + live_results.UNIT_RE = keep[3] + live_results.gene_words = lambda t: {w.upper().rstrip("._-") for w in re.findall(r"[A-Za-z0-9][A-Za-z0-9._-]*", str(t))} + control("gene_words that reads every word, in any case, as a gene", not table_problems()) + finally: + (live_results.FIXED_PANELS, live_results.FIXED_LINES, live_results.NAMES_NO_GENE, live_results.UNIT_RE) = keep + live_results.gene_words = first_words + + def overview_problems(): """What is wrong with the home page's view of a child's run (overview): a finding that names a closed result (a symptom ranking of its gene) is listed among the first findings, or the closed lead still names it.""" @@ -623,6 +706,10 @@ def tagged(sample_dir): control("a veil that tags a known deep intronic site's line by its gene", not veil_lines_problems()) finally: live_results.FIXED_LINES = fixed + bad = table_problems() + check("8. every fixed-panel card, fixed line and gene-free card of the veil's tables closes as a panel or never, " + "and only a gene symbol written as one names a gene", not bad, bad) + table_controls() bad = overview_problems() check("8. a child's home tile lists no finding that names a closed result, and its closed lead names none", not bad, bad) From 6314a5c443ed9f4e260b83562bc1480cd2995cb0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:25:51 +0100 Subject: [PATCH 027/255] test(live): open a closed result by its handle and read the map chip's opened address The page no longer finds a closed card by its id, so the map checks open a planted result by its place among the closed findings, and read an Opened in its section link (#f/opened-) as the finding it names. The medicine finding closes with its panel now, so the open-finding-over-a-closed-result check watches a carrier finding. closedActs() names its argument apart from mapClosed(), so the chip-count control plants its mistake in one place. --- assets/live/person.js | 2 +- tests/test_live_demo.py | 86 ++++++++++++++++++++++++++++++++++------- 2 files changed, 72 insertions(+), 16 deletions(-) diff --git a/assets/live/person.js b/assets/live/person.js index ddfbdbbf8..5955a95c3 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -370,7 +370,7 @@ } function veiledOf(c) { return (c.findings || []).map(findingById).filter(function (f) { return f && veiled(f); }); } // The results to act on this page keeps closed, drawn yet or not. - function closedActs() { return ((results && results.findings) || []).filter(function (f) { return f.level === 'act' && veiled(f); }); } + function closedActs() { return ((results && results.findings) || []).filter(function (x) { return x.level === 'act' && veiled(x); }); } // A card of a fixed panel (results.veil.panels, bin/live_results.py FIXED_PANELS: the same genes or loci for // everyone, known in advance): a row or dot of it closed with a result would tell the result's gene by what is // missing. So it closes whole while this page keeps any result to act on closed, behind one box with the same diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index eda441dc1..d0f75cf89 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -645,15 +645,44 @@ def planted_closed(b, base): return out +# The handle of a closed finding, as the page counts it (person.js closedOrder()): the closed findings in the order +# of the sections, then any other; a result to act on is closed until opened, a finding tagged with one while that +# result is, a finding tagged PANEL while any result to act on is, a finding tagged with no finding of the page always. +HANDLE_JS = r"""fetch('results.json', {cache: 'no-store'}).then(function (r) { return r.json(); }).then(function (r) { + var fid = %s, byId = {}, tags = (r.veil && r.veil.findings) || {}; + r.findings.forEach(function (f) { byId[f.id] = f; }); + function open(id) { return !!document.getElementById('f-' + id); } + var acts = r.findings.filter(function (f) { return f.level === 'act'; }); + var anyShut = acts.some(function (f) { return !open(f.id); }); + function shut(f) { + if (f.level === 'act') { return !open(f.id); } + var t = tags[f.id]; + if (!t || t === f.id) { return false; } + if (t === '*panel*') { return anyShut; } + return !byId[t] || !open(t); + } + var order = []; + (r.journal || []).forEach(function (s) { (s.findings || []).forEach(function (id) { + if (byId[id] && shut(byId[id]) && order.indexOf(id) < 0) { order.push(id); } }); }); + r.findings.forEach(function (f) { if (shut(f) && order.indexOf(f.id) < 0) { order.push(f.id); } }); + return order.indexOf(fid) + 1; +})""" + + def open_closed(b, fid): - """Opens the closed result FID as a reader who holds a link to it does: #f/ lands on its closed card, whose - address and element carry its handle (#f/closed-), never its id; then its "Show the result". False when the - link lands on no closed card.""" - b.eval("location.hash = %s; true" % json.dumps("#f/" + fid)) - if not b.wait("/^#f\\/closed-\\d+$/.test(location.hash) && document.getElementById('f-' + location.hash.slice(3)) " - "&& location.hash", 5): + """Opens the closed result FID as a reader does who knows its place among the page's closed findings: its handle + (#f/closed-), never its id, which the page does not use to find a closed card; then its "Show the result". + False when the handle names no closed card.""" + r = b.call("Runtime.evaluate", {"expression": HANDLE_JS % json.dumps(fid), "awaitPromise": True, + "returnByValue": True}, b.session) + n = (r.get("result") or {}).get("value") or 0 + if not n: + return False + b.eval("location.hash = %s; true" % json.dumps(f"#f/closed-{n}")) + if not b.wait("location.hash === %s && document.getElementById('f-closed-%d') && location.hash" + % (json.dumps(f"#f/closed-{n}"), n), 5): return False - b.eval("document.getElementById('f-' + location.hash.slice(3)).querySelector('.veil button').click(); true") + b.eval("document.getElementById('f-closed-%d').querySelector('.veil button').click(); true" % n) return True @@ -777,11 +806,11 @@ def write(p, text): check("browser: the home page and every finding page of the son name the planted gene and positions nowhere " "(the rail's next link and the related list included) until opened, and the symptom ranking's page " "opened on purpose shows it", not bad, bad) - a = "!!tagOf(x) && !opened[tagOf(x)]" + a = "return t === PANEL ? !opened[x.id] && actShut() : !!t && !opened[t];" if a not in keep[fj]: check("control: finding.js whose findings ignore their tags (the planted line is in finding.js)", False, a) else: - write(fj, keep[fj].replace(a, "false")) + write(fj, keep[fj].replace(a, "return false;")) control("finding.js whose findings ignore their tags", not planted_pages(b, base, out)) write(fj, keep[fj]) finally: @@ -2468,7 +2497,23 @@ def map_now(b): focus: f && f.classList.contains('gm-yes') ? 'gm-yes' : f === first ? 'first' : f === document.querySelector('.gm-canvas') ? 'canvas' : (f ? f.tagName + '.' + f.className : ''), opened: Array.prototype.map.call(document.querySelectorAll('.gm-chip.opened'), function (o) { var a = o.querySelector('a'); - return { kind: o.dataset.opened, text: o.textContent, href: a ? a.getAttribute('href') : '' }; }) }; + var raw = a ? a.getAttribute('href') : ''; + return { kind: o.dataset.opened, text: o.textContent, raw: raw, href: opened(raw) }; }) }; + // The finding an "Opened in its section" link names: #f/opened- is the n-th finding the page can close and + // shows opened, in the order of the sections (person.js openedOrder()). + function opened(raw) { + var m = /^#f\/opened-(\d+)$/.exec(raw); + if (!m) { return raw; } + if (!window.__pgpResults) { + var x = new XMLHttpRequest(); x.open('GET', 'results.json', false); x.send(); window.__pgpResults = JSON.parse(x.responseText); + } + var r = window.__pgpResults, tags = (r.veil && r.veil.findings) || {}, byId = {}, order = []; + r.findings.forEach(function (f) { byId[f.id] = f; }); + function ok(f) { return f && (f.level === 'act' || (tags[f.id] && tags[f.id] !== f.id)) && !!document.getElementById('f-' + f.id); } + (r.journal || []).forEach(function (s) { (s.findings || []).forEach(function (id) { if (ok(byId[id]) && order.indexOf(id) < 0) { order.push(id); } }); }); + r.findings.forEach(function (f) { if (ok(f) && order.indexOf(f.id) < 0) { order.push(f.id); } }); + return '#f/' + (order[Number(m[1]) - 1] || raw); + } }()))""" @@ -2507,6 +2552,9 @@ def chip_open(b): named = [o["href"] for o in after["opened"] if o["kind"] == "section" and o["href"][3:] not in after["ids"]] if named: bad.append(f"an Opened in its section chip names a result that is not open: {named}") + raw = [o["raw"] for o in after["opened"] if o["kind"] == "section" and not re.match(r"^#f/opened-\d+$", o["raw"])] + if raw: + bad.append(f"an Opened in its section chip links a result by its id: {raw}") leaked = [after["ids"][i] for i in after["closed"] if not re.match(r"^closed-\d+$", after["ids"][i] or "")] if leaked: bad.append(f"a closed card carries an id, not a handle: {leaked}") @@ -2822,8 +2870,13 @@ def act(fid, gene, viewer, title): [act("pl-iota", "PLIOTA", "chr7:117400000-117700000", "Planted iota deletion")], [{"id": "pl-kappa", "level": "review", "card": "exomiser", "gene": "CFTR", "title": "Planted kappa review", "detail": "CFTR", "text": "A planted review finding that names CFTR only."}], "pl-kappa", "chr7"), - ("the page's own CYP2D6 medicine finding over a closed variant naming another gene", - [act("pl-theta", "PLTHETA", "chr22:42125000", "Planted theta result")], [], "pgx-cyp2d6", "chr22")) + # The page's own CYP2D6 medicine finding closes with the medicine card while any result is closed (a fixed + # panel), so a carrier finding stands in for a gene-only finding of a card that is no panel. + ("a gene-only carrier finding (CYP2D6) over a closed variant naming another gene", + [act("pl-theta", "PLTHETA", "chr22:42125000", "Planted theta result")], + [{"id": "pl-lambda", "level": "carrier", "card": "carrier-list", "gene": "CYP2D6", + "title": "Planted lambda carrier", "detail": "CYP2D6", "text": "A planted carrier finding that names CYP2D6 only."}], + "pl-lambda", "chr22")) def under_problems(b, base, out, og): @@ -3007,10 +3060,13 @@ def open_order_problems(b, base, out, og): if k == 0: gone = [x["id"] for x in extra if not x["viewer"]][0] at = chips_now(b)["ids"].index(gone) if gone in chips_now(b)["ids"] else -1 - b.eval("var o = document.querySelector('.gm-chip.opened[data-opened=section] a[href=\"#f/%s\"]'); " - "o.parentNode.parentNode.querySelector('button').click(); true" % gone) + was = [o for o in chips_now(b)["opened"] if o["kind"] == "section"] + raw = [o["raw"] for o in was if o["href"] == f"#f/{gone}"] + b.eval("var o = document.querySelector('.gm-chip.opened[data-opened=section] a[href=\"%s\"]'); " + "o.parentNode.parentNode.querySelector('button').click(); true" % (raw[0] if raw else "")) after = chips_now(b) - if after["closed"] != [at] or any(o["kind"] == "section" and gone in o["href"] for o in after["opened"]): + left = [o for o in after["opened"] if o["kind"] == "section"] + if after["closed"] != [at] or len(left) != len(was) - 1: bad.append(f"{what}: Close it again on {gone}'s section chip: closed {after['closed']}, " f"opened chips {after['opened']}") finally: From 6fd5ca69a45234d3568c1869b180ca43d6b70a80 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:52:38 +0100 Subject: [PATCH 028/255] fix(map): leave a finding a closed result ties out of the map's count of findings with no place --- rust/crates/og-map/src/facts.rs | 32 +++++++++++++++++++++++++++++- rust/crates/og-map/src/findings.rs | 6 ++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/rust/crates/og-map/src/facts.rs b/rust/crates/og-map/src/facts.rs index c96539ce1..edeb7fea0 100644 --- a/rust/crates/og-map/src/facts.rs +++ b/rust/crates/og-map/src/facts.rs @@ -5,7 +5,9 @@ //! when it is male or female; else the results body's `person.inferred_sex`, //! then `person.declared_sex`; else unknown. //! - Findings: `results.findings`, one object per finding, in the page's -//! order. Each keeps the fields the map places and labels it with. +//! order. Each keeps the fields the map places and labels it with, and +//! whether `results.veil.findings` ties it to a result the page can close +//! (a carrier row in a closed deletion's second gene). //! - The gated APOE result: the traits card keeps APOE behind its own //! warning (`hidden` on the card `traits-panel`). It is not a row of //! `results.findings`, so the map adds it as one finding, `apoe`, with @@ -68,8 +70,16 @@ pub struct Finding { /// The evidence row `Genotype`, when there is one. pub genotype: Option, pub steps: Vec, + /// `results.veil.findings` ties it to a closable result (any tag but + /// the fixed-panel one, `*panel*`, which is the same whatever is closed): + /// it closes with that result, so no count of open findings takes it. + pub veiled: bool, } +/// The tag of `results.veil.findings` that names no result: a fixed-panel +/// card's own finding (bin/live_results.py `PANEL`). +pub const PANEL_TAG: &str = "*panel*"; + /// The findings part of stdin. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Findings { @@ -134,6 +144,7 @@ fn finding(v: &Value) -> Option { hgvs: evidence(f, "Variant"), genotype: evidence(f, "Genotype"), steps, + veiled: false, }) } @@ -224,6 +235,13 @@ pub fn parse(bytes: &[u8]) -> Result { None | Some((_, None)) => Findings::Missing, Some((r, Some(Value::Array(list)))) => { let mut out: Vec = list.iter().filter_map(finding).collect(); + if let Some(Value::Object(v)) = r.get("veil") + && let Some(Value::Object(tags)) = v.get("findings") + { + for f in &mut out { + f.veiled = matches!(tags.get(&f.id), Some(Value::Str(t)) if t != PANEL_TAG); + } + } let has_apoe = out .iter() .any(|f| f.id == APOE_ID || f.gate.as_deref() == Some(APOE_ID)); @@ -319,5 +337,17 @@ mod tests { panic!("not read") }; assert_eq!(f.len(), 2); + // A finding the veil ties to a closable result is veiled; the + // fixed-panel tag ties it to none. + let Findings::Read(f) = p(&body.replace( + r#""chapters""#, + r#""veil":{"findings":{"q":"sf-x","sf-x":"*panel*"}},"chapters""#, + )) else { + panic!("not read") + }; + assert_eq!( + (f[0].veiled, f[1].veiled, f[2].veiled), + (false, true, false) + ); } } diff --git a/rust/crates/og-map/src/findings.rs b/rust/crates/og-map/src/findings.rs index d65ba737b..a59a50d22 100644 --- a/rust/crates/og-map/src/findings.rs +++ b/rust/crates/og-map/src/findings.rs @@ -9,7 +9,9 @@ //! reads (`genes::parse_gtf`): medicine genes, HLA, LPA, a gene-level //! result. A name on several chromosomes takes the first in map order. //! - A finding with neither is not on the map. The layer note counts those -//! that no closed result hides; a closable one is left out of every count. +//! that no closed result hides; a closable one, and one the results' +//! veil ties to a closable result (`Finding::veiled`), is left out of +//! every count. //! - The layer's count and totals take the rows with no veil tag only: a //! closable finding, and an open one over a closable one's span, are in //! no number. @@ -197,7 +199,7 @@ pub fn read(findings: &facts::Findings, root: &Path, run: &str, reference: &Refe within((c, s, e, None)) }); let Some((chrom, s, e, pos)) = place else { - if !veil::closable(f) { + if !veil::closable(f) && !f.veiled { open_unplaced += 1; } continue; From 5d45630e29ba388cafcb3ad3a0755d14ef3ce39d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 21:52:38 +0100 Subject: [PATCH 029/255] fix(live): tag a row, finding or chart mark by its gene and place fields, never by a report code such as AR or INS --- CHANGELOG.md | 2 +- bin/live_results.py | 96 +++++++++++++++++++++++++++++---- docs/live-site.md | 2 +- tests/test_live_demo.py | 106 +++++++++++++++++++++++++++++++++++-- tests/test_live_results.py | 106 +++++++++++++++++++++++++++++++++++++ 5 files changed, 298 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb03678de..dfd5813ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. -- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; the address never holds a closable finding's id, closed (`#f/closed-`) or opened (`#f/opened-`), so no history entry keeps it, and any other `#f/` on such a page, a closed result's id or nothing, lands on the first closed finding, so a guessed id tells nothing. The home page links a child's closed result to the child's report, not to the finding's own page. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and so do their own findings (a medicine gene, a CHIP call) and the lead's words on them, the ClinVar card's lines for the known deep intronic sites, the carrier list's two-hit line and the callability card's rows, all together; a row or line closes with a result only when it names its gene as a gene symbol (never a prose word, an ontology id or a unit such as `5 MB`), and the polygenic score card is never tagged; a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels`, the tag `*panel*` in `veil.lines`, `veil.rows`, `veil.findings` and `veil.lead`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; nothing is renamed or removed. +- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; the address never holds a closable finding's id, closed (`#f/closed-`) or opened (`#f/opened-`), so no history entry keeps it, and any other `#f/` on such a page, a closed result's id or nothing, lands on the first closed finding, so a guessed id tells nothing. The home page links a child's closed result to the child's report, not to the finding's own page. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and so do their own findings (a medicine gene, a CHIP call) and the lead's words on them, the ClinVar card's lines for the known deep intronic sites, the carrier list's two-hit line and the callability card's rows, all together; a table row, a finding or a chart mark closes with a result only by its gene and place fields, never by a code such as the inheritance `AR` or the structural-variant type `INS`, and prose closes only by a word written as a gene symbol (never a prose word, an ontology id or a unit such as `5 MB`); the polygenic score card and its chart are never tagged, and the genome map no longer counts a finding a closed result ties among its findings with no place; a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels`, the tag `*panel*` in `veil.lines`, `veil.rows`, `veil.findings` and `veil.lead`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; nothing is renamed or removed. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. - Steps 21, 32 and 9b (and the CYRIUS, PYPGX and STRANGER_RFC1 modules) now run the CYP2D6 depth check and the RFC1 motif count with `og`, and `bin/cyp2d6_depth_check.py` and `bin/rfc1_motif.py` are gone. A run that reaches these steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull; without it the step stops with `og is not built`. The output files, their names and formats do not change; in CI, every e2e case that runs these steps in bash checks `og` against the deleted Python, byte for byte. diff --git a/bin/live_results.py b/bin/live_results.py index 65c542fe9..a6a7157df 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -1144,7 +1144,8 @@ def lead(found, levels, summary): # finding's own card with it, but other places can name the same gene or variant: any card's table and # lines, the charts' marks and their titles, another finding (a symptom ranking), the lead, the search index. Each such row carries the id of the # result it belongs to, so the page closes it with that result and opens it with it. The rule: a row -# belongs to a closed result when it names the result's gene, or its position falls inside the result's +# belongs to a closed result when it names the result's gene in a field that names a gene (TABLE_COLUMNS, +# finding_names(), mark_names(); words only in prose that has no such field), or its position falls inside the result's # position or span; a row that is the result's own position or span belongs to that result first, else # the first such result in finding order. The veil lives in the browser: the files still hold every row. # The cards whose rows are a fixed panel: the same genes or loci for everyone, known before the run. A row or @@ -1182,6 +1183,30 @@ def lead(found, levels, summary): # score card names its scores and their traits (assets/pgs_scores.tsv) and its words "OR 1.57", "HR 1.4", "(TG)" # are odds and hazard ratios and a trait, though HR and TG are gene symbols too. NAMES_NO_GENE = {"prs": "the polygenic scores of assets/pgs_scores.tsv"} +# Every column of every card's table (the adapters' table() calls), and whether it names a gene or a place (True) +# or is the report's own vocabulary (False): an inheritance code (AR), a structural-variant type (INS), a genotype, +# a class, a significance, a unit, a caller, a condition. veil() tags a table row by its gene and place cells only, +# because any code a report prints in capitals may also be a gene symbol: a closed result in AR would else close +# every recessive carrier row, and one in INS every insertion. A column missing here counts as naming a gene; +# tests/test_live_results.py fails until every column of every table() call is listed. +TABLE_COLUMNS = { + "Gene": True, "Genes": True, "Variant": True, "Where": True, "Locus": True, "Site": True, "Known site": True, + "Change": True, "Chromosome": True, "Start": True, "End": True, + "Status": False, "Bases read": False, "Why": False, "Genotype": False, "Significance": False, "Stars": False, + "Frequency": False, "Call": False, "Kind": False, "Rule": False, "Final class": False, "ClinVar": False, + "Rank": False, "Inheritance": False, "Score": False, "ACMG": False, "Disease": False, "Type": False, + "Length": False, "Class": False, "Support": False, "Checks": False, "Element": False, "Condition": False, + "Depth": False, "Repeats": False, "Stranger status": False, "Normal up to": False, "Pathologic from": False, + "95% interval": False, "Allele fraction": False, "VAF": False, "Alt reads": False, "Length (Mb)": False, + "Cell fraction": False, "Result": False, "Chance for a child": False, "This person": False, "Partner": False, + "Copies": False, "Detail": False, "Diplotype": False, "Phenotype (CPIC's words)": False, "Caller": False, + "CYP2D6": False, "Alleles (T1K quality)": False, "Panel haplotypes": False, "Quality": False, "Filter": False, + "Primary caller": False, "Level": False, "Group": False, "Share": False, "Bootstrap range": False, + "Branch": False, "Derived": False, "Ancestral": False, "No call": False, "Dates": False, "Country": False, + "Distance": False, "Trait": False, "Reading": False, "Percentile": False, "Variants matched": False, + "Effect": False, "System": False, "Predicted phenotype": False, "Confidence": False, "Virus": False, + "Reads": False, "Breadth": False, "Copies per cell": False, "Sizing": False, "Intermediate zone": False, +} def panel_finding(f): @@ -1254,9 +1279,10 @@ def closable_of(summary): return closable(finding_rows(summary or {})) -def veil_id(spans, text="", chrom=None, pos=None, end=None): +def veil_id(spans, text="", chrom=None, pos=None, end=None, genes=None): """The id of the closed result (spans: closable()) a row belongs to, or None. TEXT is what the row - shows (its gene names, its loci are read too); chrom, pos and end its own position when it has one.""" + shows (its gene names, its loci are read too); chrom, pos and end its own position when it has one. + GENES, when given, are the genes the row names, from its own gene field: TEXT then gives its loci only.""" if not spans: return None locs = [] @@ -1269,7 +1295,7 @@ def veil_id(spans, text="", chrom=None, pos=None, end=None): except (TypeError, ValueError): pass locs += _loci(text) - words = gene_words(text) + words = gene_words(text) if genes is None else {str(g).upper() for g in genes if g} for fid, _gene, c, a, b in spans: if c and (c, a, b) in locs: return fid @@ -1281,6 +1307,52 @@ def veil_id(spans, text="", chrom=None, pos=None, end=None): return None +def row_naming_cells(columns, row): + """The cells of a table row under a column that names a gene or a place (TABLE_COLUMNS): the only cells veil() + reads a row's genes and loci from.""" + if not isinstance(row, (list, tuple)): + return [row] + cols = list(columns or []) + return [x for i, x in enumerate(row) if i >= len(cols) or TABLE_COLUMNS.get(cols[i], True)] + + +def finding_names(f): + """(genes, loci text) a finding is tagged by: a finding with a gene field is known by its gene, its genes (a + structural variant's every gene) and the loci of its viewer, detail and evidence, never by its title's words (an + SV type such as INS, a condition); one with no gene but a place (a mosaic call) by that place alone; genes None + for one with neither (a card's own headline, a data-quality finding): its title and detail are read word by + word, the only words it has.""" + where = " ".join(str(x) for x in [f.get("viewer"), f.get("detail")] + [b for _a, b in f.get("evidence") or []] + if x not in (None, "")) + genes = [g for g in list(f.get("genes") or []) + [f.get("gene")] if g] + if genes or _loci(where): + return genes, where + return None, f"{f.get('gene') or ''} {f.get('detail') or ''} {f.get('title') or ''} {f.get('viewer') or ''}" + + +def mark_names(S, sp, m): + """(genes, loci text) of a chart mark, from the summary row it draws, never from its label: a ClinVar hit's gene + and position, a structural variant's genes, each star bar's hits (the ones its detail lists). A mark of a list + known before the run (a polygenic score, an ancestry group, a mitochondrial site) names no gene: its position + alone, if it has one.""" + hits = _items(_data(S, "clinvar").get("hits")) + mid = str(m.get("id") or "") + try: + if m.get("kind") == "hit": + h = hits[int(mid.split("-", 1)[1])] + return [h.get("gene")], "" + if m.get("kind") == "sv": + r = _items(_data(S, "annotsv").get("rows"))[int(mid.split("-", 1)[1])] + return list(r.get("genes") or []), "" + if sp.get("id") == "clinvar-stars" and mid.startswith("stars-"): + k = mid.split("-", 1)[1] + idx = [h for h in hits if isinstance(h, dict) and str(h.get("stars")) == k and not h.get("low_confidence")][:10] + return [h.get("gene") for h in idx], " ".join(f"{h.get('chrom')}:{h.get('pos')}" for h in idx) + except (IndexError, ValueError, AttributeError, TypeError): + pass + return [], "" + + def veil(summary, found, chapters, lead_parts=()): """The rows a child's page closes with a result to act on, by the result's id: each card's table rows (rows: card -> one id or None per row; the ClinVar card's by each hit's own position), each card's viewer @@ -1312,8 +1384,9 @@ def veil(summary, found, chapters, lead_parts=()): continue rows = (c.get("table") or {}).get("rows") or [] if c["id"] != "clinvar" and rows: + cols = (c.get("table") or {}).get("columns") tags = [PANEL if any(fixed_line(c["id"], x) for x in (r if isinstance(r, (list, tuple)) else [r])) - else veil_id(spans, " ".join(str(x) for x in r) if isinstance(r, (list, tuple)) else str(r)) + else veil_id(spans, " ".join(str(x) for x in row_naming_cells(cols, r))) for r in rows] if any(tags): out["rows"][c["id"]] = tags @@ -1344,8 +1417,8 @@ def veil(summary, found, chapters, lead_parts=()): for m in sp.get("marks") or []: if m.get("kind") == "roh": # a run of homozygosity is no result's source continue - fid = veil_id(spans, f"{m.get('label') or ''} {m.get('detail') or ''}", m.get("chrom"), m.get("pos"), - m.get("end")) + genes, where = mark_names(S, sp, m) + fid = veil_id(spans, where, m.get("chrom"), m.get("pos"), m.get("end"), genes=genes) if fid: out["marks"].setdefault(sp["id"], {})[m["id"]] = fid # A finding of a fixed-panel card (a medicine gene, a CHIP call, a repeat) is tagged PANEL whatever it names: it @@ -1355,14 +1428,19 @@ def veil(summary, found, chapters, lead_parts=()): panel_spans = [(PANEL, str(g).upper(), None, None, None) for f in panel_found for g in [f.get("gene")] if g] for f in found: if f.get("level") != "act": - fid = PANEL if f in panel_found else veil_id(spans, f"{f.get('gene') or ''} {f.get('detail') or ''} " - f"{f.get('title') or ''} {f.get('viewer') or ''}") + genes, where = finding_names(f) + fid = PANEL if f in panel_found else veil_id(spans, where, genes=genes) if fid: out["findings"][f["id"]] = fid + # A segment that holds findings' titles is tagged as those findings are, never by its words; the others are + # the lead's own sentences. def lead_tag(t): if veil_id(panel_spans, t) or any(f.get("title") and f["title"] in t for f in panel_found): return PANEL + named = [f for f in found if f.get("level") != "act" and f.get("title") and f["title"] in t] + if named: + return next((out["findings"][f["id"]] for f in named if f["id"] in out["findings"]), None) return veil_id(spans, t) tags = [None if len(x) > 2 else lead_tag(x[0]) for x in lead_parts or ()] out["lead"] = tags if any(tags) else [] diff --git a/docs/live-site.md b/docs/live-site.md index 7d9ea19ac..1a3e78656 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -58,7 +58,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. Two pages whose open rows are the same differ only in how many results are closed, not in which; the section that holds a closed result still says it holds one: -- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call, a repeat) close with them, whatever gene they name, and so does a segment of the lead that names one (`veil.findings` and `veil.lead` tag them `*panel*`): a medicine finding that closed only when the closed result is in its gene would tell the gene. A card can hold a fixed panel among its lines or rows: the ClinVar card prints a line for each known deep intronic site, hit or not; the carrier list prints its two-hit line, which names the copy-number callers it read; and the callability card lists the genes this reference does not hold and the hard genes the reads did not cover, nearly the same for everyone. Those lines and rows close all together while any result to act on is closed and come back once every result is opened (`FIXED_LINES` in `bin/live_results.py`; `veil.lines` and `veil.rows` tag each of them `*panel*`). A row or line closes with a result only when it names the result's gene written as a gene symbol: a prose word ("set"), an ontology id (`HP:0001250`) or a unit after a number (`5 MB`, `48-52% GC`) names no gene, and the polygenic score card, which names scores and traits only, is never tagged (`NAMES_NO_GENE`). +- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call, a repeat) close with them, whatever gene they name, and so does a segment of the lead that names one (`veil.findings` and `veil.lead` tag them `*panel*`): a medicine finding that closed only when the closed result is in its gene would tell the gene. A card can hold a fixed panel among its lines or rows: the ClinVar card prints a line for each known deep intronic site, hit or not; the carrier list prints its two-hit line, which names the copy-number callers it read; and the callability card lists the genes this reference does not hold and the hard genes the reads did not cover, nearly the same for everyone. Those lines and rows close all together while any result to act on is closed and come back once every result is opened (`FIXED_LINES` in `bin/live_results.py`; `veil.lines` and `veil.rows` tag each of them `*panel*`). A table row, a finding or a chart mark closes with a result only by the fields that name a gene or a place: a row by its gene and position columns, a finding by its gene, its genes and its loci, a chart mark by the gene and position of the summary row it draws. It never closes by a vocabulary cell (an inheritance code, a structural-variant type, a genotype, a class, a significance, a unit) or by a structural variant's title, because any code the report prints in capitals may also be a gene symbol: a closed result in AR would else close every recessive carrier row, and one in INS every insertion. `TABLE_COLUMNS` in `bin/live_results.py` says for each column of each card's table whether it names a gene or a place. A finding with no gene and no place (a card's own headline, a data-quality finding), a card's lines and facts, and the lead's own sentences are prose with no such field: they close by the words that name the gene written as a gene symbol. A prose word ("set"), an ontology id (`HP:0001250`) or a unit after a number (`5 MB`, `48-52% GC`) names no gene there. A segment of the lead that holds findings' titles closes as those findings do. The polygenic score card, which names scores and traits only, is never tagged (`NAMES_NO_GENE`), and neither are the marks of a chart whose labels come from a list known before the run (the scores, the ancestry groups, the ClinVar star bars' labels). The genome map leaves a finding the veil ties to a closed result out of its count of findings with no place. - A closed finding carries no id in the page or the address, since its id names its gene and variant. It is known by its place among the page's closed findings (`#f/closed-2`). On a page that keeps a result closed, the address never holds the id of a finding the page can close, open or closed: once opened on purpose it is `#f/opened-`, its place among the opened ones, so no history entry keeps the id. Any other `#f/` on such a page, the id of a closed result or no id at all, lands the same way, on the first closed finding (`#f/closed-1`): the page never picks a closed card by its id, so a guessed id tells nothing. On the home page, a child's closed result links to the child's report, never to the finding's own page, whose address names it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. While the traits card is closed whole on a child's page, the chip is the switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index d0f75cf89..b4afd916f 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1536,18 +1536,41 @@ def change(S): return change -def sv_over(locus, genes): - """A class 5 deletion over GENES, and a carrier row that names only its second gene.""" +def sv_over(locus, genes, carrier=True): + """A class 5 deletion over GENES, and (CARRIER) a carrier row that names only its second gene.""" def change(S): rows = S["annotsv"]["data"]["rows"] S["annotsv"]["data"]["rows"] = rows + [{"sv": locus, "type": "DEL", "gt": "0/1", "length": "-100000", "acmg_class": "5", "genes": genes, "caller": "Manta", "support": 1, "support_callers": ["Manta"]}] + if not carrier: + return S["carrier"]["data"]["carriers"] = [{"gene": genes[1], "variant": "c.100C>T", "genotype": "het", "significance": "Pathogenic", "stars": 2, "inheritance": "AR"}] return change +def recessive(change): + """CHANGE, plus two recessive carrier rows (CFTR and HBB, inheritance AR) on the carrier card.""" + def go(S): + change(S) + S["carrier"]["data"]["carriers"] = S["carrier"]["data"]["carriers"] + [ + {"gene": g, "variant": v, "genotype": "het", "significance": "Pathogenic", "stars": 2, "inheritance": "AR", + "condition": c} for g, v, c in (("CFTR", "c.1521_1523del", "Cystic fibrosis"), + ("HBB", "c.20A>T", "Sickle cell disease"))] + return go + + +def insertion(change): + """CHANGE, plus a class 4 insertion (type INS) over NKD2: a review finding with a place on the map.""" + def go(S): + change(S) + S["annotsv"]["data"]["rows"] = S["annotsv"]["data"]["rows"] + [ + {"sv": "chr5:1000000-1000300", "type": "INS", "gt": "0/1", "length": 300, "acmg_class": "4", + "genes": ["NKD2"], "caller": "Manta", "support": 1, "support_callers": ["Manta"]}] + return go + + def second_hit(S): """A second ClinVar row in the gene of the son's planted secondary finding, the card's counts kept.""" S["clinvar"]["data"]["hits"] = S["clinvar"]["data"]["hits"] + [dict( @@ -1559,7 +1582,8 @@ def second_hit(S): # a gene that a fixed-panel card's own finding names (the medicine card's CYP2D6 and NAT2) against one no card names, # a gene of the carrier list's two-hit line (SMN1) against one it does not name, a gene of the callability card's # list of genes not on this reference (GSTT1) or of hard genes not covered (NBPF1) against one not on it, a gene -# spelled like a word or unit the report prints (HR, MB) against one that is not, and one guessed address on two +# spelled like a word or unit the report prints (HR, MB) or like a code it prints (AR as an inheritance, INS as a +# structural-variant type, TG in a score's name) against one that is not, and one guessed address on two # pages where it is a closed result's id on one side only, loaded with the page and typed after it. GUESS = "#f/cpsr-brca1-c-68-69del" INFER = (("two SNV results in different genes (BRCA1, BRCA2)", cpsr("BRCA1"), cpsr("BRCA2")), @@ -1582,6 +1606,17 @@ def second_hit(S): ("a gene spelled like a word of the report (HR) against one that is not (BRCA2)", cpsr("HR"), cpsr("BRCA2")), ("a gene spelled like a unit of the report (MB) against one that is not (BRCA2)", cpsr("MB"), cpsr("BRCA2")), + ("an inheritance code (AR) against a gene (BRCA2), with recessive carrier rows", recessive(cpsr("AR")), + recessive(cpsr("BRCA2"))), + ("a deletion over AR and OPHN1 against one over BRCA1 and NBR2, with recessive carrier rows", + recessive(sv_over("chrX:67500000-67700000", ["AR", "OPHN1"], False)), + recessive(sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"], False))), + ("a deletion over AR and OPHN1 against one over BRCA1 and NBR2, each with a carrier row in its second gene " + "(OPHN1 has no place on the map, NBR2 has one)", recessive(sv_over("chrX:67500000-67700000", ["AR", "OPHN1"])), + recessive(sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"]))), + ("a structural-variant type (INS) against a gene (BRCA2), with an insertion over NKD2", insertion(cpsr("INS")), + insertion(cpsr("BRCA2"))), + ("a trait's abbreviation in a score's name (TG) against a gene (BRCA2)", cpsr("TG"), cpsr("BRCA2")), ("one guessed address, a closed result's id on one page only (BRCA1 against BRCA2 closed)", cpsr("BRCA1"), cpsr("BRCA2"), (GUESS, GUESS)), ("the same guessed address typed after the page drew (BRCA1 against BRCA2 closed)", cpsr("BRCA1"), @@ -1607,6 +1642,26 @@ def infer_problems(b, base, out, og, cases=INFER): return bad +DEEP_JS = """(function () { + var el = document.getElementById('f-closed-1'), r = el && el.getBoundingClientRect(); + var sec = el && el.closest('section.sec'), a = sec && document.querySelector('[data-toc="' + sec.id.slice(1) + '"]'); + return JSON.stringify({ found: !!el, y: Math.round(window.scrollY), inview: !!r && r.bottom > 0 && r.top < innerHeight, + current: a ? a.getAttribute('aria-current') : null }); +}())""" + + +def deep_link_problems(b, base): + """[problems]: on the son's page, #f/closed-1 and a guessed #f/ each scroll to the first closed result's card, + put it in view and mark its section in the contents.""" + bad = [] + for h in ("#f/closed-1", GUESS): + page_now(b, base, h) + got = json.loads(b.eval(DEEP_JS) or "{}") + if not (got.get("found") and got.get("y", 0) > 0 and got.get("inview") and got.get("current") == "true"): + bad.append(f"{h}: {got}") + return bad + + def sv_gene_problems(b, base, out, og): """[problems]: with a closed deletion over BRCA1 and NBR2, the page names NBR2 nowhere (a carrier row and its finding name only NBR2) before any click.""" @@ -1729,6 +1784,9 @@ def infer_checks(b, base, out): bad = infer_problems(b, base, out, og) check("browser: two pages of the son that differ only in which result is closed are the same before any click, " "HTML, attributes, address and map canvas (" + "; ".join(x[0] for x in INFER) + ")", not bad, bad) + bad = deep_link_problems(b, base) + check("browser: #f/closed-1 and a guessed #f/ scroll to the first closed result, in view, its section marked in " + "the contents", not bad, bad) bad = sv_gene_problems(b, base, out, og) check("browser: a closed deletion over two genes closes the rows that name only its second gene", not bad, bad) bad = panel_words_problems(b, base, out, og) @@ -1815,6 +1873,48 @@ def infer_checks(b, base, out): control("a veil that tags only a deletion's first gene", not sv_gene_problems(b, base, out, og)) finally: live_results.closable = first + cols, names, marks = live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names + try: + live_results.TABLE_COLUMNS = dict(cols, Inheritance=True) + control("a veil that reads a table's Inheritance column as naming a gene", + not infer_problems(b, base, out, og, [x for x in INFER if x[0].startswith("an inheritance code")])) + live_results.TABLE_COLUMNS = cols + live_results.finding_names = lambda f: (None, f"{f.get('gene') or ''} {f.get('detail') or ''} " + f"{f.get('title') or ''}") + control("a veil that reads a finding's title word by word", + not infer_problems(b, base, out, og, [x for x in INFER if "(INS)" in x[0]])) + live_results.finding_names = names + live_results.mark_names = lambda S, sp, m: (None, f"{m.get('label') or ''} {m.get('detail') or ''}") + control("a veil that reads a chart mark's label word by word", + not infer_problems(b, base, out, og, [x for x in INFER if "(TG)" in x[0]])) + finally: + live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names = cols, names, marks + og_map = build_live_demo.og_map + + def blind(og_, args, body): + """og map without results.veil.findings: it no longer knows which open findings a closed result ties.""" + d = json.loads(body) + (d["results"].get("veil") or {}).pop("findings", None) + return og_map(og_, args, json.dumps(d).encode()) + try: + build_live_demo.og_map = blind + control("og map that counts a finding a closed result ties among the open findings with no place", + not infer_problems(b, base, out, og, [x for x in INFER if "(OPHN1 has no place" in x[0]])) + finally: + build_live_demo.og_map = og_map + with open(js) as fh: + keep_js = fh.read() + route = "{ var hf = hashFinding(); target = document.getElementById(" + try: + if route not in keep_js: + check("control: person.js whose #f/closed- lands nowhere (the planted line is in person.js)", False, route) + else: + with open(js, "w") as fh: + fh.write(keep_js.replace(route, "{ var hf = null; h = 'f/nowhere'; target = document.getElementById(")) + control("person.js whose #f/closed- lands nowhere", not deep_link_problems(b, base)) + finally: + with open(js, "w") as fh: + fh.write(keep_js) print(f" infer checks: {time.time() - t0:.0f} s") diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 2562d3abb..dac6d93a7 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -586,6 +586,105 @@ def table_controls(): live_results.gene_words = first_words +# Every adapter is a fixed panel (live_results.FIXED_PANELS) or listed here as one whose rows say something about this +# genome alone: a new or renamed adapter turns this file red until someone decides which it is. +WANT_NOT_PANELS = ("qc", "callability", "variants", "clinvar", "clinical-filter", "cpsr", "slivar", "exomiser", + "structural-variants", "mito", "mosaic", "mosaic-mca", "carrier-list", "couple", "pharmcat", + "ancestry-fine", "mt-haplogroup", "y-chromosome", "archaic", "ancient", "roh", "prs", "telomere", + "viral", "consumer-export") + + +def table_columns(): + """Every column of every table() call in bin/live_results.py, read from its source.""" + import ast + with open(os.path.join(REPO, "bin", "live_results.py")) as fh: + tree = ast.parse(fh.read()) + return {e.value for n in ast.walk(tree) if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "table" + and n.args and isinstance(n.args[0], ast.List) for e in n.args[0].elts if isinstance(e, ast.Constant)} + + +def declared_problems(adapters=None): + """What the veil's tables leave undeclared: a table column not in TABLE_COLUMNS, a fixed-panel card that is no + adapter, an adapter in neither FIXED_PANELS nor WANT_NOT_PANELS.""" + adapters = live_results.ADAPTERS if adapters is None else adapters + out = [f"column {c!r} is not in TABLE_COLUMNS" for c in sorted(table_columns() - set(live_results.TABLE_COLUMNS))] + out += [f"FIXED_PANELS card {c} is no adapter" for c in live_results.FIXED_PANELS if c not in adapters] + out += [f"adapter {c} is neither a fixed panel nor in WANT_NOT_PANELS" for c in adapters + if c not in live_results.FIXED_PANELS and c not in WANT_NOT_PANELS] + return out + + +def vocabulary_summary(gene): + """A summary with two recessive carrier rows (inheritance AR), an insertion over NKD2 (type INS) and the score of + 'Triglycerides (TG)', plus a closed CPSR result in GENE.""" + car = [{"gene": g, "variant": v, "genotype": "het", "significance": "Pathogenic", "stars": 2, "inheritance": "AR", + "condition": c} for g, v, c in (("CFTR", "c.1521_1523del", "Cystic fibrosis"), + ("HBB", "c.20A>T", "Sickle cell disease"))] + return {"sections": { + "carrier": {"state": "ok", "data": {"genes_on_list": 2, "carriers": car}}, + "annotsv": {"state": "ok", "data": {"count": 1, "rows": [ + {"sv": "chr5:1000000-1000300", "type": "INS", "gt": "0/1", "length": 300, "acmg_class": "4", "caller": "Manta", + "genes": ["NKD2"], "support": 1, "support_callers": ["Manta"]}]}}, + "exomiser": {"state": "ok", "data": {"terms": [], "genes": [ + {"rank": 1, "gene": "GJB2", "moi": "AR", "exomiser_score": 0.9, "variants": [{"hgvs": "c.35del"}]}]}}, + "roh": {"state": "ok", "data": {"autosomal_over_5mb": []}}, + "prs": {"state": "ok", "data": {"adjusted": True, "scores": [ + {"pgs_id": "PGS000066", "condition": "Triglycerides (TG)", "percentile": "63.3"}]}}, + "cpsr": {"state": "ok", "data": {"plp": [{"gene": gene, "variant": "c.68_69del", "final": "Pathogenic"}]}}}} + + +def vocabulary_problems(): + """What is wrong with the veil's tags of the report's own vocabulary: a closed result in AR (an inheritance code), + INS (a structural-variant type) or TG (a trait's abbreviation in a score's name) tags nothing, while one in CFTR + tags the CFTR carrier row, one in GJB2 the symptom ranking's row and one in NKD2 the insertion's row, finding and + chart mark.""" + marks = {(sp["id"], m["id"]) for sp in rr.chart_specs(vocabulary_summary("AR")) for m in sp["marks"]} + out = [f"chart mark {m} not drawn" for m in (("genome", "sv-0"), ("prs", "pgs-PGS000066")) if m not in marks] + for g in ("AR", "INS", "TG"): + res = live_results.build("T", vocabulary_summary(g)) + v = {k: x for k, x in (res.get("veil") or {}).items() if k not in ("spans", "panels")} + v["findings"] = {k: t for k, t in (v.get("findings") or {}).items() if t != live_results.PANEL} + if any(v.values()): + out.append(f"a closed result in {g} tags {v}") + for g, where in (("CFTR", ("rows", "carrier-list")), ("GJB2", ("rows", "exomiser")), + ("NKD2", ("rows", "structural-variants")), ("NKD2", ("marks", "genome"))): + v = live_results.build("T", vocabulary_summary(g)).get("veil") or {} + if not any((v.get(where[0]) or {}).get(where[1]) or []): + out.append(f"a closed result in {g} leaves {where} untagged: {v.get(where[0])}") + v = live_results.build("T", vocabulary_summary("NKD2")).get("veil") or {} + sv = [k for k in (v.get("findings") or {}) if k.startswith("sv-")] + if not sv: + out.append(f"a closed result in NKD2 leaves the insertion's finding untagged: {v.get('findings')}") + return out + + +def vocabulary_controls(): + """Each guard of vocabulary_problems() dropped in turn: the check must fail every time.""" + cols, names, marks = live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names + try: + live_results.TABLE_COLUMNS = dict(cols, Inheritance=True) + control("TABLE_COLUMNS that reads the Inheritance column as naming a gene", not vocabulary_problems()) + live_results.TABLE_COLUMNS = dict(cols, Type=True) + control("TABLE_COLUMNS that reads the SV Type column as naming a gene", not vocabulary_problems()) + live_results.TABLE_COLUMNS = cols + live_results.finding_names = lambda f: (None, f"{f.get('gene') or ''} {f.get('detail') or ''} " + f"{f.get('title') or ''}") + control("a veil that reads a finding's title word by word", not vocabulary_problems()) + live_results.finding_names = names + live_results.mark_names = lambda S, sp, m: (None, f"{m.get('label') or ''} {m.get('detail') or ''}") + control("a veil that reads a chart mark's label word by word", not vocabulary_problems()) + finally: + live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names = cols, names, marks + try: + live_results.TABLE_COLUMNS = {k: x for k, x in cols.items() if k != "Inheritance"} + control("TABLE_COLUMNS without the Inheritance column", not declared_problems()) + finally: + live_results.TABLE_COLUMNS = cols + renamed = {("traits" if k == "traits-panel" else k): f for k, f in live_results.ADAPTERS.items()} + control("the traits-panel adapter renamed traits", not declared_problems(renamed)) + control("a new adapter nobody classified", not declared_problems(dict(live_results.ADAPTERS, newcard=None))) + + def overview_problems(): """What is wrong with the home page's view of a child's run (overview): a finding that names a closed result (a symptom ranking of its gene) is listed among the first findings, or the closed lead still names it.""" @@ -710,6 +809,13 @@ def tagged(sample_dir): check("8. every fixed-panel card, fixed line and gene-free card of the veil's tables closes as a panel or never, " "and only a gene symbol written as one names a gene", not bad, bad) table_controls() + bad = declared_problems() + check("8. every table column is declared as naming a gene or not (TABLE_COLUMNS), and every adapter as a fixed " + "panel or not", not bad, bad) + bad = vocabulary_problems() + check("8. the report's own codes (AR as an inheritance, INS as an SV type, TG in a score's name) tag nothing; a " + "row, finding or mark is tagged by its gene and place only", not bad, bad) + vocabulary_controls() bad = overview_problems() check("8. a child's home tile lists no finding that names a closed result, and its closed lead names none", not bad, bad) From 7585f763fb98526f86213ce138a0e4286c3dcb28 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:19:21 +0100 Subject: [PATCH 030/255] test(live): tie the map's planted rankings to the closed deletion by their genes, not their titles --- tests/test_live_demo.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index b4afd916f..00c6cac62 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -3036,13 +3036,16 @@ def second_open_problems(b, base, out, og): unplant(keep) -# A finding the server tags with a closed result because it names its gene (results.veil.findings: a symptom -# ranking), once with og's tag on its row too (CFTR, inside the deletion's span) and once with none (BRCA1). +# A finding the server tags with a closed result because it names its gene in its genes (results.veil.findings: +# an open structural variant over two genes), once with og's tag on its row too (CFTR, inside the deletion's span) +# and once with none (BRCA1). NAMED = (act("pl-x", "PLXGENE", "chr7:117400000-117700000", "Planted x deletion"), - {"id": "pl-q", "level": "review", "card": "exomiser", "gene": "CFTR", "title": "Symptom ranking: PLXGENE ranks 1", - "detail": "CFTR", "text": "A planted ranking that names the deletion's gene."}, - {"id": "pl-r", "level": "review", "card": "exomiser", "gene": "BRCA1", "title": "Symptom ranking: PLXGENE ranks 2", - "detail": "BRCA1", "text": "A planted ranking that names the deletion's gene."}) + {"id": "pl-q", "level": "review", "card": "exomiser", "gene": "CFTR", "genes": ["CFTR", "PLXGENE"], + "title": "Symptom ranking: PLXGENE ranks 1", "detail": "CFTR", + "text": "A planted ranking that names the deletion's gene."}, + {"id": "pl-r", "level": "review", "card": "exomiser", "gene": "BRCA1", "genes": ["BRCA1", "PLXGENE"], + "title": "Symptom ranking: PLXGENE ranks 2", "detail": "BRCA1", + "text": "A planted ranking that names the deletion's gene."}) # A row of the map with no finding in results.json (a newer finding the map holds before the page has it), # on the closed deletion's place and with its tag. GHOST = {"id": "pl-ghost", "level": "review", "gene": "GHOSTGENE", "title": "Ghost review", "href": "finding/pl-ghost/", From ab027bb093e2079c7e12bee22ac9b08b3571117f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:08:47 +0100 Subject: [PATCH 031/255] fix(live): tag each place by every closed result its fields name, and prose by the panel tag A place that named two closed results carried the first one's id only, so opening that one brought back a place naming the other. And prose was still read word by word, so the inheritance code AR on each two-hit line tied it to a closed result in AR. A tag is now the list of every closed result a place names, from its structured fields only; a line, fact, headline or finding with no such field is tagged PANEL unless its card is declared to print no gene there. --- bin/live_results.py | 448 ++++++++++++++++++++++++------------- tests/test_live_results.py | 323 +++++++++++++++++--------- 2 files changed, 517 insertions(+), 254 deletions(-) diff --git a/bin/live_results.py b/bin/live_results.py index a6a7157df..8975944e1 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -1099,55 +1099,75 @@ def screened(summary): return any((S.get(k) or {}).get("state") in ("ok", "stale") for k in ("clinvar", "clinical", "cpsr")) -def lead(found, levels, summary): +def lead(found, levels, summary, names=None): """The report's lead paragraph: [[text, bold]] segments (a third item "act" marks the names of the - findings to act on), plain sentences that name the findings by level. It never says a section is fine: with no finding of a level it says what was assessed.""" + findings to act on), plain sentences that name the findings by level. It never says a section is fine: with no finding of a level it says what was assessed. + NAMES, when a list, gets one entry per segment: the findings whose titles or genes the segment prints ([] for + one of the lead's own sentences, which print no gene). veil() tags each segment by them.""" S = summary.get("sections") or {} by = {lv: [f for f in found if f["level"] == lv] for lv in LEVELS} - out = [] + out, named = [], [] + + def seg(text, bold, shows=(), act=False): + out.append([text, bold] + (["act"] if act else [])) + named.append(list(shows)) if by["trust"]: - out += [["Read the data-quality section first: ", True], [_names(by["trust"]) + ". ", False]] + seg("Read the data-quality section first: ", True) + seg(_names(by["trust"]) + ". ", False, by["trust"][:3]) if by["act"]: n = len(by["act"]) - out += [["Of everything this genome was screened for, ", False], - [_count(n, "finding should", "findings should") + " go to a doctor", True], - [": " + _names(by["act"]) + ". ", False, "act"]] # "act": the names a child's page keeps closed + seg("Of everything this genome was screened for, ", False) + seg(_count(n, "finding should", "findings should") + " go to a doctor", True) + seg(": " + _names(by["act"]) + ". ", False, by["act"][:3], act=True) # "act": the names a child's page keeps closed elif screened(summary): - out += [["No finding in what these steps assessed needs a doctor; ", True], - ["sections 8 and 9 say what they do not see. ", False]] + seg("No finding in what these steps assessed needs a doctor; ", True) + seg("sections 8 and 9 say what they do not see. ", False) else: # no clinical screen has a result: saying nothing needs a doctor would read as an all-clear - out += [["The clinical screens have no result yet; ", True], - ["section 9 lists the steps that did not run. ", False]] + seg("The clinical screens have no result yet; ", True) + seg("section 9 lists the steps that did not run. ", False) if by["medicine"]: genes = sorted({f["gene"] for f in by["medicine"] if f["gene"]}) - out += [[_count(len(by["medicine"]), "medicine result", "medicine results").capitalize(), True], - [f" bear{'s' if len(by['medicine']) == 1 else ''} on how drugs work ({', '.join(genes)}): tell whoever prescribes. ", False]] + seg(_count(len(by["medicine"]), "medicine result", "medicine results").capitalize(), True) + seg(f" bear{'s' if len(by['medicine']) == 1 else ''} on how drugs work ({', '.join(genes)}): tell whoever prescribes. ", + False, by["medicine"]) if by["carrier"]: - out += [[f"A carrier of {_names(by['carrier'], 2)}", True], - [": carriers are not affected; it matters when a partner carries the same gene. ", False]] + seg(f"A carrier of {_names(by['carrier'], 2)}", True, by["carrier"][:2]) + seg(": carriers are not affected; it matters when a partner carries the same gene. ", False) if by["review"]: - out += [[_count(len(by["review"]), "result is", "results are").capitalize() + " worth a look", True], - [": " + _names(by["review"]) + ". ", False]] + seg(_count(len(by["review"]), "result is", "results are").capitalize() + " worth a look", True) + seg(": " + _names(by["review"]) + ". ", False, by["review"][:3]) cov = _data(S, "coverage").get("mean_depth") q = _data(S, "sample_qc") checked = q.get("contamination") == "ok" and q.get("sex_check") == "ok" # both ran and raised nothing if not by["trust"] and isinstance(cov, (int, float)): - out.append([f"Mean depth {cov:.1f}×" + ("; the identity and contamination checks raised no flag." if checked - else "; the identity and contamination checks have no result here: section 5 says why."), False]) + seg(f"Mean depth {cov:.1f}×" + ("; the identity and contamination checks raised no flag." if checked + else "; the identity and contamination checks have no result here: section 5 says why."), False) if out: out[-1][0] = out[-1][0].rstrip() + if isinstance(names, list): + names[:] = named return out # --- the veil's tags ----------------------------------------------------------------------------- # A child's page keeps each result to act on closed (assets/live/person.js veiled()). The page closes a -# finding's own card with it, but other places can name the same gene or variant: any card's table and -# lines, the charts' marks and their titles, another finding (a symptom ranking), the lead, the search index. Each such row carries the id of the -# result it belongs to, so the page closes it with that result and opens it with it. The rule: a row -# belongs to a closed result when it names the result's gene in a field that names a gene (TABLE_COLUMNS, -# finding_names(), mark_names(); words only in prose that has no such field), or its position falls inside the result's -# position or span; a row that is the result's own position or span belongs to that result first, else -# the first such result in finding order. The veil lives in the browser: the files still hold every row. +# finding's own card with it, but other places can name the same gene or variant: any card's table, lines, facts +# and headline, the charts' marks, another finding (a symptom ranking), the lead, the search index. Each such place +# carries a tag: the SET of the closed results it names, as a list of their ids. The page keeps the place closed +# while ANY of them is closed, so a place that names two closed results never comes back with the first one +# opened (person.js tagClosed()). The veil lives in the browser: the files still hold every row. +# A place is tagged from its structured fields only, never by reading its words as gene symbols: a report prints +# codes in capitals (AR as an inheritance, INS as a structural-variant type, TG in a score's name) that are gene +# symbols too, so a word rule would close a place for a result it does not name, and what closes would tell the +# gene. The structured fields: a row's gene and place cells (TABLE_COLUMNS), a finding's gene, genes and loci +# (finding_names()), a chart mark's summary row (mark_names()), a two-hit line's gene (two_hit_names()), a +# drawing's gene, the findings a lead segment names (lead()). A place belongs to a closed result when it names one +# of the result's genes in such a field, or its position falls inside the result's position or span. +# Prose has no such field: a line, a fact, a headline or a finding that has none cannot be shown to be free of a +# closed result's name. So it is tagged PANEL (closed while any result is closed, the same whatever the result), +# unless it is on one of the lists below of prose that never names a gene (LINES_NO_GENE, FACTS_NO_GENE, +# HEADLINES_NO_GENE, NAMES_NO_GENE), each pinned in tests/test_live_results.py. The cost: a child's page with a +# result closed shows less prose until every result is opened. # The cards whose rows are a fixed panel: the same genes or loci for everyone, known before the run. A row or # dot of one that closes with a result would tell which gene the result is in by what is missing, so a child's # page closes each of these cards whole while any result to act on is closed, behind one box with the same words @@ -1169,43 +1189,83 @@ def lead(found, levels, summary): } # The lines and table rows of a card that are a fixed panel inside it (card -> the words such a line, or a cell of # such a row, starts with): the ClinVar card prints one line per known deep intronic site of -# assets/deep_intronic_sites.tsv, hit or not; the carrier list prints its two-hit line, which names SMN1, HBA1 and -# HBA2 as the callers it read, whatever was found; the callability card lists the catalog genes this reference does +# assets/deep_intronic_sites.tsv, hit or not; the callability card lists the catalog genes this reference does # not hold, the same for every genome on it, and the hard genes the reads did not cover, which are nearly the same -# for every genome (segmental duplications, collapsed regions). veil() tags each of them PANEL, never by a gene, so they close all -# together while any result to act on is closed and open all together once the last one is opened -# (assets/live/person.js tagClosed()). Every other line or row says something about this genome alone. +# for every genome (segmental duplications, collapsed regions); the runs-of-homozygosity card names the imprinted +# regions of assets/imprinted_regions.tsv (their genes) on a chromosome a long run covers. (The carrier list's line of +# the two-hit callers, which names SMN1, HBA1 and HBA2, is prose of a card whose lines are not declared gene-free: +# PANEL as well.) veil() tags each of them +# PANEL, never by a gene, so they close all together while any result to act on is closed and open all together +# once the last one is opened (assets/live/person.js tagClosed()). Every other line or row says something about +# this genome alone. FIXED_LINES = {"clinvar": ("Known deep intronic site (always reported):",), - "carrier-list": ("Two hits across callers",), - "callability": ("not on this reference", "not read")} + "callability": ("not on this reference", "not read"), + "roh": ("Imprinted ",)} PANEL = "*panel*" # The cards whose rows, lines and facts name no gene, only things known before the run: veil() never tags them. The # score card names its scores and their traits (assets/pgs_scores.tsv) and its words "OR 1.57", "HR 1.4", "(TG)" # are odds and hazard ratios and a trait, though HR and TG are gene symbols too. NAMES_NO_GENE = {"prs": "the polygenic scores of assets/pgs_scores.tsv"} -# Every column of every card's table (the adapters' table() calls), and whether it names a gene or a place (True) -# or is the report's own vocabulary (False): an inheritance code (AR), a structural-variant type (INS), a genotype, -# a class, a significance, a unit, a caller, a condition. veil() tags a table row by its gene and place cells only, -# because any code a report prints in capitals may also be a gene symbol: a closed result in AR would else close -# every recessive carrier row, and one in INS every insertion. A column missing here counts as naming a gene; +# The cards whose lines never name a gene (card -> what they print), FIXED_LINES aside: veil() tags their lines by +# the places they print only. Every other card's line is prose that can name one, and is tagged PANEL. +LINES_NO_GENE = { + "qc": "the sex chromosomes' karyotype and copy numbers (render_report.karyotype_lines)", + "clinvar": "where the frequencies came from, and the places of the cDNA scan", + "mosaic": "the loss of an X or the Y, from depth and from allele fractions", + "mosaic-mca": "the cell-fraction floor, and each call's type, arm and place", + "roh": "F_ROH, its band, and the chromosome a long run covers", + "telomere": "the telomeric read counts", + "consumer-export": "the genotype counts and the file names", +} +# The cards whose facts never name a gene (card -> what they print): counts, states, versions, haplogroups, a +# partner's sample name. FACT_NAMES overrides one fact of such a card; every fact of a card on neither list is +# tagged PANEL (the fixed-panel cards' facts, closed with their card anyway). +FACTS_NO_GENE = { + "qc": "depth, read counts, sex, contamination, a matching sample", + "callability": "counts", "variants": "counts", "clinvar": "counts and the ClinVar file date", + "clinical-filter": "counts", "cpsr": "CPSR's counts", "slivar": "counts", + "exomiser": "the symptoms searched (HPO terms)", "structural-variants": "the callers' counts", + "mito": "counts", "mosaic": "the call", "mosaic-mca": "the status and counts", + "carrier-list": "counts", "couple": "the partner's sample name and counts", "pharmcat": "the report's version", + "ancestry-fine": "the groups' shares", "mt-haplogroup": "the haplogroup", "y-chromosome": "the haplogroups", + "archaic": "shares", "ancient": "counts", "roh": "lengths and F_ROH", + "telomere": "the telomere content", "viral": "the HHV-6 call and counts", "consumer-export": "counts", +} +# The facts of a FACTS_NO_GENE card that name genes: (card, label) -> the summary field that lists them, or None +# for prose (PANEL). +FACT_NAMES = {("carrier-list", "Genes with two hits"): "two_hits", + ("clinical-filter", "High-impact rare variants in its genes"): None} +# The summary keys whose headline (bin/owngenome_live.py HEADLINES) never names a gene (key -> what it prints). A +# card whose headline comes from another key (the couple screen's, which lists the shared genes) gets PANEL. +HEADLINES_NO_GENE = { + "clinvar": "the P/LP count", "carrier": "the carrier list's size and count", "roh": "F_ROH or the chromosome a run covers", + "pypgx": "CYP2D6's depth check", "annotsv": "the class 4 and 5 counts", "viral": "the HHV-6 call", + "archaic": "the Neanderthal share", "lpa": "the KIV-2 count", "cpsr": "the P/LP count and the ClinVar releases", + "mito": "the poly-C or ClinVar count", +} +# Every column of every card's table (the adapters' table() calls), and what it holds: a gene ("gene": a gene +# symbol, or a list of them), a place ("place": its loci are read, never its words) or the report's own vocabulary +# (None: an inheritance code such as AR, a structural-variant type such as INS, a genotype, a class, a +# significance, a unit, a caller, a condition), never read. A column missing here counts as a gene column; # tests/test_live_results.py fails until every column of every table() call is listed. +G, P = "gene", "place" TABLE_COLUMNS = { - "Gene": True, "Genes": True, "Variant": True, "Where": True, "Locus": True, "Site": True, "Known site": True, - "Change": True, "Chromosome": True, "Start": True, "End": True, - "Status": False, "Bases read": False, "Why": False, "Genotype": False, "Significance": False, "Stars": False, - "Frequency": False, "Call": False, "Kind": False, "Rule": False, "Final class": False, "ClinVar": False, - "Rank": False, "Inheritance": False, "Score": False, "ACMG": False, "Disease": False, "Type": False, - "Length": False, "Class": False, "Support": False, "Checks": False, "Element": False, "Condition": False, - "Depth": False, "Repeats": False, "Stranger status": False, "Normal up to": False, "Pathologic from": False, - "95% interval": False, "Allele fraction": False, "VAF": False, "Alt reads": False, "Length (Mb)": False, - "Cell fraction": False, "Result": False, "Chance for a child": False, "This person": False, "Partner": False, - "Copies": False, "Detail": False, "Diplotype": False, "Phenotype (CPIC's words)": False, "Caller": False, - "CYP2D6": False, "Alleles (T1K quality)": False, "Panel haplotypes": False, "Quality": False, "Filter": False, - "Primary caller": False, "Level": False, "Group": False, "Share": False, "Bootstrap range": False, - "Branch": False, "Derived": False, "Ancestral": False, "No call": False, "Dates": False, "Country": False, - "Distance": False, "Trait": False, "Reading": False, "Percentile": False, "Variants matched": False, - "Effect": False, "System": False, "Predicted phenotype": False, "Confidence": False, "Virus": False, - "Reads": False, "Breadth": False, "Copies per cell": False, "Sizing": False, "Intermediate zone": False, + "Gene": G, "Genes": G, "Locus": G, "Variant": P, "Where": P, "Site": P, "Known site": P, "Change": P, + "Chromosome": P, "Start": P, "End": P, + "Status": None, "Bases read": None, "Why": None, "Genotype": None, "Significance": None, "Stars": None, + "Frequency": None, "Call": None, "Kind": None, "Rule": None, "Final class": None, "ClinVar": None, + "Rank": None, "Inheritance": None, "Score": None, "ACMG": None, "Disease": None, "Type": None, + "Length": None, "Class": None, "Support": None, "Checks": None, "Element": None, "Condition": None, + "Depth": None, "Repeats": None, "Stranger status": None, "Normal up to": None, "Pathologic from": None, + "95% interval": None, "Allele fraction": None, "VAF": None, "Alt reads": None, "Length (Mb)": None, + "Cell fraction": None, "Result": None, "Chance for a child": None, "This person": None, "Partner": None, + "Copies": None, "Detail": None, "Diplotype": None, "Phenotype (CPIC's words)": None, "Caller": None, + "CYP2D6": None, "Alleles (T1K quality)": None, "Panel haplotypes": None, "Quality": None, "Filter": None, + "Primary caller": None, "Level": None, "Group": None, "Share": None, "Bootstrap range": None, + "Branch": None, "Derived": None, "Ancestral": None, "No call": None, "Dates": None, "Country": None, + "Distance": None, "Trait": None, "Reading": None, "Percentile": None, "Variants matched": None, + "Effect": None, "System": None, "Predicted phenotype": None, "Confidence": None, "Virus": None, + "Reads": None, "Breadth": None, "Copies per cell": None, "Sizing": None, "Intermediate zone": None, } @@ -1218,29 +1278,20 @@ def panel_finding(f): def fixed_line(card_id, text): """Whether TEXT, a line of the card CARD_ID or a cell of one of its rows, is one of its fixed lines (FIXED_LINES).""" return any(str(text).startswith(w) for w in FIXED_LINES.get(card_id) or ()) + + LOCUS_RE = re.compile(r"(?= a for lc, ls, le in locs): - return fid - return None + if fid in out: + continue + if (gene and gene in words) or (c and any(lc == c and ls <= b and le >= a for lc, ls, le in locs)): + out.append(fid) + return out or None + + +def veil_id(spans, text="", chrom=None, pos=None, end=None, genes=None): + """veil_tags() for a caller that has the genes a place names (GENES, from its own gene field; TEXT then gives its + loci). A place given as TEXT alone is prose: nothing proves it names no closed result, so it is tagged PANEL while + there is one to close (bin/results_index.py, whose search hits quote the summary's rows).""" + if genes is None: + return [PANEL] if spans else None + return veil_tags(spans, genes, text, chrom, pos, end) + + +def union(*tags): + """One tag naming every id of TAGS (each a tag or None), in order; None for none.""" + out = [] + for t in tags: + for x in t or (): + if x not in out: + out.append(x) + return out or None -def row_naming_cells(columns, row): - """The cells of a table row under a column that names a gene or a place (TABLE_COLUMNS): the only cells veil() - reads a row's genes and loci from.""" - if not isinstance(row, (list, tuple)): - return [row] +def row_tag(spans, columns, row, genes=()): + """A table row's tag, from its gene and place cells (TABLE_COLUMNS) and GENES its summary row names.""" + cells = list(row) if isinstance(row, (list, tuple)) else [row] cols = list(columns or []) - return [x for i, x in enumerate(row) if i >= len(cols) or TABLE_COLUMNS.get(cols[i], True)] + names, where, at = list(genes or ()), [], {} + for i, x in enumerate(cells): + kind = TABLE_COLUMNS.get(cols[i], G) if i < len(cols) else G + if kind == G: + names += gene_cell(x) + elif kind == P: + where.append(str(x)) + if i < len(cols) and cols[i] in ("Chromosome", "Start", "End"): + at[cols[i]] = x + return veil_tags(spans, names, " ".join(where), at.get("Chromosome"), at.get("Start"), at.get("End")) def finding_names(f): """(genes, loci text) a finding is tagged by: a finding with a gene field is known by its gene, its genes (a structural variant's every gene) and the loci of its viewer, detail and evidence, never by its title's words (an - SV type such as INS, a condition); one with no gene but a place (a mosaic call) by that place alone; genes None - for one with neither (a card's own headline, a data-quality finding): its title and detail are read word by - word, the only words it has.""" + SV type such as INS, a condition); one with no gene but a place (a mosaic call) by that place alone; None for one + with neither (a card's own headline, a data-quality finding, a row that could not be read): prose_finding().""" where = " ".join(str(x) for x in [f.get("viewer"), f.get("detail")] + [b for _a, b in f.get("evidence") or []] if x not in (None, "")) genes = [g for g in list(f.get("genes") or []) + [f.get("gene")] if g] if genes or _loci(where): return genes, where - return None, f"{f.get('gene') or ''} {f.get('detail') or ''} {f.get('title') or ''} {f.get('viewer') or ''}" + return None + + +def two_hit_names(S): + """{line: (gene, loci text)} of the carrier list's two-hit lines, each from its row of carrier.two_hits.genes.""" + th = _data(S, "carrier").get("two_hits") + out = {} + if not isinstance(th, dict): + return out + for x in _items(th.get("genes")): + if not isinstance(x, dict): + continue + try: + ln = rr.two_hit_lines({"genes": [x], "deletions_read": [], "deletions_not_read": []})[0] + except (KeyError, TypeError, ValueError, AttributeError): + continue + out[plain(ln)] = (x.get("gene"), f"{x.get('deletion') or ''} {x.get('variant') or ''}") + return out + + +def headline_free(card): + """Whether CARD's headline names no gene: it comes from a key of HEADLINES_NO_GENE, or from no headline helper.""" + try: + import owngenome_live + helpers = owngenome_live.HEADLINES + except Exception: # without the helpers no key is known to name no gene + helpers = {} + return all(k in HEADLINES_NO_GENE or k not in helpers for k in card.get("keys") or []) def mark_names(S, sp, m): @@ -1353,60 +1454,92 @@ def mark_names(S, sp, m): return [], "" -def veil(summary, found, chapters, lead_parts=()): - """The rows a child's page closes with a result to act on, by the result's id: each card's table rows - (rows: card -> one id or None per row; the ClinVar card's by each hit's own position), each card's viewer - links (viewer: card -> one per link), each card's lines (lines: card -> one per line; a line of a fixed - panel, such as a known deep intronic site, printed whether hit or not, is tagged PANEL: FIXED_LINES), the - marks of the report's charts (marks: chart -> {mark id: id}), every other finding that names a closed result (findings: finding id -> id; a symptom ranking can - rank the gene; PANEL for every finding of a FIXED_PANELS card) and the lead's segments that name one or name a - fixed-panel card's finding (lead: one per segment of LEAD_PARTS, the "act" names left out: the page closes those - itself), each card's facts (facts: card -> one per fact) and the items - of its drawing (widget: card -> one per CHIP call or copy-number dot), and the cards of a fixed panel - (panels: FIXED_PANELS, which close whole while any result is closed). spans is closable() itself: the family - page (assets/live/family-page.js) reads which results are closed from it, and closes every place of the child - that could carry one while one is. Empty when nothing can be closed.""" +def veil(summary, found, chapters, lead_parts=(), lead_names=()): + """Each place a child's page closes with a result to act on, with its tag (veil_tags(): the ids of every closed + result it names, or PANEL): each card's table rows (rows: card -> one tag or None per row), viewer links + (viewer), lines (lines; a line of a fixed panel, such as a known deep intronic site, printed whether hit or not, + is PANEL: FIXED_LINES), facts (facts), headline (headline: card -> tag) and the items of its drawing (widget: card + -> one per CHIP call or copy-number dot), the marks of the report's charts (marks: chart -> {mark id: tag}), every + finding not to act on that names a closed result (findings: finding id -> tag; PANEL for every finding of a + FIXED_PANELS card), the lead's segments (lead: one per segment of LEAD_PARTS, from the findings LEAD_NAMES says + each prints; the "act" names left out: the page closes those itself), and the cards of a fixed panel (panels: + FIXED_PANELS, which close whole while any result is closed). spans is closable() itself: the family page + (assets/live/family-page.js) reads which results are closed from it, and closes every place of the child that + could carry one while one is. Empty when nothing can be closed.""" spans = closable(found) if not spans: return {} - out = {"rows": {}, "viewer": {}, "lines": {}, "facts": {}, "widget": {}, "marks": {}, "findings": {}, "lead": [], - "spans": [list(s) for s in spans], + out = {"rows": {}, "viewer": {}, "lines": {}, "facts": {}, "headline": {}, "widget": {}, "marks": {}, + "findings": {}, "lead": [], "spans": [list(s) for s in spans], "panels": [c["id"] for ch in chapters for c in ch["sections"] if c["id"] in FIXED_PANELS and c.get("state") != "missing"]} S = summary.get("sections") or {} hits = [h for h in _items(_data(S, "clinvar").get("hits"))[:50]] - tags = [veil_id(spans, str(h.get("gene") or ""), h.get("chrom"), h.get("pos")) if isinstance(h, dict) else None + tags = [veil_tags(spans, [h.get("gene")], "", h.get("chrom"), h.get("pos")) if isinstance(h, dict) else None for h in hits] if any(tags): out["rows"]["clinvar"] = tags + sv_rows = [x for x in _items(_data(S, "annotsv").get("rows"))] + two = two_hit_names(S) + th = _data(S, "carrier").get("two_hits") + two_genes = [x.get("gene") if isinstance(x, dict) else x for x in _items((th or {}).get("genes"))] \ + if isinstance(th, dict) else [] + cards = {} for ch in chapters: for c in ch["sections"]: + cards[c["id"]] = c if c["id"] in NAMES_NO_GENE: continue rows = (c.get("table") or {}).get("rows") or [] if c["id"] != "clinvar" and rows: cols = (c.get("table") or {}).get("columns") - tags = [PANEL if any(fixed_line(c["id"], x) for x in (r if isinstance(r, (list, tuple)) else [r])) - else veil_id(spans, " ".join(str(x) for x in row_naming_cells(cols, r))) - for r in rows] + tags = [] + for i, r in enumerate(rows): + if any(fixed_line(c["id"], x) for x in (r if isinstance(r, (list, tuple)) else [r])): + tags.append([PANEL]) + continue + more = [] + if c["id"] == "structural-variants" and i < len(sv_rows) and isinstance(sv_rows[i], dict): + more = [g for g in sv_rows[i].get("genes") or [] if g] # the whole list the cell shortens + tags.append(row_tag(spans, cols, r, more)) if any(tags): out["rows"][c["id"]] = tags - tags = [veil_id(spans, v) for v in c.get("viewer") or []] + tags = [veil_tags(spans, (), v) for v in c.get("viewer") or []] if c["id"] == "clinvar": # one link per row: a row closed by its gene alone closes its link too rt = out["rows"].get("clinvar") or [] - tags = [(rt[i] if i < len(rt) else None) or t for i, t in enumerate(tags)] + tags = [union(rt[i] if i < len(rt) else None, t) for i, t in enumerate(tags)] if any(tags): out["viewer"][c["id"]] = tags - tags = [PANEL if fixed_line(c["id"], ln) else veil_id(spans, ln) for ln in c.get("lines") or []] + tags = [] + for ln in c.get("lines") or []: + if fixed_line(c["id"], ln): + tags.append([PANEL]) + elif c["id"] == "carrier-list" and ln in two: + tags.append(veil_tags(spans, [two[ln][0]], two[ln][1])) + elif c["id"] in LINES_NO_GENE: + tags.append(veil_tags(spans, (), ln)) + else: + tags.append([PANEL]) if any(tags): out["lines"][c["id"]] = tags - tags = [veil_id(spans, " ".join(str(x) for x in f)) for f in c.get("facts") or []] + tags = [] + for f in c.get("facts") or []: + label = f[0] if isinstance(f, (list, tuple)) and f else f + rule = FACT_NAMES.get((c["id"], label), "") + if rule == "two_hits": + tags.append(veil_tags(spans, two_genes)) + elif rule is None or c["id"] not in FACTS_NO_GENE: + tags.append([PANEL]) + else: + tags.append(None) if any(tags): out["facts"][c["id"]] = tags - # A drawing's items: the CHIP calls (gene and change) and the copy-number dots (gene). The CHIP + if c.get("headline") and not headline_free(c): + out["headline"][c["id"]] = [PANEL] + # A drawing's items: the CHIP calls (gene) and the copy-number dots (gene), by their gene field. The CHIP # chart's list of genes with their detection limits is the same panel for everyone: never tagged. w = c.get("widget") or {} items = w.get("calls") if w.get("kind") == "vaf" else w.get("genes") if w.get("kind") == "copies" else None - tags = [veil_id(spans, f"{x.get('gene') or ''} {x.get('change') or ''}") for x in items or []] + tags = [veil_tags(spans, gene_cell(x.get("gene"))) for x in items or []] if any(tags): out["widget"][c["id"]] = tags try: @@ -1418,35 +1551,45 @@ def veil(summary, found, chapters, lead_parts=()): if m.get("kind") == "roh": # a run of homozygosity is no result's source continue genes, where = mark_names(S, sp, m) - fid = veil_id(spans, where, m.get("chrom"), m.get("pos"), m.get("end"), genes=genes) - if fid: - out["marks"].setdefault(sp["id"], {})[m["id"]] = fid + t = veil_tags(spans, genes, where, m.get("chrom"), m.get("pos"), m.get("end")) + if t: + out["marks"].setdefault(sp["id"], {})[m["id"]] = t # A finding of a fixed-panel card (a medicine gene, a CHIP call, a repeat) is tagged PANEL whatever it names: it # closes with its card while any result to act on is closed, so the findings left open never tell which gene the - # closed one is in. So is each segment of the lead that names one (its gene or its title). - panel_found = [f for f in found if panel_finding(f)] - panel_spans = [(PANEL, str(g).upper(), None, None, None) for f in panel_found for g in [f.get("gene")] if g] + # closed one is in. A finding with no gene and no place is prose (prose_finding()). for f in found: - if f.get("level") != "act": - genes, where = finding_names(f) - fid = PANEL if f in panel_found else veil_id(spans, where, genes=genes) - if fid: - out["findings"][f["id"]] = fid - - # A segment that holds findings' titles is tagged as those findings are, never by its words; the others are - # the lead's own sentences. - def lead_tag(t): - if veil_id(panel_spans, t) or any(f.get("title") and f["title"] in t for f in panel_found): - return PANEL - named = [f for f in found if f.get("level") != "act" and f.get("title") and f["title"] in t] - if named: - return next((out["findings"][f["id"]] for f in named if f["id"] in out["findings"]), None) - return veil_id(spans, t) - tags = [None if len(x) > 2 else lead_tag(x[0]) for x in lead_parts or ()] + if f.get("level") == "act": + continue + if panel_finding(f): + t = [PANEL] + else: + got = finding_names(f) + t = veil_tags(spans, got[0], got[1]) if got else prose_finding(f, cards.get(f.get("card")) or {}) + if t: + out["findings"][f["id"]] = t + # A lead segment is tagged by the findings it prints (lead()): every closed result one of them names. The + # lead's own sentences print no gene. + tags = [] + for i, x in enumerate(lead_parts or ()): + shows = list(lead_names[i]) if i < len(lead_names or ()) else [] + tags.append(None if len(x) > 2 else union(*[out["findings"].get(f["id"]) for f in shows if f.get("level") != "act"])) out["lead"] = tags if any(tags) else [] return {k: v for k, v in out.items() if v} +def prose_finding(f, card): + """The tag of finding F, which has no gene and no place: None when its words are known to name no gene (a + data-quality finding of its own sentences and a sample's name, a row that could not be read, or a card's + headline finding whose headline and facts name none), else PANEL.""" + cid = f.get("card") + if cid == "qc" or str(f.get("id") or "").startswith("unreadable-"): + return None + if f.get("id") == slug("card", cid) and cid in FACTS_NO_GENE and headline_free(card) \ + and not any((cid, a) in FACT_NAMES for a, _b in card.get("facts") or []): + return None + return [PANEL] + + def build(sample, summary, files=()): """results.json of one person from a summary.json; files: the run's paths the report links (for links).""" summary = summary or {} @@ -1461,7 +1604,8 @@ def build(sample, summary, files=()): person = {"sample": sample, "declared_sex": (summary.get("run") or {}).get("declared_sex"), "inferred_sex": sexc.get("inferred_sex"), "karyotype": sexc.get("karyotype")} found, levels, sections = journal(chapters, summary) - lead_parts = lead(found, levels, summary) + lead_names = [] + lead_parts = lead(found, levels, summary, lead_names) return {"schema_version": SCHEMA, "sample": sample, "generated_utc": summary.get("generated_utc"), "person": person, "chapters": chapters, "findings": found, "levels": levels, "journal": sections, "lead": lead_parts, @@ -1471,7 +1615,7 @@ def build(sample, summary, files=()): "not_assessed": [plain(x) for x in summary.get("not_assessed") or []], "not_run": [plain(x) for x in summary.get("not_run") or []], "new_since": summary.get("new_since") or None, - "charts": CHART_SECTION, "veil": veil(summary, found, chapters, lead_parts)} + "charts": CHART_SECTION, "veil": veil(summary, found, chapters, lead_parts, lead_names)} def attention_count(summary): @@ -1488,7 +1632,7 @@ def overview(summary): # A finding that names a result to act on (veil: a symptom ranking of its gene) is not listed: a child's tile # closes only the results to act on themselves, and the result it names already comes first. # A finding tagged PANEL (a fixed-panel card's own, veil()) names no result: it stays, the same for everyone. - named = {k: v for k, v in ((r.get("veil") or {}).get("findings") or {}).items() if v != PANEL} + named = {k: v for k, v in ((r.get("veil") or {}).get("findings") or {}).items() if any(x != PANEL for x in v)} top = sorted((f for f in r["findings"] if f["level"] != "info" and f["id"] not in named), key=lambda f: order[f["level"]])[:4] tags = (r.get("veil") or {}).get("lead") or [] diff --git a/tests/test_live_results.py b/tests/test_live_results.py index dac6d93a7..6892e626f 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -32,33 +32,39 @@ report as sentences (never zero counts), an unsized "normal" locus as not sized, AnnotSV's exon-edge check, a genotype rule and a deep intronic call; 8. the veil's tags on DEMO-003 (make_demo_sample.py --sample DEMO-003, with its - planted secondary finding and class 5 deletion): the ClinVar row and viewer - link of the planted hit, its genome-chart tick and its review-star bar carry - the secondary finding's id, the deletion's span carries its own, the search - index's items that name the gene carry one of them, and nothing else is - tagged; DEMO-001, with no result to act on, has no tag; and every known - deep intronic site's line carries the fixed-panel tag, never the id of a - closed result in its gene (the card prints each site whether hit or not, - so one missing line would name the gene), a gene - named before a full stop is that gene, and a ClinVar row closed by its gene - alone closes its "See the reads at" link with it; every other card's row - that names the gene (the son's symptom ranking ranks it) carries one of - the ids and no other row does, the ranking's finding carries the secondary - finding's id, and so does a segment of the lead that names the gene; the - home page's view of the run (overview) lists no such finding among its - first ones and its closed lead names none; a CHIP call, a copy-number dot - and a fact that name a closed result's gene carry its id (the CHIP chart's - panel of genes never does), and results.json carries the closed results as - veil.spans (the family page closes a child's places while one is closed); - a closed structural variant over two genes closes the rows and the finding - that name only its second gene, and veil.panels lists the cards of a fixed - panel (live_results.FIXED_PANELS). + planted secondary finding and class 5 deletion): a tag is the list of every + closed result a place names; the ClinVar row and viewer link of the planted + hit, its genome-chart tick and its review-star bar carry the secondary + finding's id, the deletion's span its own, every search item the fixed-panel + tag (a hit quotes the summary's rows, prose), and nothing else is tagged; + DEMO-001, with no result to act on, has no tag; every known deep intronic + site's line carries the fixed-panel tag, never the id of a closed result in + its gene (the card prints each site whether hit or not, so one missing line + would name the gene); prose (a line, a fact, a headline) is tagged by the + fixed-panel tag whatever it names, unless its card is declared as printing + no gene there (LINES_NO_GENE, FACTS_NO_GENE, HEADLINES_NO_GENE), so no word + is ever read as a gene; a gene cell is read as its list of symbols; a + ClinVar row closed by its gene alone closes its "See the reads at" link with + it; every other card's row that names the gene (the son's symptom ranking + ranks it) carries one of the ids and no other row does, the ranking's + finding carries the secondary finding's id, and so does a segment of the + lead that prints the ranking, and a segment that prints two closed results' + findings carries both ids; the home page's view of the run (overview) lists + no such finding among its first ones and its closed lead names none; a CHIP + call, a copy-number dot and the carrier list's two-hit fact and line carry + the id of the closed result in their gene (the CHIP chart's panel of genes + never does), and results.json carries the closed results as veil.spans (the + family page closes a child's places while one is closed); a closed + structural variant over two genes closes the rows and the finding that name + only its second gene, a place over two closed results carries both ids, and + veil.panels lists the cards of a fixed panel (live_results.FIXED_PANELS). Each check runs again on a planted mistake and must fail there: 1 with VNTYPER's entry dropped, 2 with one card's caveat blanked, 3 with "ok" planted in a card and "All clear" in a script, 4 with the hits emptied, 5 with the results route registered without the login, 6 with a row printed as its Python dict, 7 with the Slivar adapter returning nothing, -8 with a veil rule that tags nothing (rows, lines, facts and drawings), one that tags every row and one that tags -only a structural variant's first gene. A check +8 with a veil rule that tags nothing (rows, lines, facts and drawings), one that tags every row, one that tags +only a structural variant's first gene, one that keeps only the first id of a place, one that reads prose word by +word, and each entry of the declared lists dropped. A check that cannot fail is no check. Run: python3 tests/test_live_results.py @@ -368,19 +374,21 @@ def veil_problems(res, idx): v = res.get("veil") or {} out = [] card = next(x for ch in res["chapters"] for x in ch["sections"] if x["id"] == "clinvar") + # The planted hit sits inside the planted deletion and names its gene: its places name both closed results. + both = sorted([sf, sv]) rows = (v.get("rows") or {}).get("clinvar") or [] - want = [sf if r[0] == gene and r[1].startswith(f"{c}:{p} ") else None for r in card["table"]["rows"]] - if rows != want: + want = [both if r[0] == gene and r[1].startswith(f"{c}:{p} ") else None for r in card["table"]["rows"]] + if [sorted(t) if t else None for t in rows] != want: out.append(f"ClinVar rows tagged {rows}, want {want}") view = (v.get("viewer") or {}).get("clinvar") or [] - if view != [sf if x == f"{c}:{p}" else None for x in card.get("viewer") or []]: + if [sorted(t) if t else None for t in view] != [both if x == f"{c}:{p}" else None for x in card.get("viewer") or []]: out.append(f"ClinVar viewer links tagged {view}") - marks = v.get("marks") or {} + marks = {k: {m: sorted(t) for m, t in x.items()} for k, x in (v.get("marks") or {}).items()} specs = {sp["id"]: sp for sp in rr.chart_specs(res["_summary"])} hit = [m["id"] for m in specs["genome"]["marks"] if m.get("kind") == "hit" and m["pos"] == p and m["chrom"] == c] span = [m["id"] for m in specs["genome"]["marks"] if m.get("kind") == "sv" and (m["chrom"], m["pos"], m["end"]) == (sv_c, a, b)] bar = [m["id"] for m in specs["clinvar-stars"]["marks"] if gene in (m.get("detail") or "")] - want = {"genome": {**{h: sf for h in hit}, **{s_: sv for s_ in span}}, "clinvar-stars": {x: sf for x in bar}} + want = {"genome": {**{h: both for h in hit}, **{s_: both for s_ in span}}, "clinvar-stars": {x: both for x in bar}} if not hit or not span or not bar or marks != want: out.append(f"chart marks tagged {marks}, want {want}") # Every other card's rows: a row that names the gene carries one of the two ids, a row that does not carries @@ -389,19 +397,20 @@ def veil_problems(res, idx): if cid == "clinvar" or not rows_: continue got = (v.get("rows") or {}).get(cid) or [None] * len(rows_) - bad = [(r, t) for r, t in zip(rows_, got) if (t in (sf, sv)) != (gene in " ".join(map(str, r)))] + bad = [(r, t) for r, t in zip(rows_, got) if bool(set(t or ()) & {sf, sv}) != (gene in " ".join(map(str, r)))] if bad or len(got) != len(rows_): out.append(f"{cid} rows tagged wrong: {bad or got}") exo = (v.get("rows") or {}).get("exomiser") or [] ranked = [f["id"] for f in res["findings"] if f["level"] != "act" and gene in f["title"]] # A fixed-panel card's own findings (a medicine gene) carry the panel tag, whatever the closed result. - named = {k: t for k, t in (v.get("findings") or {}).items() if t != live_results.PANEL} - if sf not in exo or not ranked or named != {i: sf for i in ranked}: + named = {k: t for k, t in (v.get("findings") or {}).items() if t != [live_results.PANEL]} + if not any(sf in (t or ()) for t in exo) or not ranked or set(named) != set(ranked) \ + or any(sf not in named[i] for i in ranked): out.append(f"the symptom ranking of {gene}: rows tagged {exo}, findings tagged {v.get('findings')}, want {ranked} -> {sf}") - tagged = {(t, it["section"]): it.get("veil") for t, items in idx["terms"].items() for it in items if it.get("veil")} - named = {(gene, it["section"]) for it in idx["terms"].get(gene) or []} - if set(tagged) != named or not named or any(x not in (sf, sv) for x in tagged.values()): - out.append(f"search items tagged {tagged}, want every item of {gene}: {sorted(named)}") + # A search item quotes the summary's rows: prose, so every one carries the fixed-panel tag. + items = [it for items in idx["terms"].values() for it in items] + if not items or any(it.get("veil") != [live_results.PANEL] for it in items): + out.append(f"search items tagged {sorted({str(it.get('veil')) for it in items})}, want every one {[live_results.PANEL]}") return out @@ -417,17 +426,21 @@ def veil_lines_problems(): lines = live_results.a_clinvar(S, {"sections": S}).get("lines") or [] found = [{"id": "sf-gla-test", "level": "act", "gene": "GLA", "detail": "GLA"}] v = live_results.veil({"sections": S}, found, [{"sections": [{"id": "clinvar", "lines": lines}]}]) - want = [live_results.PANEL if ln.startswith(live_results.FIXED_LINES["clinvar"]) else None for ln in lines] + panel = [live_results.PANEL] + want = [panel if ln.startswith(live_results.FIXED_LINES["clinvar"]) else None for ln in lines] got = (v.get("lines") or {}).get("clinvar") - out = [] if want.count(live_results.PANEL) == 2 and got == want else [f"lines {lines} tagged {got}, want {want}"] - # A gene named at the end of a sentence ("in GLA.") is that gene. - if live_results.veil_id(live_results.closable(found), "A change in GLA.") != "sf-gla-test": - out.append("'A change in GLA.' carries no tag") - # A segment of the lead that names the gene, beside the act names the page closes itself. - lead = [[": Symptom ranking: GLA ranks 1. ", False], [": Secondary finding in GLA. ", False, "act"], ["Mean depth", False]] - got = live_results.veil({"sections": S}, found, [], lead).get("lead") - if got != ["sf-gla-test", None, None]: - out.append(f"the lead's segments tagged {got}, want the names that are not the act names") + out = [] if want.count(panel) == 2 and got == want else [f"lines {lines} tagged {got}, want {want}"] + # A segment of the lead is tagged by the findings it prints (lead()'s names), beside the act names the page + # closes itself; a segment printing two closed results' findings carries both ids. + rank = {"id": "exomiser-gla", "level": "review", "card": "exomiser", "gene": "GLA", "title": "Symptom ranking: GLA ranks 1"} + two = found + [{"id": "sf-tp53-test", "level": "act", "gene": "TP53", "detail": "TP53"}, + {"id": "exomiser-tp53", "level": "review", "card": "exomiser", "gene": "TP53", + "title": "Symptom ranking: TP53 ranks 2"}] + lead = [[": Symptom ranking: GLA ranks 1 and Symptom ranking: TP53 ranks 2. ", False], + [": Secondary finding in GLA. ", False, "act"], ["Mean depth", False]] + got = live_results.veil({"sections": S}, two + [rank], [], lead, [[rank, two[2]], [found[0]], []]).get("lead") + if got != [["sf-gla-test", "sf-tp53-test"], None, None]: + out.append(f"the lead's segments tagged {got}, want both ids on the segment that prints both rankings") # A finding known by its gene alone (no position): its ClinVar row is closed by the gene, and the row's # "See the reads at" link, which shows only the position, with it. S["clinvar"]["data"]["hits"] = [{"gene": "GLA", "chrom": "chrX", "pos": 101398000, "ref": "G", "alt": "A"}, @@ -436,7 +449,7 @@ def veil_lines_problems(): gene_only = [{"id": "carrier-gla-test", "level": "act", "gene": "GLA"}] v = live_results.veil({"sections": S}, gene_only, [{"sections": [card]}]) rows, view = (v.get("rows") or {}).get("clinvar"), (v.get("viewer") or {}).get("clinvar") - if rows != ["carrier-gla-test", None] or view != rows: + if rows != [["carrier-gla-test"], None] or view != rows: out.append(f"a gene-only finding: ClinVar rows tagged {rows}, viewer links {view}, want both " "['carrier-gla-test', None]") return out @@ -462,13 +475,28 @@ def widget_problems(): par["facts"] = [["SMN1 copies", "1"], ["Genes read", "2"]] v = live_results.veil(summ, found, [{"sections": [chip, par]}]) out = [] - want = {"chip": [None, "cpsr-tp53-test"], "paralogs": ["two-hits-smn1-test", None]} + want = {"chip": [None, ["cpsr-tp53-test"]], "paralogs": [["two-hits-smn1-test"], None]} if v.get("widget") != want: out.append(f"drawings tagged {v.get('widget')}, want {want}") - if (v.get("facts") or {}) != {"paralogs": ["two-hits-smn1-test", None]}: + # A fixed-panel card's facts are prose no list declares: the panel tag, whatever they name. + if (v.get("facts") or {}) != {c: [[live_results.PANEL], [live_results.PANEL]] for c in ("chip", "paralogs")}: out.append(f"facts tagged {v.get('facts')}") if v.get("spans") != [list(x) for x in live_results.closable(found)]: out.append(f"spans {v.get('spans')}") + # The carrier list's two-hit fact and line are tagged by the two-hit row's gene field, never by their words: the + # line prints the inheritance code AR, and a closed result in AR leaves both open. + th = {"genes": [{"gene": "CFTR", "verdict": "possible", "condition": "Cystic fibrosis", "deletion": "DEL chr7:117480025-117580025", + "variant": "c.1521_1523del", "genotype": "het", "why": "a deletion and a variant"}], + "deletions_read": ["AnnotSV"], "deletions_not_read": []} + S2 = {"carrier": {"state": "ok", "data": {"genes_on_list": 2, "carriers": [], "two_hits": th}}} + car = dict(live_results.a_carrier(S2, {"sections": S2}), id="carrier-list") + for g, want in (("CFTR", ["two-hits-cftr"]), ("AR", None)): + found2 = [{"id": "two-hits-cftr", "level": "act", "gene": "CFTR"}] if g == "CFTR" else \ + [{"id": "cpsr-ar", "level": "act", "gene": "AR", "detail": "AR c.1A>G"}] + v2 = live_results.veil({"sections": S2}, found2, [{"sections": [car]}]) + facts, lines = (v2.get("facts") or {}).get("carrier-list") or [], (v2.get("lines") or {}).get("carrier-list") or [] + if (facts[2:3] or [None])[0] != want or (lines[:1] or [None])[0] != want or (lines[1:2] or [None])[0] != [live_results.PANEL]: + out.append(f"a closed result in {g}: the two-hit fact tagged {facts}, lines {lines}, want {want} and the panel tag") return out @@ -492,34 +520,65 @@ def sv_genes_problems(): out = [] if len(sv) != 1 or len(car) != 1: return [f"the planted findings: {[f['id'] for f in res['findings']]}"] - if (v.get("rows") or {}).get("clinvar") != [sv[0], None]: + if (v.get("rows") or {}).get("clinvar") != [[sv[0]], None]: out.append(f"ClinVar rows tagged {(v.get('rows') or {}).get('clinvar')}, want the deletion's id on the NBR2 row") - if (v.get("rows") or {}).get("carrier-list") != [sv[0]]: + if (v.get("rows") or {}).get("carrier-list") != [[sv[0]]]: out.append(f"carrier rows tagged {(v.get('rows') or {}).get('carrier-list')}") - if (v.get("findings") or {}).get(car[0]) != sv[0]: + if (v.get("findings") or {}).get(car[0]) != [sv[0]]: out.append(f"the NBR2 carrier finding tagged {(v.get('findings') or {}).get(car[0])}") if sorted(s[1] for s in v.get("spans") or [] if s[0] == sv[0]) != ["BRCA1", "NBR2"]: out.append(f"spans {v.get('spans')}") if v.get("panels") != ["paralogs"]: out.append(f"panels {v.get('panels')}, want the copy-number card, the only fixed panel here") + # A place that names two closed results keeps both ids: a class 4 duplication over BRCA1 and BRCA2 with a closed + # CPSR result in each, so opening one never brings back a place that still names the other. + S["annotsv"]["data"]["rows"] = [{"sv": "chr5:1000000-1300000", "type": "DUP", "length": 300000, "acmg_class": "4", + "genes": ["BRCA1", "BRCA2"], "gt": "0/1", "support": 1, "caller": "Manta", + "support_callers": ["Manta"]}] + S["cpsr"] = {"state": "ok", "data": {"plp": [{"gene": g, "variant": "c.68_69del", "final": "Pathogenic"} + for g in ("BRCA1", "BRCA2")]}} + res = live_results.build("T", {"sections": S}) + v = res.get("veil") or {} + acts = [f["id"] for f in res["findings"] if f["card"] == "cpsr"] + dup = [f["id"] for f in res["findings"] if f["card"] == "structural-variants"] + want = sorted(acts) + row = ((v.get("rows") or {}).get("structural-variants") or [None])[0] + if len(acts) != 2 or len(dup) != 1 or sorted((v.get("findings") or {}).get(dup[0]) or []) != want \ + or sorted(row or []) != want: + out.append(f"a duplication over two closed results: finding {(v.get('findings') or {}).get(dup[0] if dup else '')}, " + f"row {row}, want {want}") return out # The veil's tables, pinned here so that an entry dropped from one turns this file red (each is checked by what # veil() does with it, not by reading the table back): the fixed-panel cards, the fixed lines of a card (a sample -# line or row naming a gene, its card, and whether it is a row), the cards that name no gene, and the words that -# name no gene though a gene has their name. +# line or row naming a gene, its card, and whether it is a row), the cards that name no gene, the cards whose lines, +# facts or headline name no gene, and prose that names a gene or a word spelled like one. WANT_PANELS = ("mei", "expansions", "vntyper", "lpa", "chip", "paralogs", "cpic", "cyp2d6", "hla", "locityper", "traits-panel", "bloodtype") WANT_LINES = (("clinvar", "Known deep intronic site (always reported): GLA c.640-801G>A chrX:1 C>T: hom", False), - ("carrier-list", "Two hits across callers (a deletion plus a variant on the other copy): none found; " - "deletions read from Parascopy's SMN1 count (step 35)", False), ("callability", ["GSTT1", "not on this reference", ".", "ALT-only"], True), - ("callability", ["NBPF1", "not read", "69%", "segdup, collapsed"], True)) + ("callability", ["NBPF1", "not read", "69%", "segdup, collapsed"], True), + ("roh", "Imprinted 6q24 (PLAGL1, HYMAI): from the mother: -; from the father: Transient neonatal diabetes " + "mellitus (6q24-related) (PMID 20301706)", False)) WANT_NO_GENE = (("prs", ["Asthma", "PGS002727", "68.3 (EUR)", "2605/2872", "HR 1.57 (EUR, PPM014751)"], "HR"),) -WANT_NOT_A_GENE = (("SET", "the same set of genes"), ("MB", "0.0 MB in runs of 5 MB or more"), - ("HP", "Seizure (HP:0001250)"), ("GC", "reads per million with 48-52% GC")) -WANT_A_GENE = (("KCNH2", "A change in KCNH2."), ("C9ORF72", "C9orf72 repeat"), ("HLA-A", "HLA-A*02:01")) +WANT_LINES_NO_GENE = ("qc", "clinvar", "mosaic", "mosaic-mca", "roh", "telomere", "consumer-export") +WANT_FACTS_NO_GENE = ("qc", "callability", "variants", "clinvar", "clinical-filter", "cpsr", "slivar", "exomiser", + "structural-variants", "mito", "mosaic", "mosaic-mca", "carrier-list", "couple", "pharmcat", + "ancestry-fine", "mt-haplogroup", "y-chromosome", "archaic", "ancient", "roh", "telomere", + "viral", "consumer-export") +WANT_FACT_PROSE = (("clinical-filter", "High-impact rare variants in its genes"),) +WANT_HEADLINES_NO_GENE = ("clinvar", "carrier", "roh", "pypgx", "annotsv", "viral", "archaic", "lpa", "cpsr", "mito") +# Prose, each with the gene it names or the gene its word is spelled like: the tag must be the panel tag either way. +# The carrier list's summary of the two-hit callers (it names SMN1, HBA1 and HBA2 as the callers it read) is prose of +# a card with no gene-free lines: the panel tag, as every other line of that card but a two-hit gene's own. +WANT_PROSE_LINES = (("carrier-list", "Two hits across callers (a deletion plus a variant on the other copy): none found; " + "deletions read from Parascopy's SMN1 count (step 35)"),) +WANT_PROSE = (("KCNH2", "A change in KCNH2."), ("C9ORF72", "C9orf72 repeat"), ("HLA-A", "HLA-A*02:01"), + ("SET", "the same set of genes"), ("MB", "0.0 MB in runs of 5 MB or more"), ("AR", "CFTR (Cystic fibrosis; AR)"), + ("HP", "Seizure (HP:0001250)"), ("GC", "reads per million with 48-52% GC")) +WANT_GENE_CELLS = (("NKD2, SLC12A7 and 2 more", ["NKD2", "SLC12A7"]), ("SMN1 (Parascopy)", ["SMN1"]), + ("HLA-A", ["HLA-A"]), ("BRCA1, BRCA2", ["BRCA1", "BRCA2"]), (".", [])) def gene_of(text): @@ -531,9 +590,11 @@ def gene_of(text): def table_problems(): """What is wrong with veil()'s tables (WANT_*): a fixed-panel card missing from veil.panels, a fixed line or row - tagged by the gene it names instead of the panel tag, a card that names no gene tagged, a word that names no gene - read as one, or a gene symbol not read as one.""" + tagged by the gene it names instead of the panel tag, a card that names no gene tagged, a line, fact or headline + of a card declared gene-free there tagged by the panel tag (or one of a card not declared not), prose read word by + word, or a gene cell not read as its list of symbols.""" out = [] + panel = [live_results.PANEL] found = lambda g: [{"id": "sf-" + g.lower(), "level": "act", "gene": g, "detail": g}] chapters = [{"sections": [{"id": c, "state": "ok"} for c in WANT_PANELS]}] got = live_results.veil({"sections": {}}, found("BRCA2"), chapters).get("panels") or [] @@ -543,47 +604,83 @@ def table_problems(): sec = {"id": card, "table": {"head": ["a", "b", "c", "d"], "rows": [item]}} if is_row else {"id": card, "lines": [item]} v = live_results.veil({"sections": {}}, found(g), [{"sections": [sec]}]) tag = ((v.get("rows" if is_row else "lines") or {}).get(card) or [None])[0] - if tag != live_results.PANEL: + if tag != panel: out.append(f"{card}: {str(item)[:60]} tagged {tag} with a closed result in {g}, want the panel tag") + for card, line in WANT_PROSE_LINES: + v = live_results.veil({"sections": {}}, found("SMN1"), [{"sections": [{"id": card, "lines": [line]}]}]) + if ((v.get("lines") or {}).get(card) or [None])[0] != panel: + out.append(f"{card}: {line[:60]} tagged {(v.get('lines') or {}).get(card)}, want the panel tag") for card, row, g in WANT_NO_GENE: sec = {"id": card, "table": {"head": ["a"] * len(row), "rows": [row]}, "lines": [" ".join(row)], "facts": [row[:2]]} v = live_results.veil({"sections": {}}, found(g), [{"sections": [sec]}]) if any((v.get(k) or {}).get(card) for k in ("rows", "lines", "facts")): out.append(f"{card} names no gene, yet a closed result in {g} tags it: {v}") - for g, text in WANT_NOT_A_GENE: - if live_results.veil_id(live_results.closable(found(g)), text): - out.append(f"{text!r} read as naming {g}") - for g, text in WANT_A_GENE: - if not live_results.veil_id(live_results.closable(found(g)), text): - out.append(f"{text!r} not read as naming {g}") + # A line or fact of a card declared gene-free is untagged even when it spells the closed gene; one of a card not + # declared carries the panel tag whatever it says. A headline likewise, by the summary key it comes from. + for card in sorted(set(WANT_LINES_NO_GENE) | {"cpsr", "slivar", "archaic"}): + v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [{"id": card, "lines": ["a line, BRCA2"]}]}]) + tag = ((v.get("lines") or {}).get(card) or [None])[0] + if tag != (None if card in WANT_LINES_NO_GENE else panel): + out.append(f"the line of {card} tagged {tag}") + for card in sorted(set(WANT_FACTS_NO_GENE) | set(WANT_PANELS)): + v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [{"id": card, "facts": [["A count", "BRCA2"]]}]}]) + tag = ((v.get("facts") or {}).get(card) or [None])[0] + if tag != (None if card in WANT_FACTS_NO_GENE else panel): + out.append(f"the fact of {card} tagged {tag}") + for card, label in WANT_FACT_PROSE: + v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [{"id": card, "facts": [[label, "x"]]}]}]) + if ((v.get("facts") or {}).get(card) or [None])[0] != panel: + out.append(f"the fact {label!r} of {card} is prose, yet not tagged by the panel tag") + for key in sorted(set(WANT_HEADLINES_NO_GENE) | {"couple"}): + sec = {"id": "c", "keys": [key], "headline": "BRCA2 and more"} + v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [sec]}]) + if ((v.get("headline") or {}).get("c")) != (None if key in WANT_HEADLINES_NO_GENE else panel): + out.append(f"the headline from {key} tagged {(v.get('headline') or {}).get('c')}") + # Prose read with no gene field: the panel tag whatever it names, so the tag never tells the words apart. + for g, text in WANT_PROSE: + for spans in (live_results.closable(found(g)), live_results.closable(found("BRCA2"))): + if live_results.veil_id(spans, text) != panel: + out.append(f"{text!r} tagged {live_results.veil_id(spans, text)}, want the panel tag") + for cell, want in WANT_GENE_CELLS: + if live_results.gene_cell(cell) != want: + out.append(f"gene cell {cell!r} read as {live_results.gene_cell(cell)}, want {want}") return out def table_controls(): - """Each entry of the veil's tables dropped in turn: table_problems() must fail every time.""" - keep = (live_results.FIXED_PANELS, live_results.FIXED_LINES, live_results.NAMES_NO_GENE, live_results.UNIT_RE) - first_words = live_results.gene_words + """Each entry of the veil's tables dropped in turn, and the word rule back: table_problems() must fail every time.""" + names = ("FIXED_PANELS", "FIXED_LINES", "NAMES_NO_GENE", "LINES_NO_GENE", "FACTS_NO_GENE", "FACT_NAMES", + "HEADLINES_NO_GENE") + keep = {n: getattr(live_results, n) for n in names} + first_id = live_results.veil_id try: for c in WANT_PANELS: - live_results.FIXED_PANELS = {k: v for k, v in keep[0].items() if k != c} + live_results.FIXED_PANELS = {k: v for k, v in keep["FIXED_PANELS"].items() if k != c} control(f"FIXED_PANELS without {c}", not table_problems()) - live_results.FIXED_PANELS = keep[0] - for card, words in keep[1].items(): + live_results.FIXED_PANELS = keep["FIXED_PANELS"] + for card, words in keep["FIXED_LINES"].items(): for w in words: - live_results.FIXED_LINES = dict(keep[1], **{card: tuple(x for x in words if x != w)}) + live_results.FIXED_LINES = dict(keep["FIXED_LINES"], **{card: tuple(x for x in words if x != w)}) control(f"FIXED_LINES[{card}] without {w!r}", not table_problems()) - live_results.FIXED_LINES = keep[1] + live_results.FIXED_LINES = keep["FIXED_LINES"] live_results.NAMES_NO_GENE = {} control("NAMES_NO_GENE empty", not table_problems()) - live_results.NAMES_NO_GENE = keep[2] - live_results.UNIT_RE = re.compile(r"(?!)") - control("gene_words with no units", not table_problems()) - live_results.UNIT_RE = keep[3] - live_results.gene_words = lambda t: {w.upper().rstrip("._-") for w in re.findall(r"[A-Za-z0-9][A-Za-z0-9._-]*", str(t))} - control("gene_words that reads every word, in any case, as a gene", not table_problems()) + live_results.NAMES_NO_GENE = keep["NAMES_NO_GENE"] + for n in ("LINES_NO_GENE", "FACTS_NO_GENE", "FACT_NAMES", "HEADLINES_NO_GENE"): + for k in keep[n]: + setattr(live_results, n, {x: y for x, y in keep[n].items() if x != k}) + control(f"{n} without {k}", not (table_problems() or widget_problems())) + setattr(live_results, n, keep[n]) + live_results.LINES_NO_GENE = dict(keep["LINES_NO_GENE"], cpsr="its lines") + control("LINES_NO_GENE with the CPSR card, whose lines name genes", not table_problems()) + live_results.LINES_NO_GENE = keep["LINES_NO_GENE"] + live_results.veil_id = lambda spans, text="", *a, genes=None, **k: live_results.veil_tags( + spans, re.findall(r"[A-Za-z0-9][A-Za-z0-9-]*", str(text).upper()), text) + control("a veil that reads prose word by word, the word rule", not table_problems()) finally: - (live_results.FIXED_PANELS, live_results.FIXED_LINES, live_results.NAMES_NO_GENE, live_results.UNIT_RE) = keep - live_results.gene_words = first_words + for n in names: + setattr(live_results, n, keep[n]) + live_results.veil_id = first_id # Every adapter is a fixed panel (live_results.FIXED_PANELS) or listed here as one whose rows say something about this @@ -633,6 +730,18 @@ def vocabulary_summary(gene): "cpsr": {"state": "ok", "data": {"plp": [{"gene": gene, "variant": "c.68_69del", "final": "Pathogenic"}]}}}} +def named_tags(v): + """The veil's places (rows, lines, facts, marks, ...) whose tag names a result, never the panel tag alone: the + same nesting, emptied of the rest.""" + if v is None or (isinstance(v, list) and all(isinstance(x, str) for x in v)): # a tag + return v if v and v != [live_results.PANEL] else None + if isinstance(v, dict): + got = {k: named_tags(x) for k, x in v.items()} + return {k: x for k, x in got.items() if x} or None + got = [named_tags(x) for x in v] + return got if any(got) else None + + def vocabulary_problems(): """What is wrong with the veil's tags of the report's own vocabulary: a closed result in AR (an inheritance code), INS (a structural-variant type) or TG (a trait's abbreviation in a score's name) tags nothing, while one in CFTR @@ -643,7 +752,7 @@ def vocabulary_problems(): for g in ("AR", "INS", "TG"): res = live_results.build("T", vocabulary_summary(g)) v = {k: x for k, x in (res.get("veil") or {}).items() if k not in ("spans", "panels")} - v["findings"] = {k: t for k, t in (v.get("findings") or {}).items() if t != live_results.PANEL} + v = named_tags(v) or {} if any(v.values()): out.append(f"a closed result in {g} tags {v}") for g, where in (("CFTR", ("rows", "carrier-list")), ("GJB2", ("rows", "exomiser")), @@ -662,16 +771,15 @@ def vocabulary_controls(): """Each guard of vocabulary_problems() dropped in turn: the check must fail every time.""" cols, names, marks = live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names try: - live_results.TABLE_COLUMNS = dict(cols, Inheritance=True) + live_results.TABLE_COLUMNS = dict(cols, Inheritance="gene") control("TABLE_COLUMNS that reads the Inheritance column as naming a gene", not vocabulary_problems()) - live_results.TABLE_COLUMNS = dict(cols, Type=True) + live_results.TABLE_COLUMNS = dict(cols, Type="gene") control("TABLE_COLUMNS that reads the SV Type column as naming a gene", not vocabulary_problems()) live_results.TABLE_COLUMNS = cols - live_results.finding_names = lambda f: (None, f"{f.get('gene') or ''} {f.get('detail') or ''} " - f"{f.get('title') or ''}") + live_results.finding_names = lambda f: (re.findall(r"[A-Z0-9]+", f"{f.get('title') or ''}"), f.get("detail") or "") control("a veil that reads a finding's title word by word", not vocabulary_problems()) live_results.finding_names = names - live_results.mark_names = lambda S, sp, m: (None, f"{m.get('label') or ''} {m.get('detail') or ''}") + live_results.mark_names = lambda S, sp, m: (re.findall(r"[A-Z0-9]+", f"{m.get('label') or ''} {m.get('detail') or ''}"), "") control("a veil that reads a chart mark's label word by word", not vocabulary_problems()) finally: live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names = cols, names, marks @@ -693,7 +801,7 @@ def overview_problems(): {"id": "exomiser-gla", "level": "review", "title": "Symptom ranking: GLA ranks 1"}], "lead": [["Of everything, ", False], [": Secondary finding in GLA. ", False, "act"], ["One result is worth a look", True], [": Symptom ranking: GLA ranks 1.", False]], - "veil": {"findings": {"exomiser-gla": "sf-gla-test"}, "lead": [None, None, None, "sf-gla-test"]}} + "veil": {"findings": {"exomiser-gla": ["sf-gla-test"]}, "lead": [None, None, None, ["sf-gla-test"]]}} keep = live_results.build try: live_results.build = lambda *a, **k: r @@ -796,9 +904,9 @@ def tagged(sample_dir): r3["_summary"] = s3 return r3, results_index.build(s3, sample_dir, rr.SECTION_OF) bad = veil_lines_problems() - check("8. every known deep intronic site's line carries the fixed-panel tag, not a closed result's id, a gene " - "before a full stop is that gene, a ClinVar row closed by its gene closes its reads link, and a lead " - "segment naming the gene carries its id", not bad, bad) + check("8. every known deep intronic site's line carries the fixed-panel tag, not a closed result's id, a ClinVar " + "row closed by its gene closes its reads link, and a lead segment carries the ids of every closed result " + "its findings name", not bad, bad) fixed = live_results.FIXED_LINES try: # the card still prints its lines; veil() no longer knows they are a panel live_results.FIXED_LINES = type("Unknown", (dict,), {"get": lambda self, k, d=None: None})(fixed) @@ -807,7 +915,7 @@ def tagged(sample_dir): live_results.FIXED_LINES = fixed bad = table_problems() check("8. every fixed-panel card, fixed line and gene-free card of the veil's tables closes as a panel or never, " - "and only a gene symbol written as one names a gene", not bad, bad) + "prose is the panel tag whatever it names unless declared gene-free, and a gene cell is its symbols", not bad, bad) table_controls() bad = declared_problems() check("8. every table column is declared as naming a gene or not (TABLE_COLUMNS), and every adapter as a fixed " @@ -820,11 +928,13 @@ def tagged(sample_dir): check("8. a child's home tile lists no finding that names a closed result, and its closed lead names none", not bad, bad) bad = widget_problems() - check("8. a CHIP call, a copy-number dot and a fact that name a closed result's gene carry its id, the CHIP " - "chart's gene panel none, and results.json carries the closed results as veil.spans", not bad, bad) + check("8. a CHIP call, a copy-number dot and the two-hit fact and line carry the id of a closed result in their " + "gene field, never by their words (AR), the CHIP chart's gene panel none, a fixed-panel card's facts the " + "panel tag, and results.json carries the closed results as veil.spans", not bad, bad) bad = sv_genes_problems() check("8. a closed structural variant over two genes closes the rows and the finding that name only its second " - "gene, veil.spans lists it per gene, and veil.panels lists the fixed-panel cards", not bad, bad) + "gene, veil.spans lists it per gene, veil.panels lists the fixed-panel cards, and a duplication over two " + "closed results carries both ids", not bad, bad) first = live_results.closable try: live_results.closable = lambda found: first([{k: v for k, v in f.items() if k != "genes"} for f in found]) @@ -836,16 +946,25 @@ def tagged(sample_dir): "deletion's span its own, every other card's row of the gene and the symptom ranking's finding one of them, " "and every search item of the gene one of them", not bad, bad) # The planted mistakes: a rule that tags nothing, and one that tags every row with the first result. - keep = live_results.veil_id + keep = live_results.veil_tags try: - live_results.veil_id = lambda spans, *a, **k: None + live_results.veil_tags = lambda spans, *a, **k: None control("a veil rule that tags nothing", not veil_problems(*tagged(d3))) control("a veil rule that tags no line", not veil_lines_problems()) control("a veil rule that tags no fact or drawing", not widget_problems()) - live_results.veil_id = lambda spans, *a, **k: spans[0][0] if spans else None + live_results.veil_tags = lambda spans, *a, **k: [spans[0][0]] if spans else None control("a veil rule that tags every row", not veil_problems(*tagged(d3))) + live_results.veil_tags = lambda spans, *a, **k: (keep(spans, *a, **k) or [None])[:1] if keep(spans, *a, **k) else None + control("a veil rule that keeps only the first id of a place", not veil_problems(*tagged(d3))) + control("a veil rule that keeps only the first id of a place, two closed results", not sv_genes_problems()) + finally: + live_results.veil_tags = keep + first_union = live_results.union + try: + live_results.union = lambda *t: next((x[:1] for x in t if x), None) + control("a veil rule that keeps only the first id of a lead segment", not veil_lines_problems()) finally: - live_results.veil_id = keep + live_results.union = first_union # 3, the map documents: og map build of DEMO-001 and DEMO-003, with the results each page gets. map_docs = map_documents(root, [(SAMPLE, res), ("DEMO-003", tagged(d3)[0])]) From ef94e6355fae13a43804f45da2dfa839b163f63f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:15:30 +0100 Subject: [PATCH 032/255] fix(live): keep a place closed while any closed result it names is closed, on the page and the map person.js, finding.js and map.js read a tag as the list of every closed result a place names. og map build writes a veils column beside veil: every closable finding a row lies over. The card headline joins the tagged places. New browser checks open one result through its handle and compare the pages: the two-hit line with a closed result in AR, and a lead segment or duplication that names two closed results. --- assets/live/finding.js | 14 ++- assets/live/map.js | 12 +- assets/live/person.js | 41 ++++--- bin/live_results.py | 25 ++-- rust/crates/og-map/README.md | 6 +- rust/crates/og-map/src/doc.rs | 13 ++- rust/crates/og-map/src/facts.rs | 31 ++++- rust/crates/og-map/src/veil.rs | 65 +++++++++-- rust/crates/og-map/tests/findings.rs | 3 + tests/test_live_demo.py | 163 +++++++++++++++++++++++++-- tests/test_live_results.py | 7 +- 11 files changed, 318 insertions(+), 62 deletions(-) diff --git a/assets/live/finding.js b/assets/live/finding.js index fbcdee3c8..253957d8c 100644 --- a/assets/live/finding.js +++ b/assets/live/finding.js @@ -80,20 +80,24 @@ var child = unknown || (!!known && !me) || !!(me && (me.father || me.mother)); // A finding that is not itself a result to act on but names one (a symptom ranking can rank the gene: // results.veil.findings, bin/live_results.py veil()) stays closed with it, as on the person page. + // A tag is the list of every result it names: the finding stays closed while any of them is. var tags = (r.veil && r.veil.findings) || {}; - function tagOf(x) { var t = tags[x.id]; return t && t !== x.id ? t : null; } - // Opening this finding opens it and the result it names, nothing else: another result keeps its own veil. + function tagOf(x) { + var t = tags[x.id]; + t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (y) { return y !== x.id; }); + return t.length ? t : null; + } + // Opening this finding opens it and the results it names, nothing else: another result keeps its own veil. // A finding of a fixed-panel card (tag PANEL, bin/live_results.py FIXED_PANELS) names no result: it stays closed // while any result to act on is, whichever, as on the person page. var PANEL = '*panel*'; var opened = {}; - if (draw.open) { opened[f.id] = true; if (tagOf(f) && tagOf(f) !== PANEL) { opened[tagOf(f)] = true; } } + if (draw.open) { opened[f.id] = true; (tagOf(f) || []).forEach(function (t) { if (t !== PANEL) { opened[t] = true; } }); } function actShut() { return (r.findings || []).some(function (x) { return x.level === 'act' && !opened[x.id]; }); } function shut(x) { if (!child) { return false; } if (x.level === 'act') { return !opened[x.id]; } - var t = tagOf(x); - return t === PANEL ? !opened[x.id] && actShut() : !!t && !opened[t]; + return (tagOf(x) || []).some(function (t) { return t === PANEL ? !opened[x.id] && actShut() : !opened[t]; }); } // Closed: no title, gene or variant anywhere on the page (its title, crumb, cover, rail) until opened. var closed = shut(f); diff --git a/assets/live/map.js b/assets/live/map.js index 860382ebe..a4901128a 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -90,12 +90,14 @@ (f.id || []).forEach(function (fid, i) { if (!(f.chrom[i] in by)) { return; } rows.push({ id: fid, level: f.level[i], gate: f.gate[i], chrom: f.chrom[i], pos: f.pos[i], s: f.s[i], e: f.e[i], - gene: f.gene[i], title: f.title[i], href: f.href[i], phrase: f.phrase[i], veil: f.veil[i] >= 0 ? veilIds[f.veil[i]] : null }); + gene: f.gene[i], title: f.title[i], href: f.href[i], phrase: f.phrase[i], veil: f.veil[i] >= 0 ? veilIds[f.veil[i]] : null, + // Every closable finding the row lies over (veils); a document from before the column has its first only. + veils: ((f.veils && f.veils[i]) || (f.veil[i] >= 0 ? [f.veil[i]] : [])).map(function (k) { return veilIds[k]; }) }); }); var gated = {}; rows.forEach(function (r) { if (r.gate) { gated[r.id] = r; } }); - // What can be closed: each veil tag, and each result to act on that carries another result's tag (a row - // over two of them carries only the first one's tag, and one over APOE carries APOE's): its own id too. + // What can be closed: each veil tag, and each result to act on whose first tag is another result's (a row over + // two of them lists the first one first, and one over APOE lists APOE's first): its own id too. var closable = veilIds.slice(); rows.forEach(function (r) { if (r.veil && r.level === 'act' && !r.gate && closable.indexOf(r.id) < 0) { closable.push(r.id); } }); return { doc: d, id: id, label: person.label || id, chroms: chroms, by: by, bands: bands, roh: roh, bins: bins, @@ -170,7 +172,9 @@ // inside it. A row the page has no finding for (the map's APOE row) follows its tag too. function shownRow(r) { var own = M.closable.indexOf(r.id) < 0 || stateOf(r.id) !== 'closed'; - var tag = (opts.known && opts.known(r.id)) || !r.veil || stateOf(r.veil) !== 'closed'; + // Closed while ANY result the row lies over is closed: a row over two closed results never comes back with + // the first one opened. + var tag = (opts.known && opts.known(r.id)) || r.veils.every(function (id) { return stateOf(id) !== 'closed'; }); return own && tag && (!opts.now || opts.now(r.id)); } // The runs of homozygosity, as this page shows them now: a replay before their step ended has none yet. diff --git a/assets/live/person.js b/assets/live/person.js index 5955a95c3..c29db59f8 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -322,18 +322,22 @@ } // The findings of a card that a child's page keeps closed: while one is closed the card's own rows, // numbers and drawing stay closed too, or its table would name the gene and variant under the veil. - // A row outside its result's card (a ClinVar row, a card's line, a viewer link, a chart's mark, a search hit) - // carries the id of the result to act on it names (results.veil, bin/live_results.py veil()): it stays closed - // while that result is, and "Show the result" opens it with it. A tag naming no finding of these results stays closed. - // A line of a fixed panel (PANEL, bin/live_results.py FIXED_LINES: a known deep intronic site) names no result: - // it stays closed while any result to act on is, whichever, so what is missing never tells the gene. + // A place outside its result's card (a ClinVar row, a card's line, fact or headline, a viewer link, a chart's mark, + // a search hit) carries a tag (results.veil, bin/live_results.py veil()): the list of every result to act on it + // names. It stays closed while ANY of them is, so a place naming two closed results never comes back with the + // first one opened. A tag naming no finding of these results stays closed. PANEL (a fixed panel's line, or prose + // that cannot be shown to name no gene) names no result: it stays closed while any result to act on is, + // whichever, so what is missing never tells the gene. The ids of a tag never reach the page. var PANEL = '*panel*'; function tagClosed(id) { + if (Array.isArray(id)) { return id.some(tagClosed); } if (id === PANEL) { return closedPage() && closedActs().length > 0; } var f = findingById(id); return closedPage() && (!f || veiled(f)); } function veilTags(kind, key) { return ((results && results.veil && results.veil[kind]) || {})[key] || []; } + // A card's headline is prose: tagged PANEL unless the summary key it comes from prints no gene (veil.headline). + function headlineShut(c) { var t = ((results && results.veil && results.veil.headline) || {})[c.id]; return !!(t && tagClosed(t)); } // The charts (plugins/charts.js) ask for each mark. A closed page with no results.json yet cannot tell which // marks those are, so it leaves every mark out until it can. window.PGP.markClosed = function (chart, mark) { @@ -458,7 +462,7 @@ function () { openFinding(closedFs[0]); redraw(true); var x = document.getElementById('f-' + closedFs[0].id) || document.getElementById('card-' + c.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); } else { (c.notes || []).forEach(function (n) { add(text, el('p', { 'class': 'note-stale' }, n)); }); - if (c.headline) { add(text, el('p', { 'class': 'headline' }, c.headline)); } + if (c.headline && !headlineShut(c)) { add(text, el('p', { 'class': 'headline' }, c.headline)); } var of = openFacts(c); add(margin, U.facts(of[0])); add(text, widget(of[1])); @@ -469,7 +473,7 @@ var ot = openTable(c), tb = U.table(ot[0]); var lt = veilTags('lines', c.id); var lines = (c.lines || []).filter(function (l, i) { return !(lt[i] && tagClosed(lt[i])); }); - var shut = ot[1] + (c.lines || []).length - lines.length, items = of[2]; + var shut = ot[1] + (c.lines || []).length - lines.length + (c.headline && headlineShut(c) ? 1 : 0), items = of[2]; if (tb) { add(text, tb); } else if (c.empty_text && !ot[1]) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } if (shut || items) { // The same words however many rows or items close, and whichever result they belong to: a count would @@ -572,18 +576,27 @@ } // A finding that is not itself a result to act on but names one (a symptom ranking can rank the gene: results.veil.findings) // stays closed while that result is, and opening it opens that result. - function findingTag(f) { var t = ((results && results.veil && results.veil.findings) || {})[f.id]; return t && t !== f.id ? t : null; } + function findingTag(f) { + var t = ((results && results.veil && results.veil.findings) || {})[f.id]; + t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (x) { return x !== f.id; }); + return t.length ? t : null; + } function veiled(f) { if (!closedPage()) { return false; } if (f.level === 'act') { return !shown[f.id]; } var t = findingTag(f); return !!(t && tagClosed(t)); } - // A finding of a fixed-panel card (tag PANEL) stays closed while any result to act on is: opening it opens the next - // closed result in the page's order, as its card's box does. + // Opening a finding opens every result its tag names. One tagged PANEL (a fixed-panel card's own) stays closed while + // any result to act on is: opening it opens the next closed result in the page's order too, as its card's box does. function openFinding(f) { - var t = findingTag(f), n = t === PANEL ? nextClosed() : null; - keepHash(function () { shown[f.id] = true; if (n) { shown[n.id] = true; } else if (t && t !== PANEL) { shown[t] = true; } }); + var t = findingTag(f) || []; + keepHash(function () { + shown[f.id] = true; + t.forEach(function (x) { if (x !== PANEL) { shown[x] = true; } }); + var n = t.indexOf(PANEL) >= 0 ? nextClosed() : null; + if (n) { shown[n.id] = true; } + }); } // The address never holds the id of a finding this page can close, closed or opened: #f/closed- while it is // closed (its handle) and #f/opened- once opened on purpose (its place among the opened ones), so no history @@ -638,7 +651,7 @@ if (on) { if (f) { openFinding(f); } else { shown[id] = true; } } else { keepHash(function () { delete shown[id]; - ((results && results.findings) || []).forEach(function (x) { if (findingTag(x) === id) { delete shown[x.id]; } }); + ((results && results.findings) || []).forEach(function (x) { if ((findingTag(x) || []).indexOf(id) >= 0) { delete shown[x.id]; } }); }); } redraw(true); @@ -846,7 +859,7 @@ var g = gate(c); return g === 'show' && veiledOf(c).length ? 'A result about a child, closed until opened.' : g === 'show' && panelShut(c) ? PANEL_LINE - : g === 'show' ? (c.headline || (openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)) + : g === 'show' ? ((!headlineShut(c) && c.headline) || (openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)) : pending(g) ? 'Waiting for ' + stepWords(c) + '.' : g === 'failed' ? 'Not assessed: its step failed.' : 'Not assessed in this run.'; } // A card the report flags that has no finding of its own: on a phone it is the only place that flag diff --git a/bin/live_results.py b/bin/live_results.py index 8975944e1..544832f15 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -1356,12 +1356,19 @@ def veil_tags(spans, genes=(), where="", chrom=None, pos=None, end=None): return out or None +def prose_tag(spans, text): + """The tag of prose (TEXT: a line, a fact, a headline, a search hit's quote) that is on no list of prose naming no + gene: nothing proves it names no closed result, so PANEL while there is one to close, whatever its words. Its + words are never read: a code the report prints (AR, INS, TG) is a gene symbol too.""" + return [PANEL] if spans and str(text or "").strip() else None + + def veil_id(spans, text="", chrom=None, pos=None, end=None, genes=None): """veil_tags() for a caller that has the genes a place names (GENES, from its own gene field; TEXT then gives its - loci). A place given as TEXT alone is prose: nothing proves it names no closed result, so it is tagged PANEL while - there is one to close (bin/results_index.py, whose search hits quote the summary's rows).""" + loci). A place given as TEXT alone is prose (prose_tag(); bin/results_index.py, whose search hits quote the + summary's rows).""" if genes is None: - return [PANEL] if spans else None + return prose_tag(spans, text) return veil_tags(spans, genes, text, chrom, pos, end) @@ -1518,7 +1525,7 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): elif c["id"] in LINES_NO_GENE: tags.append(veil_tags(spans, (), ln)) else: - tags.append([PANEL]) + tags.append(prose_tag(spans, ln)) if any(tags): out["lines"][c["id"]] = tags tags = [] @@ -1528,13 +1535,13 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): if rule == "two_hits": tags.append(veil_tags(spans, two_genes)) elif rule is None or c["id"] not in FACTS_NO_GENE: - tags.append([PANEL]) + tags.append(prose_tag(spans, " ".join(map(str, f)) if isinstance(f, (list, tuple)) else f)) else: tags.append(None) if any(tags): out["facts"][c["id"]] = tags if c.get("headline") and not headline_free(c): - out["headline"][c["id"]] = [PANEL] + out["headline"][c["id"]] = prose_tag(spans, c["headline"]) # A drawing's items: the CHIP calls (gene) and the copy-number dots (gene), by their gene field. The CHIP # chart's list of genes with their detection limits is the same panel for everyone: never tagged. w = c.get("widget") or {} @@ -1564,7 +1571,7 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): t = [PANEL] else: got = finding_names(f) - t = veil_tags(spans, got[0], got[1]) if got else prose_finding(f, cards.get(f.get("card")) or {}) + t = veil_tags(spans, got[0], got[1]) if got else prose_finding(spans, f, cards.get(f.get("card")) or {}) if t: out["findings"][f["id"]] = t # A lead segment is tagged by the findings it prints (lead()): every closed result one of them names. The @@ -1577,7 +1584,7 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): return {k: v for k, v in out.items() if v} -def prose_finding(f, card): +def prose_finding(spans, f, card): """The tag of finding F, which has no gene and no place: None when its words are known to name no gene (a data-quality finding of its own sentences and a sample's name, a row that could not be read, or a card's headline finding whose headline and facts name none), else PANEL.""" @@ -1587,7 +1594,7 @@ def prose_finding(f, card): if f.get("id") == slug("card", cid) and cid in FACTS_NO_GENE and headline_free(card) \ and not any((cid, a) in FACT_NAMES for a, _b in card.get("facts") or []): return None - return [PANEL] + return prose_tag(spans, f"{f.get('title') or ''} {f.get('text') or ''}") def build(sample, summary, files=()): diff --git a/rust/crates/og-map/README.md b/rust/crates/og-map/README.md index b2906b391..5902cf646 100644 --- a/rust/crates/og-map/README.md +++ b/rust/crates/og-map/README.md @@ -59,7 +59,7 @@ One gzip member with mtime 0 and no file name, holding JSON written as Python's "per": { "DEMO-001": { "findings": {"id": [], "level": [], "gate": [], "chrom": [], "pos": [], "s": [], "e": [], "gene": [], "title": [], - "hgvs": [], "zyg": [], "copy": [], "steps": [], "href": [], "reads": [], "phrase": [], "veil": []}, + "hgvs": [], "zyg": [], "copy": [], "steps": [], "href": [], "reads": [], "phrase": [], "veil": [], "veils": []}, "roh": {"chr1": {"ds": [], "l": [], "markers": [], "q": [], "long": [], "art": []}}, "roh_bins": {"size": 1000000, "chr1": []} } @@ -73,8 +73,8 @@ One gzip member with mtime 0 and no file name, holding JSON written as Python's - `ref.chroms`: chr1-22, X and Y. `len` is the chromosome's last band end, `cen` the span of its `acen` bands, `masked` its `acen`, `gvar` and `stalk` bands with touching neighbours merged. - `L`, a layer entry: `state` (`ok`, `missing` or `unreadable`), `step`, `step_name`, `source` (the file read or looked for, relative to GENOME_DIR), `note` (why it is missing, or `file:line: reason` when unreadable), `count` (rows on the map), `totals` (`all` rows read and `kept` rows on the map, when the file was read), `caveat`, and `empty` (the sentence for a read file with no rows). A missing input is `missing`, an input that cannot be read is `unreadable`, and a read input with no rows is `ok` with `count` 0: three answers, never a blank lane. -- `findings`: one flat column per field, rows sorted by chromosome then start. A finding's `viewer` (`chrom:pos` or `chrom:start-end`, 1-based, `chr` optional) places it: a position gives `pos` and `[pos - 1, pos)`, a span gives `[start - 1, end)` and `pos` null. Without a usable `viewer`, a finding that names a gene sits on that gene's span from the GENCODE genes file (the first in map order when a name is on several chromosomes). A finding with neither is not a row. `gate` is set for a finding the page keeps behind its own warning: the traits card holds APOE that way, so the map adds the finding `apoe` on the APOE gene, with `gate` `apoe`; the hidden text, which names the genotype, never enters the document. `zyg` (`het`, `hom`, `hemi` or `unknown`) comes from the finding's own `Genotype` evidence; no VCF is read. `hgvs` is its `Variant` evidence. `copy` is null until the family map. `href` is the finding's page relative to the person's page, `reads` the place as the reads viewer takes it (1-based), `phrase` the sentence the map shows. `veil` is the index in `dict.veil` of the first closable finding (in `dict.veil` order) whose span the row overlaps: its own for a closable finding with no earlier overlap, else -1. So an open finding over a closable one's span is tagged too. The layer's `count` and `totals.kept` are the rows whose `veil` is -1; `totals.all` adds the findings that are not closable and have no place, and the `note` counts those ("N findings with no place on the map stay on the page"). -- **The veil.** The document is the same for every reader. Whether a page is closed is the page's own rule; the document says only what a closed result could reveal. Closable findings are those of level `act` and every gated one (APOE closes on every page, an adult's too). Each closable finding with a place veils its span, and `dict.veil` lists their ids: gated ones first, then the act ones in the page's order. A row of a layer that can be a finding's own source (structural variants, mosaic calls and ClinVar hits, as their layers arrive), and every findings row, that overlaps a veiled span carries the index of the first such finding in `dict.veil` in a `veil` column, else -1. A row holds one tag only. Gated findings come first, so a row over both APOE and an act finding carries APOE, which is closed on every page; a row over two act findings carries the first, and both close under the same rule. The page decides its own findings with its own veil, and the tag binds only the rows that are not one of the page's findings (APOE, or a row the page's results do not have yet): the page hides such a row while its result is closed. Counts, totals and density bins cover untagged rows only, so no number counts a closed row. Runs of homozygosity and bands are never a finding's source and carry no tag. +- `findings`: one flat column per field, rows sorted by chromosome then start. A finding's `viewer` (`chrom:pos` or `chrom:start-end`, 1-based, `chr` optional) places it: a position gives `pos` and `[pos - 1, pos)`, a span gives `[start - 1, end)` and `pos` null. Without a usable `viewer`, a finding that names a gene sits on that gene's span from the GENCODE genes file (the first in map order when a name is on several chromosomes). A finding with neither is not a row. `gate` is set for a finding the page keeps behind its own warning: the traits card holds APOE that way, so the map adds the finding `apoe` on the APOE gene, with `gate` `apoe`; the hidden text, which names the genotype, never enters the document. `zyg` (`het`, `hom`, `hemi` or `unknown`) comes from the finding's own `Genotype` evidence; no VCF is read. `hgvs` is its `Variant` evidence. `copy` is null until the family map. `href` is the finding's page relative to the person's page, `reads` the place as the reads viewer takes it (1-based), `phrase` the sentence the map shows. `veil` is the index in `dict.veil` of the first closable finding (in `dict.veil` order) whose span the row overlaps: its own for a closable finding with no earlier overlap, else -1. `veils` is the list of every such index, smallest first (`[]` for none), so its first item is `veil`. So an open finding over a closable one's span is tagged too. The layer's `count` and `totals.kept` are the rows whose `veil` is -1; `totals.all` adds the findings that are not closable and have no place, and the `note` counts those ("N findings with no place on the map stay on the page"). +- **The veil.** The document is the same for every reader. Whether a page is closed is the page's own rule; the document says only what a closed result could reveal. Closable findings are those of level `act` and every gated one (APOE closes on every page, an adult's too). Each closable finding with a place veils its span, and `dict.veil` lists their ids: gated ones first, then the act ones in the page's order. A row of a layer that can be a finding's own source (structural variants, mosaic calls and ClinVar hits, as their layers arrive), and every findings row, that overlaps a veiled span carries the index of the first such finding in `dict.veil` in a `veil` column, else -1, and the index of every such finding in a `veils` column. Gated findings come first, so a row's `veil` over both APOE and an act finding is APOE, which is closed on every page. The page keeps a row closed while any finding of its `veils` is closed: a row over two act findings never comes back with the first one opened while it still lies over the second. The page decides its own findings with its own veil, and the tag binds only the rows that are not one of the page's findings (APOE, or a row the page's results do not have yet): the page hides such a row while its result is closed. Counts, totals and density bins cover untagged rows only, so no number counts a closed row. Runs of homozygosity and bands are never a finding's source and carry no tag. - `roh`: the `RG` rows of step 11's `bcftools roh` text (`{run}/vcf/{run}_roh.txt`, else `{run}/roh/{run}_roh.txt`). bcftools writes 1-based inclusive positions, so `s = start - 1` and `e = end`. `ST` lines are skipped; rows on chrM and other contigs count in `totals.all` only. `markers` and `q` are bcftools' columns 7 and 8. `art` indexes `dict.art`: `one X` on a male's chrX, `one Y` on a male's chrY, else `centromere` when more than half the run lies in masked spans. An unknown sex sets neither flag. `long` is 1 for a run of 5 Mb or more with no `art`. - `roh_bins`: bases covered by any run per 1 Mb bin, for each chromosome with a run. - `text`: the sentences the page shows, in pieces; `b` marks bold words. `text.findings` is the count of findings on the map in three forms (`none`, `one`, `many`): the page counts the rows it draws: each of its own findings that its veil shows at this moment, whatever the row's tag, and each other row whose tag is -1 or names a result that is open; it picks the form and puts the number in place of `{n}`. diff --git a/rust/crates/og-map/src/doc.rs b/rust/crates/og-map/src/doc.rs index 1df5e65de..be511ed1f 100644 --- a/rust/crates/og-map/src/doc.rs +++ b/rust/crates/og-map/src/doc.rs @@ -188,7 +188,7 @@ fn strs<'a>(xs: impl IntoIterator) -> Value { /// The person's findings on the map: one flat column per field, rows in /// map order. `veil` is the index in `dict.veil` of the first closable -/// finding whose span the row overlaps, else -1. +/// finding whose span the row overlaps, else -1; `veils` every such index. fn findings_columns(m: &Model) -> Value { let rows = &m.findings.rows; let tags = m.findings.veil.tags(rows); @@ -255,6 +255,17 @@ fn findings_columns(m: &Model) -> Value { Value::Array(rows.iter().map(|r| Value::Str(r.phrase())).collect()), ), ("veil", ints(veil::column(&tags))), + ( + "veils", + Value::Array( + m.findings + .veil + .all_tags(rows) + .into_iter() + .map(ints) + .collect(), + ), + ), ]) } diff --git a/rust/crates/og-map/src/facts.rs b/rust/crates/og-map/src/facts.rs index edeb7fea0..18ba81fd5 100644 --- a/rust/crates/og-map/src/facts.rs +++ b/rust/crates/og-map/src/facts.rs @@ -70,9 +70,10 @@ pub struct Finding { /// The evidence row `Genotype`, when there is one. pub genotype: Option, pub steps: Vec, - /// `results.veil.findings` ties it to a closable result (any tag but - /// the fixed-panel one, `*panel*`, which is the same whatever is closed): - /// it closes with that result, so no count of open findings takes it. + /// `results.veil.findings` ties it to a closable result (a tag, the list + /// of the ids it names, holding any id but the fixed-panel one, + /// `*panel*`, which is the same whatever is closed): it closes with that + /// result, so no count of open findings takes it. pub veiled: bool, } @@ -239,7 +240,13 @@ pub fn parse(bytes: &[u8]) -> Result { && let Some(Value::Object(tags)) = v.get("findings") { for f in &mut out { - f.veiled = matches!(tags.get(&f.id), Some(Value::Str(t)) if t != PANEL_TAG); + f.veiled = match tags.get(&f.id) { + Some(Value::Str(t)) => t != PANEL_TAG, + Some(Value::Array(ts)) => ts + .iter() + .any(|t| matches!(t, Value::Str(t) if t != PANEL_TAG)), + _ => false, + }; } } let has_apoe = out @@ -349,5 +356,21 @@ mod tests { (f[0].veiled, f[1].veiled, f[2].veiled), (false, true, false) ); + // A tag is the list of the ids it names: veiled while it holds any id + // but the fixed-panel one. + for (tag, want) in [ + (r#"["sf-x","sf-y"]"#, true), + (r#"["*panel*","sf-y"]"#, true), + (r#"["*panel*"]"#, false), + ("[]", false), + ] { + let Findings::Read(f) = p(&body.replace( + r#""chapters""#, + &format!(r#""veil":{{"findings":{{"q":{tag}}}}},"chapters""#), + )) else { + panic!("not read") + }; + assert_eq!(f[1].veiled, want, "{tag}"); + } } } diff --git a/rust/crates/og-map/src/veil.rs b/rust/crates/og-map/src/veil.rs index 70187d37a..39202d5be 100644 --- a/rust/crates/og-map/src/veil.rs +++ b/rust/crates/og-map/src/veil.rs @@ -14,13 +14,15 @@ //! its span, or its gene's span when it has no position. //! 3. Row tags: a row of a layer that can be a finding's own source (a //! structural variant, a mosaic call, a ClinVar hit) that overlaps a -//! veiled span gets that finding's index in `dict.veil`, else -1. A row -//! holds one tag: when several spans overlap it, the first in -//! `dict.veil`, where gated findings come first (closed on every page), -//! then the act ones in the page's order. A finding's own row is tagged -//! the same way, so an open finding over a closable one's span is -//! tagged. Runs of homozygosity and bands are never a finding's source -//! and carry no tag. +//! veiled span gets that finding's index in `dict.veil`, else -1: the +//! first in `dict.veil` when several spans overlap it, where gated +//! findings come first (closed on every page), then the act ones in the +//! page's order. Its `veils` holds every such index, so a page keeps the +//! row closed while any of them is (`Veil::all`): a row over two closed +//! results never comes back with the first one opened. A finding's own +//! row is tagged the same way, so an open finding over a closable one's +//! span is tagged. Runs of homozygosity and bands are never a finding's +//! source and carry no tag. //! 4. Counts, totals and density bins cover untagged rows only, so no number on //! the map counts a closed row. @@ -94,6 +96,31 @@ impl Veil { pub fn tags(&self, rows: &[R]) -> Vec> { rows.iter().map(|r| self.tag(r.chrom(), r.span())).collect() } + + /// The index in `ids` of every closable finding whose span `row` + /// overlaps, smallest first, each once: the first of them is + /// [`Veil::tag`]. + pub fn all(&self, chrom: usize, span: (u64, u64)) -> Vec { + let (s, e) = nonempty(span); + let mut out: Vec = self + .spans + .get(chrom) + .map(|v| { + v.iter() + .filter(|&&(vs, ve, _)| s < ve && vs < e) + .map(|x| x.2) + .collect() + }) + .unwrap_or_default(); + out.sort_unstable(); + out.dedup(); + out + } + + /// [`Veil::all`] of every row, in order. + pub fn all_tags(&self, rows: &[R]) -> Vec> { + rows.iter().map(|r| self.all(r.chrom(), r.span())).collect() + } } /// A tag column as the document writes it: the index, or [`NONE`]. @@ -161,4 +188,28 @@ mod tests { assert_eq!(column(&tags), [NONE, 0, NONE]); assert_eq!(untagged(&tags), 2); } + + #[test] + fn every_id_a_row_overlaps() { + // A row over two closable findings keeps both, smallest index first, + // whatever order their spans were placed in; the first is `tag`. + let v = Veil::new([ + (2, "c", 0, (100, 101)), + (0, "a", 0, (90, 300)), + (1, "b", 0, (100, 150)), + (3, "d", 1, (100, 101)), + ]); + assert_eq!(v.all(0, (100, 101)), [0, 1, 2]); + assert_eq!(v.all(0, (120, 130)), [0, 1]); + assert_eq!(v.all(0, (200, 250)), [0]); + assert_eq!(v.all(0, (0, 10)), Vec::::new()); + assert_eq!(v.all(1, (100, 101)), [3]); + assert_eq!(v.all(5, (100, 101)), Vec::::new()); + let rows = [R(0, 100, 101), R(0, 0, 10), R(1, 100, 100)]; + let all = v.all_tags(&rows); + assert_eq!(all, [vec![0, 1, 2], vec![], vec![3]]); + // `veil` is the first of `veils`, -1 for none. + let first: Vec> = all.iter().map(|x| x.first().copied()).collect(); + assert_eq!(first, v.tags(&rows)); + } } diff --git a/rust/crates/og-map/tests/findings.rs b/rust/crates/og-map/tests/findings.rs index ad448a374..5deaa5351 100644 --- a/rust/crates/og-map/tests/findings.rs +++ b/rust/crates/og-map/tests/findings.rs @@ -124,6 +124,9 @@ fn findings_land_on_their_place_and_the_rest_are_counted() { // APOE is gated, so it comes first in dict.veil. The open ClinVar row // at the act finding's base carries that finding's tag. assert_eq!(get(f, &["veil"]), &js("[3,1,1,2,-1,0,-1]")); + // `veils` holds every closable finding a row overlaps (one each here; + // src/veil.rs tests a row over several), its first being `veil`. + assert_eq!(get(f, &["veils"]), &js("[[3],[1],[1],[2],[],[0],[]]")); assert_eq!( get(f, &["href"]), &js( diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 00c6cac62..8c9ceaf35 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -657,9 +657,8 @@ def planted_closed(b, base): function shut(f) { if (f.level === 'act') { return !open(f.id); } var t = tags[f.id]; - if (!t || t === f.id) { return false; } - if (t === '*panel*') { return anyShut; } - return !byId[t] || !open(t); + t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (x) { return x !== f.id; }); + return t.some(function (x) { return x === '*panel*' ? anyShut : !byId[x] || !open(x); }); } var order = []; (r.journal || []).forEach(function (s) { (s.findings || []).forEach(function (id) { @@ -713,9 +712,10 @@ def planted_problems(b, base, out_dir): if not ranked: out.append(f"opening the secondary finding did not bring back the symptom ranking of {gene}") out += [f"{w} shown after opening only the secondary finding" for w in sv if w in shown] + # A search hit quotes the summary's rows (prose): it stays closed while any result is, the same whichever. hits, _ = son_search(b, gene) - if hits in (None, "0"): - out.append(f"the search for {gene} finds nothing once its result is opened") + if hits not in ("0", None): + out.append(f"the search for {gene} finds {hits} hits while the deletion is still closed") if not open_closed(b, del_): return out + [f"a link to {del_} lands on no closed card"] time.sleep(0.8) @@ -723,6 +723,9 @@ def planted_problems(b, base, out_dir): spans = b.eval("document.querySelectorAll('figure[data-chart=genome] a[data-mark^=sv-]').length") if sv[0] not in shown or spans != 2: out.append(f"opening the deletion did not bring back its span on the genome chart: {spans} spans") + hits, _ = son_search(b, gene) + if hits in (None, "0"): + out.append(f"the search for {gene} finds nothing once every result is opened") out += [f"after a reload: {x}" for x in planted_closed(b, base)] return out @@ -806,7 +809,7 @@ def write(p, text): check("browser: the home page and every finding page of the son name the planted gene and positions nowhere " "(the rail's next link and the related list included) until opened, and the symptom ranking's page " "opened on purpose shows it", not bad, bad) - a = "return t === PANEL ? !opened[x.id] && actShut() : !!t && !opened[t];" + a = "return (tagOf(x) || []).some(function (t) { return t === PANEL ? !opened[x.id] && actShut() : !opened[t]; });" if a not in keep[fj]: check("control: finding.js whose findings ignore their tags (the planted line is in finding.js)", False, a) else: @@ -1771,6 +1774,120 @@ def home_problems(b, base, out, cases=HOME): return bad +def two_hits(change): + """CHANGE, plus a two-hit gene in CFTR on the carrier list (a result to act on, two-hits-cftr), whose line prints + the inheritance code AR.""" + def go(S): + change(S) + th = S["carrier"]["data"].setdefault("two_hits", {"deletions_read": [], "deletions_not_read": []}) + th["genes"] = [{"gene": "CFTR", "verdict": "possible", "condition": "Cystic fibrosis", + "deletion": "DEL chr7:117480025-117580025", "variant": "c.1521_1523del", "genotype": "het", + "why": "a deletion and a variant on the other copy"}] + return go + + +def cpsrs(*genes): + """A closed CPSR result in each of GENES, the same variant in each.""" + def change(S): + S["cpsr"] = {"state": "ok", "data": {"call_floor_dp": 10, "low_confidence": 0, "plp": [ + {"gene": g, "variant": "c.68_69del", "final": "Pathogenic", "clinvar": "Pathogenic"} for g in genes]}} + return change + + +def dup_over(change, locus, genes): + """CHANGE, plus a class 4 duplication over GENES (a finding worth a look, with a place on the map).""" + def go(S): + change(S) + S["annotsv"]["data"]["rows"] = S["annotsv"]["data"]["rows"] + [ + {"sv": locus, "type": "DUP", "gt": "0/1", "length": 300000, "acmg_class": "4", "genes": genes, + "caller": "Manta", "support": 1, "support_callers": ["Manta"]}] + return go + + +def cid(gene): + """The id of cpsr(GENE)'s result.""" + return f"cpsr-{gene.lower()}-c-68-69del" + + +# Pairs of the son's pages that differ only in which result is closed, compared after the same click: the result +# each side opens through its handle (its place among the closed cards, as a reader who never saw its id opens it). +# A place tied to a closed result must open with the results it names and with nothing else, so the page after a +# click is again a function of the open results and the count of closed ones. The two-hit line prints the +# inheritance code AR: a closed result in AR (a CPSR result, or a deletion over AR) must not hold it closed. +CLICKS = (("a two-hit result in CFTR opened, the other closed result in AR against BRCA2", two_hits(cpsr("AR")), + two_hits(cpsr("BRCA2")), ("two-hits-cftr", "two-hits-cftr")), + ("a two-hit result in CFTR opened, a closed deletion over AR and OPHN1 against one over BRCA1 and NBR2", + two_hits(sv_over("chrX:67500000-67700000", ["AR", "OPHN1"], False)), + two_hits(sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"], False)), ("two-hits-cftr", "two-hits-cftr"))) + + +def opened_page(b, base, fid): + """The son's page after FID is opened through its handle: (lines of its HTML, its address, its canvas hash).""" + page_now(b, base) + if not open_closed(b, fid): + return None + time.sleep(0.8) + got = json.loads(b.eval(PAGE_JS) or "{}") + return (got.get("html") or "").replace("><", ">\n<").split("\n"), got.get("href") or "", \ + hashlib.sha256((got.get("canvas") or "").encode()).hexdigest()[:12] + + +def click_problems(b, base, out, og, cases=CLICKS): + """[problems]: for each pair of CASES, the son's page after each side opens its result differs (its HTML, an + attribute, the address or the map's canvas), with the number of differing lines and the first ones.""" + bad = [] + for what, one, two, fids in cases: + got = [] + for change, fid in zip((one, two), fids): + keep = plant_summary(out, og, change) + try: + got.append(opened_page(b, base, fid)) + finally: + unplant(keep) + if None in got: + bad.append(f"{what}: a side has no closed card for {fids}") + continue + (h1, a1, c1), (h2, a2, c2) = got + diff = [x for x in difflib.unified_diff(h1, h2, lineterm="", n=0) if x[:1] in "+-" and x[:3] not in ("+++", "---")] + if diff or a1 != a2 or c1 != c2: + bad.append(f"{what}: {len(diff)} lines differ, address {a1 == a2}, canvas {c1 == c2}: {diff[:6]}") + return bad + + +# Two closed results, one opened (each in turn): the other's gene must be on the page no more often than before the +# click (the search box's example names BRCA2 on every page). A place that names both (the lead's segment of the +# findings worth a look, which prints the ClinVar findings in DEMOC and DEMOA; a duplication over BRCA1 and BRCA2) +# stays closed while either is. +TWO_CLOSED = (("two CPSR results named in one segment of the lead, DEMOC opened", cpsrs("DEMOC", "DEMOA"), cid("DEMOC"), + "DEMOA"), + ("the same, DEMOA opened", cpsrs("DEMOC", "DEMOA"), cid("DEMOA"), "DEMOC"), + ("a class 4 duplication over BRCA1 and BRCA2, both closed, BRCA1 opened", + dup_over(cpsrs("BRCA1", "BRCA2"), "chr5:1000000-1300000", ["BRCA1", "BRCA2"]), cid("BRCA1"), "BRCA2"), + ("the same, BRCA2 opened", dup_over(cpsrs("BRCA1", "BRCA2"), "chr5:1000000-1300000", ["BRCA1", "BRCA2"]), + cid("BRCA2"), "BRCA1")) + + +def two_closed_problems(b, base, out, og, cases=TWO_CLOSED): + """[problems]: for each of CASES, the still closed result's gene is on the page more often after the click.""" + bad = [] + for what, change, fid, still in cases: + keep = plant_summary(out, og, change) + try: + before = "\n".join(page_now(b, base)[0]).count(still) + after = opened_page(b, base, fid) + finally: + unplant(keep) + if after is None: + bad.append(f"{what}: no closed card for {fid}") + continue + html = "\n".join(after[0]) + if html.count(still) > before: + at = html.find(still, 0) + bad.append(f"{what}: {still} {before} times before, {html.count(still)} after: " + f"...{html[max(0, at - 160):at + 40]}...") + return bad + + def infer_checks(b, base, out): """The son's page is a function of his open results and the count of his closed ones: two pages that differ only in which result is closed are the same before any click (INFER), and a closed deletion's second gene is named @@ -1784,6 +1901,12 @@ def infer_checks(b, base, out): bad = infer_problems(b, base, out, og) check("browser: two pages of the son that differ only in which result is closed are the same before any click, " "HTML, attributes, address and map canvas (" + "; ".join(x[0] for x in INFER) + ")", not bad, bad) + bad = click_problems(b, base, out, og) + check("browser: two pages of the son that differ only in which result is closed are the same after the same " + "result is opened through its handle (" + "; ".join(x[0] for x in CLICKS) + ")", not bad, bad) + bad = two_closed_problems(b, base, out, og) + check("browser: a place that names two closed results stays closed while either is: opening one never shows the " + "other's gene (" + "; ".join(x[0] for x in TWO_CLOSED) + ")", not bad, bad) bad = deep_link_problems(b, base) check("browser: #f/closed-1 and a guessed #f/ scroll to the first closed result, in view, its section marked in " "the contents", not bad, bad) @@ -1819,13 +1942,25 @@ def infer_checks(b, base, out): not infer_problems(b, base, out, og, [x for x in INFER if "(NAT2)" in x[0]])) finally: live_results.panel_finding = first - fixed = live_results.FIXED_LINES - try: - live_results.FIXED_LINES = {k: v for k, v in fixed.items() if k != "carrier-list"} - control("a veil that tags the carrier list's two-hit line by its genes", + prose, two_names = live_results.prose_tag, live_results.two_hit_names + try: # the word rule back: prose tagged by its words read as gene symbols + live_results.prose_tag = lambda spans, text: live_results.veil_tags( + spans, re.findall(r"[A-Za-z0-9][A-Za-z0-9-]*", str(text).upper()), text) + control("a veil that tags the carrier list's two-hit callers line by its words", not infer_problems(b, base, out, og, [x for x in INFER if "two-hit" in x[0]])) + live_results.two_hit_names = lambda S: {} + control("a veil that tags the two-hit gene lines by their words (AR)", not click_problems(b, base, out, og, CLICKS[:1])) + finally: + live_results.prose_tag = prose + live_results.two_hit_names = two_names + tags_, union_ = live_results.veil_tags, live_results.union + try: # one id per place, as before: the first closed result a place names + live_results.veil_tags = lambda *a, **k: (tags_(*a, **k) or [None])[:1] if tags_(*a, **k) else None + live_results.union = lambda *t: next((x[:1] for x in t if x), None) + control("a veil that keeps one id per place (the lead's segment)", not two_closed_problems(b, base, out, og, TWO_CLOSED[:1])) + control("a veil that keeps one id per place (the duplication)", not two_closed_problems(b, base, out, og, TWO_CLOSED[2:3])) finally: - live_results.FIXED_LINES = fixed + live_results.veil_tags, live_results.union = tags_, union_ js = os.path.join(out, "assets", "person.js") with open(js) as fh: keep_js = fh.read() @@ -1855,6 +1990,9 @@ def infer_checks(b, base, out): for desc, a, z, still in ( ("person.js whose fixed-panel card says 'This section holds a result' first", "} else if (panelShut(c)) {", "} else if (panelShut(c) && !closedFs.length) {", lambda: not panel_words_problems(b, base, out, og)), + ("person.js that reads only the first id of a tag", "if (Array.isArray(id)) { return id.some(tagClosed); }", + "if (Array.isArray(id)) { return tagClosed(id[0]); }", + lambda: not two_closed_problems(b, base, out, og, TWO_CLOSED[2:3])), ("person.js that writes an opened result's id into the address", "function addr(f) { return !closable(f) ? f.id : veiled(f) ? handle(f) : 'opened-' + (openedOrder().indexOf(f.id) + 1); }", "function addr(f) { return veiled(f) ? handle(f) : f.id; }", lambda: not address_problems(b, base, out))): @@ -3063,7 +3201,7 @@ def named_problems(b, base, out, og): try: with open(os.path.join(out, "runs", son, "results.json")) as fh: tags = (json.load(fh).get("veil") or {}).get("findings") or {} - if tags.get("pl-q") != "pl-x" or tags.get("pl-r") != "pl-x" or row_tag(out, "pl-r"): + if tags.get("pl-q") != ["pl-x"] or tags.get("pl-r") != ["pl-x"] or row_tag(out, "pl-r"): return [f"the server's tags {tags}, og's tag on pl-r {row_tag(out, 'pl-r')}: the planting tests nothing"] path = os.path.join(out, "runs", son, "map.json") doc = read_map(path) @@ -3077,6 +3215,7 @@ def named_problems(b, base, out, og): for k, v in GHOST.items(): f[k][-1] = v f["veil"][-1] = doc["dict"]["veil"].index("pl-x") + f["veils"][-1] = [f["veil"][-1]] write_map(path, doc) c, fig = son_figure(b, base) bad = [] diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 6892e626f..bc0d0596d 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -652,7 +652,7 @@ def table_controls(): names = ("FIXED_PANELS", "FIXED_LINES", "NAMES_NO_GENE", "LINES_NO_GENE", "FACTS_NO_GENE", "FACT_NAMES", "HEADLINES_NO_GENE") keep = {n: getattr(live_results, n) for n in names} - first_id = live_results.veil_id + first_prose = live_results.prose_tag try: for c in WANT_PANELS: live_results.FIXED_PANELS = {k: v for k, v in keep["FIXED_PANELS"].items() if k != c} @@ -674,13 +674,14 @@ def table_controls(): live_results.LINES_NO_GENE = dict(keep["LINES_NO_GENE"], cpsr="its lines") control("LINES_NO_GENE with the CPSR card, whose lines name genes", not table_problems()) live_results.LINES_NO_GENE = keep["LINES_NO_GENE"] - live_results.veil_id = lambda spans, text="", *a, genes=None, **k: live_results.veil_tags( + live_results.prose_tag = lambda spans, text: live_results.veil_tags( spans, re.findall(r"[A-Za-z0-9][A-Za-z0-9-]*", str(text).upper()), text) control("a veil that reads prose word by word, the word rule", not table_problems()) + control("a veil that reads prose word by word, the two-hit line's AR", not widget_problems()) finally: for n in names: setattr(live_results, n, keep[n]) - live_results.veil_id = first_id + live_results.prose_tag = first_prose # Every adapter is a fixed panel (live_results.FIXED_PANELS) or listed here as one whose rows say something about this From 1628a7f5dac81d2e6070a41edc30052b63fd815e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:51:54 +0100 Subject: [PATCH 033/255] test(live): load every planted file fresh, and check places that name two closed results --- tests/test_live_demo.py | 55 +++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 8c9ceaf35..27c3a53af 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -49,8 +49,12 @@ term of an open card is still found; the son's planted secondary finding and class 5 deletion (make_demo_sample.py PLANTED) name their gene and positions in no card, chart, chart title, link, symptom ranking or search - suggestion until each is opened, opening one brings back its own rows, - marks and ranking only, and a reload closes them again; the home page and + suggestion until each is opened, a row, mark or ranking that names both + comes back only once both are opened (the search too), and a reload + closes them again; a place that names two closed results (a lead segment, + a duplication over two genes) stays closed until both are opened, and the + page after a click is the same whichever closed result shares the opened + one's place; the home page and every finding page of the son (an open finding's rail and related list too) name them nowhere, and the symptom ranking's own page shows the gene once opened on purpose; on the family page, while the son has a planted @@ -271,11 +275,17 @@ def located_term(out): class Static(SimpleHTTPRequestHandler): - """A static host: files only, no Content-Security-Policy header.""" + """A static host: files only, no Content-Security-Policy header. Nothing is cached: the checks rewrite + results.json, map.json and the scripts between page loads, and a heuristically fresh copy in the browser's cache + (10% of its age since Last-Modified) would load the old file and let a control pass on the planted mistake.""" def log_message(self, *a): pass + def end_headers(self): + self.send_header("Cache-Control", "no-store") + super().end_headers() + def browser_checks(out): chrome = find_chrome() @@ -697,20 +707,24 @@ def planted_problems(b, base, out_dir): del_ = next((i for i in ids.split() if i.startswith("sv-") and start in i), None) if not sf or not del_: return out + [f"the planted results are not among the son's results to act on: {ids}"] + rows_js = ("Array.prototype.filter.call(document.querySelectorAll('#card-clinvar tbody tr'), function (r) { " + "return r.textContent.indexOf(%s) >= 0; }).length" % json.dumps(gene)) + tick_js = ("Array.prototype.filter.call(document.querySelectorAll('figure[data-chart=genome] a[data-mark^=hit-] " + "title'), function (t) { return t.textContent.indexOf(%s) >= 0; }).length" % json.dumps(gene)) + ranked_js = ("Array.prototype.filter.call(document.querySelectorAll('[data-finding^=exomiser-]'), function (e) { " + "return e.textContent.indexOf(%s) >= 0; }).length" % json.dumps(gene)) if not open_closed(b, sf): return out + [f"a link to {sf} lands on no closed card"] time.sleep(0.8) shown = b.eval(SHOWN_JS) or "" - rows = b.eval("Array.prototype.filter.call(document.querySelectorAll('#card-clinvar tbody tr'), function (r) { " - "return r.textContent.indexOf(%s) >= 0; }).length" % json.dumps(gene)) - tick = b.eval("Array.prototype.filter.call(document.querySelectorAll('figure[data-chart=genome] a[data-mark^=hit-] " - "title'), function (t) { return t.textContent.indexOf(%s) >= 0; }).length" % json.dumps(gene)) - if rows != 1 or not tick or not all(w in shown for w in [gene, hit[0]]): - out.append(f"opening the secondary finding did not bring back its ClinVar row and tick: {rows} rows, {tick} ticks") - ranked = b.eval("Array.prototype.filter.call(document.querySelectorAll('[data-finding^=exomiser-]'), function (e) { " - "return e.textContent.indexOf(%s) >= 0; }).length" % json.dumps(gene)) - if not ranked: - out.append(f"opening the secondary finding did not bring back the symptom ranking of {gene}") + # The planted hit lies inside the planted deletion, in its gene: its ClinVar row, its tick and the symptom + # ranking of the gene name both results, so they stay closed until the deletion is opened too. + rows, tick, ranked = b.eval(rows_js), b.eval(tick_js), b.eval(ranked_js) + if not all(w in shown for w in [gene, hit[0]]): + out.append(f"opening the secondary finding did not bring back its own card ({gene}, {hit[0]})") + if rows or tick or ranked: + out.append(f"opening the secondary finding alone brought back places that name the closed deletion too: " + f"{rows} ClinVar rows, {tick} ticks, {ranked} rankings") out += [f"{w} shown after opening only the secondary finding" for w in sv if w in shown] # A search hit quotes the summary's rows (prose): it stays closed while any result is, the same whichever. hits, _ = son_search(b, gene) @@ -726,6 +740,10 @@ def planted_problems(b, base, out_dir): hits, _ = son_search(b, gene) if hits in (None, "0"): out.append(f"the search for {gene} finds nothing once every result is opened") + rows, tick, ranked = b.eval(rows_js), b.eval(tick_js), b.eval(ranked_js) + if rows != 1 or not tick or not ranked: + out.append(f"opening both results did not bring back the hit's ClinVar row, tick and ranking: {rows} rows, " + f"{tick} ticks, {ranked} rankings") out += [f"after a reload: {x}" for x in planted_closed(b, base)] return out @@ -787,8 +805,8 @@ def write(p, text): try: bad = planted_problems(b, base, out) check("browser: the son's planted secondary finding and class 5 deletion name their gene and positions nowhere " - "(cards, charts, titles, links, search) until each is opened, each opens only its own rows and marks, " - "and a reload closes them", not bad, bad) + "(cards, charts, titles, links, search) until each is opened, each opens only its own rows and marks (a " + "place that names both once both are), and a reload closes them", not bad, bad) res = json.loads(keep[rj]) res["veil"] = {} write(rj, json.dumps(res)) @@ -2013,16 +2031,15 @@ def infer_checks(b, base, out): live_results.closable = first cols, names, marks = live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names try: - live_results.TABLE_COLUMNS = dict(cols, Inheritance=True) + live_results.TABLE_COLUMNS = dict(cols, Inheritance="gene") control("a veil that reads a table's Inheritance column as naming a gene", not infer_problems(b, base, out, og, [x for x in INFER if x[0].startswith("an inheritance code")])) live_results.TABLE_COLUMNS = cols - live_results.finding_names = lambda f: (None, f"{f.get('gene') or ''} {f.get('detail') or ''} " - f"{f.get('title') or ''}") + live_results.finding_names = lambda f: (re.findall(r"[A-Z0-9]+", f"{f.get('title') or ''}"), f.get("detail") or "") control("a veil that reads a finding's title word by word", not infer_problems(b, base, out, og, [x for x in INFER if "(INS)" in x[0]])) live_results.finding_names = names - live_results.mark_names = lambda S, sp, m: (None, f"{m.get('label') or ''} {m.get('detail') or ''}") + live_results.mark_names = lambda S, sp, m: (re.findall(r"[A-Z0-9]+", f"{m.get('label') or ''} {m.get('detail') or ''}"), "") control("a veil that reads a chart mark's label word by word", not infer_problems(b, base, out, og, [x for x in INFER if "(TG)" in x[0]])) finally: From be95e199a2fe3d61327cd5f74779890a982d4ce5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:52:47 +0100 Subject: [PATCH 034/255] docs(live): say that a place closes with every closed result it names and prose closes whole --- CHANGELOG.md | 2 +- docs/live-site.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfd5813ef..0f5bc06db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. -- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; the address never holds a closable finding's id, closed (`#f/closed-`) or opened (`#f/opened-`), so no history entry keeps it, and any other `#f/` on such a page, a closed result's id or nothing, lands on the first closed finding, so a guessed id tells nothing. The home page links a child's closed result to the child's report, not to the finding's own page. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and so do their own findings (a medicine gene, a CHIP call) and the lead's words on them, the ClinVar card's lines for the known deep intronic sites, the carrier list's two-hit line and the callability card's rows, all together; a table row, a finding or a chart mark closes with a result only by its gene and place fields, never by a code such as the inheritance `AR` or the structural-variant type `INS`, and prose closes only by a word written as a gene symbol (never a prose word, an ontology id or a unit such as `5 MB`); the polygenic score card and its chart are never tagged, and the genome map no longer counts a finding a closed result ties among its findings with no place; a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels`, the tag `*panel*` in `veil.lines`, `veil.rows`, `veil.findings` and `veil.lead`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; nothing is renamed or removed. +- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; the address never holds a closable finding's id, closed (`#f/closed-`) or opened (`#f/opened-`), so no history entry keeps it, and any other `#f/` on such a page, a closed result's id or nothing, lands on the first closed finding, so a guessed id tells nothing. The home page links a child's closed result to the child's report, not to the finding's own page. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and so do their own findings (a medicine gene, a CHIP call) and the lead's words on them, the ClinVar card's lines for the known deep intronic sites, the carrier list's two-hit line and the callability card's rows, all together; a table row, a finding or a chart mark closes with a result only by its gene and place fields, never by a code such as the inheritance `AR` or the structural-variant type `INS`, and prose is never read word by word against gene symbols: a card's headline, its lines and facts, a finding with no gene and no place, and the lead's own sentences close whole while any result is closed, unless they are structured (the carrier list's two-hit lines close by the gene each is built from) or on a list of prose that names no gene (the QC, ClinVar, mosaic, ROH and telomere lines, most cards' facts, the headlines built from counts), so a child's page with a result closed shows less prose, and no search hit, until the results are opened; a place that names two closed results (a segment of the lead, a duplication over two genes) stays closed until both are opened; the polygenic score card and its chart are never tagged, and the genome map no longer counts a finding a closed result ties among its findings with no place; a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels` and `veil.headline`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; each tag in `veil.rows`, `veil.lines`, `veil.facts`, `veil.findings`, `veil.lead`, `veil.marks`, `veil.viewer` and `veil.widget` is now a list of closed-result ids (or `*panel*`) instead of one id. The genome map's findings layer gains a `veils` column, every closed result a row overlaps; `veil` stays. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. - Steps 21, 32 and 9b (and the CYRIUS, PYPGX and STRANGER_RFC1 modules) now run the CYP2D6 depth check and the RFC1 motif count with `og`, and `bin/cyp2d6_depth_check.py` and `bin/rfc1_motif.py` are gone. A run that reaches these steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull; without it the step stops with `og is not built`. The output files, their names and formats do not change; in CI, every e2e case that runs these steps in bash checks `og` against the deleted Python, byte for byte. diff --git a/docs/live-site.md b/docs/live-site.md index 1a3e78656..ee3005c9a 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -58,10 +58,10 @@ Every result carries one of six levels, with a word, a shape and a colour, so th A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. Two pages whose open rows are the same differ only in how many results are closed, not in which; the section that holds a closed result still says it holds one: -- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call, a repeat) close with them, whatever gene they name, and so does a segment of the lead that names one (`veil.findings` and `veil.lead` tag them `*panel*`): a medicine finding that closed only when the closed result is in its gene would tell the gene. A card can hold a fixed panel among its lines or rows: the ClinVar card prints a line for each known deep intronic site, hit or not; the carrier list prints its two-hit line, which names the copy-number callers it read; and the callability card lists the genes this reference does not hold and the hard genes the reads did not cover, nearly the same for everyone. Those lines and rows close all together while any result to act on is closed and come back once every result is opened (`FIXED_LINES` in `bin/live_results.py`; `veil.lines` and `veil.rows` tag each of them `*panel*`). A table row, a finding or a chart mark closes with a result only by the fields that name a gene or a place: a row by its gene and position columns, a finding by its gene, its genes and its loci, a chart mark by the gene and position of the summary row it draws. It never closes by a vocabulary cell (an inheritance code, a structural-variant type, a genotype, a class, a significance, a unit) or by a structural variant's title, because any code the report prints in capitals may also be a gene symbol: a closed result in AR would else close every recessive carrier row, and one in INS every insertion. `TABLE_COLUMNS` in `bin/live_results.py` says for each column of each card's table whether it names a gene or a place. A finding with no gene and no place (a card's own headline, a data-quality finding), a card's lines and facts, and the lead's own sentences are prose with no such field: they close by the words that name the gene written as a gene symbol. A prose word ("set"), an ontology id (`HP:0001250`) or a unit after a number (`5 MB`, `48-52% GC`) names no gene there. A segment of the lead that holds findings' titles closes as those findings do. The polygenic score card, which names scores and traits only, is never tagged (`NAMES_NO_GENE`), and neither are the marks of a chart whose labels come from a list known before the run (the scores, the ancestry groups, the ClinVar star bars' labels). The genome map leaves a finding the veil ties to a closed result out of its count of findings with no place. +- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call, a repeat) close with them, whatever gene they name, and so does a segment of the lead that names one (`veil.findings` and `veil.lead` tag them `*panel*`): a medicine finding that closed only when the closed result is in its gene would tell the gene. A card can hold a fixed panel among its lines or rows: the ClinVar card prints a line for each known deep intronic site, hit or not; the carrier list prints its two-hit line, which names the copy-number callers it read; and the callability card lists the genes this reference does not hold and the hard genes the reads did not cover, nearly the same for everyone. Those lines and rows close all together while any result to act on is closed and come back once every result is opened (`FIXED_LINES` in `bin/live_results.py`; `veil.lines` and `veil.rows` tag each of them `*panel*`). A table row, a finding or a chart mark closes with a result only by the fields that name a gene or a place: a row by its gene and position columns, a finding by its gene, its genes and its loci, a chart mark by the gene and position of the summary row it draws. It never closes by a vocabulary cell (an inheritance code, a structural-variant type, a genotype, a class, a significance, a unit) or by a structural variant's title, because any code the report prints in capitals may also be a gene symbol: a closed result in AR would else close every recessive carrier row, and one in INS every insertion. `TABLE_COLUMNS` in `bin/live_results.py` says for each column of each card's table whether it names a gene or a place. A place that names two closed results closes with both: its tag is the list of every closed result it names, never the first one only, and it stays closed while any of them is closed. A segment of the lead that names two closed results, or a duplication over two closed genes, comes back only once both are opened, so opening one never tells the other. The page never matches the words of prose against gene symbols: any word the report prints in capitals may be a gene symbol (`AR` in "Cystic fibrosis; AR"), and a line that closes by such a word tells the gene. Prose with no field that names a gene closes whole (`*panel*`) while any result to act on is closed: a card's headline, its lines and facts, a finding with no gene and no place, and the lead's own sentences. The exceptions are structured or declared free of genes, each list pinned in `tests/test_live_results.py`. Structured: the carrier list's two-hit lines close by the gene and variant each line is built from, and its two-hit count by the two-hit genes (`FACT_NAMES`); a segment of the lead closes with the findings it prints (`veil.lead`); a line that gives a position closes by that position. Declared free of genes: the lines of the QC, ClinVar, mosaic, MoChA, ROH, telomere and consumer-export cards (`LINES_NO_GENE`), the facts of the cards in `FACTS_NO_GENE`, the headlines of the cards in `HEADLINES_NO_GENE` (built from counts and fixed words), a data-quality finding, and a card's own finding when its facts and headline are free. The cost: a child's page with any result closed shows less prose (the headline and lines of a card that can name a gene, the search's hits) until the results are opened. The polygenic score card, which names scores and traits only, is never tagged (`NAMES_NO_GENE`), and neither are the marks of a chart whose labels come from a list known before the run (the scores, the ancestry groups, the ClinVar star bars' labels). The genome map leaves a finding the veil ties to a closed result out of its count of findings with no place. - A closed finding carries no id in the page or the address, since its id names its gene and variant. It is known by its place among the page's closed findings (`#f/closed-2`). On a page that keeps a result closed, the address never holds the id of a finding the page can close, open or closed: once opened on purpose it is `#f/opened-`, its place among the opened ones, so no history entry keeps the id. Any other `#f/` on such a page, the id of a closed result or no id at all, lands the same way, on the first closed finding (`#f/closed-1`): the page never picks a closed card by its id, so a guessed id tells nothing. On the home page, a child's closed result links to the child's report, never to the finding's own page, whose address names it. -Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: a hit from a card the page keeps closed, or one that names a closed result, is left out, and so is a suggested gene that only such hits name. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. While the traits card is closed whole on a child's page, the chip is the switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with that result's id. +Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: its index is prose, so while any result to act on is closed it shows no hit and suggests no gene, and its hits come back once every result is opened. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. While the traits card is closed whole on a child's page, the chip is the switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with the ids of every closed result it overlaps (the `veils` column; `veil` keeps the first). A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. @@ -95,7 +95,7 @@ The reads cross the network to the browser that logged in: only the byte ranges `GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. -The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the rows outside a result's own card that close with it; `spans`, the closed results with their position, one row per gene they name, from which the family page knows whether a child has a result still closed; and `panels`, the cards of a fixed panel). A structural variant's finding also lists every gene it covers in `genes`. It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. +The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the places outside a result's own card that close with it, each tagged with the list of closed results it names, or `*panel*`; `spans`, the closed results with their position, one row per gene they name, from which the family page knows whether a child has a result still closed; and `panels`, the cards of a fixed panel). A structural variant's finding also lists every gene it covers in `genes`. It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. ## Privacy and what is stored From 029dedd2c8664b614ba64599879a789c0988f3d8 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:34:09 +0100 Subject: [PATCH 035/255] refactor(ancestry): og archaic and og ancient in Rust, with a chain liftover og archaic table|strict|genotypes|report and og ancient select|lift|person-geno|align|report port bin/archaic_report.py and bin/ancient_report.py. og-intervals gains chain, the UCSC chain reader with a single-position lift. Bead: pgp-bs9u.21 --- rust/Cargo.lock | 1 + rust/crates/og-ancestry/Cargo.toml | 3 +- rust/crates/og-ancestry/src/ancient.rs | 1569 +++++++++++++++++++++ rust/crates/og-ancestry/src/archaic.rs | 1728 ++++++++++++++++++++++++ rust/crates/og-ancestry/src/lib.rs | 2 + rust/crates/og-intervals/src/chain.rs | 376 ++++++ rust/crates/og-intervals/src/lib.rs | 3 + rust/og/src/main.rs | 12 + 8 files changed, 3693 insertions(+), 1 deletion(-) create mode 100644 rust/crates/og-ancestry/src/ancient.rs create mode 100644 rust/crates/og-ancestry/src/archaic.rs create mode 100644 rust/crates/og-intervals/src/chain.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 86841ba6c..c25f174d4 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -601,6 +601,7 @@ name = "og-ancestry" version = "0.1.0" dependencies = [ "flate2", + "og-intervals", "og-io", "og-tsv", "pycompat", diff --git a/rust/crates/og-ancestry/Cargo.toml b/rust/crates/og-ancestry/Cargo.toml index b27b4bbfa..4406d01ba 100644 --- a/rust/crates/og-ancestry/Cargo.toml +++ b/rust/crates/og-ancestry/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "og-ancestry" -description = "Step 26b, fine-grained ancestry: og ancestry-fine panel-map, check-map, order, sample-bed, chroms and report" +description = "Ancestry steps: og ancestry-fine (26b), og archaic (43) and og ancient (44)" version.workspace = true edition.workspace = true license.workspace = true @@ -9,6 +9,7 @@ publish.workspace = true [dependencies] pycompat.workspace = true og-io.workspace = true +og-intervals.workspace = true og-tsv.workspace = true flate2.workspace = true diff --git a/rust/crates/og-ancestry/src/ancient.rs b/rust/crates/og-ancestry/src/ancient.rs new file mode 100644 index 000000000..612a71212 --- /dev/null +++ b/rust/crates/og-ancestry/src/ancient.rs @@ -0,0 +1,1569 @@ +//! Step 44, ancient similarity (`docs/44-ancient.md`): where a genome falls +//! among ancient genomes on a PCA of present-day West Eurasians (smartpca, +//! EIGENSOFT), with the ancient samples of the Allen Ancient DNA Resource +//! (AADR) projected onto the same axes, as `og ancient `. +//! +//! - `select` (setup): reads the AADR `.anno` and `.ind` files and chooses +//! the people: the present-day West Eurasians of [`MODERN_GROUPS`] build +//! the axes, the ancient West Eurasians with enough sites are projected. +//! - `lift` (setup): moves the 1240K sites of the AADR `.snp` file from +//! GRCh37 to GRCh38 with UCSC's chain file ([`og_intervals::chain`]) and +//! keeps a site only when one of its two alleles is the GRCh38 base. +//! - `person-geno` (step 44): the person's genotype at each site, appended +//! as one more column to the present-day panel's EIGENSTRAT `.geno`, +//! streamed row by row. +//! - `align` (setup, step 44): checks that the axes of two smartpca runs on +//! the same panel agree, up to the sign of each. +//! - `report` (step 44): the nearest ancient populations, the summary and +//! the plot. +//! +//! Each command writes what `bin/ancient_report.py` wrote (checked by +//! `tests/parity`). Deliberate changes: every output is written under a +//! temporary name and renamed into place (the Python wrote the population +//! list, `ancients.tsv`, the stats and the text report in place); a gzip +//! input is told by its first bytes, not its name; a `.fai` that does not +//! match its FASTA stops `lift`; and malformed input is an `ERROR:` line +//! and exit 1, never a traceback. + +use super::{ArgError, Args, Fail, Pending, Result, Spec, cannot_read, int, repr, split_ws}; +use og_intervals::chain::Chain; +use og_intervals::fasta::Indexed; +use og_io::{Lines, Utf8}; +use pycompat::fmt::{fixed, int_grouped, int_str, ljust, rjust, round_int}; +use pycompat::text::{html_escape, strip, upper}; +use std::collections::{BTreeMap, HashMap}; +use std::io::{BufRead, Write}; +use std::path::{Path, PathBuf}; + +/// Present-day groups of the AADR 1240K file (Group ID, as written in the +/// `.anno` file of v66.p1) that build the axes: Europe, the Caucasus and +/// the Near East. North African groups and groups with recent East +/// Eurasian ancestry are left out, so the axes describe West Eurasian +/// variation only. +pub const MODERN_GROUPS: [&str; 41] = [ + "Abkhasian", + "Adygei", + "Albanian", + "Armenian", + "Basque", + "BedouinA", + "BedouinB", + "Bulgarian", + "CEU", + "Chechen", + "Czech", + "Druze", + "English", + "Estonian", + "FIN", + "Finnish", + "French", + "GBR", + "Georgian", + "Greek", + "Greek_Crete", + "Hungarian", + "IBS", + "Icelandic", + "Iranian", + "Italian_North", + "Jew_Iraqi", + "Jordanian", + "Lezgin", + "Norwegian", + "Orcadian", + "Ossetian", + "Palestinian", + "Polish", + "Russian", + "Sardinian", + "Scottish", + "Spanish", + "Spanish_North", + "TSI", + "Turkish", +]; +const MODERN_DATA_TYPE: &str = "Shotgun.diploid"; +/// Ancient samples projected: dated, passing the AADR's own assessment, +/// with this many of the 1240K autosomal sites covered, inside West Eurasia. +const MIN_ANCIENT_SNPS: i64 = 100_000; +const LAT: (f64, f64) = (25.0, 75.0); +const LON: (f64, f64) = (-25.0, 75.0); +const PASSING: [&str; 3] = ["PASS", "PROVISIONAL_PASS", "MERGE_PASS"]; +/// The label of a projected ancient person in the `.ind` and `.evec` files. +pub const ANCIENT_LABEL: &str = "Ancient"; +/// The label of the person in the run's `.evec`. +pub const PERSON_LABEL: &str = "Person"; +/// PCs the distance uses, each weighted by the square root of its eigenvalue. +const DISTANCE_PCS: i64 = 4; +const TOP: usize = 20; +/// Two runs on the same panel give the same axes; a PC whose present-day +/// coordinates correlate less than this between the two is a mismatch. +const MIN_AXIS_CORRELATION: f64 = 0.999; +const CAVEAT: &str = "Closeness on a PCA is genetic similarity, not descent: a genome near an ancient group \ +shares ancestry with it, it does not descend from those people."; +const QPADM_NOTE: &str = "qpAdm models of ancestry proportions are not part of this step."; + +/// The `.anno` columns read, matched by the start of the header cell (the +/// AADR writes long explanations after the name). +const ANNO_COLUMNS: [(Col, &str); 13] = [ + (Col::Id, "Genetic ID"), + (Col::Individual, "Individual ID"), + (Col::DateBp, "Date mean in BP"), + (Col::DateSd, "Date standard deviation in BP"), + (Col::FullDate, "Full Date"), + (Col::Group, "Group ID"), + (Col::Locality, "Locality"), + (Col::Country, "Political Entity"), + (Col::Lat, "Latitude"), + (Col::Lon, "Longitude"), + (Col::DataType, "Data type"), + ( + Col::Snps, + "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", + ), + (Col::Assessment, "ASSESSMENT"), +]; + +/// A column of the `.anno` file this tool reads. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Col { + Id, + Individual, + DateBp, + DateSd, + FullDate, + Group, + Locality, + Country, + Lat, + Lon, + DataType, + Snps, + Assessment, +} + +/// One `.anno` row: the 13 cells read, stripped, in [`ANNO_COLUMNS`] order. +#[derive(Debug, Clone)] +struct Anno([String; 13]); + +impl Anno { + fn get(&self, c: Col) -> &str { + &self.0[c as usize] + } +} + +/// A text file read line by line as Python's text mode reads it. +struct Text { + path: PathBuf, + lines: Lines>, + line: String, +} + +impl Text { + fn open(path: &str, utf8: Utf8) -> Result { + let path = PathBuf::from(path); + let input = og_io::open(&path).map_err(|e| cannot_read(&path, &e))?; + Ok(Text { + path, + lines: Lines::new(input, utf8), + line: String::new(), + }) + } + + /// The next line with its `\n`, or `None` at the end. + fn raw(&mut self) -> Result> { + let more = self + .lines + .read_line(&mut self.line) + .map_err(|e| cannot_read(&self.path, &e))?; + Ok(more.then_some(self.line.as_str())) + } + + /// The next line without its `\n`. + fn next(&mut self) -> Result> { + Ok(self.raw()?.map(|l| l.strip_suffix('\n').unwrap_or(l))) + } +} + +/// Python's `float(s)` of a finite value, or `None`. +fn to_float(s: &str) -> Option { + pycompat::num::parse_float(s).ok().filter(|v| v.is_finite()) +} + +/// Python's `float(s)`, or the input error that names the file. +fn need_float(s: &str, path: &Path) -> Result { + pycompat::num::parse_float(s) + .map_err(|_| Fail(format!("{}: {} is not a number", path.display(), repr(s)))) +} + +/// Python's `f >= i` for a float and an int: exact, whatever their sizes. +fn float_ge_int(f: f64, i: i64) -> bool { + let fi = i as f64; + if f != fi { + return f > fi; + } + // f equals the float nearest i, so it is a whole number. + (f as i128) >= i128::from(i) +} + +/// `" ".join(str(s).replace("\t", " ").split()) or "."`. +fn clean(s: &str) -> String { + let out = split_ws(s).collect::>().join(" "); + if out.is_empty() { ".".to_owned() } else { out } +} + +// --- select ----------------------------------------------------------------------- + +/// The index of each read column in the `.anno` header. +fn anno_columns(header: &[&str]) -> Result<[usize; 13]> { + let cells: Vec<&str> = header + .iter() + .map(|c| strip(strip(strip(c, None), Some("\"")), None)) + .collect(); + let mut out = [0usize; 13]; + for (k, (col, name)) in ANNO_COLUMNS.iter().enumerate() { + // ASSESSMENT must not match ASSESSMENT WARNINGS: the exact cell first. + let exact = cells.iter().position(|c| c == name); + let hit = if *col == Col::Assessment { + exact + } else { + exact.or_else(|| cells.iter().position(|c| c.starts_with(name))) + }; + out[k] = hit.ok_or_else(|| Fail(format!("the .anno file has no column '{name}'")))?; + } + Ok(out) +} + +fn passing(assessment: &str) -> bool { + let a = upper(strip(assessment, None)); + PASSING.iter().any(|p| a.starts_with(p)) +} + +/// What [`choose`] keeps of one genetic id. +#[derive(Debug, Clone)] +enum Chosen { + /// A present-day reference person, with their group. + Modern(String), + /// An ancient person to project, with their row. + Ancient(Box), +} + +/// The present-day reference people and the ancient people to project, +/// by genetic id, one genetic id per individual (the one with the most +/// sites). Rows are streamed from the `.anno` file. +fn choose(path: &str, min_snps: i64) -> Result> { + let mut t = Text::open(path, Utf8::Replace)?; + let Some(head) = t.next()? else { + return Err(Fail(format!("{path} is empty"))); + }; + let header: Vec<&str> = head.split('\t').collect(); + let col = anno_columns(&header)?; + let widest = *col.iter().max().expect("13 columns"); + // (kind is modern, individual) -> (sites, row), in first-seen order. + let mut index: HashMap<(bool, String), usize> = HashMap::new(); + let mut best: Vec<(f64, Anno, bool)> = Vec::new(); + while let Some(line) = t.next()? { + let cells: Vec<&str> = line.split('\t').collect(); + if line.is_empty() || cells.len() <= widest { + continue; + } + let r = Anno(col.map(|i| strip(cells[i], None).to_owned())); + if !passing(r.get(Col::Assessment)) { + continue; + } + let bp = to_float(r.get(Col::DateBp)); + let snps = to_float(r.get(Col::Snps)).unwrap_or(0.0); + let modern = if bp == Some(0.0) + && MODERN_GROUPS.contains(&r.get(Col::Group)) + && r.get(Col::DataType) == MODERN_DATA_TYPE + { + true + } else if bp.is_some_and(|b| b > 0.0) && float_ge_int(snps, min_snps) { + let (Some(lat), Some(lon)) = (to_float(r.get(Col::Lat)), to_float(r.get(Col::Lon))) + else { + continue; + }; + if !(LAT.0 <= lat && lat <= LAT.1 && LON.0 <= lon && lon <= LON.1) { + continue; + } + false + } else { + continue; + }; + let who = if r.get(Col::Individual).is_empty() { + r.get(Col::Id) + } else { + r.get(Col::Individual) + }; + let key = (modern, who.to_owned()); + match index.get(&key) { + None => { + index.insert(key, best.len()); + best.push((snps, r, modern)); + } + Some(&i) if snps > best[i].0 => best[i] = (snps, r, modern), + Some(_) => {} + } + } + let mut out = HashMap::new(); + for (_, r, modern) in best { + let id = r.get(Col::Id).to_owned(); + let c = if modern { + Chosen::Modern(r.get(Col::Group).to_owned()) + } else { + Chosen::Ancient(Box::new(r)) + }; + out.insert(id, c); + } + Ok(out) +} + +const SELECT: Spec = Spec { + verb: "select", + usage: "usage: og ancient select --anno ANNO --ind IND --out-ind OUT_IND --poplist POPLIST --ancients ANCIENTS [--min-snps MIN_SNPS]\n", + options: &[ + "--anno", + "--ind", + "--out-ind", + "--poplist", + "--ancients", + "--min-snps", + ], + required: &["--anno", "--ind", "--out-ind", "--poplist", "--ancients"], +}; + +fn cmd_select(a: &Args, out: &mut dyn Write) -> Result<()> { + let min_snps = a + .int("--min-snps", MIN_ANCIENT_SNPS) + .unwrap_or(MIN_ANCIENT_SNPS); + let chosen = choose(a.req("--anno"), min_snps)?; + let (mut n_mod, mut n_anc) = (0u64, 0u64); + let mut groups: std::collections::BTreeSet = Default::default(); + let mut ind = String::new(); + let mut t = Text::open(a.req("--ind"), Utf8::Strict)?; + while let Some(line) = t.next()? { + let c: Vec<&str> = split_ws(line).collect(); + if c.len() < 3 { + continue; + } + let label = match chosen.get(c[0]) { + Some(Chosen::Modern(g)) => { + n_mod += 1; + groups.insert(g.clone()); + g.as_str() + } + Some(Chosen::Ancient(_)) => { + n_anc += 1; + ANCIENT_LABEL + } + None => "Ignore", + }; + ind += &format!("{}\t{}\t{label}\n", c[0], c[1]); + } + let missing: Vec<&str> = { + let mut m: Vec<&str> = MODERN_GROUPS + .iter() + .copied() + .filter(|g| !groups.contains(*g)) + .collect(); + m.sort_unstable(); + m + }; + if n_mod < 50 || n_anc < 1 { + return Err(Fail(format!( + "the .ind file matches {n_mod} present-day reference people and {n_anc} ancient ones; \ + is it the .ind file of the same release as the .anno file?" + ))); + } + let mut files = Pending::new(); + files.put(PathBuf::from(a.req("--out-ind")), ind.as_bytes())?; + let pops: String = groups.iter().map(|g| format!("{g}\n")).collect(); + files.put(PathBuf::from(a.req("--poplist")), pops.as_bytes())?; + let mut anc = + String::from("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n"); + let sorted: BTreeMap<&String, &Chosen> = chosen.iter().collect(); + for c in sorted.values() { + let Chosen::Ancient(r) = c else { continue }; + let cells: Vec = [ + Col::Id, + Col::Group, + Col::DateBp, + Col::DateSd, + Col::FullDate, + Col::Locality, + Col::Country, + Col::Lat, + Col::Lon, + Col::Snps, + ] + .iter() + .map(|k| clean(r.get(*k))) + .collect(); + anc += &cells.join("\t"); + anc.push('\n'); + } + files.put(PathBuf::from(a.req("--ancients")), anc.as_bytes())?; + files.commit_all()?; + let suffix = if !missing.is_empty() && missing.len() <= 10 { + format!(" (not in this file: {})", missing.join(", ")) + } else if !missing.is_empty() { + format!( + " ({} of the {} groups are not in this file)", + missing.len(), + MODERN_GROUPS.len() + ) + } else { + String::new() + }; + let _ = writeln!( + out, + "Present-day reference people: {n_mod} in {} groups{suffix}", + groups.len() + ); + let _ = writeln!( + out, + "Ancient people projected: {n_anc} (at least {} sites, West Eurasia)", + int_grouped(i128::from(min_snps)) + ); + Ok(()) +} + +// --- lift ------------------------------------------------------------------------- + +/// A base's complement, for the four bases. +fn complement(b: &str) -> Option<&'static str> { + match b { + "A" => Some("T"), + "C" => Some("G"), + "G" => Some("C"), + "T" => Some("A"), + _ => None, + } +} + +/// The chromosome column of an EIGENSOFT `.snp` line: 1 to 22, or `None`. +fn eigen_chrom(c: &str) -> Option { + int(c).filter(|n| (1..=22).contains(n)) +} + +const LIFT: Spec = Spec { + verb: "lift", + usage: "usage: og ancient lift --snp SNP --chain CHAIN --fasta FASTA --out OUT\n", + options: &["--snp", "--chain", "--fasta", "--out"], + required: &["--snp", "--chain", "--fasta", "--out"], +}; + +fn cmd_lift(a: &Args, out: &mut dyn Write) -> Result<()> { + let chain_path = a.req("--chain"); + let input = og_io::open(chain_path).map_err(|e| cannot_read(Path::new(chain_path), &e))?; + let chain = Chain::read(input).map_err(|e| Fail(format!("{chain_path}: {e}")))?; + let fasta_path = a.req("--fasta"); + let mut fasta = Indexed::open(fasta_path).map_err(|e| Fail(format!("{fasta_path}: {e}")))?; + let mut kept: Vec<(i64, i64, String, &'static str, &'static str)> = Vec::new(); + let mut dropped: BTreeMap<&str, u64> = BTreeMap::new(); + let mut n: u64 = 0; + let snp_path = a.req("--snp"); + let mut t = Text::open(snp_path, Utf8::Replace)?; + let mut lineno = 0u64; + while let Some(line) = t.next()? { + lineno += 1; + let c: Vec<&str> = split_ws(line).collect(); + if c.len() < 6 { + continue; + } + n += 1; + let Some(chrom) = eigen_chrom(c[1]) else { + *dropped.entry("not an autosome").or_default() += 1; + continue; + }; + let (u1, u2) = (upper(c[4]), upper(c[5])); + let (Some(k1), Some(k2)) = (complement(&u1), complement(&u2)) else { + *dropped + .entry("alleles not two different bases") + .or_default() += 1; + continue; + }; + if u1 == u2 { + *dropped + .entry("alleles not two different bases") + .or_default() += 1; + continue; + } + let pos = pycompat::num::parse_float(c[3]) + .ok() + .filter(|p| p.is_finite()) + .map(f64::trunc) + .filter(|p| p.abs() < 9.0e18) + .ok_or_else(|| { + Fail(format!( + "{snp_path}: line {lineno}: position {} is not a number", + repr(c[3]) + )) + })? as i64; + let name = format!("chr{chrom}"); + let Some(hit) = chain.lift(&name, pos) else { + *dropped.entry("no single place on GRCh38").or_default() += 1; + continue; + }; + if hit.chrom != name { + *dropped.entry("moves to another chromosome").or_default() += 1; + continue; + } + let (a1, a2) = if hit.strand == "-" { + (k1, k2) + } else { + ( + complement(k1).expect("a base"), + complement(k2).expect("a base"), + ) + }; + let q_pos = hit.pos; + let base = if q_pos >= 1 { + fasta + .base(hit.chrom, (q_pos - 1) as u64) + .map_err(|e| Fail(format!("{fasta_path}: {e}")))? + } else { + None + }; + let base = base.map(|b| b.to_ascii_uppercase()); + if !base.is_some_and(|b| b == a1.as_bytes()[0] || b == a2.as_bytes()[0]) { + *dropped + .entry("neither allele is the GRCh38 base") + .or_default() += 1; + continue; + } + kept.push((chrom, q_pos, c[0].to_owned(), a1, a2)); + } + kept.sort_unstable(); + let mut text = String::from("snp\tchrom\tpos\tallele1\tallele2\n"); + for (chrom, pos, sid, a1, a2) in &kept { + text += &format!("{sid}\tchr{chrom}\t{pos}\t{a1}\t{a2}\n"); + } + let mut files = Pending::new(); + files.put(PathBuf::from(a.req("--out")), text.as_bytes())?; + files.commit_all()?; + let _ = writeln!( + out, + "Sites of the .snp file: {}; on GRCh38: {}", + int_grouped(i128::from(n)), + int_grouped(kept.len() as i128) + ); + for (why, k) in &dropped { + let _ = writeln!(out, " dropped, {why}: {}", int_grouped(i128::from(*k))); + } + if kept.is_empty() { + return Err(Fail( + "no site could be placed on GRCh38: check the chain file and the reference".into(), + )); + } + Ok(()) +} + +// --- person-geno ------------------------------------------------------------------ + +/// Copies of allele1 in a bcftools `%TGT` genotype, or `None` (missing, a +/// haploid or half call, or an allele that is neither of the two). +fn dosage(gt: &str, a1: &str, a2: &str) -> Option { + let mut parts = gt.split(['/', '|']); + let (Some(x), Some(y), None) = (parts.next(), parts.next(), parts.next()) else { + return None; + }; + let mut d = 0; + for al in [x, y] { + if al == a1 { + d += 1; + } else if al != a2 { + return None; + } + } + Some(d) +} + +/// The panel's GRCh38 sites: per (chromosome, position), the sites there +/// (id, allele1, allele2), packed in one array. +struct Sites { + chroms: HashMap, + at: HashMap<(u32, i64), Vec>, + sites: Vec<(Box, Box, Box)>, +} + +fn read_sites(path: &str) -> Result { + let mut s = Sites { + chroms: HashMap::new(), + at: HashMap::new(), + sites: Vec::new(), + }; + let mut t = Text::open(path, Utf8::Strict)?; + let _ = t.next()?; + let mut lineno = 1u64; + while let Some(line) = t.next()? { + lineno += 1; + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 { + continue; + } + let pos = int(c[2]).ok_or_else(|| { + Fail(format!( + "{path}: line {lineno}: position {} is not a whole number", + repr(c[2]) + )) + })?; + let next = s.chroms.len() as u32; + let chrom = *s.chroms.entry(c[1].to_owned()).or_insert(next); + s.at.entry((chrom, pos)) + .or_default() + .push(s.sites.len() as u32); + s.sites.push((c[0].into(), c[3].into(), c[4].into())); + } + Ok(s) +} + +const PERSON_GENO: Spec = Spec { + verb: "person-geno", + usage: "usage: og ancient person-geno --sites SITES --genotypes GENOTYPES --snp SNP --moderns-geno MODERNS_GENO --out OUT --stats STATS\n", + options: &[ + "--sites", + "--genotypes", + "--snp", + "--moderns-geno", + "--out", + "--stats", + ], + required: &[ + "--sites", + "--genotypes", + "--snp", + "--moderns-geno", + "--out", + "--stats", + ], +}; + +fn cmd_person_geno(a: &Args, out: &mut dyn Write) -> Result<()> { + let sites = read_sites(a.req("--sites"))?; + // The person's dosage by site id: a later line for the same site wins. + let mut dos: HashMap<&str, u8> = HashMap::new(); + let (mut called, mut other) = (0u64, 0u64); + let gpath = a.req("--genotypes"); + let mut t = Text::open(gpath, Utf8::Strict)?; + let mut lineno = 0u64; + while let Some(line) = t.next()? { + lineno += 1; + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 { + continue; + } + let pos = int(c[1]).ok_or_else(|| { + Fail(format!( + "{gpath}: line {lineno}: position {} is not a whole number", + repr(c[1]) + )) + })?; + let Some(&chrom) = sites.chroms.get(c[0]) else { + continue; + }; + let Some(here) = sites.at.get(&(chrom, pos)) else { + continue; + }; + let gt = upper(c[4]); + // quirk: a position the AADR holds two SNPs at is counted once per + // SNP in sites_genotyped, as the Python counted it. + for &i in here { + let (sid, a1, a2) = &sites.sites[i as usize]; + match dosage(>, a1, a2) { + None => other += 1, + Some(d) => { + dos.insert(sid, d); + called += 1; + } + } + } + } + let snp_path = a.req("--snp"); + let geno_path = a.req("--moderns-geno"); + let mut snp = Text::open(snp_path, Utf8::Strict)?; + let mut geno = Text::open(geno_path, Utf8::Strict)?; + let mut files = Pending::new(); + let out_path = PathBuf::from(a.req("--out")); + let shown = out_path.clone(); + let o = files.create(out_path)?; + let (mut n, mut used) = (0u64, 0u64); + let mut width: Option = None; + let mut row = String::new(); + while let Some(sline) = snp.next()? { + let Some(id) = split_ws(sline).next() else { + continue; + }; + let d = dos.get(id).copied(); + let g = geno.next()?.unwrap_or(""); + if g.is_empty() { + return Err(Fail(format!("{geno_path} has fewer lines than {snp_path}"))); + } + let w = g.chars().count(); + match width { + None => width = Some(w), + Some(want) if w != want => { + return Err(Fail(format!( + "{geno_path} line {} has {w} genotypes, not {want}", + n + 1 + ))); + } + Some(_) => {} + } + if d.is_some() { + used += 1; + } + row.clear(); + row.push_str(g); + match d { + None => row.push('9'), + Some(d) => row.push(char::from(b'0' + d)), + } + row.push('\n'); + o.write_all(row.as_bytes()) + .map_err(|e| super::cannot_write(&shown, &e))?; + n += 1; + } + if geno.raw()?.is_some() { + return Err(Fail(format!("{geno_path} has more lines than {snp_path}"))); + } + let genotyped = called + other; + files.put( + PathBuf::from(a.req("--stats")), + format!( + "panel_sites\t{n}\nsites_genotyped\t{genotyped}\nother_allele\t{other}\nsites_used\t{used}\n" + ) + .as_bytes(), + )?; + files.commit_all()?; + let _ = writeln!( + out, + "Sites of the panel: {}; genotyped from the gVCF: {}; used: {} (an allele that is neither of a site's two: {})", + int_grouped(i128::from(n)), + int_grouped(i128::from(genotyped)), + int_grouped(i128::from(used)), + int_grouped(i128::from(other)) + ); + if used == 0 { + return Err(Fail( + "none of the panel's sites was genotyped: is the gVCF on GRCh38 with chr names?".into(), + )); + } + Ok(()) +} + +// --- evec, align ------------------------------------------------------------------ + +/// A smartpca `.evec`: the eigenvalues, and each row's label and PCs in +/// file order (a repeated id keeps its first place and its last values). +struct Evec { + evals: Vec, + rows: Vec<(String, String, Vec)>, + index: HashMap, +} + +impl Evec { + fn get(&self, id: &str) -> Option<(&str, &[f64])> { + self.index.get(id).map(|&i| { + let r = &self.rows[i]; + (r.1.as_str(), r.2.as_slice()) + }) + } +} + +fn read_evec(path: &str) -> Result { + let mut e = Evec { + evals: Vec::new(), + rows: Vec::new(), + index: HashMap::new(), + }; + let p = Path::new(path); + let mut t = Text::open(path, Utf8::Strict)?; + while let Some(line) = t.next()? { + let c: Vec<&str> = split_ws(line).collect(); + let Some(first) = c.first() else { continue }; + if first.starts_with('#') { + if *first == "#eigvals:" { + e.evals = c[1..] + .iter() + .map(|x| need_float(x, p)) + .collect::>()?; + } + continue; + } + let label = c[c.len() - 1].to_owned(); + let pcs: Vec = if c.len() > 1 { + c[1..c.len() - 1] + .iter() + .map(|x| need_float(x, p)) + .collect::>()? + } else { + Vec::new() + }; + match e.index.get(*first) { + Some(&i) => e.rows[i] = ((*first).to_owned(), label, pcs), + None => { + e.index.insert((*first).to_owned(), e.rows.len()); + e.rows.push(((*first).to_owned(), label, pcs)); + } + } + } + if e.evals.is_empty() || e.rows.is_empty() { + return Err(Fail(format!( + "{path} is not a smartpca .evec file (no #eigvals: line or no rows)" + ))); + } + Ok(e) +} + +/// The Pearson correlation of two equal-length series, 0 when either is +/// flat; means by `fmean`, sums by the builtin `sum`. +fn correlation(x: &[f64], y: &[f64]) -> f64 { + let mx = pycompat::stats::fmean(x.iter().copied()).unwrap_or(f64::NAN); + let my = pycompat::stats::fmean(y.iter().copied()).unwrap_or(f64::NAN); + let sxy = pycompat::num::sum(x.iter().zip(y).map(|(a, b)| (a - mx) * (b - my))); + let sx = pycompat::num::sum(x.iter().map(|a| (a - mx) * (a - mx))).sqrt(); + let sy = pycompat::num::sum(y.iter().map(|b| (b - my) * (b - my))).sqrt(); + if sx > 0.0 && sy > 0.0 { + sxy / (sx * sy) + } else { + 0.0 + } +} + +/// The k-th PC of a row, or the error that names the row. +fn pc(pcs: &[f64], k: usize, id: &str, path: &str) -> Result { + pcs.get(k) + .copied() + .ok_or_else(|| Fail(format!("{path}: {id} has no PC{}", k + 1))) +} + +/// The sign that turns each of the run's PCs into the panel's, from the +/// present-day people both runs used to build the axes. +fn axis_signs( + panel: &Evec, + panel_path: &str, + run: &Evec, + run_path: &str, + n_pcs: usize, +) -> Result> { + let mut shared: Vec<&str> = run + .rows + .iter() + .filter(|(id, lab, _)| { + lab != ANCIENT_LABEL + && lab != PERSON_LABEL + && panel.get(id).is_some_and(|(pl, _)| pl == lab) + }) + .map(|(id, ..)| id.as_str()) + .collect(); + shared.sort_unstable(); + if shared.len() < 10 { + return Err(Fail(format!( + "only {} present-day people are in both runs; the panel and this run do not match", + shared.len() + ))); + } + let mut signs = Vec::with_capacity(n_pcs); + for k in 0..n_pcs { + let mut x = Vec::with_capacity(shared.len()); + let mut y = Vec::with_capacity(shared.len()); + for id in &shared { + x.push(pc(panel.get(id).expect("shared").1, k, id, panel_path)?); + y.push(pc(run.get(id).expect("shared").1, k, id, run_path)?); + } + let r = correlation(&x, &y); + if r.abs() < MIN_AXIS_CORRELATION { + return Err(Fail(format!( + "PC{} of this run correlates {} with the panel's over {} present-day people: \ + the axes differ. Install the panel again (setup.sh --ancient-panel).", + k + 1, + fixed(r, 4), + shared.len() + ))); + } + signs.push(if r > 0.0 { 1.0 } else { -1.0 }); + } + Ok(signs) +} + +const ALIGN: Spec = Spec { + verb: "align", + usage: "usage: og ancient align --panel-evec PANEL_EVEC --run-evec RUN_EVEC [--pcs PCS]\n", + options: &["--panel-evec", "--run-evec", "--pcs"], + required: &["--panel-evec", "--run-evec"], +}; + +fn cmd_align(a: &Args, out: &mut dyn Write) -> Result<()> { + let (pp, rp) = (a.req("--panel-evec"), a.req("--run-evec")); + let panel = read_evec(pp)?; + let run = read_evec(rp)?; + let pcs = a.int("--pcs", DISTANCE_PCS).unwrap_or(DISTANCE_PCS); + let signs = axis_signs(&panel, pp, &run, rp, usize::try_from(pcs).unwrap_or(0))?; + let words: Vec = signs + .iter() + .enumerate() + .map(|(i, s)| format!("PC{}{}", i + 1, if *s > 0.0 { '+' } else { '-' })) + .collect(); + let _ = writeln!(out, "Axes agree with the panel's: {}", words.join(" ")); + Ok(()) +} + +// --- report ----------------------------------------------------------------------- + +/// A whole float as Python prints the int of it. +fn whole(x: f64) -> String { + int_str(x) +} + +/// `int(round(x))` printed. +fn rounded(x: f64) -> String { + whole(round_int(x).unwrap_or(f64::NAN)) +} + +/// A date in years before 1950 as `N BCE` or `N CE`, the AADR's own +/// reading: BCE = BP - 1950 and CE = 1950 - BP, with 1950 BP as 1 BCE. +fn year_label(bp: f64) -> String { + let y = 1950.0 - round_int(bp).unwrap_or(0.0); + if y >= 1.0 { + format!("{} CE", whole(y)) + } else { + format!("{} BCE", whole(if -y > 1.0 { -y } else { 1.0 })) + } +} + +/// The span of a population's dates, oldest first: `5300-4900 BCE`, +/// `200-450 CE`, or `300 BCE-200 CE`. +fn range_label(old_bp: f64, young_bp: f64) -> String { + let (a, b) = (year_label(old_bp), year_label(young_bp)); + if a == b { + return a; + } + let (na, ea) = a.split_once(' ').expect("N ERA"); + let (nb, eb) = b.split_once(' ').expect("N ERA"); + if ea == eb { + format!("{na}-{nb} {ea}") + } else { + format!("{a}-{b}") + } +} + +/// An `ancients.tsv` row as `csv.DictReader` gives it: a short row's +/// missing cells are `None`. +struct AncientRow { + group: String, + date_bp: Option, + country: Option, + locality: Option, +} + +/// `ancients.tsv` by id, in file order (a repeated id keeps its first +/// place and its last values). +fn read_ancients(path: &str) -> Result> { + let mut t = Text::open(path, Utf8::Strict)?; + let mut rows: Vec<(String, AncientRow)> = Vec::new(); + let mut index: HashMap = HashMap::new(); + // DictReader: the first non-empty row is the header. + let mut names: Option> = None; + while let Some(line) = t.next()? { + if line.is_empty() { + continue; + } + let cells: Vec<&str> = line.split('\t').collect(); + let Some(names) = names.as_ref() else { + names = Some(cells.iter().map(|s| (*s).to_owned()).collect()); + continue; + }; + // The last column of a repeated name wins, as in a dict. + let field = |key: &str| -> Option { + let k = names.iter().rposition(|n| n == key)?; + cells.get(k).map(|s| (*s).to_owned()) + }; + if !names.iter().any(|n| n == "id") { + return Err(Fail(format!("{path} has no id column"))); + } + let id = field("id").unwrap_or_else(|| "None".to_owned()); + let r = AncientRow { + group: field("group").unwrap_or_else(|| "None".to_owned()), + date_bp: field("date_bp"), + country: field("country"), + locality: field("locality"), + }; + match index.get(&id) { + Some(&i) => rows[i].1 = r, + None => { + index.insert(id.clone(), rows.len()); + rows.push((id, r)); + } + } + } + Ok(rows) +} + +/// The most common non-empty value other than `.`, the first in sort +/// order on a tie; `.` when there is none. +fn most_common<'a>(values: impl Iterator>) -> String { + let mut counts: BTreeMap<&str, u64> = BTreeMap::new(); + for v in values.flatten() { + if !v.is_empty() && v != "." { + *counts.entry(v).or_default() += 1; + } + } + let top = counts.values().copied().max().unwrap_or(0); + counts + .into_iter() + .find(|(_, k)| *k == top) + .map_or_else(|| ".".to_owned(), |(v, _)| v.to_owned()) +} + +/// One ancient group, nearest first. +struct Near { + rank: usize, + group: String, + n: usize, + distance: f64, + pc1: f64, + pc2: f64, + date_bp: Option, + dates: String, + country: String, + locality: String, +} + +/// The ancient groups ranked by the weighted distance from the person to +/// the group's mean position on the first `n_pcs` PCs; the top [`TOP`] and +/// the number of groups. +fn nearest_populations( + person: &[f64], + panel: &Evec, + ancients: &[(String, AncientRow)], + n_pcs: usize, +) -> Result<(Vec, usize)> { + let w: Vec = panel + .evals + .iter() + .take(n_pcs) + .map(|v| if 0.0 > *v { 0.0 } else { *v }.sqrt()) + .collect(); + // Groups in first-seen order, each with its members. + let mut order: Vec<&str> = Vec::new(); + let mut groups: HashMap<&str, Vec<(&[f64], &AncientRow)>> = HashMap::new(); + for (gid, r) in ancients { + if let Some((label, pcs)) = panel.get(gid) + && label == ANCIENT_LABEL + { + let g = groups.entry(r.group.as_str()).or_insert_with(|| { + order.push(r.group.as_str()); + Vec::new() + }); + g.push((pcs, r)); + } + } + let mut rows = Vec::with_capacity(order.len()); + for g in order { + let members = &groups[g]; + let k = members[0].0.len().min(n_pcs); + let mut centre = Vec::with_capacity(k); + for i in 0..k { + let mut xs = Vec::with_capacity(members.len()); + for (pcs, _) in members { + xs.push(*pcs.get(i).ok_or_else(|| { + Fail(format!( + "a member of the ancient group {g} has no PC{}", + i + 1 + )) + })?); + } + centre.push(pycompat::stats::fmean(xs).map_err(|e| Fail(e.to_string()))?); + } + let Some(&pc1) = centre.first() else { + return Err(Fail(format!("the ancient group {g} has no PCs"))); + }; + let distance = pycompat::num::sum(person.iter().take(n_pcs).zip(¢re).zip(&w).map( + |((a, b), wi)| { + let d = wi * (a - b); + d * d + }, + )) + .sqrt(); + let dates: Vec = members + .iter() + .filter_map(|(_, r)| r.date_bp.as_deref().and_then(to_float)) + .collect(); + let (date_bp, dates_text) = if dates.is_empty() { + (None, ".".to_owned()) + } else { + let old = dates + .iter() + .copied() + .fold(dates[0], |m, d| if d > m { d } else { m }); + let young = dates + .iter() + .copied() + .fold(dates[0], |m, d| if d < m { d } else { m }); + ( + Some( + pycompat::stats::median(dates.iter().copied()) + .map_err(|e| Fail(e.to_string()))?, + ), + range_label(old, young), + ) + }; + rows.push(Near { + rank: 0, + group: g.to_owned(), + n: members.len(), + distance, + pc1, + pc2: centre.get(1).copied().unwrap_or(0.0), + date_bp, + dates: dates_text, + country: most_common(members.iter().map(|(_, r)| r.country.as_deref())), + locality: most_common(members.iter().map(|(_, r)| r.locality.as_deref())), + }); + } + rows.sort_by(|a, b| { + a.distance + .partial_cmp(&b.distance) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.group.cmp(&b.group)) + }); + for (i, r) in rows.iter_mut().enumerate() { + r.rank = i + 1; + } + let n = rows.len(); + rows.truncate(TOP); + Ok((rows, n)) +} + +/// Age bins of the plot: (oldest BP, colour, legend), oldest first. +const AGE_BINS: [(f64, &str, &str); 4] = [ + (f64::INFINITY, "#5aa9e6", "before 8000 BCE"), + (9950.0, "#7fc97f", "8000-4000 BCE"), + (5950.0, "#f0a35e", "4000-1000 BCE"), + (2950.0, "#e377c2", "after 1000 BCE"), +]; + +fn age_colour(bp: Option) -> &'static str { + let mut colour = AGE_BINS[0].1; + for (limit, c, _) in AGE_BINS { + if bp.is_some_and(|b| b < limit) { + colour = c; + } + } + colour +} + +/// Python's `min()` and `max()` of a non-empty list of floats. +fn py_min_max(xs: &[f64]) -> (f64, f64) { + let (mut lo, mut hi) = (xs[0], xs[0]); + for &x in &xs[1..] { + if x < lo { + lo = x; + } + if x > hi { + hi = x; + } + } + (lo, hi) +} + +/// The PC1/PC2 plane: present-day people grey, ancient people coloured by +/// age, the person a large ring, on a dark background. +fn svg_plot( + person: &[f64], + panel: &Evec, + ancients: &HashMap, + sample: &str, + nearest: &[Near], +) -> Result { + const WIDTH: i64 = 900; + const HEIGHT: i64 = 720; + const PAD: f64 = 0.04; + const LEFT: i64 = 60; + const RIGHT: i64 = 20; + const TOPM: i64 = 50; + const BOTTOM: i64 = 110; + let two = |id: &str, pcs: &[f64]| -> Result<(f64, f64)> { + match pcs { + [x, y, ..] => Ok((*x, *y)), + _ => Err(Fail(format!("{id} has fewer than two PCs to plot"))), + } + }; + let mut xs = Vec::with_capacity(panel.rows.len() + 1); + let mut ys = Vec::with_capacity(panel.rows.len() + 1); + for (id, _, pcs) in &panel.rows { + let (x, y) = two(id, pcs)?; + xs.push(x); + ys.push(y); + } + let (px0, py0) = two(sample, person)?; + xs.push(px0); + ys.push(py0); + let (x0, x1) = py_min_max(&xs); + let (y0, y1) = py_min_max(&ys); + let or_one = |d: f64| if d == 0.0 { 1.0 } else { d }; + let (dx, dy) = (or_one(x1 - x0), or_one(y1 - y0)); + let (pw, ph) = (WIDTH - LEFT - RIGHT, HEIGHT - TOPM - BOTTOM); + let (left, top, pwf, phf) = (LEFT as f64, TOPM as f64, pw as f64, ph as f64); + let x = |v: f64| left + ((v - x0) / dx * (1.0 - 2.0 * PAD) + PAD) * pwf; + let y = |v: f64| top + (1.0 - ((v - y0) / dy * (1.0 - 2.0 * PAD) + PAD)) * phf; + let e = |s: &str| html_escape(s).into_owned(); + let mut out = vec![ + format!( + "" + ), + "" + .to_owned(), + format!(""), + format!( + "{} among ancient West Eurasians (AADR, projected onto present-day PCs)", + e(sample) + ), + format!( + "" + ), + ]; + for (_, label, pcs) in &panel.rows { + if label != ANCIENT_LABEL { + out.push(format!( + "", + fixed(x(pcs[0]), 1), + fixed(y(pcs[1]), 1) + )); + } + } + for (gid, label, pcs) in &panel.rows { + if label == ANCIENT_LABEL + && let Some(r) = ancients.get(gid) + { + let c = age_colour(r.date_bp.as_deref().and_then(to_float)); + out.push(format!( + "", + fixed(x(pcs[0]), 1), + fixed(y(pcs[1]), 1) + )); + } + } + for r in nearest.iter().take(5) { + out.push(format!( + "{}", + fixed(x(r.pc1) + 6.0, 1), + fixed(y(r.pc2) - 4.0, 1), + e(&r.group) + )); + } + let (px, py) = (x(px0), y(py0)); + out.push(format!( + "", + fixed(px, 1), + fixed(py, 1) + )); + out.push(format!( + "", + fixed(px, 1), + fixed(py, 1) + )); + out.push(format!( + "{}", + fixed(px + 12.0, 1), + fixed(py + 4.0, 1), + e(sample) + )); + out.push(format!( + "PC1", + fixed(left + pwf / 2.0, 0), + TOPM + ph + 22 + )); + let mid = fixed(top + phf / 2.0, 0); + out.push(format!( + "PC2" + )); + let (mut lx, ly) = (LEFT, TOPM + ph + 48); + out.push(format!( + "present day", + lx + 5, + ly - 4, + lx + 14 + )); + lx += 120; + for (_, c, name) in AGE_BINS { + out.push(format!( + "{}", + lx + 5, + ly - 4, + lx + 14, + e(name) + )); + lx += 150; + } + out.push(format!( + "{}", + ly + 26, + e(CAVEAT) + )); + out.push("".to_owned()); + Ok(out.join("\n") + "\n") +} + +const REPORT: Spec = Spec { + verb: "report", + usage: "usage: og ancient report --panel-evec PANEL_EVEC --run-evec RUN_EVEC --ancients ANCIENTS --sample SAMPLE --release RELEASE [--stats STATS] --out-tsv OUT_TSV --out-info OUT_INFO --out-txt OUT_TXT --out-svg OUT_SVG\n", + options: &[ + "--panel-evec", + "--run-evec", + "--ancients", + "--sample", + "--release", + "--stats", + "--out-tsv", + "--out-info", + "--out-txt", + "--out-svg", + ], + required: &[ + "--panel-evec", + "--run-evec", + "--ancients", + "--sample", + "--release", + "--out-tsv", + "--out-info", + "--out-txt", + "--out-svg", + ], +}; + +fn cmd_report(a: &Args, out: &mut dyn Write) -> Result<()> { + let (pp, rp) = (a.req("--panel-evec"), a.req("--run-evec")); + let panel = read_evec(pp)?; + let run = read_evec(rp)?; + let sample = a.req("--sample"); + let Some((PERSON_LABEL, pcs)) = run.get(sample) else { + return Err(Fail(format!( + "{rp} has no row for {sample} labelled {PERSON_LABEL}" + ))); + }; + let n_pcs = (DISTANCE_PCS as usize) + .min(panel.evals.len()) + .min(pcs.len()); + let signs = axis_signs(&panel, pp, &run, rp, n_pcs.max(2))?; + let person: Vec = signs.iter().zip(pcs).map(|(s, v)| s * v).collect(); + let ancients = read_ancients(a.req("--ancients"))?; + let (nearest, n_groups) = nearest_populations(&person, &panel, &ancients, n_pcs)?; + let mut stats: HashMap = HashMap::new(); + if let Some(sp) = a.get("--stats").filter(|s| !s.is_empty()) { + let mut t = Text::open(sp, Utf8::Strict)?; + while let Some(line) = t.next()? { + let (k, v) = line.split_once('\t').unwrap_or((line, "")); + stats.insert(k.to_owned(), v.to_owned()); + } + } + let stat = |k: &str| stats.get(k).map_or(".", String::as_str).to_owned(); + let n_anc = panel.rows.iter().filter(|r| r.1 == ANCIENT_LABEL).count(); + let n_mod = panel + .rows + .iter() + .filter(|r| r.1 != ANCIENT_LABEL && r.1 != PERSON_LABEL) + .count(); + let mut files = Pending::new(); + let mut tsv = + String::from("rank\tgroup\tn\tdistance\tpc1\tpc2\tdate_bp\tdates\tcountry\tlocality\n"); + for r in &nearest { + tsv += &format!( + "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n", + r.rank, + clean(&r.group), + r.n, + fixed(r.distance, 5), + fixed(r.pc1, 5), + fixed(r.pc2, 5), + r.date_bp.map_or_else(|| ".".to_owned(), rounded), + r.dates, + clean(&r.country), + clean(&r.locality) + ); + } + files.put(PathBuf::from(a.req("--out-tsv")), tsv.as_bytes())?; + let release = a.req("--release"); + let p1 = person.first().copied().unwrap_or(f64::NAN); + let p2 = person.get(1).copied().unwrap_or(f64::NAN); + let info: [(&str, String); 9] = [ + ("release", release.to_owned()), + ("sites_used", stat("sites_used")), + ("panel_sites", stat("panel_sites")), + ("present_day_people", n_mod.to_string()), + ("ancient_people", n_anc.to_string()), + ("ancient_groups", n_groups.to_string()), + ("pcs_in_distance", n_pcs.to_string()), + ("pc1", fixed(p1, 5)), + ("pc2", fixed(p2, 5)), + ]; + let mut text = String::from("key\tvalue\n"); + for (k, v) in &info { + text += &format!("{k}\t{v}\n"); + } + files.put(PathBuf::from(a.req("--out-info")), text.as_bytes())?; + let by_id: HashMap = + ancients.iter().map(|(id, r)| (id.clone(), r)).collect(); + let svg = svg_plot(&person, &panel, &by_id, sample, &nearest)?; + let mut lines = vec![ + format!( + "Ancient populations nearest to {sample} (AADR {release}, smartpca with lsqproject)" + ), + format!( + "Sites used: {} of {}; axes from {n_mod} present-day West Eurasians; {n_anc} ancient people in {n_groups} groups projected.", + stat("sites_used"), + stat("panel_sites") + ), + format!( + "Distance: the first {n_pcs} PCs, each weighted by the square root of its eigenvalue." + ), + String::new(), + ]; + for r in nearest.iter().take(10) { + lines.push(format!( + " {}. {} {} {} n={} d={}", + rjust(&r.rank.to_string(), 2), + ljust(&r.group, 45), + ljust(&r.dates, 22), + ljust(&r.country, 20), + ljust(&r.n.to_string(), 4), + fixed(r.distance, 4) + )); + } + lines.extend([String::new(), CAVEAT.to_owned(), QPADM_NOTE.to_owned()]); + let body = lines.join("\n") + "\n"; + files.put(PathBuf::from(a.req("--out-svg")), svg.as_bytes())?; + files.put(PathBuf::from(a.req("--out-txt")), body.as_bytes())?; + files.commit_all()?; + let _ = out.write_all(body.as_bytes()); + Ok(()) +} + +// --- the command line ------------------------------------------------------------- + +const USAGE: &str = "usage: og ancient {select,lift,person-geno,align,report} ...\n"; + +type Run = fn(&Args, &mut dyn Write) -> Result<()>; + +/// `og ancient [args]`: the exit code. Results go to `out`, +/// messages to `err`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let Some(verb) = args.first() else { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og ancient: error: the following arguments are required: verb" + ); + return 2; + }; + let (spec, run): (&Spec, Run) = match verb.as_str() { + "select" => (&SELECT, cmd_select), + "lift" => (&LIFT, cmd_lift), + "person-geno" => (&PERSON_GENO, cmd_person_geno), + "align" => (&ALIGN, cmd_align), + "report" => (&REPORT, cmd_report), + "-h" | "--help" => { + let _ = write!(out, "{USAGE}"); + return 0; + } + other => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og ancient: error: argument verb: invalid choice: {}", + repr(other) + ); + return 2; + } + }; + let parsed = Args::parse(spec, &args[1..]).and_then(|a| { + for name in ["--min-snps", "--pcs"] { + if spec.options.contains(&name) { + a.int(name, 0)?; + } + } + Ok(a) + }); + let a = match parsed { + Ok(a) => a, + Err(ArgError::Help) => { + let _ = write!(out, "{}", spec.usage); + return 0; + } + Err(ArgError::Usage(msg)) => { + let _ = write!(err, "{}", spec.usage); + let _ = writeln!(err, "og ancient {}: error: {msg}", spec.verb); + return 2; + } + }; + match run(&a, out) { + Ok(()) => 0, + Err(Fail(msg)) => { + let _ = out.flush(); + let _ = writeln!(err, "ERROR: {msg}"); + 1 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dosages() { + assert_eq!(dosage("A/G", "A", "G"), Some(1)); + assert_eq!(dosage("A|A", "A", "G"), Some(2)); + assert_eq!(dosage("G/G", "A", "G"), Some(0)); + assert_eq!(dosage("A/T", "A", "G"), None); + assert_eq!(dosage("./.", "A", "G"), None); + assert_eq!(dosage("A", "A", "G"), None); + } + + #[test] + fn year_labels() { + assert_eq!(year_label(4250.0), "2300 BCE"); + assert_eq!(year_label(1950.0), "1 BCE"); + assert_eq!(year_label(1949.0), "1 CE"); + assert_eq!(year_label(1750.5), "200 CE"); + assert_eq!(range_label(7250.0, 6850.0), "5300-4900 BCE"); + assert_eq!(range_label(1750.0, 1500.0), "200-450 CE"); + assert_eq!(range_label(2250.0, 1750.0), "300 BCE-200 CE"); + assert_eq!(range_label(2250.0, 2250.0), "300 BCE"); + } + + #[test] + fn most_common_breaks_ties_by_name() { + let v = [ + Some("b"), + Some("a"), + Some("b"), + Some("a"), + Some("."), + None, + Some(""), + ]; + assert_eq!(most_common(v.into_iter()), "a"); + assert_eq!(most_common([Some("."), None].into_iter()), "."); + } + + #[test] + fn exact_int_compare() { + assert!(float_ge_int(100_000.0, 100_000)); + assert!(!float_ge_int(99_999.5, 100_000)); + assert!(!float_ge_int( + 9_007_199_254_740_992.0, + 9_007_199_254_740_993 + )); + assert!(!float_ge_int(f64::NAN, 0)); + } + + #[test] + fn clean_cells() { + assert_eq!(clean(" a\t b "), "a b"); + assert_eq!(clean(" \t "), "."); + } +} diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs new file mode 100644 index 000000000..37bdf7624 --- /dev/null +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -0,0 +1,1728 @@ +//! Step 43, archaic ancestry (`docs/43-archaic.md`): the commands around +//! IBDmix (Chen et al. 2020), as `og archaic `. +//! +//! IBDmix compares a modern genome with an archaic one site by site and +//! adds up a log-odds score (LOD) that a stretch of the modern genome shares +//! a recent ancestor with the archaic genome. It takes each site's allele +//! frequency from the modern people in its input, and one genome alone +//! gives no frequency, so `genotypes` adds stand-in columns whose genotypes +//! carry the 1000 Genomes allele count of the chosen population. +//! +//! - `table` (setup): one GRCh37 chromosome of the archaic table, a +//! sites-only VCF that Picard lifts to GRCh38. The 1000 Genomes SNVs and +//! the three archaic genomes' calls are merged by position in one pass; +//! each genome is read inside its own mask, and a call outside it (or at +//! an excluded site) is kept as masked, code 10 plus its ALT count. +//! - `strict` (setup): the lifted rows of one chromosome, once per +//! position, with every call outside the 1000 Genomes strict mask marked +//! masked. +//! - `genotypes` (step 43): IBDmix's genotype file of each chromosome, each +//! archaic genome's mask of masked positions, and the spans table. +//! - `report` (step 43): the segments at LOD 4 and 50 kb or longer, merged +//! into Neanderthal and Denisovan ones, the shares, the ten longest and +//! their protein-coding genes. +//! +//! Each command writes what `bin/archaic_report.py` wrote (checked by +//! `tests/parity`). Deliberate changes: every output is written under a +//! temporary name and renamed into place; a gzip input is told by its first +//! bytes, not its name; `strict` reads its rows in position order and stops +//! (exit 2) on a row that goes back, instead of keeping every position of a +//! chromosome in memory; and malformed input is an `ERROR:` line, never a +//! traceback. +//! +//! The table's bytes are part of the archaic data version: a change to what +//! `table` or `strict` writes must change `ARCHAIC_TABLE_FORMAT` in +//! `scripts/setup.sh`, so existing installs rebuild their table. + +use super::{ArgError, Args, Fail, Result, Spec, cannot_read, cannot_write, int, repr, split_ws}; +use flate2::Compression; +use flate2::write::GzEncoder; +use og_io::{AtomicFile, Lines, Utf8}; +use pycompat::fmt::{fixed, general, zero_pad}; +use std::cmp::Reverse; +use std::collections::BinaryHeap; +use std::io::{self, BufRead, Write}; +use std::path::{Path, PathBuf}; + +pub const ARCHAICS: [&str; 3] = ["Altai", "Vindija", "Denisova"]; +const NEANDERTHALS: [&str; 2] = ["Altai", "Vindija"]; +const POPS: [&str; 6] = ["ALL", "AFR", "AMR", "EAS", "EUR", "SAS"]; +const TABLE_HEADER: [&str; 13] = [ + "chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", "AF_ALL", "AF_AFR", "AF_AMR", + "AF_EAS", "AF_EUR", "AF_SAS", +]; +/// Table code: MASKED plus the ALT count, a call outside its genome's mask. +const MASKED: u8 = 10; +/// Stand-in modern people: 200 alleles, about one 1000 Genomes population. +const PSEUDO: i64 = 100; +/// Chen et al. 2020: LOD 4 and 50 kb. +const MIN_LOD: f64 = 4.0; +const MIN_LENGTH: i64 = 50_000; +const TOP: usize = 10; +const SEGMENT_HEADER: &str = "chrom\tstart\tend\tlength_bp\tancestry\tmatched\tlod\tgenes"; +const CAVEAT: &str = "IBDmix finds stretches where your genome matches a Neanderthal or the Denisovan genome more \ +closely than the allele frequencies of a modern population explain. A stretch counts when either \ +of your two copies carries it, and the share is of the analysed span of the autosomes. Some of what \ +it finds in anyone is old ancestry shared with the archaic genomes rather than gene flow from them: \ +Chen et al. 2020 find Neanderthal sequence in every population they studied, Africans included. \ +The numbers move with the cutoffs (LOD 4, 50 kb) and with the frequencies used. An estimate, not \ +a measurement."; + +/// Why a command stopped: its exit code (2 for a wrong option value the +/// command checks itself, 1 otherwise) and the message after `ERROR: `. +#[derive(Debug)] +struct Stop(u8, String); + +impl From for Stop { + fn from(f: Fail) -> Self { + Stop(1, f.0) + } +} + +fn die(msg: impl Into) -> std::result::Result { + Err(Stop(1, msg.into())) +} + +fn usage(msg: impl Into) -> std::result::Result { + Err(Stop(2, msg.into())) +} + +type Out = std::result::Result; + +/// A text file read line by line as Python's text mode reads it (universal +/// newlines, strict UTF-8; gzip read through), keeping the raw line for +/// messages that quote it. +struct Text { + path: PathBuf, + lines: Lines>, + line: String, +} + +impl Text { + fn open(path: &str) -> Result { + let path = PathBuf::from(path); + let input = og_io::open(&path).map_err(|e| cannot_read(&path, &e))?; + Ok(Text { + path, + lines: Lines::new(input, Utf8::Strict), + line: String::new(), + }) + } + + /// The next line with its `\n`, or `None` at the end. + fn raw(&mut self) -> Result> { + let more = self + .lines + .read_line(&mut self.line) + .map_err(|e| cannot_read(&self.path, &e))?; + Ok(more.then_some(self.line.as_str())) + } + + /// The next line without its `\n` (`line.rstrip("\n")`). + fn next(&mut self) -> Result> { + Ok(self.raw()?.map(|l| l.strip_suffix('\n').unwrap_or(l))) + } +} + +/// Python's `int(s)`, or the message of the ValueError it raises. +fn need_int(s: &str) -> Out { + int(s).ok_or_else(|| { + Stop( + 1, + format!("invalid literal for int() with base 10: {}", repr(s)), + ) + }) +} + +/// Python's `float(s)`, or the message of the ValueError it raises. +fn need_float(s: &str) -> Out { + pycompat::num::parse_float(s) + .map_err(|_| Stop(1, format!("could not convert string to float: {}", repr(s)))) +} + +/// `repr(line[:80])`. +fn head80(line: &str) -> String { + let end = line.char_indices().nth(80).map_or(line.len(), |(i, _)| i); + repr(&line[..end]) +} + +/// `os.path.join(a, b)` for a relative `b`. +fn join(a: &str, b: &str) -> String { + if a.is_empty() || a.ends_with('/') { + format!("{a}{b}") + } else { + format!("{a}/{b}") + } +} + +/// `'0/1'`, `'1|1'`, `'0/0'` to 1, 2, 0; a missing, partial or +/// multi-allelic call to `None`. +fn gt_code(gt: &str) -> Option { + let mut parts = gt.split(['/', '|']); + let (Some(a), Some(b), None) = (parts.next(), parts.next(), parts.next()) else { + return None; + }; + let allele = |x: &str| match x { + "0" => Some(0), + "1" => Some(1), + _ => None, + }; + Some(allele(a)? + allele(b)?) +} + +/// Python's `max(a, b)` and `min(a, b)` of floats: the first unless the +/// second compares greater (smaller), so a NaN first stays. +fn py_max(a: f64, b: f64) -> f64 { + if b > a { b } else { a } +} + +fn py_min(a: f64, b: f64) -> f64 { + if b < a { b } else { a } +} + +// --- table (setup) ---------------------------------------------------------------- + +/// One mask's intervals on one chromosome, sorted and merged: 0-based, +/// half-open, so `[s, e)` holds the 1-based positions `s + 1` to `e`. The +/// chromosome is matched with and without the `chr` prefix. +fn read_mask(path: &str, chrom: &str) -> Out> { + let num = chrom.strip_prefix("chr").unwrap_or(chrom); + let with_chr = format!("chr{num}"); + let mut iv = Vec::new(); + let mut t = Text::open(path)?; + while let Some(line) = t.raw()? { + if line.starts_with('#') || line.starts_with("track") || line.starts_with("browser") { + continue; + } + let c: Vec<&str> = split_ws(line).collect(); + if c.len() < 3 || (c[0] != num && c[0] != with_chr) { + continue; + } + let (s, e) = (need_int(c[1])?, need_int(c[2])?); + if e <= s { + continue; // an empty or reversed interval holds no position + } + iv.push((s, e)); + } + iv.sort_unstable(); + let mut out: Vec<(i64, i64)> = Vec::with_capacity(iv.len()); + for (s, e) in iv { + match out.last_mut() { + Some(last) if s <= last.1 => last.1 = last.1.max(e), + _ => out.push((s, e)), + } + } + Ok(out) +} + +/// Whether a 1-based position is inside a mask, for positions asked in +/// ascending order: the walk over the intervals never goes back. +struct Mask { + iv: Vec<(i64, i64)>, + i: usize, +} + +impl Mask { + fn holds(&mut self, pos: i64) -> bool { + let p = pos - 1; + while self.i < self.iv.len() && self.iv[self.i].1 <= p { + self.i += 1; + } + self.i < self.iv.len() && self.iv[self.i].0 <= p + } +} + +/// One record of a table input: a 1000 Genomes SNV (REF, ALT and the six +/// frequencies as written) or an archaic call (REF, ALT, GT). +#[derive(Debug, Clone)] +enum Rec { + Modern { + r#ref: String, + alt: String, + afs: Vec, + }, + Archaic { + r#ref: String, + alt: String, + gt: String, + }, +} + +/// One input of the merge: its lines, read on demand. +struct Stream { + text: Text, + /// 0 for the 1000 Genomes lines, 1 + i for archaic genome i. + src: usize, +} + +impl Stream { + /// The next record, `(pos, rec)`; 1000 Genomes lines that are not + /// SNVs are skipped. + fn next(&mut self) -> Out> { + loop { + let path = self.text.path.display().to_string(); + let Some(raw) = self.text.raw()? else { + return Ok(None); + }; + let line = raw.strip_suffix('\n').unwrap_or(raw); + let c: Vec<&str> = line.split('\t').collect(); + if self.src == 0 { + if c.len() < 9 { + return die(format!( + "{path}: a line has {} columns, want 9: {}", + c.len(), + head80(raw) + )); + } + if c[1].chars().count() != 1 || c[2].chars().count() != 1 { + continue; + } + let pos = need_int(c[0])?; + return Ok(Some(( + pos, + Rec::Modern { + r#ref: c[1].to_owned(), + alt: c[2].to_owned(), + afs: c[3..9].iter().map(|s| (*s).to_owned()).collect(), + }, + ))); + } + if c.len() < 4 { + return die(format!( + "{path}: a line has {} columns, want 4: {}", + c.len(), + head80(raw) + )); + } + let pos = need_int(c[0])?; + return Ok(Some(( + pos, + Rec::Archaic { + r#ref: c[1].to_owned(), + alt: c[2].to_owned(), + gt: c[3].to_owned(), + }, + ))); + } + } +} + +/// An archaic call at one position: (REF, ALT, GT). +type Call<'a> = Option<(&'a str, &'a str, &'a str)>; + +/// The ALT count of one archaic record at a row with this REF and ALT, or 9. +fn archaic_code(rec: Call<'_>, r: &str, alt: &str) -> u8 { + let Some((aref, aalt, gt)) = rec else { + return 9; + }; + if aref != r { + return 9; + } + let Some(code) = gt_code(gt) else { + return 9; + }; + if aalt == "." || aalt == "<*>" || code == 0 { + return if code == 0 { 0 } else { 9 }; + } + if aalt == alt { code } else { 9 } +} + +/// The row of one position: REF, ALT, the three codes and the six +/// frequencies as written, or `None` when nothing is known there. +fn table_row<'a>( + modern: Option<(&'a str, &'a str, &'a [String])>, + arch: [Call<'a>; 3], +) -> Option<(&'a str, &'a str, [u8; 3], Vec)> { + const ZEROS: [&str; 6] = ["0"; 6]; + let (r, alt, afs): (&str, &str, Vec<&str>) = match modern { + Some((r, alt, afs)) => (r, alt, afs.iter().map(String::as_str).collect()), + None => { + // A site no modern person varies at: the ALT of the first + // archaic genome with a non-reference call, every frequency 0. + let (r, alt, _) = arch.iter().flatten().find(|(r, a, gt)| { + !matches!(gt_code(gt), None | Some(0)) + && r.chars().count() == 1 + && a.chars().count() == 1 + })?; + (r, alt, ZEROS.to_vec()) + } + }; + let codes = arch.map(|a| archaic_code(a, r, alt)); + if codes.iter().all(|&c| c == 9) { + return None; + } + if modern.is_none() && !codes.iter().any(|&c| c == 1 || c == 2) { + return None; + } + let clean = afs + .iter() + .map(|v| { + let x = pycompat::num::parse_float(v).unwrap_or(0.0); + general(py_min(py_max(x, 0.0), 1.0), 6) + }) + .collect(); + Some((r, alt, codes, clean)) +} + +/// `NAME=FILE` with NAME one of the archaic genomes. +fn named<'a>(spec: &'a str, option: &str) -> Out<(&'a str, &'a str)> { + let (name, path) = spec.split_once('=').unwrap_or((spec, "")); + if !ARCHAICS.contains(&name) || path.is_empty() { + return usage(format!( + "{option} takes NAME=FILE with NAME one of {}, got {}", + ARCHAICS.join(", "), + repr(spec) + )); + } + Ok((name, path)) +} + +fn cmd_table(a: &Args, err: &mut dyn Write) -> Out<()> { + let chrom_arg = a.req("--chrom"); + let mut names = Vec::new(); + let mut paths = Vec::new(); + for spec in a.all("--archaic") { + let (name, path) = named(spec, "--archaic")?; + names.push(name); + paths.push(path); + } + if names != ARCHAICS { + return usage(format!( + "--archaic must name {} in that order, got {}", + ARCHAICS.join(", "), + names.join(", ") + )); + } + let mut masks: [Option; 3] = [None, None, None]; + for spec in a.all("--mask") { + let (name, path) = named(spec, "--mask")?; + let k = ARCHAICS.iter().position(|n| *n == name).expect("checked"); + if masks[k].is_some() { + return usage(format!("--mask names {name} twice")); + } + let iv = read_mask(path, chrom_arg)?; + if iv.is_empty() { + return die(format!( + "{path} holds no interval on chromosome {chrom_arg}: a mask that matches nothing \ + would make every site unknown" + )); + } + masks[k] = Some(Mask { iv, i: 0 }); + } + let mut excluded = Vec::new(); + for path in a.all("--exclude") { + let iv = read_mask(path, chrom_arg)?; + if iv.is_empty() { + return die(format!( + "{path} holds no interval on chromosome {chrom_arg}: an empty exclusion list is a \ + download that went wrong, not a chromosome without segmental duplications" + )); + } + excluded.push(Mask { iv, i: 0 }); + } + let chrom = if chrom_arg.starts_with("chr") { + chrom_arg.to_owned() + } else { + format!("chr{chrom_arg}") + }; + let out_path = PathBuf::from(a.req("--out")); + let mut out = AtomicFile::create(&out_path).map_err(|e| cannot_write(&out_path, &e))?; + let mut head = format!("##fileformat=VCFv4.2\n##contig=\n"); + for n in ARCHAICS { + head += &format!( + "##INFO=\n" + ); + } + for p in POPS { + head += &format!( + "##INFO=\n" + ); + } + head += "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n"; + let w = |out: &mut AtomicFile, s: &str| { + out.write_all(s.as_bytes()) + .map_err(|e| Stop::from(cannot_write(&out_path, &e))) + }; + w(&mut out, &head)?; + + // The k-way merge of heapq.merge: the head of each input on a heap + // keyed (position, input), each input advanced only when its head is + // taken, so the order is the same as the Python's on any input. + let mut streams = vec![Stream { + text: Text::open(a.req("--modern"))?, + src: 0, + }]; + for (i, p) in paths.iter().enumerate() { + streams.push(Stream { + text: Text::open(p)?, + src: 1 + i, + }); + } + let mut heads: Vec> = vec![None; streams.len()]; + let mut heap = BinaryHeap::new(); + for (i, s) in streams.iter_mut().enumerate() { + if let Some((pos, rec)) = s.next()? { + heads[i] = Some(rec); + heap.push(Reverse((pos, i))); + } + } + let mut rows: u64 = 0; + let mut cur: Option = None; + let mut modern: Option = None; + let mut arch: [Option; 3] = [None, None, None]; + let mut last = -1i64; + let mut flush = |cur: i64, + modern: &Option, + arch: &[Option; 3], + out: &mut AtomicFile| + -> Out<()> { + // Every mask is walked for every position, before any early + // return, because the walk only goes forward. + let mut excl = false; + for m in excluded.iter_mut() { + excl |= m.holds(cur); + } + let mut outside = [false; 3]; + for (k, m) in masks.iter_mut().enumerate() { + outside[k] = excl || m.as_mut().is_some_and(|m| !m.holds(cur)); + } + let modern = modern.as_ref().map(|m| match m { + Rec::Modern { r#ref, alt, afs } => (r#ref.as_str(), alt.as_str(), afs.as_slice()), + Rec::Archaic { .. } => unreachable!("only the modern input gives Modern"), + }); + let calls = [0, 1, 2].map(|k| match &arch[k] { + Some(Rec::Archaic { r#ref, alt, gt }) => { + Some((r#ref.as_str(), alt.as_str(), gt.as_str())) + } + _ => None, + }); + let Some((r, alt, codes, afs)) = table_row(modern, calls) else { + return Ok(()); + }; + // A call outside its genome's mask, or at an excluded site, is kept + // but marked masked: step 43 hands IBDmix the call and a mask of + // those positions, as Chen et al. do. + let mut info = String::new(); + for k in 0..3 { + let c = codes[k]; + let c = if outside[k] && c != 9 { c + MASKED } else { c }; + info += &format!("{}={c};", ARCHAICS[k]); + } + for (k, p) in POPS.iter().enumerate() { + info += &format!("AF_{p}={}", afs[k]); + if k + 1 < POPS.len() { + info.push(';'); + } + } + out.write_all(format!("{chrom}\t{cur}\t.\t{r}\t{alt}\t.\t.\t{info}\n").as_bytes()) + .map_err(|e| Stop::from(cannot_write(&out_path, &e)))?; + rows += 1; + Ok(()) + }; + while let Some(Reverse((pos, i))) = heap.pop() { + let rec = heads[i].take().expect("a head for each heap entry"); + if pos < last { + return die(format!("input not sorted by position at {pos}")); + } + if cur != Some(pos) { + if let Some(c) = cur { + flush(c, &modern, &arch, &mut out)?; + } + cur = Some(pos); + modern = None; + arch = [None, None, None]; + } + last = pos; + match rec { + Rec::Modern { .. } => { + // A second modern record at one position (a split + // multi-allelic site) makes the row ambiguous: keep neither. + modern = Some(match modern { + None => rec, + Some(_) => Rec::Modern { + r#ref: "N".into(), + alt: "N".into(), + afs: vec!["0".into(); POPS.len()], + }, + }); + } + Rec::Archaic { .. } => arch[streams[i].src - 1] = Some(rec), + } + // As heapq.merge does, the input whose record was taken is read on + // only now. + if let Some((p, r)) = streams[i].next()? { + heads[i] = Some(r); + heap.push(Reverse((p, i))); + } + } + if let Some(c) = cur { + flush(c, &modern, &arch, &mut out)?; + } + out.commit().map_err(|e| cannot_write(&out_path, &e))?; + let _ = writeln!(err, "{chrom}: {rows} rows"); + Ok(()) +} + +// --- strict (setup) --------------------------------------------------------------- + +/// The lifted GRCh38 rows of one chromosome (no header, the 13 table +/// columns), once per position, with the calls of every row outside the +/// 1000 Genomes strict mask marked masked (MASKED plus the ALT count; 9 +/// stays 9). The strict mask is the same for the three genomes, so all +/// three are marked. +fn cmd_strict(a: &Args, err: &mut dyn Write) -> Out<()> { + let chrom = a.req("--chrom"); + let strict = a.req("--strict"); + let iv = read_mask(strict, chrom)?; + if iv.is_empty() { + return die(format!( + "{strict} holds no interval on chromosome {chrom}: a mask that matches nothing \ + would mark every site masked" + )); + } + let rows_path = a.req("--rows"); + let mut t = Text::open(rows_path)?; + let out_path = PathBuf::from(a.req("--out")); + let mut out = AtomicFile::create(&out_path).map_err(|e| cannot_write(&out_path, &e))?; + let (mut rows, mut marked) = (0u64, 0u64); + let mut last: Option = None; + let mut n = 0u64; + while let Some(raw) = t.raw()? { + n += 1; + let line = raw.strip_suffix('\n').unwrap_or(raw); + let mut c: Vec = line.split('\t').map(str::to_owned).collect(); + if c.len() != TABLE_HEADER.len() { + return die(format!( + "{rows_path}: a row has {} columns, want {}: {}", + c.len(), + TABLE_HEADER.len(), + head80(raw) + )); + } + let pos = need_int(&c[1])?; + // The rows come sorted (Picard sorts its output), so a position + // seen before is the one just before: a repeat keeps its first row. + match last { + Some(l) if pos == l => continue, + Some(l) if pos < l => { + return usage(format!( + "{rows_path}: line {n}: position {pos} comes after {l}; the lifted rows must be sorted by position" + )); + } + _ => {} + } + last = Some(pos); + let p = pos - 1; + let i = iv.partition_point(|&(s, _)| s <= p); + let inside = i > 0 && p < iv[i - 1].1; + if !inside { + let mut hit = false; + for cell in &mut c[4..7] { + if let Some(v) = ["0", "1", "2"].iter().position(|x| x == cell) { + *cell = (v as u8 + MASKED).to_string(); + hit = true; + } + } + marked += u64::from(hit); + } + out.write_all((c.join("\t") + "\n").as_bytes()) + .map_err(|e| cannot_write(&out_path, &e))?; + rows += 1; + } + out.commit().map_err(|e| cannot_write(&out_path, &e))?; + let _ = writeln!( + err, + "{chrom}: {rows} rows, {marked} outside the strict mask and marked masked" + ); + Ok(()) +} + +// --- genotypes (step 43) ---------------------------------------------------------- + +/// The stand-in columns for each ALT allele count 0..2n: as many 2s and +/// one 1 as the count needs, then 0s. +fn pseudo_columns(n: usize) -> Vec { + (0..=2 * n) + .map(|ac| { + let mut g = vec!["2"; ac / 2]; + if ac % 2 == 1 { + g.push("1"); + } + g.resize(n, "0"); + g.join("\t") + }) + .collect() +} + +/// A call of the sample's VCF: REF, ALT (`None` for an insertion, a +/// deletion or a multi-allelic record) and the code (0, 1, 2, or 9 when it +/// cannot be read). +#[derive(Debug, Clone, PartialEq, Eq)] +struct Sample { + r#ref: String, + alt: Option, + code: u8, +} + +/// What [`pick_call`] keeps. +#[derive(Debug, PartialEq, Eq)] +enum Picked { + Call(Sample), + Clash, +} + +/// The call to keep when a position comes twice, as in a VCF split with +/// bcftools norm -m-. An SNV wins over an insertion or a multi-allelic +/// record, in either order. Of one SNV twice, a usable record wins over +/// code 9, and two different usable codes clash. Of two different SNVs, +/// the one that carries its ALT wins over a 0/0 or unusable one; both 0/0 +/// is the reference; both carrying their ALT, or neither readable, clash +/// (unknown, 9). +fn pick_call(prev: Option, call: Sample) -> Picked { + let Some(prev) = prev else { + return Picked::Call(call); + }; + if prev.alt.is_none() || call.alt.is_none() { + return Picked::Call(if prev.alt.is_none() && call.alt.is_some() { + call + } else { + prev + }); + } + if call.alt == prev.alt { + if call.code == 9 || call.code == prev.code { + return Picked::Call(prev); + } + return if prev.code == 9 { + Picked::Call(call) + } else { + Picked::Clash + }; + } + let carries = |c: u8| c == 1 || c == 2; + let (has_prev, has_call) = (carries(prev.code), carries(call.code)); + if has_prev != has_call { + return Picked::Call(if has_prev { prev } else { call }); + } + if !has_prev && prev.code == 0 && call.code == 0 { + return Picked::Call(Sample { + r#ref: prev.r#ref, + alt: None, + code: 0, + }); + } + Picked::Clash +} + +/// One record of the sample's VCF that `genotypes` reads. +struct VcfRec { + chrom: String, + pos: i64, + call: Sample, +} + +/// The biallelic SNVs (and the other records, as code 9) of a one-sample +/// VCF, in file order. A record whose FILTER is not PASS or `.` gives +/// code 9 (not read), so a DeepVariant RefCall is neither your allele nor +/// the reference. +struct Vcf { + text: Text, +} + +impl Vcf { + fn next(&mut self) -> Out> { + loop { + let path = self.text.path.display().to_string(); + let Some(raw) = self.text.raw()? else { + return Ok(None); + }; + if raw.starts_with("##") { + continue; + } + let line = raw.strip_suffix('\n').unwrap_or(raw); + let c: Vec<&str> = line.split('\t').collect(); + if raw.starts_with('#') { + if c.len() != 10 { + return die(format!( + "{path} holds {} samples; step 43 reads a one-sample VCF \ + (keep one with bcftools view -s ).", + c.len().saturating_sub(9) + )); + } + continue; + } + if c.len() < 10 || c[3].chars().count() != 1 || c[4] == "." { + continue; + } + if c[4].contains(',') || c[4].chars().count() != 1 { + return Ok(Some(VcfRec { + chrom: c[0].to_owned(), + pos: need_int(c[1])?, + call: Sample { + r#ref: c[3].to_owned(), + alt: None, + code: 9, + }, + })); + } + let fmt: Vec<&str> = c[8].split(':').collect(); + let Some(k) = fmt.iter().position(|f| *f == "GT") else { + continue; + }; + let gt = c[9].split(':').nth(k).unwrap_or("."); + let mut code = gt_code(gt).unwrap_or(9); + if c[6] != "PASS" && c[6] != "." { + code = 9; + } + return Ok(Some(VcfRec { + chrom: c[0].to_owned(), + pos: need_int(c[1])?, + call: Sample { + r#ref: c[3].to_owned(), + alt: Some(c[4].to_owned()), + code, + }, + })); + } + } +} + +/// The sample's calls on one chromosome, one per position after +/// [`pick_call`], sorted by position: a compact array, not a map. +fn settle(mut recs: Vec<(i64, Sample)>) -> Vec<(i64, Sample)> { + // A stable sort keeps the records of one position in file order. + recs.sort_by_key(|(p, _)| *p); + let mut out: Vec<(i64, Sample)> = Vec::new(); + let mut clashed = false; + for (pos, call) in recs { + match out.last_mut() { + Some((p, kept)) if *p == pos => { + if clashed { + continue; + } + let r#ref = call.r#ref.clone(); + match pick_call(Some(kept.clone()), call) { + Picked::Call(c) => *kept = c, + Picked::Clash => { + *kept = Sample { + r#ref, + alt: None, + code: 9, + }; + clashed = true; + } + } + } + _ => { + clashed = false; + out.push((pos, call)); + } + } + } + out +} + +/// `chrN.tsv.gz` with N from 1 to 22: the chromosome's name. +fn table_chrom(name: &str) -> Option<&str> { + let chrom = name.strip_suffix(".tsv.gz")?; + let num = chrom.strip_prefix("chr")?; + let ok = match num.len() { + 1 => matches!(num.as_bytes()[0], b'1'..=b'9'), + 2 => { + let (a, b) = (num.as_bytes()[0], num.as_bytes()[1]); + (a == b'1' && b.is_ascii_digit()) || (a == b'2' && (b'0'..=b'2').contains(&b)) + } + _ => false, + }; + ok.then_some(chrom) +} + +/// A file written whole through a temporary name. +fn put(path: &Path, data: &[u8]) -> Result<()> { + let mut f = AtomicFile::create(path).map_err(|e| cannot_write(path, &e))?; + f.write_all(data).map_err(|e| cannot_write(path, &e))?; + f.commit().map_err(|e| cannot_write(path, &e)) +} + +fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { + let pseudo_n = a + .int("--pseudo", PSEUDO) + .map_err(|_| Stop(2, String::new()))?; + if pseudo_n < 1 { + return usage("--pseudo must be 1 or more"); + } + let pop = a.get("--pop").unwrap_or("ALL"); + let Some(af_col) = POPS.iter().position(|p| *p == pop).map(|k| 7 + k) else { + return usage(format!( + "--pop must be one of {}, got {}", + POPS.join(", "), + repr(pop) + )); + }; + let table_dir = a.req("--table-dir"); + let rd = std::fs::read_dir(table_dir).map_err(|e| cannot_read(Path::new(table_dir), &e))?; + let mut names: Vec = Vec::new(); + for e in rd { + let e = e.map_err(|e| cannot_read(Path::new(table_dir), &e))?; + if let Ok(n) = e.file_name().into_string() { + names.push(n); + } + } + names.sort(); + let tables: Vec<(String, String)> = names + .iter() + .filter_map(|n| table_chrom(n).map(|c| (c.to_owned(), join(table_dir, n)))) + .collect(); + if tables.is_empty() { + return die(format!( + "{table_dir} holds no chrN.tsv.gz: install the archaic data with scripts/setup.sh --archaic" + )); + } + let out_dir = a.req("--out-dir"); + std::fs::create_dir_all(out_dir).map_err(|e| cannot_write(Path::new(out_dir), &e))?; + let n = pseudo_n as usize; + let pseudo = pseudo_columns(n); + let two_n = (2 * n) as f64; + let mut head = String::from("chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tSAMPLE"); + let mut samples = String::from("SAMPLE\n"); + for i in 1..=n { + let id = format!("P{}", zero_pad(i as i128, 3)); + head += &format!("\t{id}"); + samples += &format!("{id}\n"); + } + head.push('\n'); + put(&Path::new(out_dir).join("samples.txt"), samples.as_bytes())?; + + let vcf_path = a.req("--vcf"); + let mut vcf = Vcf { + text: Text::open(vcf_path)?, + }; + let mut spans: Vec = Vec::new(); + let mut done: Vec = Vec::new(); + let mut pending = vcf.next()?; + while let Some(first) = pending.take() { + let chrom = first.chrom.clone(); + if done.contains(&chrom) { + return die(format!( + "{vcf_path} is not sorted: {chrom} comes back after another chromosome" + )); + } + done.push(chrom.clone()); + let table = tables.iter().find(|(c, _)| *c == chrom).map(|(_, t)| t); + // The sample's calls on this chromosome; on a chromosome the table + // does not hold they are read past. + let mut recs = Vec::new(); + if table.is_some() { + recs.push((first.pos, first.call)); + } + loop { + match vcf.next()? { + Some(r) if r.chrom == chrom => { + if table.is_some() { + recs.push((r.pos, r.call)); + } + } + other => { + pending = other; + break; + } + } + } + let Some(table) = table else { + continue; + }; + let calls = settle(recs); + if let Some(span) = write_chrom( + &chrom, table, &calls, &head, &pseudo, af_col, two_n, out_dir, + )? { + spans.push(span); + } + } + let mut text = + String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); + for (c, f, l, n, carried, dropped) in &spans { + text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); + } + put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; + if spans.is_empty() { + let names: Vec<&str> = tables.iter().map(|(c, _)| c.as_str()).collect(); + let mut sorted = names.clone(); + sorted.sort_unstable(); + return die(format!( + "{vcf_path} has no records on a chromosome of the archaic table ({})", + sorted.join(", ") + )); + } + let sites: u64 = spans.iter().map(|s| s.3).sum(); + let _ = writeln!(err, "{} chromosomes, {sites} sites", spans.len()); + Ok(()) +} + +/// A chromosome's row of `spans.tsv`: chromosome, first and last +/// position, sites, sites with your ALT, sites with another allele. +type Span = (String, i64, i64, u64, u64, u64); + +/// IBDmix's genotype file of one chromosome, and each archaic genome's +/// mask; the chromosome's span row, or `None` when no row was written. +#[allow(clippy::too_many_arguments)] +fn write_chrom( + chrom: &str, + table: &str, + calls: &[(i64, Sample)], + head: &str, + pseudo: &[String], + af_col: usize, + two_n: f64, + out_dir: &str, +) -> Out> { + let num = &chrom[3..]; + let path = Path::new(out_dir).join(format!("{chrom}.gt.gz")); + let file = AtomicFile::create(&path).map_err(|e| cannot_write(&path, &e))?; + let mut gz = GzEncoder::new(file, Compression::new(1)); + let werr = |e: io::Error| Stop::from(cannot_write(&path, &e)); + gz.write_all(head.as_bytes()).map_err(werr)?; + let mut t = Text::open(table)?; + let header = t.next()?.unwrap_or(""); + if header.split('\t').ne(TABLE_HEADER.iter().copied()) { + return die(format!( + "{table} does not start with the table header ({})", + TABLE_HEADER.join(" ") + )); + } + let (mut n, mut first, mut last, mut dropped, mut carried) = (0u64, 0i64, 0i64, 0u64, 0u64); + // Per archaic genome, the runs of written rows whose call is masked: + // [first, last] positions, closed at the next unmasked row. + let mut runs: [Vec<(i64, i64)>; 3] = [Vec::new(), Vec::new(), Vec::new()]; + let mut open_run: [Option<(i64, i64)>; 3] = [None; 3]; + let mut row = String::new(); + while let Some(line) = t.next()? { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 7 || c.len() <= af_col { + return die(format!( + "{table}: a row has {} columns, want {}", + c.len(), + TABLE_HEADER.len() + )); + } + let pos = need_int(c[1])?; + let (r, alt) = (c[2], c[3]); + let mut arch = [0u8; 3]; + for k in 0..3 { + let v = c[4 + k]; + arch[k] = match v { + "0" | "1" | "2" | "9" | "10" | "11" | "12" => v.parse().expect("a code"), + _ => { + return die(format!( + "{table}: {chrom}:{pos} has {} code {}; the table holds 0, 1, 2, 9 or {MASKED} to {}", + ARCHAICS[k], + repr(v), + MASKED + 2 + )); + } + }; + } + let code = match calls.binary_search_by_key(&pos, |(p, _)| *p) { + Err(_) => 0, + Ok(i) => { + let call = &calls[i].1; + if call.r#ref != r || call.alt.as_ref().is_some_and(|x| x != alt) { + // Another allele than the row's: IBDmix's generate_gt + // drops the line. + dropped += 1; + continue; + } + call.code + } + }; + if code == 1 || code == 2 { + carried += 1; + } + let ac = match pycompat::num::parse_float(c[af_col]) { + Err(_) => 0.0, + Ok(x) => match pycompat::fmt::round_int(x * two_n) { + Ok(v) => v, + Err(pycompat::fmt::RoundError::Nan) => 0.0, + Err(e) => return die(format!("{table}: {chrom}:{pos}: {e}")), + }, + }; + let ac = ac.clamp(0.0, two_n) as usize; + for k in 0..3 { + if arch[k] >= MASKED { + match &mut open_run[k] { + None => open_run[k] = Some((pos, pos)), + Some(run) => run.1 = pos, + } + } else if let Some(run) = open_run[k].take() { + runs[k].push(run); + } + } + row.clear(); + row += &format!("{num}\t{pos}\t{r}\t{alt}"); + for v in arch { + let v = if v >= MASKED { v - MASKED } else { v }; + row += &format!("\t{v}"); + } + row += &format!("\t{code}\t{}\n", pseudo[ac]); + gz.write_all(row.as_bytes()).map_err(werr)?; + if n == 0 { + first = pos; + } + last = pos; + n += 1; + } + if n == 0 { + drop(gz); + match std::fs::remove_file(&path) { + Err(e) if e.kind() != io::ErrorKind::NotFound => { + return Err(Stop::from(cannot_write(&path, &e))); + } + _ => {} + } + return Ok(None); + } + gz.finish().and_then(AtomicFile::commit).map_err(werr)?; + // IBDmix's mask (-r): the chromosome as in the genotype file, and a + // line covers the positions start + 1 to end, as in a BED. + for k in 0..3 { + if let Some(run) = open_run[k] { + runs[k].push(run); + } + let text: String = runs[k] + .iter() + .map(|(s, e)| format!("{num}\t{}\t{e}\n", s - 1)) + .collect(); + put( + &Path::new(out_dir).join(format!("{chrom}.{}.mask.bed", ARCHAICS[k])), + text.as_bytes(), + )?; + } + Ok(Some((chrom.to_owned(), first, last, n, carried, dropped))) +} + +// --- report (step 43) ------------------------------------------------------------- + +/// A segment: chromosome, start, end, LOD. +type Seg = (String, i64, i64, f64); + +/// IBDmix's output rows for SAMPLE, kept at LOD 4 and 50 kb or longer. +fn read_segments(path: &str) -> Out> { + let mut t = Text::open(path)?; + let head: Vec = t + .next()? + .unwrap_or("") + .split('\t') + .map(str::to_owned) + .collect(); + let first5: Vec<&str> = head.iter().take(5).map(String::as_str).collect(); + if first5 != ["ID", "chrom", "start", "end", "slod"] { + return die(format!( + "{path} is not IBDmix output (header {})", + first5.join(" ") + )); + } + let mut segs = Vec::new(); + while let Some(line) = t.next()? { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 5 || c[0] != "SAMPLE" { + continue; + } + let (s, e, lod) = (need_int(c[2])?, need_int(c[3])?, need_float(c[4])?); + if lod >= MIN_LOD && e - s >= MIN_LENGTH { + let chrom = if c[1].starts_with("chr") { + c[1].to_owned() + } else { + format!("chr{}", c[1]) + }; + segs.push((chrom, s, e, lod)); + } + } + Ok(segs) +} + +/// The order of chromosomes in the report: the number, or 99 for any +/// other name (so chrX and chrY sort together, by position). +fn chrom_key(c: &str) -> i128 { + let c = c.strip_prefix("chr").unwrap_or(c); + if pycompat::text::isdigit(c) { + pycompat::num::parse_int(c).unwrap_or(99) + } else { + 99 + } +} + +/// A merged segment: chromosome, start, end, the highest LOD, and which +/// archaic genomes it holds. +type Merged = (String, i64, i64, f64, Vec<&'static str>); + +/// Overlapping or touching segments of one class as one. +fn merge(mut segs: Vec<(Seg, &'static str)>) -> Vec { + segs.sort_by(|(a, _), (b, _)| (chrom_key(&a.0), a.1, a.2).cmp(&(chrom_key(&b.0), b.1, b.2))); + let mut out: Vec = Vec::new(); + for ((chrom, start, end, lod), who) in segs { + match out.last_mut() { + Some(m) if m.0 == chrom && start <= m.2 => { + m.2 = m.2.max(end); + m.3 = py_max(m.3, lod); + if !m.4.contains(&who) { + m.4.push(who); + } + } + _ => out.push((chrom, start, end, lod, vec![who])), + } + } + out +} + +fn overlaps(chrom: &str, s: i64, e: i64, others: &[Merged]) -> bool { + others.iter().any(|o| o.0 == chrom && o.1 < e && s < o.2) +} + +/// A row of the report: chromosome, start, end, ancestry, matched, LOD. +type Row = (String, i64, i64, &'static str, Vec<&'static str>, f64); + +/// Neanderthal: the union of the Altai and Vindija segments. Denisovan: +/// the Denisova segments that overlap no Neanderthal one (a stretch both +/// match is counted as Neanderthal, the more common source outside +/// Oceania). +fn classify(per: &[Vec; 3]) -> Vec { + let nea = merge( + NEANDERTHALS + .iter() + .enumerate() + .flat_map(|(k, n)| per[k].iter().map(move |s| (s.clone(), *n))) + .collect(), + ); + let den = merge(per[2].iter().map(|s| (s.clone(), "Denisova")).collect()); + let mut rows: Vec = Vec::new(); + for (c, s, e, l, mut who) in nea.iter().cloned() { + who.sort_unstable(); + if overlaps(&c, s, e, &den) { + who.push("Denisova"); + } + rows.push((c, s, e, "Neanderthal", who, l)); + } + for (c, s, e, l, _) in den.iter().cloned() { + if !overlaps(&c, s, e, &nea) { + rows.push((c, s, e, "Denisovan", vec!["Denisova"], l)); + } + } + rows.sort_by(|a, b| (chrom_key(&a.0), a.1).cmp(&(chrom_key(&b.0), b.1))); + rows +} + +/// Genes per chromosome: (start, end, name), sorted. +type Genes = std::collections::HashMap>; + +/// Protein-coding genes of a GENCODE gene GTF, per chromosome, sorted by +/// (start, end, name). +fn read_genes(path: &str) -> Out { + let mut genes = Genes::new(); + let mut t = Text::open(path)?; + while let Some(line) = t.next()? { + if line.starts_with('#') { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 9 || c[2] != "gene" || !c[8].contains("gene_type \"protein_coding\"") { + continue; + } + if let Some(name) = gene_name(c[8]) { + let (s, e) = (need_int(c[3])?, need_int(c[4])?); + genes + .entry(c[0].to_owned()) + .or_default() + .push((s, e, name.to_owned())); + } + } + for v in genes.values_mut() { + v.sort(); + } + Ok(genes) +} + +/// The first `gene_name "X"` with a non-empty X (the regex +/// `gene_name "([^"]+)"`). +fn gene_name(attrs: &str) -> Option<&str> { + const KEY: &str = "gene_name \""; + attrs.match_indices(KEY).find_map(|(i, _)| { + let rest = &attrs[i + KEY.len()..]; + let end = rest.find('"')?; + (end > 0).then(|| &rest[..end]) + }) +} + +/// The names of the genes overlapping `[start, end)`, once each, in order. +/// IBDmix's start is the segment's first site and its end the next site, +/// past the segment; a GTF gene is `[s, e]`, 1-based. +fn genes_in<'a>(genes: &'a Genes, chrom: &str, start: i64, end: i64) -> Vec<&'a str> { + let name = if chrom.starts_with("chr") { + chrom.to_owned() + } else { + format!("chr{chrom}") + }; + let mut seen: Vec<&str> = Vec::new(); + for (s, e, n) in genes.get(&name).map_or(&[][..], Vec::as_slice) { + if *s >= end { + break; // sorted by start: no later gene starts before the end + } + if *e >= start && !seen.contains(&n.as_str()) { + seen.push(n); + } + } + seen +} + +fn cmd_report(a: &Args, out: &mut dyn Write) -> Out<()> { + let spans_path = a.req("--spans"); + let mut t = Text::open(spans_path)?; + let head = t.next()?.unwrap_or("").to_owned(); + if head + .split('\t') + .take(4) + .ne(["chrom", "first", "last", "sites"]) + { + return die(format!( + "{spans_path} is not the spans table of the genotypes step" + )); + } + let mut spans: Vec<(String, i64, i64, i64)> = Vec::new(); + while let Some(line) = t.next()? { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 4 { + return die(format!( + "{spans_path}: a row has {} columns, want 4", + c.len() + )); + } + spans.push(( + c[0].to_owned(), + need_int(c[1])?, + need_int(c[2])?, + need_int(c[3])?, + )); + } + if spans.is_empty() { + return die(format!("{spans_path} lists no chromosome")); + } + let spans_bp: i64 = spans.iter().map(|(_, f, l, _)| l - f + 1).sum(); + let ibdmix = a.req("--ibdmix-dir"); + let mut per: [Vec; 3] = [Vec::new(), Vec::new(), Vec::new()]; + for (chrom, ..) in &spans { + for (k, n) in ARCHAICS.iter().enumerate() { + let p = join(ibdmix, &format!("{chrom}.{n}.txt")); + if !Path::new(&p).is_file() { + return die(format!("no IBDmix output {p}")); + } + per[k].extend(read_segments(&p)?); + } + } + let rows = classify(&per); + let genes_arg = a.get("--genes").unwrap_or(""); + let genes = if genes_arg.is_empty() { + Default::default() + } else { + read_genes(genes_arg)? + }; + let genes_of = |c: &str, s: i64, e: i64| -> Vec<&str> { + if genes_arg.is_empty() { + Vec::new() + } else { + genes_in(&genes, c, s, e) + } + }; + let sample = a.req("--sample"); + let pop = a.get("--pop").unwrap_or("ALL"); + let out_dir = a.req("--out-dir"); + std::fs::create_dir_all(out_dir).map_err(|e| cannot_write(Path::new(out_dir), &e))?; + + let seg_name = format!("{sample}_archaic_segments.tsv"); + let mut text = format!("{SEGMENT_HEADER}\n"); + for (c, s, e, anc, who, lod) in &rows { + let g = genes_of(c, *s, *e); + text += &format!( + "{c}\t{s}\t{e}\t{}\t{anc}\t{}\t{}\t{}\n", + e - s, + who.join(","), + fixed(*lod, 1), + if g.is_empty() { + ".".to_owned() + } else { + g.join(",") + } + ); + } + put(&Path::new(out_dir).join(&seg_name), text.as_bytes())?; + + // The summary of each ancestry. + let mut summary: Vec<(String, String)> = Vec::new(); + for anc in ["Neanderthal", "Denisovan"] { + let r: Vec<&Row> = rows.iter().filter(|x| x.3 == anc).collect(); + let bp: i64 = r.iter().map(|x| x.2 - x.1).sum(); + let key = anc.to_lowercase(); + summary.push((format!("{key}_segments"), r.len().to_string())); + summary.push((format!("{key}_bp"), bp.to_string())); + summary.push(( + format!("{key}_percent"), + if spans_bp != 0 { + fixed(100.0 * bp as f64 / spans_bp as f64, 2) + } else { + "0.00".to_owned() + }, + )); + summary.push(( + format!("{key}_longest_bp"), + r.iter().map(|x| x.2 - x.1).max().unwrap_or(0).to_string(), + )); + } + let sv = |k: &str| { + summary + .iter() + .find(|(key, _)| key == k) + .map(|(_, v)| v.as_str()) + .expect("a summary key") + }; + let mut longest: Vec<&Row> = rows.iter().collect(); + longest.sort_by(|a, b| { + (-(a.2 - a.1), chrom_key(&a.0), a.1).cmp(&(-(b.2 - b.1), chrom_key(&b.0), b.1)) + }); + longest.truncate(TOP); + let sites: i64 = spans.iter().map(|s| s.3).sum(); + let mut text = String::from("key\tvalue\n"); + let meta: [(&str, String); 10] = [ + ("sample", sample.to_owned()), + ("method", "IBDmix".to_owned()), + ( + "archaic_genomes", + "Altai Neanderthal, Vindija 33.19 Neanderthal, Altai Denisovan".to_owned(), + ), + ("population", pop.to_owned()), + ("min_lod", general(MIN_LOD, 6)), + ("min_length_bp", MIN_LENGTH.to_string()), + ("analysed_bp", spans_bp.to_string()), + ("analysed_chromosomes", spans.len().to_string()), + ("analysed_sites", sites.to_string()), + ( + "genes", + if genes_arg.is_empty() { + "none (no gene annotation installed)" + } else { + "GENCODE protein-coding" + } + .to_owned(), + ), + ]; + for (k, v) in meta.iter() { + text += &format!("{k}\t{v}\n"); + } + for (k, v) in &summary { + text += &format!("{k}\t{v}\n"); + } + let genes_text = |g: &[&str]| { + if g.is_empty() { + "none".to_owned() + } else { + g.join(", ") + } + }; + for (i, (c, s, e, anc, who, _)) in longest.iter().enumerate() { + let g = genes_of(c, *s, *e); + text += &format!( + "longest_{}\t{c}:{s}-{e} {anc} {} Mb ({}); genes: {}\n", + i + 1, + fixed((e - s) as f64 / 1e6, 2), + who.join("/"), + genes_text(&g) + ); + } + put( + &Path::new(out_dir).join(format!("{sample}_archaic.tsv")), + text.as_bytes(), + )?; + let mb = |k: &str| fixed(need_int(sv(k)).unwrap_or(0) as f64 / 1e6, 1); + let mut lines = vec![ + format!( + "Archaic ancestry of {sample} (IBDmix against the Altai and Vindija Neanderthals and the Altai Denisovan)" + ), + format!( + " Analysed: {} autosomes, {} Mb, {sites} sites; modern frequencies: 1000 Genomes {pop}", + spans.len(), + fixed(spans_bp as f64 / 1e6, 1) + ), + format!( + " Neanderthal: {}% ({} Mb in {} segments)", + sv("neanderthal_percent"), + mb("neanderthal_bp"), + sv("neanderthal_segments") + ), + format!( + " Denisovan: {}% ({} Mb in {} segments)", + sv("denisovan_percent"), + mb("denisovan_bp"), + sv("denisovan_segments") + ), + ]; + if !longest.is_empty() { + lines.push(format!(" The {} longest segments:", longest.len())); + for (c, s, e, anc, who, lod) in &longest { + let g = genes_of(c, *s, *e); + lines.push(format!( + " {c}:{s}-{e} {} Mb {anc} ({}, LOD {}) genes: {}", + fixed((e - s) as f64 / 1e6, 2), + who.join("/"), + fixed(*lod, 0), + genes_text(&g) + )); + } + } + lines.push(format!(" {CAVEAT}")); + lines.push(format!(" Every segment: {seg_name}")); + let body = lines.join("\n") + "\n"; + put( + &Path::new(out_dir).join(format!("{sample}_archaic.txt")), + body.as_bytes(), + )?; + let _ = out.write_all(body.as_bytes()); + Ok(()) +} + +// --- the command line ------------------------------------------------------------- + +const USAGE: &str = "usage: og archaic {table,strict,genotypes,report} ...\n"; + +const TABLE: Spec = Spec { + verb: "table", + usage: "usage: og archaic table --chrom CHROM --modern MODERN [--archaic NAME=FILE] [--mask NAME=BED] [--exclude BED] --out OUT\n", + options: &[ + "--chrom", + "--modern", + "--archaic", + "--mask", + "--exclude", + "--out", + ], + required: &["--chrom", "--modern", "--out"], +}; +const STRICT: Spec = Spec { + verb: "strict", + usage: "usage: og archaic strict --chrom CHROM --strict STRICT --rows ROWS --out OUT\n", + options: &["--chrom", "--strict", "--rows", "--out"], + required: &["--chrom", "--strict", "--rows", "--out"], +}; +const GENOTYPES: Spec = Spec { + verb: "genotypes", + usage: "usage: og archaic genotypes --table-dir TABLE_DIR --vcf VCF [--pop POP] [--pseudo PSEUDO] --out-dir OUT_DIR\n", + options: &["--table-dir", "--vcf", "--pop", "--pseudo", "--out-dir"], + required: &["--table-dir", "--vcf", "--out-dir"], +}; +const REPORT: Spec = Spec { + verb: "report", + usage: "usage: og archaic report --sample SAMPLE --ibdmix-dir IBDMIX_DIR --spans SPANS [--genes GENES] [--pop POP] --out-dir OUT_DIR\n", + options: &[ + "--sample", + "--ibdmix-dir", + "--spans", + "--genes", + "--pop", + "--out-dir", + ], + required: &["--sample", "--ibdmix-dir", "--spans", "--out-dir"], +}; + +/// `og archaic [args]`: the exit code. Results go to `out`, +/// messages to `err`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let Some(verb) = args.first() else { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og archaic: error: the following arguments are required: verb" + ); + return 2; + }; + let spec = match verb.as_str() { + "table" => &TABLE, + "strict" => &STRICT, + "genotypes" => &GENOTYPES, + "report" => &REPORT, + "-h" | "--help" => { + let _ = write!(out, "{USAGE}"); + return 0; + } + other => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og archaic: error: argument verb: invalid choice: {}", + repr(other) + ); + return 2; + } + }; + let a = match Args::parse(spec, &args[1..]).and_then(|a| { + if spec.verb == "genotypes" { + a.int("--pseudo", PSEUDO)?; + } + Ok(a) + }) { + Ok(a) => a, + Err(ArgError::Help) => { + let _ = write!(out, "{}", spec.usage); + return 0; + } + Err(ArgError::Usage(msg)) => { + let _ = write!(err, "{}", spec.usage); + let _ = writeln!(err, "og archaic {}: error: {msg}", spec.verb); + return 2; + } + }; + let done = match spec.verb { + "table" => cmd_table(&a, err), + "strict" => cmd_strict(&a, err), + "genotypes" => cmd_genotypes(&a, err), + _ => cmd_report(&a, out), + }; + match done { + Ok(()) => 0, + Err(Stop(code, msg)) => { + let _ = out.flush(); + let _ = writeln!(err, "ERROR: {msg}"); + code + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s(r: &str, alt: Option<&str>, code: u8) -> Sample { + Sample { + r#ref: r.into(), + alt: alt.map(Into::into), + code, + } + } + + fn code(p: Picked) -> u8 { + match p { + Picked::Call(c) => c.code, + Picked::Clash => 9, + } + } + + #[test] + fn gt_codes() { + assert_eq!(gt_code("0/1"), Some(1)); + assert_eq!(gt_code("1|1"), Some(2)); + assert_eq!(gt_code("0/0"), Some(0)); + assert_eq!(gt_code("./."), None); + assert_eq!(gt_code("1/2"), None); + assert_eq!(gt_code("1"), None); + assert_eq!(gt_code("0/1/1"), None); + } + + #[test] + fn pick_call_is_symmetric_in_its_code() { + let pairs = [ + (s("A", Some("G"), 1), s("A", None, 9)), + (s("A", Some("G"), 1), s("A", Some("T"), 0)), + (s("A", Some("G"), 9), s("A", Some("G"), 2)), + (s("A", Some("G"), 0), s("A", Some("T"), 0)), + (s("A", Some("G"), 1), s("A", Some("T"), 2)), + (s("A", Some("G"), 9), s("A", Some("T"), 0)), + (s("A", Some("G"), 1), s("A", Some("G"), 2)), + ]; + for (x, y) in pairs { + assert_eq!( + code(pick_call(Some(x.clone()), y.clone())), + code(pick_call(Some(y), x)) + ); + } + assert_eq!( + pick_call(Some(s("A", Some("G"), 0)), s("A", Some("T"), 0)), + Picked::Call(s("A", None, 0)) + ); + } + + #[test] + fn a_clash_ends_its_position() { + let recs = vec![ + (4000, s("T", Some("C"), 1)), + (4000, s("T", Some("G"), 1)), + (4000, s("T", Some("C"), 1)), + (10, s("A", Some("G"), 2)), + ]; + assert_eq!( + settle(recs), + vec![(10, s("A", Some("G"), 2)), (4000, s("T", None, 9))] + ); + } + + #[test] + fn pseudo_columns_carry_the_count() { + let p = pseudo_columns(2); + assert_eq!(p, ["0\t0", "1\t0", "2\t0", "2\t1", "2\t2"]); + } + + #[test] + fn table_names() { + assert_eq!(table_chrom("chr1.tsv.gz"), Some("chr1")); + assert_eq!(table_chrom("chr22.tsv.gz"), Some("chr22")); + assert_eq!(table_chrom("chr19.tsv.gz"), Some("chr19")); + assert_eq!(table_chrom("chr23.tsv.gz"), None); + assert_eq!(table_chrom("chr0.tsv.gz"), None); + assert_eq!(table_chrom("chrX.tsv.gz"), None); + assert_eq!(table_chrom("chr01.tsv.gz"), None); + assert_eq!(table_chrom("chr1.tsv"), None); + } + + #[test] + fn frequencies_clamp_like_python() { + let afs: Vec = ["1e-05", "nan", "inf", "-0.0", "x", "1.5"] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + let (_, _, _, clean) = table_row( + Some(("A", "G", &afs)), + [Some(("A", "G", "0/1")), None, None], + ) + .unwrap(); + assert_eq!(clean, ["1e-05", "nan", "1", "-0", "0", "1"]); + } + + #[test] + fn genes_touching_the_ends() { + let mut genes = std::collections::HashMap::new(); + let mut v = vec![ + (90, 100, "ENDS_AT_START".to_owned()), + (100, 120, "INSIDE".to_owned()), + (200, 210, "STARTS_AT_END".to_owned()), + (50, 99, "BEFORE".to_owned()), + ]; + v.sort(); + genes.insert("chr22".to_owned(), v); + assert_eq!( + genes_in(&genes, "22", 100, 200), + ["ENDS_AT_START", "INSIDE"] + ); + } + + #[test] + fn gene_names() { + assert_eq!(gene_name("gene_id \"G\"; gene_name \"ABC\";"), Some("ABC")); + assert_eq!(gene_name("gene_name \"\"; gene_name \"B\";"), Some("B")); + assert_eq!(gene_name("gene_name \"open"), None); + } + + #[test] + fn merge_keeps_the_highest_lod() { + let m = merge(vec![ + (("chr22".into(), 1_000_000, 1_300_000, 40.0), "Altai"), + (("chr22".into(), 1_200_000, 1_500_000, 50.0), "Vindija"), + (("chr22".into(), 1_500_000, 1_600_000, 7.0), "Altai"), + ]); + assert_eq!(m.len(), 1); + assert_eq!((m[0].1, m[0].2, m[0].3), (1_000_000, 1_600_000, 50.0)); + assert_eq!(m[0].4, ["Altai", "Vindija"]); + } +} diff --git a/rust/crates/og-ancestry/src/lib.rs b/rust/crates/og-ancestry/src/lib.rs index 0594164a9..74febd042 100644 --- a/rust/crates/og-ancestry/src/lib.rs +++ b/rust/crates/og-ancestry/src/lib.rs @@ -22,6 +22,8 @@ //! A group name is ours: a set of reference populations we put together //! and named, not a nationality. +pub mod ancient; +pub mod archaic; pub mod chroms; pub mod order; pub mod panel; diff --git a/rust/crates/og-intervals/src/chain.rs b/rust/crates/og-intervals/src/chain.rs new file mode 100644 index 000000000..559a4d71e --- /dev/null +++ b/rust/crates/og-intervals/src/chain.rs @@ -0,0 +1,376 @@ +//! UCSC chain files, for lifting single positions from one assembly to +//! another (`og ancient lift`). +//! +//! A chain file is a list of chains. Each starts with a header line, +//! `chain score tName tSize tStrand tStart tEnd qName qSize qStrand qStart +//! qEnd id`, followed by block lines `size dt dq` (the last one `size` +//! alone) and a blank line. The `t` side is the assembly a position is +//! lifted from (the source), the `q` side the one it is lifted to. Each +//! block is an ungapped stretch of `size` bases; `dt` and `dq` are the gaps +//! before the next block on each side. +//! +//! [`Chain::lift`] maps a position only when the blocks that cover it all +//! belong to one chain; a position in a gap, or covered by two chains, has +//! no single place and gives `None`. + +use og_io::{Lines, Utf8}; +use pycompat::num::{parse_int, py_isspace}; +use std::collections::HashMap; +use std::fmt; +use std::io::{self, BufRead}; + +/// Why a chain file could not be read. +#[derive(Debug)] +pub enum ChainError { + Io(io::Error), + /// A chain whose source strand is not `+`: the chain's id (or its + /// 0-based number when the header has no id) and the strand. + SourceStrand { + chain: String, + strand: String, + }, + /// A block line before any chain header, or after a chain's last block. + Orphan(String), + /// A line that cannot be used: its number (1-based) and why. + Line { + line: usize, + why: String, + }, +} + +impl fmt::Display for ChainError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ChainError::Io(e) => write!(f, "{e}"), + ChainError::SourceStrand { chain, strand } => { + write!(f, "chain {chain} has source strand {strand}") + } + ChainError::Orphan(line) => write!(f, "a block line outside a chain: {line}"), + ChainError::Line { line, why } => write!(f, "line {line}: {why}"), + } + } +} + +impl std::error::Error for ChainError {} + +impl From for ChainError { + fn from(e: io::Error) -> Self { + ChainError::Io(e) + } +} + +/// One ungapped block: `[start, end)` on the source, 0-based. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Block { + start: u32, + end: u32, + /// The chain's number in the file (0-based). + chain: u32, + /// The target sequence's name, an index into [`Chain::names`]. + name: u32, + /// The block's first base on the target, counted on its strand. + to_start: u32, + /// The target strand, an index into [`Chain::names`]. + strand: u32, + /// The target sequence's length. + to_size: u32, +} + +/// The blocks of one source sequence, sorted, with the furthest end of +/// each prefix, so a lookup stops where no earlier block can reach. +#[derive(Debug, Default)] +struct Source { + blocks: Vec, + reach: Vec, +} + +/// A chain file, indexed by source sequence. +#[derive(Debug, Default)] +pub struct Chain { + /// Target names and strands, interned. + names: Vec, + sources: HashMap, +} + +/// Where [`Chain::lift`] puts a position. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Lifted<'a> { + /// The target sequence. + pub chrom: &'a str, + /// The 1-based position on it. + pub pos: i64, + /// The target strand as the file writes it (`+` or `-`). + pub strand: &'a str, +} + +/// The header fields a block line needs, kept while the chain's blocks run. +struct Open { + source: String, + at: i64, + name: u32, + to_at: i64, + strand: u32, + to_size: i64, + chain: u32, +} + +impl Chain { + /// Reads a chain file (universal newlines; invalid UTF-8 is replaced). + /// Fields are split on whitespace, so a header with tabs reads too. + pub fn read(input: impl BufRead) -> Result { + let mut lines = Lines::new(input, Utf8::Replace); + let mut text = String::new(); + let mut out = Chain::default(); + let mut interned: HashMap = HashMap::new(); + let mut intern = |names: &mut Vec, s: &str| -> u32 { + if let Some(&i) = interned.get(s) { + return i; + } + let i = names.len() as u32; + names.push(s.to_owned()); + interned.insert(s.to_owned(), i); + i + }; + let mut cur: Option = None; + let mut chains: u32 = 0; + let mut n = 0; + while lines.read_line(&mut text)? { + n += 1; + let c: Vec<&str> = text.split(py_isspace).filter(|f| !f.is_empty()).collect(); + let bad = |why: String| ChainError::Line { line: n, why }; + if c.is_empty() { + cur = None; + continue; + } + if c[0] == "chain" { + if c.len() < 11 { + return Err(bad(format!( + "a chain header of {} fields, want 13", + c.len() + ))); + } + let number = chains; + chains += 1; + if c[4] != "+" { + return Err(ChainError::SourceStrand { + chain: c + .get(12) + .map_or_else(|| number.to_string(), |s| (*s).to_owned()), + strand: c[4].to_owned(), + }); + } + let num = |k: usize| coord(c[k], n, k + 1); + cur = Some(Open { + source: c[2].to_owned(), + at: num(5)?, + name: intern(&mut out.names, c[7]), + to_size: num(8)?, + strand: intern(&mut out.names, c[9]), + to_at: num(10)?, + chain: number, + }); + continue; + } + let Some(open) = cur.as_mut() else { + return Err(ChainError::Orphan( + pycompat::num::py_strip(&text).to_owned(), + )); + }; + let size = coord(c[0], n, 1)?; + let fit = |v: i64| { + u32::try_from(v).map_err(|_| bad(format!("position {v} does not fit 32 bits"))) + }; + let block = Block { + start: fit(open.at)?, + end: fit(open.at + size)?, + chain: open.chain, + name: open.name, + to_start: fit(open.to_at)?, + strand: open.strand, + to_size: fit(open.to_size)?, + }; + out.sources + .entry(open.source.clone()) + .or_default() + .blocks + .push(block); + if c.len() >= 3 { + open.at += size + coord(c[1], n, 2)?; + open.to_at += size + coord(c[2], n, 3)?; + } else { + cur = None; + } + } + for s in out.sources.values_mut() { + let names = &out.names; + s.blocks.sort_by(|a, b| { + (a.start, a.end, a.chain) + .cmp(&(b.start, b.end, b.chain)) + .then_with(|| names[a.name as usize].cmp(&names[b.name as usize])) + .then_with(|| a.to_start.cmp(&b.to_start)) + .then_with(|| names[a.strand as usize].cmp(&names[b.strand as usize])) + .then_with(|| a.to_size.cmp(&b.to_size)) + }); + let mut far = 0; + s.reach = s + .blocks + .iter() + .map(|b| { + far = far.max(b.end); + far + }) + .collect(); + } + Ok(out) + } + + /// How many blocks were read. + pub fn blocks(&self) -> usize { + self.sources.values().map(|s| s.blocks.len()).sum() + } + + /// The 1-based position `pos` of source sequence `chrom` on the target, + /// or `None` when no block covers it, or blocks of more than one chain + /// do. On a `-` target strand the position is mirrored: + /// `size - 1 - offset`, 0-based. + pub fn lift(&self, chrom: &str, pos: i64) -> Option> { + let src = self.sources.get(chrom)?; + let p = pos - 1; + // Blocks that start at or before p, walked back while one of them + // may still reach past p. + let mut j = src.blocks.partition_point(|b| i64::from(b.start) <= p); + let mut first: Option<&Block> = None; + while j > 0 && i64::from(src.reach[j - 1]) > p { + j -= 1; + let b = &src.blocks[j]; + if i64::from(b.end) > p { + match first { + None => first = Some(b), + Some(f) if f.chain != b.chain => return None, + Some(_) => {} + } + } + } + let b = first?; + let mut q = i64::from(b.to_start) + (p - i64::from(b.start)); + let strand = self.names[b.strand as usize].as_str(); + if strand == "-" { + q = i64::from(b.to_size) - 1 - q; + } + Some(Lifted { + chrom: &self.names[b.name as usize], + pos: q + 1, + strand, + }) + } +} + +/// A whole-number field of line `line`, column `col`: Python's `int()`. +fn coord(field: &str, line: usize, col: usize) -> Result { + parse_int(field) + .ok() + .and_then(|v| i64::try_from(v).ok()) + .ok_or_else(|| ChainError::Line { + line, + why: format!("field {col} '{field}' is not a whole number"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The chain of tests/test_ancient_report.py: chr1 with a gap, chr2 on + /// the minus strand, chr3 covered twice. + const TEST: &str = "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1 +100 50 50 +50 + +chain 90 chr2 50000 + 500 600 chr2 10000 - 300 400 2 +100 + +chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3 +100 + +chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4 +100 +"; + + fn chain() -> Chain { + Chain::read(TEST.as_bytes()).unwrap() + } + + fn lift(c: &Chain, chrom: &str, pos: i64) -> Option<(String, i64, String)> { + c.lift(chrom, pos) + .map(|l| (l.chrom.to_owned(), l.pos, l.strand.to_owned())) + } + + fn at(chrom: &str, pos: i64, strand: &str) -> Option<(String, i64, String)> { + Some((chrom.to_owned(), pos, strand.to_owned())) + } + + #[test] + fn blocks_move_by_their_offset() { + let c = chain(); + assert_eq!(c.blocks(), 5); + assert_eq!(lift(&c, "chr1", 1051), at("chr1", 2051, "+")); + assert_eq!(lift(&c, "chr1", 1160), at("chr1", 2160, "+")); + assert_eq!(lift(&c, "chr1", 1120), None, "a gap has no place"); + assert_eq!(lift(&c, "chr9", 1051), None, "an unknown sequence"); + } + + #[test] + fn block_ends_are_half_open() { + let c = chain(); + // The first block holds 0-based 1000..1100: 1-based 1001 to 1100. + assert_eq!(lift(&c, "chr1", 1001), at("chr1", 2001, "+")); + assert_eq!(lift(&c, "chr1", 1100), at("chr1", 2100, "+")); + assert_eq!(lift(&c, "chr1", 1000), None); + assert_eq!(lift(&c, "chr1", 1101), None); + assert_eq!(lift(&c, "chr1", 1151), at("chr1", 2151, "+")); + assert_eq!(lift(&c, "chr1", 1200), at("chr1", 2200, "+")); + assert_eq!(lift(&c, "chr1", 1201), None); + assert_eq!(lift(&c, "chr1", 0), None); + assert_eq!(lift(&c, "chr1", -5), None); + } + + #[test] + fn minus_strand_mirrors() { + // 0-based 510 is 310 on the reverse strand: 10000 - 1 - 310 = 9689. + assert_eq!(lift(&chain(), "chr2", 511), at("chr2", 9690, "-")); + } + + #[test] + fn two_chains_over_a_position_give_none() { + let c = chain(); + assert_eq!(lift(&c, "chr3", 171), None); + assert_eq!(lift(&c, "chr3", 121), at("chr3", 121, "+")); + assert_eq!(lift(&c, "chr3", 221), at("chr3", 5221, "+")); + } + + #[test] + fn a_long_block_far_back_still_counts() { + // A long block of chain 1 starts long before a short block of + // chain 2; a position inside both is covered twice. + let text = "chain 1 c 9000 + 0 5000 c 9000 + 0 5000 1\n5000\n\n\ + chain 1 c 9000 + 4000 4010 d 9000 + 0 10 2\n10\n"; + let c = Chain::read(text.as_bytes()).unwrap(); + assert_eq!(lift(&c, "c", 4005), None); + assert_eq!(lift(&c, "c", 3000), at("c", 3000, "+")); + } + + #[test] + fn errors() { + let e = Chain::read("chain 1 c 10 - 0 5 c 10 + 0 5 7\n5\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "chain 7 has source strand -"); + let e = Chain::read("chain 1 c 10 - 0 5 c 10 + 0 5\n5\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "chain 0 has source strand -"); + let e = Chain::read(" 5 1 1 \n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "a block line outside a chain: 5 1 1"); + // After a block line with one field the chain is over. + let e = Chain::read("chain 1 c 10 + 0 5 c 10 + 0 5 1\n5\n6\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "a block line outside a chain: 6"); + let e = Chain::read("chain 1 c 10 + x 5 c 10 + 0 5 1\n".as_bytes()).unwrap_err(); + assert_eq!(e.to_string(), "line 1: field 6 'x' is not a whole number"); + } +} diff --git a/rust/crates/og-intervals/src/lib.rs b/rust/crates/og-intervals/src/lib.rs index 3e3104345..2a5008bd1 100644 --- a/rust/crates/og-intervals/src/lib.rs +++ b/rust/crates/og-intervals/src/lib.rs @@ -10,8 +10,11 @@ //! search, BED files such as the PAR regions read into them) and an //! overlap index with reciprocal overlap. //! - [`fasta`]: random access into a FASTA through its `.fai`. +//! - [`chain`]: UCSC chain files, to lift single positions to another +//! assembly. pub mod bed; +pub mod chain; pub mod fasta; pub mod mask; pub mod sets; diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index dd69a5991..35edc30d6 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -57,6 +57,8 @@ const USAGE: &str = "usage: og [args] og family ibd [args] og consumer build-sites|export|check [args] og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report [args] + og archaic table|strict|genotypes|report [args] + og ancient select|lift|person-geno|align|report [args] og vntyper report [args] og locityper report [args] og mocha report [args] @@ -252,6 +254,16 @@ fn main() -> ExitCode { &mut std::io::stdout().lock(), &mut std::io::stderr().lock(), )), + "archaic" => ExitCode::from(og_ancestry::archaic::cli( + rest, + &mut std::io::stdout().lock(), + &mut std::io::stderr().lock(), + )), + "ancient" => ExitCode::from(og_ancestry::ancient::cli( + rest, + &mut std::io::stdout().lock(), + &mut std::io::stderr().lock(), + )), "str" => match rest.first().map(String::as_str) { #[cfg(feature = "bam")] Some("motif") => ExitCode::from(og_bam::motif::cli( From f1d1aaeb9a116a35e7de7820468dedd4dc0f0971 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 11:45:48 +0100 Subject: [PATCH 036/255] refactor(ancestry): steps 43 and 44 and their setup run og archaic and og ancient The parity cases compare og with bin/archaic_report.py and bin/ancient_report.py on synthetic inputs. setup.sh stamps the archaic table with a format id instead of the Python's hash, and takes a table the last Python built as current. Bead: pgp-bs9u.21 --- modules/local/ancient/main.nf | 8 +- modules/local/archaic/main.nf | 8 +- scripts/43-archaic.sh | 8 +- scripts/44-ancient.sh | 4 +- scripts/setup.sh | 32 ++- tests/parity/cases.tsv | 51 +++++ tests/parity/gen/ancient-inputs.sh | 211 ++++++++++++++++++ tests/parity/gen/archaic-inputs.sh | 187 ++++++++++++++++ .../long-geno/o.geno.tmp.absent | 0 .../short-geno/o.geno.tmp.absent | 0 .../wide-geno/o.geno.tmp.absent | 0 .../bad-code/gt/chr22.gt.gz.absent | 0 .../golden/archaic-strict/unsorted/exit | 1 + .../archaic-strict/unsorted/s.tsv.absent | 0 .../golden/archaic-strict/unsorted/stderr | 1 + .../archaic-table/unsorted/t.vcf.absent | 0 tests/parity/oracle.ref | 9 + 17 files changed, 499 insertions(+), 21 deletions(-) create mode 100644 tests/parity/gen/ancient-inputs.sh create mode 100644 tests/parity/gen/archaic-inputs.sh create mode 100644 tests/parity/golden/ancient-person-geno/long-geno/o.geno.tmp.absent create mode 100644 tests/parity/golden/ancient-person-geno/short-geno/o.geno.tmp.absent create mode 100644 tests/parity/golden/ancient-person-geno/wide-geno/o.geno.tmp.absent create mode 100644 tests/parity/golden/archaic-genotypes/bad-code/gt/chr22.gt.gz.absent create mode 100644 tests/parity/golden/archaic-strict/unsorted/exit create mode 100644 tests/parity/golden/archaic-strict/unsorted/s.tsv.absent create mode 100644 tests/parity/golden/archaic-strict/unsorted/stderr create mode 100644 tests/parity/golden/archaic-table/unsorted/t.vcf.absent diff --git a/modules/local/ancient/main.nf b/modules/local/ancient/main.nf index 839f6ace7..a5b83de3b 100644 --- a/modules/local/ancient/main.nf +++ b/modules/local/ancient/main.nf @@ -6,11 +6,11 @@ reference block over a site becomes a record with the reference base, and a site without reads has no line ANCIENT_GENO the sample as one more column of the panel's present-day - EIGENSTRAT genotypes (bin/ancient_report.py person-geno) + EIGENSTRAT genotypes (og ancient person-geno) ANCIENT_PCA smartpca: axes from the present-day West Eurasians, the sample projected (lsqproject) ANCIENT_REPORT the nearest ancient groups, the summary and the plot, on - the panel's axes (bin/ancient_report.py report) + the panel's axes (og ancient report) The panel is the folder setup.sh --ancient-panel writes (reference/ancient/aadr_). Runs for the samples with a gVCF only; @@ -78,7 +78,7 @@ process ANCIENT_GENO { script: """ - ancient_report.py person-geno --sites ${panel}/sites_grch38.tsv --genotypes ${genotypes} \\ + og ancient person-geno --sites ${panel}/sites_grch38.tsv --genotypes ${genotypes} \\ --snp ${panel}/moderns.snp --moderns-geno ${panel}/moderns.geno --out merged.geno --stats stats.tsv { cat ${panel}/moderns.ind; printf '%s\\tU\\tPerson\\n' '${meta.id}'; } > merged.ind @@ -148,7 +148,7 @@ process ANCIENT_REPORT { script: """ release=\$(awk -F'\\t' '\$1 == "release" {print \$2}' ${panel}/DONE) - ancient_report.py report --panel-evec ${panel}/panel.evec --run-evec ${evec} --ancients ${panel}/ancients.tsv \\ + og ancient report --panel-evec ${panel}/panel.evec --run-evec ${evec} --ancients ${panel}/ancients.tsv \\ --sample ${meta.id} --release "\$release" --stats ${stats} \\ --out-tsv ${meta.id}_ancient.tsv --out-info ${meta.id}_ancient_info.tsv \\ --out-txt ${meta.id}_ancient.txt --out-svg ${meta.id}_ancient_pca.svg diff --git a/modules/local/archaic/main.nf b/modules/local/archaic/main.nf index ec31864df..bf68a4653 100644 --- a/modules/local/archaic/main.nf +++ b/modules/local/archaic/main.nf @@ -4,13 +4,13 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Opt-in ('archaic' in --tools, with --archaic_data). Three processes, as in scripts/43-archaic.sh: - ARCHAIC_GENOTYPES bin/archaic_report.py genotypes: IBDmix's genotype file of + ARCHAIC_GENOTYPES og archaic genotypes: IBDmix's genotype file of each chromosome (the archaic genomes, the sample's VCF and the 1000 Genomes stand-ins of --archaic_pop), and each archaic genome's mask ARCHAIC_IBDMIX IBDmix against each archaic genome, LOD 4, the options of Chen et al. 2020, with that genome's mask - ARCHAIC_REPORT bin/archaic_report.py report: the segments of 50 kb or + ARCHAIC_REPORT og archaic report: the segments of 50 kb or more, the share, the ten longest with their genes Equivalent to: scripts/43-archaic.sh @@ -34,7 +34,7 @@ process ARCHAIC_GENOTYPES { script: """ - archaic_report.py genotypes --table-dir ${archaic_data} --vcf ${vcf} --pop ${params.archaic_pop} --out-dir gt + og archaic genotypes --table-dir ${archaic_data} --vcf ${vcf} --pop ${params.archaic_pop} --out-dir gt cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -126,7 +126,7 @@ process ARCHAIC_REPORT { script: def genes_arg = genes ? "--genes ${genes}" : '' """ - archaic_report.py report --sample ${meta.id} --ibdmix-dir ${ibdmix} --spans ${ibdmix}/spans.tsv \\ + og archaic report --sample ${meta.id} --ibdmix-dir ${ibdmix} --spans ${ibdmix}/spans.tsv \\ --pop ${params.archaic_pop} ${genes_arg} --out-dir . cat <<-END_VERSIONS > versions.yml diff --git a/scripts/43-archaic.sh b/scripts/43-archaic.sh index c6bcb62a3..9a98b9392 100755 --- a/scripts/43-archaic.sh +++ b/scripts/43-archaic.sh @@ -6,14 +6,14 @@ # high-coverage archaic genomes, the Altai and Vindija 33.19 Neanderthals and # the Altai Denisovan, and calls the stretches you share with one of them. # It needs the archaic data setup.sh --archaic builds (reference/archaic/): -# 1. bin/archaic_report.py genotypes writes IBDmix's genotype file of each +# 1. og archaic genotypes writes IBDmix's genotype file of each # chromosome: the archaic genotypes, yours from the VCF, and 100 # stand-ins carrying the 1000 Genomes allele count of ARCHAIC_POP at each # site (IBDmix takes the modern frequency from the modern columns, and # one genome alone gives it none; docs/43-archaic.md says why this is # the same score); # 2. IBDmix scores your column against each archaic genome (LOD 4); -# 3. bin/archaic_report.py report keeps the segments of 50 kb or more, +# 3. og archaic report keeps the segments of 50 kb or more, # merges the two Neanderthals, and writes the share, the segments and # the ten longest with their protein-coding genes. # @@ -61,7 +61,7 @@ W=$(cpath "$WORK") echo "[1/3] Writing IBDmix's genotype files (your VCF, the archaic genomes, the stand-ins)..." run_in --cpus 1 --memory 4g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py genotypes --table-dir "$D" --vcf "/genome/${SAMPLE}/vcf/${SAMPLE}.vcf.gz" \ + /pgp-bin/og archaic genotypes --table-dir "$D" --vcf "/genome/${SAMPLE}/vcf/${SAMPLE}.vcf.gz" \ --pop "$POP" --out-dir "${W}/gt" # One IBDmix run per chromosome and archaic genome, THREADS at a time, with the @@ -101,7 +101,7 @@ else echo " No GENCODE gene file (${GENES}): the segments are listed without their genes (setup.sh installs it)." fi run_in --cpus 1 --memory 2g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py report --sample "$SAMPLE" --ibdmix-dir "${W}/ibdmix" \ + /pgp-bin/og archaic report --sample "$SAMPLE" --ibdmix-dir "${W}/ibdmix" \ --spans "${W}/gt/spans.tsv" --pop "$POP" ${GARG[@]+"${GARG[@]}"} --out-dir "$(cpath "$OUT")" for f in "${OUT}/${SAMPLE}_archaic.tsv" "${OUT}/${SAMPLE}_archaic_segments.tsv"; do [ -s "$f" ] || { echo "ERROR: step 43 wrote no ${f}" >&2; exit 1; } diff --git a/scripts/44-ancient.sh b/scripts/44-ancient.sh index 28cc87a79..3206fa8f4 100755 --- a/scripts/44-ancient.sh +++ b/scripts/44-ancient.sh @@ -78,7 +78,7 @@ run_in --cpus 2 --memory 4g "$BCFTOOLS_IMAGE" \ echo "[2/4] Adding ${SAMPLE} to the present-day panel..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/ancient_report.py person-geno --sites "${P}/sites_grch38.tsv" --genotypes "${W}/genotypes.tsv" \ + /pgp-bin/og ancient person-geno --sites "${P}/sites_grch38.tsv" --genotypes "${W}/genotypes.tsv" \ --snp "${P}/moderns.snp" --moderns-geno "${P}/moderns.geno" --out "${W}/merged.geno" --stats "${W}/stats.tsv" { cat "${PANEL}/moderns.ind"; printf '%s\tU\tPerson\n' "$SAMPLE"; } > "${WORK}/merged.ind" @@ -93,7 +93,7 @@ run_in --cpus "$THREADS" --memory 8g "$EIGENSOFT_IMAGE" smartpca -p "${W}/run.pa echo "[4/4] The nearest ancient groups..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/ancient_report.py report --panel-evec "${P}/panel.evec" --run-evec "${W}/run.evec" \ + /pgp-bin/og ancient report --panel-evec "${P}/panel.evec" --run-evec "${W}/run.evec" \ --ancients "${P}/ancients.tsv" --sample "$SAMPLE" --release "$RELEASE" --stats "${W}/stats.tsv" \ --out-tsv "${O}/${SAMPLE}_ancient.tsv" --out-info "${O}/${SAMPLE}_ancient_info.tsv" \ --out-txt "${O}/${SAMPLE}_ancient.txt" --out-svg "${O}/${SAMPLE}_ancient_pca.svg" diff --git a/scripts/setup.sh b/scripts/setup.sh index 88b6c4943..326dfeddd 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -801,7 +801,7 @@ consumer_sites_pin_check() { # id, each checked against the md5 Dataverse publishes) become the panel of # step 44 in reference/ancient/aadr_: # panel.ind, poplist.txt, ancients.tsv who builds the axes (the present-day -# West Eurasians of bin/ancient_report.py's MODERN_GROUPS) and who is +# West Eurasians of og ancient's MODERN_GROUPS) and who is # projected (dated West Eurasians with 100,000 sites or more); # sites_grch38.tsv the autosomal sites on GRCh38 (UCSC's hg19ToHg38 chain, # each kept only when one of its alleles is the GRCh38 base); @@ -830,6 +830,7 @@ install_ancient_panel() { done echo "Installing the ancient populations panel, AADR ${AADR_VERSION} 1240K (step 44) in ${dir}" ensure_image EIGENSOFT_IMAGE 44 || return 1 + need_og "chooses the ancient panel's people and places its sites" || return 1 mkdir -p "$raw" for e in anno ind snp geno; do f="${raw}/aadr.${e}" @@ -868,9 +869,9 @@ install_ancient_panel() { # shellcheck disable=SC2016 # $1 to $4 belong to the inner bash if ! run_in --rw "$part" -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 4g "$PYTHON_IMAGE" \ bash -euo pipefail -c ' - python3 /pgp-bin/ancient_report.py select --anno "$1/aadr.anno" --ind "$1/aadr.ind" \ + /pgp-bin/og ancient select --anno "$1/aadr.anno" --ind "$1/aadr.ind" \ --out-ind "$2/panel.ind" --poplist "$2/poplist.txt" --ancients "$2/ancients.tsv" - python3 /pgp-bin/ancient_report.py lift --snp "$1/aadr.snp" --chain "$3" --fasta "$4" --out "$2/sites_grch38.tsv"' \ + /pgp-bin/og ancient lift --snp "$1/aadr.snp" --chain "$3" --fasta "$4" --out "$2/sites_grch38.tsv"' \ _ "$R" "$P" "$C" "$REF_FASTA_C"; then echo "[WARN] Could not choose the panel's people or place its sites (the lines above say why)." return 1 @@ -894,7 +895,7 @@ install_ancient_panel() { return 1 fi if ! run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - python3 /pgp-bin/ancient_report.py align --panel-evec "${P}/panel.evec" --run-evec "${P}/check.evec"; then + /pgp-bin/og ancient align --panel-evec "${P}/panel.evec" --run-evec "${P}/check.evec"; then echo "[WARN] The two smartpca runs on the same present-day people gave different axes (logs in ${part})." return 1 fi @@ -1836,6 +1837,13 @@ archaic_filterbed_sha() { return 1 } +# The version of the table og archaic table and strict write: a change to +# their bytes changes it, and every install rebuilds its table. +ARCHAIC_TABLE_FORMAT=archaic-table-v1 +# What code= held when bin/archaic_report.py built the table: the sha256 of +# its part above "# --- genotypes (step 43)", at its last version. +ARCHAIC_TABLE_PYTHON_SHA256=6bdc4c99a8d80d96179551e37022a3cd5324f650577b9fd64e27d84383e0f1d7 + # install_archaic: the table of step 43 in reference/archaic/ARCHAIC_DATA_VERSION # (ARCHAIC_DIR: another folder inside GENOME_DIR), one chrN.tsv.gz per # autosome, GRCh38: position, REF, ALT, the ALT count of the Altai, Vindija @@ -1858,6 +1866,7 @@ install_archaic() { cpath "$dir" >/dev/null || return 1 echo "Installing the archaic data ${ARCHAIC_DATA_VERSION} (step 43) in ${dir}" for v in BCFTOOLS_IMAGE PICARD_IMAGE PYTHON_IMAGE IBDMIX_IMAGE; do ensure_image "$v" 43 || return 1; done + need_og "builds the archaic table" || return 1 [ -s "$REF_DICT" ] || { echo "[WARN] ${REF_DICT} is missing: run $0 ${GENOME_DIR} first (it writes the sequence dictionary)"; return 1; } mkdir -p "${dir}/md5" "${dir}/download" "${dir}/work" "${dir}/mask" "${dir}/exclude" for v in "Altai|${ARCHAIC_ALTAI_MD5SUMS_SHA256}" "Vindija33.19|${ARCHAIC_VINDIJA_MD5SUMS_SHA256}" "Denisova|${ARCHAIC_DENISOVA_MD5SUMS_SHA256}"; do @@ -1893,9 +1902,18 @@ install_archaic() { chroms=$(seq 1 22 | tr '\n' ' ') fi [ -n "${chroms// /}" ] || { echo "[WARN] ARCHAIC_REGIONS=${regions} names no autosome (1 to 22)"; return 1; } - stamp="data=${ARCHAIC_DATA_VERSION} regions=${rstamp} bcftools=${BCFTOOLS_IMAGE} picard=${PICARD_IMAGE} masks=$(_digest sha256 <(printf '%s\n%s\n%s\n%s\n%s\n' "$ARCHAIC_STRICT_MASK_SHA256" "$ARCHAIC_SEGDUPS_SHA256" "$ARCHAIC_FILTERBED_SHA256_ALTAI" "$ARCHAIC_FILTERBED_SHA256_VINDIJA" "$ARCHAIC_FILTERBED_SHA256_DENISOVA")) code=$(_digest sha256 <(sed '/^# --- genotypes (step 43)/,$d' "${PGP_ROOT}/bin/archaic_report.py"))" + stamp="data=${ARCHAIC_DATA_VERSION} regions=${rstamp} bcftools=${BCFTOOLS_IMAGE} picard=${PICARD_IMAGE} masks=$(_digest sha256 <(printf '%s\n%s\n%s\n%s\n%s\n' "$ARCHAIC_STRICT_MASK_SHA256" "$ARCHAIC_SEGDUPS_SHA256" "$ARCHAIC_FILTERBED_SHA256_ALTAI" "$ARCHAIC_FILTERBED_SHA256_VINDIJA" "$ARCHAIC_FILTERBED_SHA256_DENISOVA")) code=${ARCHAIC_TABLE_FORMAT}" + # Until og archaic replaced bin/archaic_report.py, code= held the sha256 of + # that file's table half. The last Python wrote the same table bytes as + # og archaic table and strict (tests/parity), so a table it built counts as + # built here: its stamp is accepted once and rewritten. Remove this one + # release after 1.0.0. + local legacy="${stamp% code=*} code=${ARCHAIC_TABLE_PYTHON_SHA256}" local have=1 for c in $chroms; do [ -s "${dir}/chr${c}.tsv.gz" ] || have=0; done + for v in "${dir}/DONE" "${dir}"/chr*.stamp; do + [ "$(cat "$v" 2>/dev/null)" = "$legacy" ] && echo "$stamp" > "$v" + done if [ "$have" = 1 ] && [ "$(cat "${dir}/DONE" 2>/dev/null)" = "$stamp" ]; then echo "[OK] archaic data ${ARCHAIC_DATA_VERSION} (step 43): ${dir}" return 0 @@ -1975,7 +1993,7 @@ install_archaic() { echo "[WARN] Could not read the archaic genotypes of chr${c}"; return 1 fi if ! run_in --rw "$dir" -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py table --chrom "$c" --modern "$(cpath "${wk}/modern.tsv")" \ + /pgp-bin/og archaic table --chrom "$c" --modern "$(cpath "${wk}/modern.tsv")" \ --archaic "Altai=$(cpath "${wk}/Altai.tsv")" --archaic "Vindija=$(cpath "${wk}/Vindija33.19.tsv")" \ --archaic "Denisova=$(cpath "${wk}/Denisova.tsv")" \ --mask "Altai=$(cpath "${dir}/mask/Altai.chr${c}.bed.gz")" \ @@ -2009,7 +2027,7 @@ install_archaic() { echo "[WARN] the GRCh38 strict mask has no interval on chr${c}: ${dir}/strict_GRCh38.bed"; return 1 fi if ! run_in --rw "$dir" -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ - python3 /pgp-bin/archaic_report.py strict --chrom "$c" --strict "$(cpath "${wk}/strict.bed")" \ + /pgp-bin/og archaic strict --chrom "$c" --strict "$(cpath "${wk}/strict.bed")" \ --rows "$(cpath "${wk}/lifted.tsv")" --out "$(cpath "${wk}/marked.tsv")"; then echo "[WARN] Could not mark the strict mask on chr${c}"; return 1 fi diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 586dfda38..6ec8e03fa 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -917,3 +917,54 @@ family-ibd mapbp gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree mapbp/pedigree.ts family-ibd nomap gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree nomap/pedigree.tsv --chroms nomap/chroms.txt --maps nomap/maps --phased nomap/phased --ibd nomap/ibd --out-dir out - content PYTHON_IMAGE family-ibd short gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree short/pedigree.tsv --chroms short/chroms.txt --maps short/maps --phased short/phased --ibd short/ibd --out-dir out - content PYTHON_IMAGE family-ibd twice gen:ibd-oracle.sh,gen:ibd-cases.sh --pedigree twice/pedigree.tsv --chroms twice/chroms.txt --maps twice/maps --phased twice/phased --ibd twice/ibd --out-dir out - content PYTHON_IMAGE +# Step 43, og archaic (bin/archaic_report.py) +archaic-table masked gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --mask Altai=altai.bed --mask Vindija=vindija.bed --exclude excl.bed --out t.vcf t.vcf bytes PYTHON_IMAGE +archaic-table plain gen:archaic-inputs.sh --chrom chr22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --out t.vcf t.vcf bytes PYTHON_IMAGE +archaic-table gz gen:archaic-inputs.sh --chrom 22 --modern modern.tsv.gz --archaic Altai=altai.tsv.gz --archaic Vindija=vindija.tsv.gz --archaic Denisova=denisova.tsv.gz --mask Denisova=altai.bed --exclude excl.bed --exclude vindija.bed --out t.vcf t.vcf bytes PYTHON_IMAGE +archaic-table unsorted gen:archaic-inputs.sh --chrom 22 --modern unsorted.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --out t.vcf - bytes PYTHON_IMAGE +archaic-table order gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Vindija=vindija.tsv --archaic Altai=altai.tsv --archaic Denisova=denisova.tsv --out t.vcf - bytes PYTHON_IMAGE +archaic-table bad-name gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Neanderthal=altai.tsv --out t.vcf - bytes PYTHON_IMAGE +archaic-table mask-twice gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --mask Altai=altai.bed --mask Altai=vindija.bed --out t.vcf - bytes PYTHON_IMAGE +archaic-table empty-mask gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --mask Altai=empty.bed --out t.vcf - bytes PYTHON_IMAGE +archaic-table empty-exclude gen:archaic-inputs.sh --chrom 22 --modern modern.tsv --archaic Altai=altai.tsv --archaic Vindija=vindija.tsv --archaic Denisova=denisova.tsv --exclude empty.bed --out t.vcf - bytes PYTHON_IMAGE +archaic-strict plain gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows.tsv --out s.tsv s.tsv bytes PYTHON_IMAGE +archaic-strict unsorted gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows-unsorted.tsv --out s.tsv - bytes PYTHON_IMAGE +archaic-strict empty-mask gen:archaic-inputs.sh --chrom 22 --strict empty.bed --rows rows.tsv --out s.tsv - bytes PYTHON_IMAGE +archaic-genotypes plain gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop EUR --pseudo 3 --out-dir gt gt/samples.txt,gt/spans.tsv,gt/chr21.gt.gz,gt/chr22.gt.gz,gt/chr22.Altai.mask.bed *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes default gen:archaic-inputs.sh --table-dir . --vcf s.vcf.gz --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes afr-one gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop AFR --pseudo 1 --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes two-samples gen:archaic-inputs.sh --table-dir . --vcf two.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes comes-back gen:archaic-inputs.sh --table-dir . --vcf back.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes no-table-chrom gen:archaic-inputs.sh --table-dir . --vcf none.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes bad-code gen:archaic-inputs.sh --table-dir badcode --vcf s.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes bad-pop gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop XYZ --out-dir gt - bytes PYTHON_IMAGE +archaic-genotypes bad-pseudo gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pseudo 0 --out-dir gt - bytes PYTHON_IMAGE +archaic-genotypes no-table gen:archaic-inputs.sh --table-dir emptydir --vcf s.vcf --out-dir gt - bytes PYTHON_IMAGE +archaic-report genes gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans spans.tsv --genes genes.gtf --pop EUR --out-dir rep rep/S_archaic.tsv,rep/S_archaic.txt,rep/S_archaic_segments.tsv bytes PYTHON_IMAGE +archaic-report no-genes gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans spans.tsv --out-dir rep rep/S_archaic.tsv,rep/S_archaic.txt,rep/S_archaic_segments.tsv bytes PYTHON_IMAGE +archaic-report missing-output gen:archaic-inputs.sh --sample S --ibdmix-dir rep --spans spans.tsv --out-dir rep - bytes PYTHON_IMAGE +archaic-report not-spans gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans genes.gtf --out-dir rep - bytes PYTHON_IMAGE +# Step 44, og ancient (bin/ancient_report.py) +ancient-select plain gen:ancient-inputs.sh --anno aadr.anno --ind aadr.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv o.ind,pops.txt,anc.tsv bytes PYTHON_IMAGE +ancient-select min-snps gen:ancient-inputs.sh --anno aadr.anno --ind aadr.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv --min-snps 140000 o.ind,pops.txt,anc.tsv bytes PYTHON_IMAGE +ancient-select too-few gen:ancient-inputs.sh --anno aadr.anno --ind few.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv - bytes PYTHON_IMAGE +ancient-select no-column gen:ancient-inputs.sh --anno aadr.ind --ind aadr.ind --out-ind o.ind --poplist pops.txt --ancients anc.tsv - bytes PYTHON_IMAGE +ancient-lift plain gen:ancient-inputs.sh --snp aadr.snp --chain hg19.chain --fasta hg38.fa --out sites.out.tsv sites.out.tsv bytes PYTHON_IMAGE +ancient-lift gz-chain gen:ancient-inputs.sh --snp aadr.snp --chain hg19.chain.gz --fasta hg38.fa --out sites.out.tsv sites.out.tsv bytes PYTHON_IMAGE +ancient-lift minus-source gen:ancient-inputs.sh --snp aadr.snp --chain minus-source.chain --fasta hg38.fa --out sites.out.tsv - bytes PYTHON_IMAGE +ancient-lift orphan-block gen:ancient-inputs.sh --snp aadr.snp --chain orphan.chain --fasta hg38.fa --out sites.out.tsv - bytes PYTHON_IMAGE +ancient-lift nothing-placed gen:ancient-inputs.sh --snp panel.geno --chain hg19.chain --fasta hg38.fa --out sites.out.tsv - bytes PYTHON_IMAGE +ancient-person-geno plain gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno panel.geno --out o.geno --stats o.stats o.geno,o.stats bytes PYTHON_IMAGE +ancient-person-geno short-geno gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno short.geno --out o.geno --stats o.stats - bytes PYTHON_IMAGE +ancient-person-geno long-geno gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno long.geno --out o.geno --stats o.stats - bytes PYTHON_IMAGE +ancient-person-geno wide-geno gen:ancient-inputs.sh --sites sites.tsv --genotypes genotypes.tsv --snp panel.snp --moderns-geno wide.geno --out o.geno --stats o.stats - bytes PYTHON_IMAGE +ancient-person-geno none-used gen:ancient-inputs.sh --sites sites.tsv --genotypes nowhere.tsv --snp panel.snp --moderns-geno panel.geno --out o.geno --stats o.stats o.geno,o.stats bytes PYTHON_IMAGE +ancient-align flipped gen:ancient-inputs.sh --panel-evec panel.evec --run-evec flipped.evec - bytes PYTHON_IMAGE +ancient-align five-pcs gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --pcs 5 - bytes PYTHON_IMAGE +ancient-align differ gen:ancient-inputs.sh --panel-evec panel.evec --run-evec bad.evec - bytes PYTHON_IMAGE +ancient-align too-few gen:ancient-inputs.sh --panel-evec panel.evec --run-evec few.evec - bytes PYTHON_IMAGE +ancient-align not-evec gen:ancient-inputs.sh --panel-evec stats.tsv --run-evec run.evec - bytes PYTHON_IMAGE +ancient-report plain gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --ancients ancients.tsv --sample S1 --release v66.p1 --stats stats.tsv --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg o.tsv,o.info,o.txt,o.svg bytes PYTHON_IMAGE +ancient-report no-stats gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --ancients ancients.tsv --sample S1 --release v66 --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg o.tsv,o.info,o.txt,o.svg bytes PYTHON_IMAGE +ancient-report not-person gen:ancient-inputs.sh --panel-evec panel.evec --run-evec run.evec --ancients ancients.tsv --sample S2 --release v66 --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg - bytes PYTHON_IMAGE +ancient-report axes-differ gen:ancient-inputs.sh --panel-evec panel.evec --run-evec bad.evec --ancients ancients.tsv --sample S1 --release v66 --out-tsv o.tsv --out-info o.info --out-txt o.txt --out-svg o.svg - bytes PYTHON_IMAGE diff --git a/tests/parity/gen/ancient-inputs.sh b/tests/parity/gen/ancient-inputs.sh new file mode 100644 index 000000000..74d6a9a35 --- /dev/null +++ b/tests/parity/gen/ancient-inputs.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# The inputs of the ancient cases of step 44 (ancient-select, ancient-lift, +# ancient-person-geno, ancient-align, ancient-report), all synthetic, +# written with awk: +# aadr.anno, aadr.ind, few.ind +# an AADR-style annotation (long header cells, a quoted one, +# ASSESSMENT WARNINGS before ASSESSMENT) with present-day people +# of the reference groups, one individual under two genetic +# ids, ancient people inside and outside West Eurasia, below the +# site floor, undated, failing the assessment; the .ind file +# (unknown ids, short lines) and one that matches too few +# hg19.chain, hg19.chain.gz, minus-source.chain, orphan.chain, hg38.fa(.fai), stale.fa(.fai) +# a chain with a gap, a minus-strand block, two chains over one +# stretch and a block that moves to another chromosome; a +# reference with lower-case lines; one whose .fai is wrong +# aadr.snp sites in blocks, in gaps, on block edges, on the minus +# strand, off the autosomes, with equal or non-base alleles and +# with a fractional position +# sites.tsv, genotypes.tsv, panel.snp, panel.geno, short.geno, +# long.geno, wide.geno, nowhere.tsv +# person-geno: two SNPs at one position, other alleles, missing +# and haploid calls; a .geno that is short, long or uneven +# panel.evec, run.evec, flipped.evec, bad.evec, few.evec, ancients.tsv, stats.tsv +# two smartpca runs on one panel (the second with PC2's sign +# flipped), one whose axes differ, one with too few present-day +# people, and the ancient people's table (names to escape, a +# long name, undated people, dates on both sides of year 1) +set -euo pipefail + +# --- select ------------------------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + print "Genetic ID", "Master ID", "Individual ID (PubMed)", \ + "Date mean in BP in years before 1950 CE [OxCal mu for a direct radiocarbon date]", \ + "Date standard deviation in BP [OxCal sigma]", "Full Date One of two formats", \ + "Group ID", "\"Locality\"", "Political Entity", "Latitude", "Longitude", "Data type", \ + "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", \ + "ASSESSMENT WARNINGS (Xcontam interval is listed if lower bound is >0.005)", " ASSESSMENT " > "aadr.anno" + split("CEU French Sardinian Basque GBR IBS TSI Druze Armenian Polish Mozabite Saami", M, " ") + for (i = 1; i <= 90; i++) { + g = M[i % 12 + 1] + a = (i % 9 == 0) ? "QUESTIONABLE" : (i % 10 == 0 ? " pass (low coverage)" : "PASS") + dt = (i % 15 == 0) ? "1240K" : "Shotgun.diploid" + print "M" i, "M" i, "Ind_M" i, (i % 17 == 0 ? "-0.0" : "0"), "0", "present", g, "Paris\tFrance", "48.8", "2.3", dt, 900000 + i, "", a >> "aadr.anno" + } + print "M91", "M91", "Ind_M5", "0", "0", "present", "CEU", "Utah", "USA", "40", "-111", "Shotgun.diploid", 999999999, "", "PASS" >> "aadr.anno" + split("Germany_CordedWare England_BellBeaker Turkey_N Italy_Imperial_Rome Hungary_LBA_with_a_very_long_name_that_runs_past_45 Türkiye_Neolithic", G, " ") + for (i = 1; i <= 60; i++) { + g = G[i % 7 + 1] + bp = (i % 13 == 0) ? ".." : 500 + 137 * i + lat = (i % 11 == 0) ? 10 : 30 + i % 40 + lon = (i % 19 == 0) ? "" : -20 + i + snps = (i % 8 == 0) ? 99999 : 100000 + 1000 * i + a = (i % 14 == 0) ? "FAIL" : (i % 6 == 0 ? "PROVISIONAL_PASS" : "MERGE_PASS") + ind = (i % 10 == 0) ? "" : "Ind_A" int(i / 2) + print "A" i, "A" i, ind, bp, "50", " 2500-2000 BCE ", g, "Site " (i % 3), (i % 4 == 0 ? "" : "Country" (i % 2)), lat, lon, "1240K", snps, "x", a >> "aadr.anno" + } + print "short", "row" >> "aadr.anno" + for (i = 1; i <= 91; i++) print "M" i, (i % 2 ? "M" : "F"), "Pop" > "aadr.ind" + for (i = 1; i <= 60; i++) print " A" i, "U", "Pop" >> "aadr.ind" + print "X1 M" >> "aadr.ind" + print "Unknown", "M", "Pop" >> "aadr.ind" + for (i = 1; i <= 20; i++) print "M" i, "M", "Pop" > "few.ind" +}' + +# --- lift --------------------------------------------------------------------------- +cat > hg19.chain <<'EOF' +chain 1000 chr1 100000 + 1000 1600 chr1 3000 + 2000 2600 1 +200 100 100 +300 + +chain 900 chr2 50000 + 500 900 chr2 3000 - 100 500 2 +400 + +chain 800 chr3 50000 + 100 400 chr3 3000 + 100 400 3 +300 + +chain 700 chr3 50000 + 300 500 chr3 3000 + 1300 1500 4 +200 + +chain 600 chr4 50000 + 0 500 chr5 3000 + 0 500 5 +500 + +chain 500 chr5 50000 + 100 1100 chr5 3000 + 1100 2100 6 +1000 +EOF +gzip -n -c hg19.chain > hg19.chain.gz +printf 'chain 1 chr1 100 - 0 10 chr1 100 + 0 10 9\n10\n' > minus-source.chain +printf 'chain 1 chr1 100 + 0 10 chr1 100 + 0 10 9\n10\n5\n' > orphan.chain +awk 'BEGIN { + s = "ACGTTGCAGC"; off = 0 + for (c = 1; c <= 5; c++) { + h = ">chr" c; printf "%s\n", h > "hg38.fa"; off += length(h) + 1 + printf "chr%d\t3000\t%d\t60\t61\n", c, off > "hg38.fa.fai" + printf "chr%d\t3000\t%d\t60\t61\n", c, off + 1 > "stale.fa.fai" + line = "" + for (i = 0; i < 3000; i++) { + line = line substr(s, (i * 7 + c * 3) % 10 + 1, 1) + if (length(line) == 60) { + if (int(i / 60) % 3 == 1) line = tolower(line) + printf "%s\n", line > "hg38.fa"; off += 61; line = "" + } + } + } +}' +cp hg38.fa stale.fa +awk 'BEGIN { + OFS = " " + split("A C A G A T C G C T G T", P, " ") + n = 0 + pos[1] = "999 1000 1001 1100 1200 1201 1250 1300 1301 1450 1600 1601" + pos[2] = "500 501 600 750 900 901" + pos[3] = "100 101 250 300 301 350 400 401 450 500 501" + pos[4] = "1 250 500" + pos[5] = "101 500 1100 1101" + for (c = 1; c <= 5; c++) { + k = split(pos[c], Q, " ") + for (j = 1; j <= k; j++) { + for (p = 1; p <= 6; p++) { + n++ + a1 = P[2 * p - 1]; a2 = P[2 * p] + if (n % 5 == 0) { t = a1; a1 = a2; a2 = t } + if (n % 23 == 0) { a1 = tolower(a1) } + q = (n % 29 == 0) ? Q[j] ".0" : Q[j] + printf " rs%d\t%d 0.0123 %s %s %s\n", n, c, q, a1, a2 > "aadr.snp" + } + } + } + print "rsA", 1, "0", 1150, "A", "A" > "aadr.snp" + print "rsN", 1, "0", 1150, "N", "G" > "aadr.snp" + print "rsX", 23, "0", 1150, "A", "G" > "aadr.snp" + print "rsY", "X", "0", 1150, "A", "G" > "aadr.snp" + print "rs0", 0, "0", 1150, "A", "G" > "aadr.snp" + print "short", 1, "0", 1150, "A" > "aadr.snp" + print "rsDup", 1, "0", 1150, "A", "G" > "aadr.snp" + print "rsDup0", 1, "0", 1150, "A", "G" > "aadr.snp" +}' + +# --- person-geno -------------------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + split("A G C T A C G T", P, " ") + print "snp", "chrom", "pos", "allele1", "allele2" > "sites.tsv" + for (i = 1; i <= 200; i++) { + pos = (i == 51) ? 500 : 10 * i + print "rs" i, "chr1", pos, P[i % 4 * 2 + 1], P[i % 4 * 2 + 2] > "sites.tsv" + } + print "rs999", "chr1" > "sites.tsv" + split("A/G A|A G/G ./. C/T A a/g G|A A/G/G", T, " ") + for (i = 1; i <= 220; i += 3) { + a1 = P[i % 4 * 2 + 1]; a2 = P[i % 4 * 2 + 2] + g = T[i % 10 + 1] + gsub(/A/, "1", g); gsub(/G/, "2", g); gsub(/a/, "3", g); gsub(/g/, "4", g) + gsub(/1/, a1, g); gsub(/2/, a2, g); gsub(/3/, tolower(a1), g); gsub(/4/, tolower(a2), g) + print "chr1", 10 * i, a1, a2, g > "genotypes.tsv" + } + print "chr1", 500, "A", "G", "A/G" > "genotypes.tsv" + print "chr1", 510, "C", "T", "T/T" > "genotypes.tsv" + print "chr1", 510, "C", "T", "C/T" > "genotypes.tsv" + print "1", 20, "A", "G", "A/G" > "genotypes.tsv" + print "chr1", 30 > "genotypes.tsv" + print "chr2", 10, "A", "G", "A/G" > "nowhere.tsv" + for (i = 1; i <= 230; i++) { + if (i == 100) print "" > "panel.snp" + print "rs" i, 1, "0.0", 10 * i, "A", "G" > "panel.snp" + g = sprintf("%d%d%d%d%d", i % 3, (i + 1) % 3, 9, i % 2, 2) + print g > "panel.geno" + if (i < 230) print g > "short.geno" + print g > "long.geno" + print (i == 77 ? g "0" : g) > "wide.geno" + } + print "" > "long.geno" +}' + +# --- align and report --------------------------------------------------------------- +awk 'BEGIN { + split("French Sardinian CEU Basque GBR", M, " ") + printf " #eigvals: 5.512 3.201 2.100 1.400 0.900 -0.100\n" > "panel.evec" + printf "#eigvals: 5.512 3.201 2.100 1.400 0.900 -0.100\n" > "run.evec" + printf "#eigvals: 5.512 3.201 2.100 1.400 0.900 -0.100\n" > "bad.evec" + printf "#eigvals: 5.512 3.201\n" > "few.evec" + for (i = 1; i <= 40; i++) { + p1 = sin(i * 1.7) * 0.05; p2 = cos(i * 0.9) * 0.04; p3 = sin(i * 0.31) * 0.03; p4 = cos(i * 2.3) * 0.02; p5 = sin(i) * 0.01 + lab = M[i % 5 + 1] + printf "%12s %10.6f %10.6f %10.6f %10.6f %10.6f %15s\n", "M" i, p1, p2, p3, p4, p5, lab > "panel.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "M" i, p1, -p2, p3, -p4, p5, lab > "run.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "M" i, (i % 2 ? p1 : -p1), p2, p3, p4, p5, lab > "bad.evec" + if (i <= 9) printf "%s %.6f %.6f %s\n", "M" i, p1, p2, lab > "few.evec" + } + for (i = 1; i <= 60; i++) { + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "A" i, sin(i * 0.77) * 0.06, cos(i * 1.31) * 0.05, sin(i * 0.5) * 0.02, cos(i) * 0.02, 0.001, "Ancient" > "panel.evec" + } + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "M3", 0.01, 0.02, 0.03, 0.04, 0.05, "Other" > "panel.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "S1", 0.0123, -0.0045, 0.002, 0.001, 0.0, "Person" > "run.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "S1", 0.0123, -0.0045, 0.002, 0.001, 0.0, "Person" > "bad.evec" + printf "%s %.6f %.6f %s\n", "S1", 0.0123, -0.0045, "Person" > "few.evec" + printf "%s %.6f %.6f %.6f %.6f %.6f %s\n", "S2", 0.0123, -0.0045, 0.002, 0.001, 0.0, "Ancient" > "run.evec" + OFS = "\t" + print "id", "group", "date_bp", "date_sd", "full_date", "locality", "country", "lat", "lon", "snps" > "ancients.tsv" + split("Germany_CordedWare England_BellBeaker Turkey_N Italy_Imperial_Rome Hungary_LBA_with_a_very_long_name_that_runs_past_45 Türkiye_Neolithic", G, " ") + split("1650 1950 1949 2450.5 4250 7300 9960 6000 3000 .", D, " ") + for (i = 1; i <= 60; i++) { + loc = (i % 3 == 0) ? "." : "Site " (i % 4) + print "A" i, G[i % 7 + 1], D[i % 10 + 1], "50", ".", loc, "Country" (i % 2), "45", "10", 200000 > "ancients.tsv" + } + print "A99", "Nowhere", "5000", "50", ".", ".", ".", "45", "10", 200000 > "ancients.tsv" + print "A7", "Germany_CordedWare", "5100", "50", ".", "Halle", "Germany" > "ancients.tsv" + print "" > "ancients.tsv" + printf "panel_sites\t230\nsites_used\t61\nother\n" > "stats.tsv" +}' +cp run.evec flipped.evec diff --git a/tests/parity/gen/archaic-inputs.sh b/tests/parity/gen/archaic-inputs.sh new file mode 100644 index 000000000..5d179c532 --- /dev/null +++ b/tests/parity/gen/archaic-inputs.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +# The inputs of the archaic cases of step 43 (archaic-table, archaic-strict, +# archaic-genotypes, archaic-report), all synthetic, written with awk: +# modern.tsv, altai.tsv, vindija.tsv, denisova.tsv +# bcftools query lines of one GRCh37 chromosome (22): 1000 Genomes +# SNVs with their six frequencies (indels, a split multi-allelic +# site, frequencies that need :.6g's exponent form, NaN, out of +# range and unreadable values) and the three genomes' calls +# (missing, partial, multi-allelic, gVCF <*> and REF-mismatch +# calls, and sites only an archaic genome carries), also gzipped +# altai.bed, vindija.bed, excl.bed, empty.bed +# the masks: chromosome 22 named with and without chr, unsorted, +# overlapping, reversed and empty intervals, track lines +# unsorted.tsv a modern file that goes back +# rows.tsv, rows-unsorted.tsv, strict.bed +# lifted GRCh38 rows (repeated positions) and the strict mask +# chr21.tsv.gz, chr22.tsv.gz, chrX.tsv.gz +# the GRCh38 table step 43 reads (chrX is not a table name), +# with masked codes and frequencies that round half to even +# s.vcf, s.vcf.gz, two.vcf, back.vcf, none.vcf, badcode/ +# the sample's VCF: split records, clashes, indels, a +# multi-allelic record, RefCall, missing GT, a chromosome the +# table lacks; a two-sample VCF, one whose chromosome comes +# back, one with no table chromosome +# chr21.Altai.txt ... chr22.Denisova.txt, spans.tsv, genes.gtf +# IBDmix's segments (short, weak, overlapping and touching +# ones, other IDs) and a GENCODE-style gene GTF +set -euo pipefail + +# --- table (GRCh37 inputs) ---------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + split("A C G T", B, " ") + split("0.5 1e-05 0.000123 . nan 1.5 -0.1 0.123456789 1 0 2.5e-7 x", F, " ") + for (i = 1; i <= 60; i++) { + pos = 1000 + 10 * i + r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1] + if (i % 11 == 0) { print pos, r "T", a, F[1], F[2], F[3], F[4], F[5], F[6] > "modern.tsv"; continue } + if (i % 13 == 0) { + print pos, r, a, "0.1", "0.2", "0.3", "0.4", "0.5", "0.6" > "modern.tsv" + print pos, r, B[(i + 2) % 4 + 1], "0.1", "0.2", "0.3", "0.4", "0.5", "0.6" > "modern.tsv" + continue + } + if (i % 7 == 3) continue + print pos, r, a, F[i % 12 + 1], F[(i + 1) % 12 + 1], F[(i + 2) % 12 + 1], F[(i + 3) % 12 + 1], F[(i + 4) % 12 + 1], F[(i + 5) % 12 + 1] > "modern.tsv" + } + split("0/0 0/1 1/1 ./. 1/2 0|1 1 0/1/1 . 1|1", G, " ") + for (i = 1; i <= 64; i++) { + pos = 1000 + 10 * i + (i % 9 == 0 ? 5 : 0) + r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1] + if (i % 3 != 0) print pos, r, (i % 10 == 4 ? "<*>" : a), G[i % 10 + 1] > "altai.tsv" + if (i % 4 != 1) print pos, (i % 17 == 0 ? "C" : r), (i % 6 == 0 ? "." : a), G[(i + 3) % 10 + 1] > "vindija.tsv" + if (i % 5 != 2) print pos, r, (i % 8 == 0 ? B[(i + 2) % 4 + 1] : a), G[(i + 7) % 10 + 1] > "denisova.tsv" + } +}' +for f in modern altai vindija denisova; do gzip -n -c "${f}.tsv" > "${f}.tsv.gz"; done +printf '1200\tA\tG\t0.1\t0.1\t0.1\t0.1\t0.1\t0.1\n1100\tA\tG\t0.1\t0.1\t0.1\t0.1\t0.1\t0.1\n' > unsorted.tsv +cat > altai.bed <<'EOF' +track name=altai +# a comment +22 1300 1500 +22 1000 1100 +22 1080 1250 +22 1600 1600 +22 1700 1650 +chr21 0 100000 +EOF +printf 'browser position chr22\nchr22\t1000\t1400\nchr22\t1450\t1700\n' > vindija.bed +printf 'chr22\t1195\t1215\n22\t1500\t1520\n' > excl.bed +printf 'chr21\t0\t1000\n' > empty.bed + +# --- strict (lifted rows) ----------------------------------------------------------- +awk 'BEGIN { + OFS = "\t" + split("0 1 2 9 10 11 12", C, " ") + for (i = 1; i <= 40; i++) { + pos = 5000 + 7 * i + row = "chr22" OFS pos OFS "A" OFS "G" OFS C[i % 7 + 1] OFS C[(i + 2) % 7 + 1] OFS C[(i + 4) % 7 + 1] OFS "0.1" OFS "0.2" OFS "0.3" OFS "0.4" OFS "0.5" OFS "0.6" + print row > "rows.tsv" + if (i % 9 == 0) print "chr22" OFS pos OFS "C" OFS "T" OFS "1" OFS "1" OFS "1" OFS "0" OFS "0" OFS "0" OFS "0" OFS "0" OFS "0" > "rows.tsv" + } +}' +{ head -5 rows.tsv; printf 'chr22\t5001\tA\tG\t1\t1\t1\t0\t0\t0\t0\t0\t0\n'; tail -3 rows.tsv; } > rows-unsorted.tsv +printf 'chr22\t5010\t5100\n22\t5150\t5200\nchr22\t5240\t5260\n' > strict.bed + +# --- genotypes (the GRCh38 table and the sample) ------------------------------------ +HEAD='chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tAF_ALL\tAF_AFR\tAF_AMR\tAF_EAS\tAF_EUR\tAF_SAS' +table() { # CHROM START N + awk -v c="$1" -v s="$2" -v n="$3" 'BEGIN { + OFS = "\t" + split("A C G T", B, " ") + split("0 1 2 9 10 11 12", C, " ") + split("0.0025 0.0075 0.5 1e-05 . nan 0.99 1.2 -0.3 0.0125 0.333333", F, " ") + for (i = 1; i <= n; i++) { + pos = s + 10 * i + print c, pos, B[i % 4 + 1], B[(i + 1) % 4 + 1], C[i % 7 + 1], C[(i + 3) % 7 + 1], C[(i * 5) % 7 + 1], \ + F[i % 11 + 1], F[(i + 1) % 11 + 1], F[(i + 2) % 11 + 1], F[(i + 3) % 11 + 1], F[(i + 4) % 11 + 1], F[(i + 5) % 11 + 1] + } + }' +} +{ printf '%b\n' "$HEAD"; table chr21 1000 30; } | gzip -n > chr21.tsv.gz +{ printf '%b\n' "$HEAD"; table chr22 2000 50; } | gzip -n > chr22.tsv.gz +{ printf '%b\n' "$HEAD"; table chrX 3000 5; } | gzip -n > chrX.tsv.gz +mkdir -p badcode emptydir +{ printf '%b\n' "$HEAD"; table chr22 2000 4 | awk -F'\t' -v OFS='\t' 'NR == 3 { $5 = 3 } { print }'; } | gzip -n > badcode/chr22.tsv.gz +vcf_head() { + printf '##fileformat=VCFv4.2\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t%s\n' "$1" +} +{ + vcf_head S + awk 'BEGIN { + OFS = "\t" + split("A C G T", B, " ") + split("0/1 1/1 0/0 ./. 0|1 1|1 0/2 1", G, " ") + print "chr1", 50, ".", "A", "G", ".", "PASS", ".", "GT", "0/1" + for (i = 1; i <= 30; i += 2) { + pos = 1000 + 10 * i; r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1] + print "chr21", pos, ".", r, a, ".", (i % 5 == 0 ? "RefCall" : "PASS"), ".", "GT:GQ", G[i % 8 + 1] ":30" + } + for (i = 1; i <= 50; i++) { + pos = 2000 + 10 * i; r = B[i % 4 + 1]; a = B[(i + 1) % 4 + 1]; o = B[(i + 2) % 4 + 1] + k = i % 12 + if (k == 0) print "chr22", pos, ".", r, a, ".", ".", ".", "GT", G[i % 8 + 1] + else if (k == 1) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "0/0" } + else if (k == 2) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/0"; print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "0/0" } + else if (k == 3) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "1/1" } + else if (k == 4) { print "chr22", pos, ".", r, a "," o, ".", "PASS", ".", "GT", "1/2" } + else if (k == 5) { print "chr22", pos, ".", r "T", r, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "1/1" } + else if (k == 6) { print "chr22", pos, ".", r, o, ".", "PASS", ".", "GT", "1/1" } + else if (k == 7) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GQ", "20" } + else if (k == 8) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT:DP", "./.:3" } + else if (k == 9) { print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "0/1"; print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT", "./." } + else if (k == 10) { print "chr22", pos, ".", r, ".", ".", "PASS", ".", "GT", "0/0" } + else print "chr22", pos, ".", r, a, ".", "PASS", ".", "GT:AD", "1/1:0,9" + } + print "chr22", 9999, ".", "A", "G", ".", "PASS", ".", "GT", "0/1" + }' +} > s.vcf +gzip -n -c s.vcf > s.vcf.gz +{ printf '##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n'; printf 'chr22\t2010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\t0/0\n'; } > two.vcf +{ vcf_head S; printf 'chr22\t2010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\nchr21\t1010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\nchr22\t2020\t.\tG\tT\t.\tPASS\t.\tGT\t0/1\n'; } > back.vcf +{ vcf_head S; printf 'chr1\t2010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\nchrX\t3010\t.\tC\tG\t.\tPASS\t.\tGT\t0/1\n'; } > none.vcf + +# --- report (IBDmix output) --------------------------------------------------------- +printf 'chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\nchr21\t1000000\t46000000\t3000\t400\t2\nchr22\t16000000\t50800000\t2500\t300\t1\n' > spans.tsv +seg() { printf 'ID\tchrom\tstart\tend\tslod\n'; cat; } +seg > chr21.Altai.txt <<'EOF' +SAMPLE 21 1000000 1200000 12.5 +SAMPLE 21 1150000 1400000 7.25 +SAMPLE 21 2000000 2040000 30 +SAMPLE 21 3000000 3100000 3.99 +P001 21 5000000 9000000 99 +SAMPLE 21 9000000 9500000 4 +EOF +seg > chr21.Vindija.txt <<'EOF' +SAMPLE 21 1400000 1600000 8.5 +SAMPLE 21 9000000 9300000 45.5 +SAMPLE chr21 20000000 20060000 4.5 +EOF +seg > chr21.Denisova.txt <<'EOF' +SAMPLE 21 1100000 1300000 6 +SAMPLE 21 30000000 30500000 14.5 +SAMPLE 21 30400000 30700000 5.5 +EOF +seg > chr22.Altai.txt <<'EOF' +SAMPLE 22 17000000 17800000 22.5 +SAMPLE 22 40000000 40050000 9 +EOF +seg > chr22.Vindija.txt <<'EOF' +SAMPLE 22 17500000 18100000 10.5 +EOF +seg > chr22.Denisova.txt <<'EOF' +SAMPLE 22 25000000 25900000 8.5 +SAMPLE 22 17000000 17050000 15 +EOF +cat > genes.gtf <<'EOF' +##description: synthetic +chr21 HAVANA gene 1000 1000000 . + . gene_id "G1"; gene_type "protein_coding"; gene_name "TOUCHES_START"; +chr21 HAVANA gene 1199999 1250000 . + . gene_id "G2"; gene_type "protein_coding"; gene_name "INSIDE"; +chr21 HAVANA gene 1100000 1300000 . + . gene_id "G3"; gene_type "lncRNA"; gene_name "NOT_CODING"; +chr21 HAVANA transcript 1100000 1300000 . + . gene_id "G2"; gene_type "protein_coding"; gene_name "INSIDE"; +chr21 HAVANA gene 1600000 1700000 . + . gene_id "G4"; gene_type "protein_coding"; gene_name "AT_END"; +chr21 HAVANA gene 30500000 30600000 . - . gene_id "G5"; gene_type "protein_coding"; gene_name "DENI"; +chr21 HAVANA gene 1300000 1350000 . - . gene_id "G6"; gene_type "protein_coding"; gene_name "INSIDE"; +chr22 HAVANA gene 17700000 17900000 . + . gene_id "G7"; gene_type "protein_coding"; gene_name "ON22"; +chr22 HAVANA gene 17000000 17100000 . + . gene_id "G8"; gene_type "protein_coding"; gene_id_only "x"; +EOF diff --git a/tests/parity/golden/ancient-person-geno/long-geno/o.geno.tmp.absent b/tests/parity/golden/ancient-person-geno/long-geno/o.geno.tmp.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/ancient-person-geno/short-geno/o.geno.tmp.absent b/tests/parity/golden/ancient-person-geno/short-geno/o.geno.tmp.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/ancient-person-geno/wide-geno/o.geno.tmp.absent b/tests/parity/golden/ancient-person-geno/wide-geno/o.geno.tmp.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/bad-code/gt/chr22.gt.gz.absent b/tests/parity/golden/archaic-genotypes/bad-code/gt/chr22.gt.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-strict/unsorted/exit b/tests/parity/golden/archaic-strict/unsorted/exit new file mode 100644 index 000000000..0cfbf0888 --- /dev/null +++ b/tests/parity/golden/archaic-strict/unsorted/exit @@ -0,0 +1 @@ +2 diff --git a/tests/parity/golden/archaic-strict/unsorted/s.tsv.absent b/tests/parity/golden/archaic-strict/unsorted/s.tsv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-strict/unsorted/stderr b/tests/parity/golden/archaic-strict/unsorted/stderr new file mode 100644 index 000000000..075caa55f --- /dev/null +++ b/tests/parity/golden/archaic-strict/unsorted/stderr @@ -0,0 +1 @@ +ERROR: rows-unsorted.tsv: line 6: position 5001 comes after 5035; the lifted rows must be sorted by position diff --git a/tests/parity/golden/archaic-table/unsorted/t.vcf.absent b/tests/parity/golden/archaic-table/unsorted/t.vcf.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/oracle.ref b/tests/parity/oracle.ref index 1604b265c..481cec78f 100644 --- a/tests/parity/oracle.ref +++ b/tests/parity/oracle.ref @@ -62,3 +62,12 @@ y-yleaf-positions 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/yleaf_run.py lookup 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py python3 lookup lookup-query 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/genome_lookup.py bash ./oracle.sh lookup query family-ibd 0baddb45aa3871a57469cf20295bdd832f72f5c0 file:bin/ibd_report.py sh ./ibd_oracle.sh family ibd +archaic-table fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" table "$@"' archaic table +archaic-strict fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" strict "$@"' archaic strict +archaic-genotypes fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" genotypes "$@"' archaic genotypes +archaic-report fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/archaic_report.py sh -c 'exec python3 "$0" report "$@"' archaic report +ancient-select fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" select "$@"' ancient select +ancient-lift fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" lift "$@"' ancient lift +ancient-person-geno fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" person-geno "$@"' ancient person-geno +ancient-align fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" align "$@"' ancient align +ancient-report fe6b67e09cef81c396c2277472df10407c56d7ac file:bin/ancient_report.py sh -c 'exec python3 "$0" report "$@"' ancient report From cae6e3e598e1a09541f4f4a0ece847490853f564 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:24:21 +0100 Subject: [PATCH 037/255] refactor(ancestry): delete archaic_report.py and ancient_report.py; tests, demo and evidence run og The step 43 and 44 tests drive og archaic and og ancient as the pipeline does and join Guard's OG_REQUIRED list; their unit checks of internals moved to the Rust tests. The docs demo writes step 44 with og ancient report, archaic_compare.py keeps its own copy of the IBDmix reader, and the archaic-share evidence job builds og. The table and VCF readers no longer build a path string per line. Bead: pgp-bs9u.21 --- .github/workflows/guard.yml | 2 +- .github/workflows/public-evidence.yml | 12 +- bin/ancient_report.py | 753 ------------------------- bin/archaic_compare.py | 26 +- bin/archaic_report.py | 745 ------------------------ bin/collect_summary.py | 2 +- rust/crates/og-ancestry/src/ancient.rs | 3 + rust/crates/og-ancestry/src/archaic.rs | 19 +- scripts/ci/e2e-case-paths.tsv | 4 +- tests/demo/make_demo_sample.py | 81 +-- tests/e2e/ancient-similarity-1.sh | 2 +- tests/test_ancient_report.py | 228 ++++---- tests/test_archaic_report.py | 62 +- 13 files changed, 237 insertions(+), 1702 deletions(-) delete mode 100755 bin/ancient_report.py delete mode 100755 bin/archaic_report.py diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 5a209a286..e22eba988 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -327,7 +327,7 @@ jobs: run: | fail=0 for t in tests/test_annotsv_sv_inputs.py tests/test_pgx_refblock_fill.py tests/test_pgx_mito_merge.py tests/test_run_delta.py \ - tests/test_ancestry_fine_report.py \ + tests/test_ancestry_fine_report.py tests/test_archaic_report.py tests/test_ancient_report.py \ tests/test_annotsv_sv_inputs.sh tests/test_annotsv_postfilter.sh tests/test_annotsv_benign_mane.sh \ tests/test_str_catalog.py tests/test_str_ranges.py tests/test_og_setup_zip.py tests/test_setup_fine_maps.sh \ tests/test_render_report.py tests/test_collect_summary.py tests/test_chip_report.py \ diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index 11b39d5a4..d44d44de8 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -227,7 +227,7 @@ jobs: # The positions of this part where HG002 has two records or more, one # of them an SNV, as GIAB writes the VCF and split with bcftools # norm -m-, and how many of them are rows of the table (the rows the - # same-position rule of archaic_report.py decides). + # same-position rule of og archaic genotypes decides). docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ bcftools norm -m- -Oz -o /g/HG002/split.vcf.gz /g/HG002/vcf/HG002.vcf.gz python3 -I - "${G}/HG002/vcf/HG002.vcf.gz" "${G}/HG002/split.vcf.gz" "$DATA" "$OUT/affected.tsv" <<'PY' @@ -470,7 +470,7 @@ jobs: archaic-share: needs: [archaic-sites, archaic-hg002] runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 40 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -492,6 +492,12 @@ jobs: mv parts "lone-${c}" && mkdir parts && mv "lone-${c}" "parts/archaic-part-${c}" fi ls parts + - name: Build og (og archaic report) + working-directory: rust + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal + OG_TREE_HASH=$(./og.sh tree-hash) cargo build --release --locked -p og - name: One report per population, over the autosomes that ran env: WANT: ${{ needs.archaic-sites.outputs.count }} @@ -517,7 +523,7 @@ jobs: mkdir -p "${d}/ibdmix" cp parts/archaic-part-*/"${pop}"/chr*.txt "${d}/ibdmix/" { printf 'chrom\tfirst\tlast\tsites\n'; for s in parts/archaic-part-*/"${pop}"/spans.tsv; do tail -n +2 "$s" | cut -f1-4; done; } > "${d}/spans.tsv" - python3 bin/archaic_report.py report --sample HG002 --ibdmix-dir "${d}/ibdmix" --spans "${d}/spans.tsv" \ + rust/target/release/og archaic report --sample HG002 --ibdmix-dir "${d}/ibdmix" --spans "${d}/spans.tsv" \ --pop "$pop" --out-dir "${d}/out" kv() { awk -F'\t' -v k="$1" '$1 == k {print $2}' "${d}/out/HG002_archaic.tsv"; } [ "$(kv analysed_chromosomes)" -eq "$WANT" ] || { echo "${pop}: $(kv analysed_chromosomes) chromosomes, want ${WANT}"; exit 1; } diff --git a/bin/ancient_report.py b/bin/ancient_report.py deleted file mode 100755 index 82f0002f0..000000000 --- a/bin/ancient_report.py +++ /dev/null @@ -1,753 +0,0 @@ -#!/usr/bin/env python3 -"""Step 44: where a genome falls among ancient genomes on a PCA of present-day -West Eurasians (smartpca, EIGENSOFT), with the ancient samples of the Allen -Ancient DNA Resource (AADR) projected onto the same axes. - -Subcommands, in the order they run: - - select (setup.sh) read the AADR .anno and .ind files and choose the - people: the present-day West Eurasians of MODERN_GROUPS build - the axes, and the ancient West Eurasians with enough sites - covered are projected. Writes the relabelled .ind (everyone - else is Ignore), the population list and ancients.tsv. - lift (setup.sh) move the 1240K sites of the AADR .snp file from - GRCh37 to GRCh38 with UCSC's chain file, and keep a site only - when one of its two alleles is the GRCh38 reference base. - person-geno (step 44) the person's genotype at each site, from bcftools' - output on the gVCF, appended as one more column to the - present-day panel's EIGENSTRAT genotype file. - align (setup.sh, step 44) check that the axes of two smartpca runs - on the same present-day panel agree, up to the sign of each. - report (step 44) the nearest ancient populations, the summary and - the plot. - -Only the Python standard library is used. Every number the report shows comes -from smartpca's .evec files and the AADR annotation. -""" -import argparse -import bisect -import collections -import csv -import gzip -import html -import math -import os -import statistics -import sys - -# Present-day groups of the AADR 1240K file (Group ID, as written in the -# .anno file of v66.p1) that build the axes: Europe, the Caucasus and the Near -# East, from the 1000 Genomes Project, HGDP and SGDP shotgun genomes. North -# African groups (Mozabite) and groups with recent East Eurasian ancestry -# (Saami, Bashkir, Tatar, Yakut) are left out, so the axes describe West -# Eurasian variation only. -MODERN_GROUPS = ( - "Abkhasian", "Adygei", "Albanian", "Armenian", "Basque", "BedouinA", "BedouinB", - "Bulgarian", "CEU", "Chechen", "Czech", "Druze", "English", "Estonian", "FIN", - "Finnish", "French", "GBR", "Georgian", "Greek", "Greek_Crete", "Hungarian", "IBS", - "Icelandic", "Iranian", "Italian_North", "Jew_Iraqi", "Jordanian", "Lezgin", - "Norwegian", "Orcadian", "Ossetian", "Palestinian", "Polish", "Russian", "Sardinian", - "Scottish", "Spanish", "Spanish_North", "TSI", "Turkish", -) -MODERN_DATA_TYPE = "Shotgun.diploid" -# Ancient samples projected: dated, passing the AADR's own assessment, with -# this many of the 1240K autosomal sites covered, inside this box. -MIN_ANCIENT_SNPS = 100_000 -WEST_EURASIA = {"lat": (25.0, 75.0), "lon": (-25.0, 75.0)} -PASSING = ("PASS", "PROVISIONAL_PASS", "MERGE_PASS") -ANCIENT_LABEL = "Ancient" -PERSON_LABEL = "Person" -# PCs the distance uses, each weighted by the square root of its eigenvalue. -DISTANCE_PCS = 4 -TOP = 20 -# Two runs on the same panel give the same axes; any PC whose present-day -# coordinates correlate less than this between the two is a panel mismatch. -MIN_AXIS_CORRELATION = 0.999 -CAVEAT = ("Closeness on a PCA is genetic similarity, not descent: a genome near an ancient group " - "shares ancestry with it, it does not descend from those people.") -QPADM_NOTE = "qpAdm models of ancestry proportions are not part of this step." - -# Header cells of the .anno file this script reads, matched by their start -# (the AADR writes long explanations after the name). -ANNO_COLUMNS = { - "id": "Genetic ID", - "individual": "Individual ID", - "date_bp": "Date mean in BP", - "date_sd": "Date standard deviation in BP", - "full_date": "Full Date", - "group": "Group ID", - "locality": "Locality", - "country": "Political Entity", - "lat": "Latitude", - "lon": "Longitude", - "data_type": "Data type", - "snps": "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", - "assessment": "ASSESSMENT", -} -COMPLEMENT = {"A": "T", "C": "G", "G": "C", "T": "A"} - - -class Fail(Exception): - pass - - -def opener(path): - return gzip.open(path, "rt", errors="replace") if path.endswith(".gz") else open(path, errors="replace") - - -# --- select --------------------------------------------------------------------- - -def anno_columns(header): - cells = [c.strip().strip('"').strip() for c in header] - out = {} - for key, name in ANNO_COLUMNS.items(): - # ASSESSMENT must not match ASSESSMENT WARNINGS: the exact cell first. - hits = [i for i, c in enumerate(cells) if c == name] or [i for i, c in enumerate(cells) if c.startswith(name)] - if key == "assessment": - hits = [i for i in hits if cells[i] == name] - if not hits: - raise Fail(f"the .anno file has no column '{name}'") - out[key] = hits[0] - return out - - -def to_float(s): - try: - v = float(s) - except (TypeError, ValueError): - return None - return v if math.isfinite(v) else None - - -def passing(assessment): - return assessment.strip().upper().startswith(PASSING) - - -def read_anno(path): - with opener(path) as f: - rows = list(csv.reader(f, delimiter="\t", quoting=csv.QUOTE_NONE)) - if not rows: - raise Fail(f"{path} is empty") - col = anno_columns(rows[0]) - out = [] - for r in rows[1:]: - if len(r) <= max(col.values()): - continue - out.append({k: r[i].strip() for k, i in col.items()}) - return out - - -def choose(anno, min_snps=MIN_ANCIENT_SNPS): - """Return {genetic id: ('modern', group) | ('ancient', row)}: the - present-day reference people and the ancient people to project, one - genetic id per individual (the one with the most sites).""" - modern_groups = set(MODERN_GROUPS) - best = {} - for r in anno: - if not passing(r["assessment"]): - continue - bp = to_float(r["date_bp"]) - snps = to_float(r["snps"]) or 0 - if bp == 0 and r["group"] in modern_groups and r["data_type"] == MODERN_DATA_TYPE: - kind = "modern" - elif bp is not None and bp > 0 and snps >= min_snps: - lat, lon = to_float(r["lat"]), to_float(r["lon"]) - if lat is None or lon is None: - continue - if not (WEST_EURASIA["lat"][0] <= lat <= WEST_EURASIA["lat"][1] - and WEST_EURASIA["lon"][0] <= lon <= WEST_EURASIA["lon"][1]): - continue - kind = "ancient" - else: - continue - key = (kind, r["individual"] or r["id"]) - if key not in best or snps > best[key][0]: - best[key] = (snps, r) - out = {} - for (kind, _), (_, r) in best.items(): - out[r["id"]] = (kind, r["group"] if kind == "modern" else r) - return out - - -def cmd_select(a): - anno = read_anno(a.anno) - chosen = choose(anno, a.min_snps) - n_mod = n_anc = 0 - groups = set() - with open(a.ind) as f, open(a.out_ind + ".tmp", "w") as o: - for line in f: - c = line.split() - if len(c) < 3: - continue - kind, val = chosen.get(c[0], (None, None)) - if kind == "modern": - label = val - n_mod += 1 - groups.add(val) - elif kind == "ancient": - label = ANCIENT_LABEL - n_anc += 1 - else: - label = "Ignore" - o.write(f"{c[0]}\t{c[1]}\t{label}\n") - missing = sorted(set(MODERN_GROUPS) - groups) - if n_mod < 50 or n_anc < 1: - os.remove(a.out_ind + ".tmp") - raise Fail(f"the .ind file matches {n_mod} present-day reference people and {n_anc} ancient ones; " - "is it the .ind file of the same release as the .anno file?") - os.replace(a.out_ind + ".tmp", a.out_ind) - with open(a.poplist, "w") as o: - for g in sorted(groups): - o.write(g + "\n") - with open(a.ancients, "w") as o: - o.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") - for gid, (kind, r) in sorted(chosen.items()): - if kind != "ancient": - continue - o.write("\t".join(clean(r[k]) for k in ("id", "group", "date_bp", "date_sd", "full_date", - "locality", "country", "lat", "lon", "snps")) + "\n") - print(f"Present-day reference people: {n_mod} in {len(groups)} groups" - + (f" (not in this file: {', '.join(missing)})" if 0 < len(missing) <= 10 else - f" ({len(missing)} of the {len(MODERN_GROUPS)} groups are not in this file)" if missing else "")) - print(f"Ancient people projected: {n_anc} (at least {a.min_snps:,} sites, West Eurasia)") - return 0 - - -def clean(s): - return " ".join(str(s).replace("\t", " ").split()) or "." - - -# --- lift ----------------------------------------------------------------------- - -def read_chain(path): - """UCSC chain file -> {source chrom: sorted [(start, end, chain no, target - chrom, target start, target strand, target size)]}, 0-based half-open - ungapped blocks.""" - blocks = collections.defaultdict(list) - n = -1 - with opener(path) as f: - cur = None - for line in f: - c = line.split() - if not c: - cur = None - continue - if c[0] == "chain": - n += 1 - t_name, t_strand, t_start = c[2], c[4], int(c[5]) - q_name, q_size, q_strand, q_start = c[7], int(c[8]), c[9], int(c[10]) - if t_strand != "+": - raise Fail(f"{path}: chain {c[12] if len(c) > 12 else n} has source strand {t_strand}") - cur = [t_name, t_start, q_name, q_start, q_strand, q_size, n] - continue - if cur is None: - raise Fail(f"{path}: a block line outside a chain: {line.strip()}") - size = int(c[0]) - t_name, t_pos, q_name, q_pos, q_strand, q_size, k = cur - blocks[t_name].append((t_pos, t_pos + size, k, q_name, q_pos, q_strand, q_size)) - if len(c) >= 3: - cur[1] = t_pos + size + int(c[1]) - cur[3] = q_pos + size + int(c[2]) - else: - cur = None - for v in blocks.values(): - v.sort() - return blocks - - -def index_blocks(blocks): - """read_chain's blocks -> {chrom: (blocks, their starts, longest block)}.""" - return {k: (v, [x[0] for x in v], max(x[1] - x[0] for x in v)) for k, v in blocks.items()} - - -def lift_point(index, chrom, pos1): - """1-based GRCh37 position -> (chrom, 1-based position, strand) on the - target, or None when no block, or blocks of more than one chain, cover it.""" - if chrom not in index: - return None - b, starts, longest = index[chrom] - p = pos1 - 1 - hits = [] - # Blocks of different chains can overlap: every block that starts at or - # before p, back to the longest block's length, may still reach it. - j = bisect.bisect_right(starts, p) - 1 - while j >= 0 and b[j][0] > p - longest: - if b[j][1] > p: - hits.append(b[j]) - j -= 1 - if len({h[2] for h in hits}) != 1: - return None - s, _, _, q_name, q_pos, q_strand, q_size = hits[0] - q = q_pos + (p - s) - if q_strand == "-": - q = q_size - 1 - q - return q_name, q + 1, q_strand - - -class Fasta: - """Random access to a FASTA with a samtools .fai index.""" - - def __init__(self, path): - self.f = open(path, "rb") - self.idx = {} - with open(path + ".fai") as fai: - for line in fai: - c = line.split("\t") - self.idx[c[0]] = (int(c[1]), int(c[2]), int(c[3]), int(c[4])) - - def base(self, chrom, pos1): - if chrom not in self.idx: - return None - length, offset, line_bases, line_width = self.idx[chrom] - if not 1 <= pos1 <= length: - return None - p = pos1 - 1 - self.f.seek(offset + (p // line_bases) * line_width + p % line_bases) - return self.f.read(1).decode("ascii", "replace").upper() - - -def eigen_chrom(c): - """The chromosome column of an EIGENSOFT .snp line -> 1..22, or None.""" - try: - n = int(c) - except ValueError: - return None - return n if 1 <= n <= 22 else None - - -def cmd_lift(a): - blocks = index_blocks(read_chain(a.chain)) - fasta = Fasta(a.fasta) - kept = [] - dropped = collections.Counter() - n = 0 - with opener(a.snp) as f: - for line in f: - c = line.split() - if len(c) < 6: - continue - n += 1 - chrom = eigen_chrom(c[1]) - a1, a2 = c[4].upper(), c[5].upper() - if chrom is None: - dropped["not an autosome"] += 1 - continue - if a1 not in COMPLEMENT or a2 not in COMPLEMENT or a1 == a2: - dropped["alleles not two different bases"] += 1 - continue - hit = lift_point(blocks, f"chr{chrom}", int(float(c[3]))) - if hit is None: - dropped["no single place on GRCh38"] += 1 - continue - q_chrom, q_pos, strand = hit - if q_chrom != f"chr{chrom}": - dropped["moves to another chromosome"] += 1 - continue - if strand == "-": - a1, a2 = COMPLEMENT[a1], COMPLEMENT[a2] - ref = fasta.base(q_chrom, q_pos) - if ref not in (a1, a2): - dropped["neither allele is the GRCh38 base"] += 1 - continue - kept.append((chrom, q_pos, c[0], a1, a2)) - kept.sort() - with open(a.out + ".tmp", "w") as o: - o.write("snp\tchrom\tpos\tallele1\tallele2\n") - for chrom, pos, sid, a1, a2 in kept: - o.write(f"{sid}\tchr{chrom}\t{pos}\t{a1}\t{a2}\n") - os.replace(a.out + ".tmp", a.out) - print(f"Sites of the .snp file: {n:,}; on GRCh38: {len(kept):,}") - for why, k in sorted(dropped.items()): - print(f" dropped, {why}: {k:,}") - if not kept: - raise Fail("no site could be placed on GRCh38: check the chain file and the reference") - return 0 - - -# --- person-geno ------------------------------------------------------------------ - -def read_sites(path): - """sites_grch38.tsv -> {(chrom, pos): [(snp, allele1, allele2)]}.""" - out = collections.defaultdict(list) - with open(path) as f: - next(f, None) - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) >= 5: - out[(c[1], int(c[2]))].append((c[0], c[3], c[4])) - return out - - -def dosage(gt, a1, a2): - """Copies of allele1 in a bcftools %TGT genotype, or None (missing, a - haploid or half call, or an allele that is neither of the two).""" - alleles = gt.replace("|", "/").split("/") - if len(alleles) != 2 or any(x not in (a1, a2) for x in alleles): - return None - return sum(1 for x in alleles if x == a1) - - -def person_dosages(sites, genotype_lines): - """bcftools query lines (CHROM POS REF ALT GT) -> ({snp: dosage}, counts).""" - out = {} - counts = collections.Counter() - for line in genotype_lines: - c = line.rstrip("\n").split("\t") - if len(c) < 5: - continue - here = sites.get((c[0], int(c[1]))) - if not here: - continue - for sid, a1, a2 in here: - d = dosage(c[4].upper(), a1, a2) - if d is None: - counts["other_allele"] += 1 - else: - out[sid] = d - counts["called"] += 1 - return out, counts - - -def cmd_person_geno(a): - sites = read_sites(a.sites) - with open(a.genotypes) as f: - dos, counts = person_dosages(sites, f) - n = used = 0 - with open(a.snp) as snp, open(a.moderns_geno) as geno, open(a.out + ".tmp", "w") as o: - width = None - for sline in snp: - c = sline.split() - if not c: - continue - g = geno.readline().rstrip("\n") - if not g: - raise Fail(f"{a.moderns_geno} has fewer lines than {a.snp}") - if width is None: - width = len(g) - elif len(g) != width: - raise Fail(f"{a.moderns_geno} line {n + 1} has {len(g)} genotypes, not {width}") - d = dos.get(c[0]) - if d is not None: - used += 1 - o.write(g + ("9" if d is None else str(d)) + "\n") - n += 1 - if geno.readline(): - raise Fail(f"{a.moderns_geno} has more lines than {a.snp}") - os.replace(a.out + ".tmp", a.out) - with open(a.stats, "w") as o: - o.write(f"panel_sites\t{n}\nsites_genotyped\t{counts['called'] + counts['other_allele']}\n" - f"other_allele\t{counts['other_allele']}\nsites_used\t{used}\n") - print(f"Sites of the panel: {n:,}; genotyped from the gVCF: {counts['called'] + counts['other_allele']:,}; " - f"used: {used:,} (an allele that is neither of a site's two: {counts['other_allele']:,})") - if used == 0: - raise Fail("none of the panel's sites was genotyped: is the gVCF on GRCh38 with chr names?") - return 0 - - -# --- evec, align ------------------------------------------------------------------ - -def read_evec(path): - """smartpca .evec -> (eigenvalues, {id: (label, [pc...])}).""" - evals, rows = [], {} - with open(path) as f: - for line in f: - c = line.split() - if not c: - continue - if c[0].startswith("#"): - if c[0] == "#eigvals:": - evals = [float(x) for x in c[1:]] - continue - rows[c[0]] = (c[-1], [float(x) for x in c[1:-1]]) - if not evals or not rows: - raise Fail(f"{path} is not a smartpca .evec file (no #eigvals: line or no rows)") - return evals, rows - - -def correlation(x, y): - mx, my = statistics.fmean(x), statistics.fmean(y) - sxy = sum((a - mx) * (b - my) for a, b in zip(x, y)) - sx = math.sqrt(sum((a - mx) ** 2 for a in x)) - sy = math.sqrt(sum((b - my) ** 2 for b in y)) - return sxy / (sx * sy) if sx > 0 and sy > 0 else 0.0 - - -def axis_signs(panel_rows, run_rows, n_pcs): - """The sign that turns each of the run's PCs into the panel's, from the - present-day people both runs used to build the axes. A correlation below - MIN_AXIS_CORRELATION on any PC stops: the two are not the same axes.""" - shared = sorted(k for k, (lab, _) in run_rows.items() - if k in panel_rows and lab not in (ANCIENT_LABEL, PERSON_LABEL) - and panel_rows[k][0] == lab) - if len(shared) < 10: - raise Fail(f"only {len(shared)} present-day people are in both runs; the panel and this run do not match") - signs = [] - for k in range(n_pcs): - r = correlation([panel_rows[i][1][k] for i in shared], [run_rows[i][1][k] for i in shared]) - if abs(r) < MIN_AXIS_CORRELATION: - raise Fail(f"PC{k + 1} of this run correlates {r:.4f} with the panel's over {len(shared)} present-day " - "people: the axes differ. Install the panel again (setup.sh --ancient-panel).") - signs.append(1.0 if r > 0 else -1.0) - return signs - - -def cmd_align(a): - _, panel = read_evec(a.panel_evec) - _, run = read_evec(a.run_evec) - signs = axis_signs(panel, run, a.pcs) - print("Axes agree with the panel's: " + " ".join(f"PC{i + 1}{'+' if s > 0 else '-'}" for i, s in enumerate(signs))) - return 0 - - -# --- report ----------------------------------------------------------------------- - -def year_label(bp): - """A date in years before 1950 -> 'N BCE' or 'N CE', the AADR's own - reading (its 2600-2000 BCE has a mean of 4250 BP): BCE = BP - 1950 and - CE = 1950 - BP, with 1950 BP read as 1 BCE.""" - y = 1950 - int(round(bp)) - return f"{y} CE" if y >= 1 else f"{max(-y, 1)} BCE" - - -def range_label(old_bp, young_bp): - """The span of a population's dates, oldest first, e.g. '5300-4900 BCE', - '200-450 CE', or '300 BCE-200 CE'.""" - a, b = year_label(old_bp), year_label(young_bp) - if a == b: - return a - (na, ea), (nb, eb) = a.split(" "), b.split(" ") - return f"{na}-{nb} {ea}" if ea == eb else f"{a}-{b}" - - -def read_ancients(path): - out = {} - with open(path) as f: - for r in csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE): - out[r["id"]] = r - return out - - -def weights(evals, n): - return [math.sqrt(max(v, 0.0)) for v in evals[:n]] - - -def distance(p, q, w): - return math.sqrt(sum((wi * (a - b)) ** 2 for a, b, wi in zip(p, q, w))) - - -def most_common(values): - vals = [v for v in values if v and v != "."] - if not vals: - return "." - c = collections.Counter(vals).most_common() - top = [v for v, k in c if k == c[0][1]] - return sorted(top)[0] - - -def nearest_populations(person, panel_rows, ancients, evals, n_pcs=DISTANCE_PCS, top=TOP): - """Rank the ancient groups by the weighted distance from the person to the - group's mean position on the first n_pcs PCs.""" - w = weights(evals, n_pcs) - groups = collections.defaultdict(list) - for gid, r in ancients.items(): - if gid in panel_rows and panel_rows[gid][0] == ANCIENT_LABEL: - groups[r["group"]].append((panel_rows[gid][1], r)) - rows = [] - for g, members in groups.items(): - k = len(members[0][0]) - centre = [statistics.fmean(m[0][i] for m in members) for i in range(min(k, n_pcs))] - dates = [to_float(m[1]["date_bp"]) for m in members] - dates = [d for d in dates if d is not None] - rows.append({ - "group": g, - "n": len(members), - "distance": distance(person[:n_pcs], centre, w), - "pc1": centre[0], - "pc2": centre[1] if len(centre) > 1 else 0.0, - "date_bp": statistics.median(dates) if dates else None, - "dates": range_label(max(dates), min(dates)) if dates else ".", - "country": most_common(m[1]["country"] for m in members), - "locality": most_common(m[1]["locality"] for m in members), - }) - rows.sort(key=lambda r: (r["distance"], r["group"])) - for i, r in enumerate(rows, 1): - r["rank"] = i - return rows[:top], len(rows) - - -AGE_BINS = ( # (oldest BP, colour, legend), oldest first; dark-background palette - (math.inf, "#5aa9e6", "before 8000 BCE"), - (9950, "#7fc97f", "8000-4000 BCE"), - (5950, "#f0a35e", "4000-1000 BCE"), - (2950, "#e377c2", "after 1000 BCE"), -) - - -def age_colour(bp): - colour = AGE_BINS[0][1] - for limit, c, _ in AGE_BINS: - if bp is not None and bp < limit: - colour = c - return colour - - -def svg_plot(person, panel_rows, ancients, sample, nearest, width=900, height=720): - """The PC1/PC2 plane: present-day people grey, ancient people coloured by - age, the person a large ring. Dark background, so it reads in a dark - report and on its own.""" - pts = [v[1][:2] for v in panel_rows.values()] + [person[:2]] - xs, ys = [p[0] for p in pts], [p[1] for p in pts] - pad = 0.04 - x0, x1 = min(xs), max(xs) - y0, y1 = min(ys), max(ys) - dx, dy = (x1 - x0) or 1.0, (y1 - y0) or 1.0 - left, right, top, bottom = 60, 20, 50, 110 - pw, ph = width - left - right, height - top - bottom - - def X(v): - return left + ((v - x0) / dx * (1 - 2 * pad) + pad) * pw - - def Y(v): - return top + (1 - ((v - y0) / dy * (1 - 2 * pad) + pad)) * ph - - e = html.escape - out = [f'', - '', - f'', - f'{e(sample)} among ancient West Eurasians ' - '(AADR, projected onto present-day PCs)', - f''] - for label, pcs in panel_rows.values(): - if label != ANCIENT_LABEL: - out.append(f'') - for gid, (label, pcs) in panel_rows.items(): - if label == ANCIENT_LABEL and gid in ancients: - c = age_colour(to_float(ancients[gid]["date_bp"])) - out.append(f'') - for r in nearest[:5]: - out.append(f'{e(r["group"])}') - px, py = X(person[0]), Y(person[1]) - out.append(f'') - out.append(f'') - out.append(f'{e(sample)}') - out.append(f'PC1') - out.append(f'PC2') - lx, ly = left, top + ph + 48 - out.append(f'present day') - lx += 120 - for _, c, name in AGE_BINS: - out.append(f'{e(name)}') - lx += 150 - out.append(f'{e(CAVEAT)}') - out.append("") - return "\n".join(out) + "\n" - - -def cmd_report(a): - evals, panel = read_evec(a.panel_evec) - _, run = read_evec(a.run_evec) - if a.sample not in run or run[a.sample][0] != PERSON_LABEL: - raise Fail(f"{a.run_evec} has no row for {a.sample} labelled {PERSON_LABEL}") - n_pcs = min(DISTANCE_PCS, len(evals), len(run[a.sample][1])) - signs = axis_signs(panel, run, max(2, n_pcs)) - person = [s * v for s, v in zip(signs, run[a.sample][1])] - ancients = read_ancients(a.ancients) - nearest, n_groups = nearest_populations(person, panel, ancients, evals, n_pcs) - stats = {} - if a.stats: - with open(a.stats) as f: - for line in f: - k, _, v = line.rstrip("\n").partition("\t") - stats[k] = v - n_anc = sum(1 for lab, _ in panel.values() if lab == ANCIENT_LABEL) - n_mod = sum(1 for lab, _ in panel.values() if lab not in (ANCIENT_LABEL, PERSON_LABEL)) - with open(a.out_tsv + ".tmp", "w") as o: - o.write("rank\tgroup\tn\tdistance\tpc1\tpc2\tdate_bp\tdates\tcountry\tlocality\n") - for r in nearest: - o.write(f"{r['rank']}\t{clean(r['group'])}\t{r['n']}\t{r['distance']:.5f}\t{r['pc1']:.5f}\t{r['pc2']:.5f}\t" - f"{'.' if r['date_bp'] is None else int(round(r['date_bp']))}\t{r['dates']}\t" - f"{clean(r['country'])}\t{clean(r['locality'])}\n") - os.replace(a.out_tsv + ".tmp", a.out_tsv) - info = [ - ("release", a.release), - ("sites_used", stats.get("sites_used", ".")), - ("panel_sites", stats.get("panel_sites", ".")), - ("present_day_people", n_mod), - ("ancient_people", n_anc), - ("ancient_groups", n_groups), - ("pcs_in_distance", n_pcs), - ("pc1", f"{person[0]:.5f}"), - ("pc2", f"{person[1]:.5f}"), - ] - with open(a.out_info + ".tmp", "w") as o: - o.write("key\tvalue\n") - for k, v in info: - o.write(f"{k}\t{v}\n") - os.replace(a.out_info + ".tmp", a.out_info) - with open(a.out_svg + ".tmp", "w") as o: - o.write(svg_plot(person, panel, ancients, a.sample, nearest)) - os.replace(a.out_svg + ".tmp", a.out_svg) - lines = [f"Ancient populations nearest to {a.sample} (AADR {a.release}, smartpca with lsqproject)", - f"Sites used: {stats.get('sites_used', '.')} of {stats.get('panel_sites', '.')}; axes from {n_mod} " - f"present-day West Eurasians; {n_anc} ancient people in {n_groups} groups projected.", - f"Distance: the first {n_pcs} PCs, each weighted by the square root of its eigenvalue.", ""] - for r in nearest[:10]: - lines.append(f" {r['rank']:>2}. {r['group']:<45} {r['dates']:<22} {r['country']:<20} n={r['n']:<4} " - f"d={r['distance']:.4f}") - lines += ["", CAVEAT, QPADM_NOTE] - with open(a.out_txt, "w") as o: - o.write("\n".join(lines) + "\n") - print("\n".join(lines)) - return 0 - - -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - sub = p.add_subparsers(dest="cmd", required=True) - s = sub.add_parser("select") - s.add_argument("--anno", required=True) - s.add_argument("--ind", required=True) - s.add_argument("--out-ind", required=True) - s.add_argument("--poplist", required=True) - s.add_argument("--ancients", required=True) - s.add_argument("--min-snps", type=int, default=MIN_ANCIENT_SNPS) - s = sub.add_parser("lift") - s.add_argument("--snp", required=True) - s.add_argument("--chain", required=True) - s.add_argument("--fasta", required=True) - s.add_argument("--out", required=True) - s = sub.add_parser("person-geno") - s.add_argument("--sites", required=True) - s.add_argument("--genotypes", required=True) - s.add_argument("--snp", required=True) - s.add_argument("--moderns-geno", required=True) - s.add_argument("--out", required=True) - s.add_argument("--stats", required=True) - s = sub.add_parser("align") - s.add_argument("--panel-evec", required=True) - s.add_argument("--run-evec", required=True) - s.add_argument("--pcs", type=int, default=DISTANCE_PCS) - s = sub.add_parser("report") - s.add_argument("--panel-evec", required=True) - s.add_argument("--run-evec", required=True) - s.add_argument("--ancients", required=True) - s.add_argument("--sample", required=True) - s.add_argument("--release", required=True) - s.add_argument("--stats") - s.add_argument("--out-tsv", required=True) - s.add_argument("--out-info", required=True) - s.add_argument("--out-txt", required=True) - s.add_argument("--out-svg", required=True) - a = p.parse_args(argv) - try: - return {"select": cmd_select, "lift": cmd_lift, "person-geno": cmd_person_geno, - "align": cmd_align, "report": cmd_report}[a.cmd](a) - except Fail as e: - print(f"ERROR: {e}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/archaic_compare.py b/bin/archaic_compare.py index 88cde4e31..7c6ebdd7d 100644 --- a/bin/archaic_compare.py +++ b/bin/archaic_compare.py @@ -24,8 +24,25 @@ import statistics import sys -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import archaic_report # noqa: E402 (the same reader and cutoffs as step 43) +ARCHAICS = ("Altai", "Vindija", "Denisova") + + +def ibdmix_segments(path, min_lod=4.0, min_length=50000): + """IBDmix's segments for SAMPLE in one of its output files, kept as step 43 + keeps them (og archaic report: LOD 4 and 50 kb or longer), as (start, end).""" + out = [] + with open(path) as f: + head = f.readline().rstrip("\n").split("\t") + if head[:5] != ["ID", "chrom", "start", "end", "slod"]: + sys.exit(f"ERROR: {path} is not IBDmix output (header {' '.join(head[:5])})") + for line in f: + c = line.rstrip("\n").split("\t") + if len(c) < 5 or c[0] != "SAMPLE": + continue + s, e, lod = int(c[2]), int(c[3]), float(c[4]) + if lod >= min_lod and e - s >= min_length: + out.append((s, e)) + return out def norm(c): @@ -170,9 +187,8 @@ def cmd_genomes(a): chroms = list(dict.fromkeys(norm(line.split("\t")[0]) for line in f if line.strip())) per = collections.defaultdict(lambda: collections.defaultdict(list)) for c in chroms: - for n in archaic_report.ARCHAICS: - segs = archaic_report.keep(archaic_report.read_segments(os.path.join(a.ibdmix_dir, f"chr{c}.{n}.txt"))) - per[n][c] += [(s, e) for _, s, e, _ in segs] + for n in ARCHAICS: + per[n][c] += ibdmix_segments(os.path.join(a.ibdmix_dir, f"chr{c}.{n}.txt")) per["Altai+Vindija"] = {c: per["Altai"][c] + per["Vindija"][c] for c in chroms} released = sum(merged_bp(calls[a.sample].get(c, [])) for c in chroms) print(f"| IBDmix output | bp | under the released {a.sample} calls | only here | only released |") diff --git a/bin/archaic_report.py b/bin/archaic_report.py deleted file mode 100755 index 6ada76bd2..000000000 --- a/bin/archaic_report.py +++ /dev/null @@ -1,745 +0,0 @@ -#!/usr/bin/env python3 -"""archaic_report.py: Neanderthal and Denisovan segments in one genome, with IBDmix. - -IBDmix (Chen et al. 2020, Cell 180:677, doi:10.1016/j.cell.2020.01.012) compares -a modern genome with an archaic one site by site and adds up a log-odds score -(LOD) that a stretch of the modern genome shares a recent ancestor with the -archaic genome, against the chance of the same genotypes in the modern -population. It needs no unadmixed reference population, but it does need the -allele frequency of each site in the modern population, and it takes that from -the modern people in its input. With one genome as the only modern input the -frequency is that person's own genotype and no site scores (find_frequency in -IBDmix's src/Genotype_Reader.cc drops every site below its minor allele count), -so `genotypes` adds PSEUDO stand-ins whose genotypes carry the 1000 Genomes -allele count of the chosen population at each site. IBDmix scores each modern -column from the site's frequency and that column's genotype only, so your rows -are the ones it would give you inside a population sample of that frequency; -the stand-ins' own rows are dropped. - -Subcommands: - - table (setup.sh --archaic) one GRCh37 chromosome: the 1000 Genomes - phase 3 biallelic SNVs with their allele frequencies, and the - genotypes of the Altai Neanderthal, the Vindija 33.19 Neanderthal - and the Altai Denisovan at those sites and wherever one of the - three carries a non-reference allele, as a sites-only VCF that - Picard lifts to GRCh38. Each archaic genome is read inside its - own mask (--mask NAME=BED, the MPI-EVA FilterBed of that - genome): a call outside it is kept as masked, code 10 plus its - ALT count. Every call at a site inside an excluded region - (--exclude BED: segmental duplications, the 5 bp around a modern - indel) is masked the same way. That is how Chen et al. filter: - the site stays in IBDmix's input and IBDmix's --mask stops it - scoring, except that a homozygous mismatch there still counts - against a segment. - strict (setup.sh --archaic) the lifted GRCh38 rows of one chromosome with - every row outside the 1000 Genomes strict mask marked masked in - all three archaic columns (the same code 10 to 12), instead of - dropped, so IBDmix still scores a mismatch there. - genotypes (step 43) IBDmix's genotype file of each chromosome: the three - archaic columns, SAMPLE (yours, from the VCF) and the stand-ins; - the mask of each archaic genome (chrN..mask.bed, the - positions where its call is masked); and spans.tsv, the span and - site count of each chromosome. - report (step 43) IBDmix's segments for SAMPLE against each archaic genome, - kept at LOD 4 and 50 kb or longer (the cutoffs of Chen et al.), - merged into Neanderthal and Denisovan segments, with the share of - the analysed span, the ten longest segments and the protein-coding - genes in them. - -Codes: an archaic or modern genotype is the number of ALT alleles (0, 1, 2), or -9 when it is unknown, the coding IBDmix's genotype file uses. In the table an -archaic call outside that genome's mask is 10, 11 or 12 (MASKED plus the ALT -count); `genotypes` writes it to IBDmix as 0, 1 or 2 and lists the position in -that genome's mask file (chrN..mask.bed), which step 43 passes to -IBDmix with -r. - -Used by scripts/setup.sh, scripts/43-archaic.sh and the ARCHAIC_* processes -(bin/ is on the task PATH); tests/test_archaic_report.py checks it. Standard -library only. -""" -import argparse -import bisect -import gzip -import heapq -import os -import re -import sys - -ARCHAICS = ("Altai", "Vindija", "Denisova") -NEANDERTHALS = ("Altai", "Vindija") -POPS = ("ALL", "AFR", "AMR", "EAS", "EUR", "SAS") -TABLE_HEADER = ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova"] + [f"AF_{p}" for p in POPS] -MASKED = 10 # table code: MASKED + ALT count, a call outside the genome's mask -PSEUDO = 100 # stand-in modern people: 200 alleles, about one 1000 Genomes population -MIN_LOD = 4.0 # Chen et al. 2020: LOD 4 -MIN_LENGTH = 50000 # and 50 kb -TOP = 10 -SEGMENT_HEADER = ["chrom", "start", "end", "length_bp", "ancestry", "matched", "lod", "genes"] -CAVEAT = ("IBDmix finds stretches where your genome matches a Neanderthal or the Denisovan genome more " - "closely than the allele frequencies of a modern population explain. A stretch counts when either " - "of your two copies carries it, and the share is of the analysed span of the autosomes. Some of what " - "it finds in anyone is old ancestry shared with the archaic genomes rather than gene flow from them: " - "Chen et al. 2020 find Neanderthal sequence in every population they studied, Africans included. " - "The numbers move with the cutoffs (LOD 4, 50 kb) and with the frequencies used. An estimate, not " - "a measurement.") - - -def die(msg, code=1): - print(f"ERROR: {msg}", file=sys.stderr) - sys.exit(code) - - -def opener(path): - return gzip.open(path, "rt") if path.endswith(".gz") else open(path) - - -def gt_code(gt): - """'0/1', '1|1', '0/0' -> 1, 2, 0; a missing, partial or multi-allelic call -> None.""" - a = re.split(r"[/|]", gt) - if len(a) != 2 or any(x not in ("0", "1") for x in a): - return None - return int(a[0]) + int(a[1]) - - -# --- table (setup) ------------------------------------------------------------------ - -def read_modern(path): - """1000 Genomes lines: POS REF ALT AF AFR_AF AMR_AF EAS_AF EUR_AF SAS_AF (bcftools query).""" - with opener(path) as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 9: - die(f"{path}: a line has {len(c)} columns, want 9: {line[:80]!r}") - if len(c[1]) != 1 or len(c[2]) != 1: - continue - yield int(c[0]), 0, (c[1], c[2], c[3:9]) - - -def read_archaic(path, idx): - """Archaic lines: POS REF ALT GT (bcftools query).""" - with opener(path) as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 4: - die(f"{path}: a line has {len(c)} columns, want 4: {line[:80]!r}") - yield int(c[0]), 1 + idx, (c[1], c[2], c[3]) - - -def read_mask(path, chrom): - """The intervals of one chromosome of a mask BED, sorted and merged. - A BED is 0-based and half-open, so [start, end) holds the 1-based - positions start + 1 to end. The chromosome is matched with and without - the chr prefix, because the archaic masks name it 22 and others chr22.""" - num = str(chrom)[3:] if str(chrom).startswith("chr") else str(chrom) - want = {num, f"chr{num}"} - iv = [] - with opener(path) as f: - for line in f: - if line.startswith(("#", "track", "browser")): - continue - c = line.split() - if len(c) < 3 or c[0] not in want: - continue - s, e = int(c[1]), int(c[2]) - if e <= s: - continue # an empty or reversed interval holds no position - iv.append((s, e)) - iv.sort() - out = [] - for s, e in iv: - if out and s <= out[-1][1]: - out[-1][1] = max(out[-1][1], e) - else: - out.append([s, e]) - return out - - -class Mask: - """Whether a 1-based position is inside a mask. Positions must come in - ascending order, as they do in the table build: the walk over the - intervals never goes back.""" - - def __init__(self, intervals): - self.iv = intervals - self.i = 0 - - def holds(self, pos): - p = pos - 1 - while self.i < len(self.iv) and self.iv[self.i][1] <= p: - self.i += 1 - return self.i < len(self.iv) and self.iv[self.i][0] <= p - - -def archaic_code(rec, ref, alt): - """The ALT count of one archaic record at a row with this REF and ALT, or 9.""" - if rec is None: - return 9 - aref, aalt, gt = rec - if aref != ref: - return 9 - code = gt_code(gt) - if code is None: - return 9 - if aalt in (".", "<*>") or code == 0: - return 0 if code == 0 else 9 - return code if aalt == alt else 9 - - -def table_row(pos, modern, arch): - """The row of one position, or None. modern: (REF, ALT, AFs) or None; - arch: one (REF, ALT, GT) or None per archaic genome.""" - if modern is not None: - ref, alt, afs = modern - else: - # A site no modern person varies at: the ALT of the first archaic - # genome with a non-reference call, every population frequency 0. - cand = [r for r in arch if r is not None and gt_code(r[2]) not in (None, 0) - and len(r[0]) == 1 and len(r[1]) == 1] - if not cand: - return None - ref, alt, afs = cand[0][0], cand[0][1], ["0"] * len(POPS) - codes = [archaic_code(r, ref, alt) for r in arch] - if all(c == 9 for c in codes): - return None - if modern is None and not any(c in (1, 2) for c in codes): - return None - clean = [] - for v in afs: - try: - x = float(v) - except ValueError: - x = 0.0 - clean.append(f"{min(max(x, 0.0), 1.0):.6g}") - return ref, alt, codes, clean - - -def cmd_table(a): - names = [] - streams = [read_modern(a.modern)] - for i, spec in enumerate(a.archaic): - name, _, path = spec.partition("=") - if name not in ARCHAICS or not path: - die(f"--archaic takes NAME=FILE with NAME one of {', '.join(ARCHAICS)}, got {spec!r}", 2) - names.append(name) - streams.append(read_archaic(path, i)) - if names != list(ARCHAICS): - die(f"--archaic must name {', '.join(ARCHAICS)} in that order, got {', '.join(names)}", 2) - masks = {} - for spec in a.mask: - name, _, path = spec.partition("=") - if name not in ARCHAICS or not path: - die(f"--mask takes NAME=FILE with NAME one of {', '.join(ARCHAICS)}, got {spec!r}", 2) - if name in masks: - die(f"--mask names {name} twice", 2) - iv = read_mask(path, a.chrom) - if not iv: - die(f"{path} holds no interval on chromosome {a.chrom}: a mask that matches nothing " - "would make every site unknown") - masks[name] = Mask(iv) - excluded = [] - for path in a.exclude: - iv = read_mask(path, a.chrom) - if not iv: - die(f"{path} holds no interval on chromosome {a.chrom}: an empty exclusion list is a " - "download that went wrong, not a chromosome without segmental duplications") - excluded.append(Mask(iv)) - chrom = a.chrom if a.chrom.startswith("chr") else f"chr{a.chrom}" - out = open(a.out, "w") - out.write("##fileformat=VCFv4.2\n") - out.write(f"##contig=\n") - for n in ARCHAICS: - out.write(f'##INFO=\n') - for p in POPS: - out.write(f'##INFO=\n') - out.write("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n") - rows = 0 - cur, modern, arch, last = None, None, [None] * len(ARCHAICS), -1 - - def flush(): - nonlocal rows - if cur is None: - return - # Every Mask is walked for every position, before any early return, - # because the walk only goes forward. - excl = any([m.holds(cur) for m in excluded]) - outside = [excl or (n in masks and not masks[n].holds(cur)) for n in ARCHAICS] - r = table_row(cur, modern, arch) - if r is None: - return - ref, alt, codes, afs = r - # A genome's call outside its mask, or at an excluded site, is kept but - # marked masked (MASKED + code). Step 43 hands IBDmix the call and a - # mask of those positions, as Chen et al. do: IBDmix scores nothing - # there except a homozygous mismatch, which still counts against a - # segment. Dropping the call instead (code 9) let segments run through - # masked sequence nothing could contradict. - codes = [c + MASKED if (out and c != 9) else c for c, out in zip(codes, outside)] - info = ";".join([f"{n}={c}" for n, c in zip(ARCHAICS, codes)] + [f"AF_{p}={v}" for p, v in zip(POPS, afs)]) - out.write(f"{chrom}\t{cur}\t.\t{ref}\t{alt}\t.\t.\t{info}\n") - rows += 1 - - for pos, src, rec in heapq.merge(*streams, key=lambda t: (t[0], t[1])): - if pos < last: - die(f"input not sorted by position at {pos}") - if pos != cur: - flush() - cur, modern, arch = pos, None, [None] * len(ARCHAICS) - last = pos - if src == 0: - # A second modern record at one position (a split multi-allelic - # site) makes the row ambiguous: keep neither. - modern = rec if modern is None else ("N", "N", ["0"] * len(POPS)) - else: - arch[src - 1] = rec - flush() - out.close() - print(f"{chrom}: {rows} rows", file=sys.stderr) - - -def cmd_strict(a): - """The lifted GRCh38 rows of one chromosome (no header, the 13 table - columns), once per position, with the calls of every row outside the 1000 - Genomes strict mask marked masked (MASKED plus the ALT count, 9 stays 9). - The strict mask is the same for the three genomes, so all three are - marked; step 43 then hands IBDmix those positions as its mask.""" - iv = read_mask(a.strict, a.chrom) - if not iv: - die(f"{a.strict} holds no interval on chromosome {a.chrom}: a mask that matches nothing " - "would mark every site masked") - starts = [s for s, _ in iv] - seen = set() - rows = marked = 0 - with open(a.rows) as f, open(a.out, "w") as out: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) != len(TABLE_HEADER): - die(f"{a.rows}: a row has {len(c)} columns, want {len(TABLE_HEADER)}: {line[:80]!r}") - pos = int(c[1]) - if pos in seen: - continue - seen.add(pos) - i = bisect.bisect_right(starts, pos - 1) - 1 - if not (i >= 0 and pos - 1 < iv[i][1]): - hit = False - for k in (4, 5, 6): - if c[k] in ("0", "1", "2"): - c[k] = str(int(c[k]) + MASKED) - hit = True - marked += hit - out.write("\t".join(c) + "\n") - rows += 1 - print(f"{a.chrom}: {rows} rows, {marked} outside the strict mask and marked masked", file=sys.stderr) - - -# setup.sh --archaic stamps its table with the sha256 of everything above this -# section, so an edit below it does not rebuild the table (about 165 GB of downloads). - -# --- genotypes (step 43) ---------------------------------------------------------- - -def pseudo_columns(n): - """The stand-in columns for each ALT allele count 0..2n: as many 2s and - one 1 as the count needs, then 0s.""" - cols = [] - for ac in range(2 * n + 1): - g = ["2"] * (ac // 2) + (["1"] if ac % 2 else []) - g += ["0"] * (n - len(g)) - cols.append("\t".join(g)) - return cols - - -def read_vcf(path): - """(chrom, pos, ref, alt, code) of the biallelic SNVs of a one-sample VCF. - A record whose FILTER is not PASS or '.' gives code 9 (not read), so a - DeepVariant RefCall is neither your allele nor the reference.""" - with opener(path) as f: - for line in f: - if line.startswith("##"): - continue - c = line.rstrip("\n").split("\t") - if line.startswith("#"): - if len(c) != 10: - die(f"{path} holds {max(len(c) - 9, 0)} samples; step 43 reads a one-sample VCF " - "(keep one with bcftools view -s ).") - continue - if len(c) < 10 or len(c[3]) != 1 or c[4] == ".": - continue - if "," in c[4] or len(c[4]) != 1: - yield c[0], int(c[1]), c[3], None, 9 - continue - fmt = c[8].split(":") - if "GT" not in fmt: - continue - vals = c[9].split(":") - gt = vals[fmt.index("GT")] if fmt.index("GT") < len(vals) else "." - code = gt_code(gt) - if code is None or c[6] not in ("PASS", "."): - code = 9 - yield c[0], int(c[1]), c[3], c[4], code - - -CLASH = object() -ARCHAIC_CODES = {str(v) for v in (0, 1, 2, 9, MASKED, MASKED + 1, MASKED + 2)} - - -def pick_call(prev, call): - """The call to keep when a position comes twice, as in a VCF split with - bcftools norm -m-: each is (REF, ALT, code), ALT None for an insertion or a - multi-allelic record. An SNV wins over such a record, in either order. Of - one SNV twice, a usable record wins over code 9, and two different usable - codes are CLASH. Of two different SNVs, the - one that carries its ALT wins over a 0/0 or unusable one (a split 0/1 at - A>G,T is A>G 0/1 and A>T 0/0); both 0/0 is the reference; both carrying - their ALT, or neither readable, is CLASH (unknown, 9).""" - if prev is None: - return call - if prev[1] is None or call[1] is None: - return call if prev[1] is None and call[1] is not None else prev - if call[1] == prev[1]: - if call[2] == 9 or call[2] == prev[2]: - return prev - return call if prev[2] == 9 else CLASH - has_prev, has_call = prev[2] in (1, 2), call[2] in (1, 2) - if has_prev != has_call: - return prev if has_prev else call - if not has_prev and prev[2] == 0 and call[2] == 0: - return (prev[0], None, 0) - return CLASH - - -def read_table(path): - """Rows of one chromosome of the GRCh38 table, with the header checked.""" - with opener(path) as f: - head = f.readline().rstrip("\n").split("\t") - if head != TABLE_HEADER: - die(f"{path} does not start with the table header ({' '.join(TABLE_HEADER)})") - for line in f: - c = line.rstrip("\n").split("\t") - yield c - - -def cmd_genotypes(a): - if a.pop not in POPS: - die(f"--pop must be one of {', '.join(POPS)}, got {a.pop!r}", 2) - af_col = TABLE_HEADER.index(f"AF_{a.pop}") - tables = {} - for name in sorted(os.listdir(a.table_dir)): - m = re.fullmatch(r"(chr(?:[1-9]|1[0-9]|2[0-2]))\.tsv\.gz", name) - if m: - tables[m.group(1)] = os.path.join(a.table_dir, name) - if not tables: - die(f"{a.table_dir} holds no chrN.tsv.gz: install the archaic data with scripts/setup.sh --archaic") - os.makedirs(a.out_dir, exist_ok=True) - pseudo = pseudo_columns(a.pseudo) - two_n = 2 * a.pseudo - head = "\t".join(["chrom", "pos", "ref", "alt", *ARCHAICS, "SAMPLE"] + [f"P{i:03d}" for i in range(1, a.pseudo + 1)]) - with open(os.path.join(a.out_dir, "samples.txt"), "w") as f: - f.write("SAMPLE\n" + "".join(f"P{i:03d}\n" for i in range(1, a.pseudo + 1))) - spans = [] - vcf = read_vcf(a.vcf) - pending = next(vcf, None) - done = set() - while pending is not None: - chrom = pending[0] - if chrom in done: - die(f"{a.vcf} is not sorted: {chrom} comes back after another chromosome") - done.add(chrom) - if chrom not in tables: - while pending is not None and pending[0] == chrom: - pending = next(vcf, None) - continue - # The sample's calls on this chromosome, merged with the table by position. - calls, clash = {}, set() - while pending is not None and pending[0] == chrom: - pos = pending[1] - if pos not in clash: - calls[pos] = pick_call(calls.get(pos), pending[2:]) - if calls[pos] is CLASH: - calls[pos] = (pending[2], None, 9) - clash.add(pos) - pending = next(vcf, None) - n = first = last = dropped = carried = 0 - num = chrom[3:] - path = os.path.join(a.out_dir, f"{chrom}.gt.gz") - # Per archaic genome, the runs of written rows whose call is masked: - # [first, last] positions, closed at the next unmasked row. - runs = {g: [] for g in ARCHAICS} - open_run = {g: None for g in ARCHAICS} - with gzip.open(path, "wt", compresslevel=1) as out: - out.write(head + "\n") - for c in read_table(tables[chrom]): - pos, ref, alt = int(c[1]), c[2], c[3] - arch = [] - for g, v in zip(ARCHAICS, c[4:7]): - if v not in ARCHAIC_CODES: - die(f"{tables[chrom]}: {chrom}:{pos} has {g} code {v!r}; the table holds 0, 1, 2, 9 " - f"or {MASKED} to {MASKED + 2}") - arch.append(int(v)) - call = calls.get(pos) - if call is None: - code = 0 - elif call[0] != ref or (call[1] is not None and call[1] != alt): - # Another allele than the row's: IBDmix's generate_gt drops the line. - dropped += 1 - continue - else: - code = call[2] - if code in (1, 2): - carried += 1 - try: - ac = int(round(float(c[af_col]) * two_n)) - except ValueError: - ac = 0 - ac = min(max(ac, 0), two_n) - for g, v in zip(ARCHAICS, arch): - if v >= MASKED: - if open_run[g] is None: - open_run[g] = [pos, pos] - else: - open_run[g][1] = pos - elif open_run[g] is not None: - runs[g].append(open_run[g]) - open_run[g] = None - gts = "\t".join(str(v - MASKED if v >= MASKED else v) for v in arch) - out.write(f"{num}\t{pos}\t{ref}\t{alt}\t{gts}\t{code}\t{pseudo[ac]}\n") - if n == 0: - first = pos - last = pos - n += 1 - if n == 0: - os.remove(path) - continue - # IBDmix's mask (-r): chromosome as in the genotype file, and a line - # covers the positions start + 1 to end, as in a BED. - for g in ARCHAICS: - if open_run[g] is not None: - runs[g].append(open_run[g]) - with open(os.path.join(a.out_dir, f"{chrom}.{g}.mask.bed"), "w") as f: - f.writelines(f"{num}\t{s0 - 1}\t{e0}\n" for s0, e0 in runs[g]) - spans.append((chrom, first, last, n, carried, dropped)) - with open(os.path.join(a.out_dir, "spans.tsv"), "w") as f: - f.write("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n") - for s in spans: - f.write("\t".join(map(str, s)) + "\n") - if not spans: - die(f"{a.vcf} has no records on a chromosome of the archaic table ({', '.join(sorted(tables))})") - print(f"{len(spans)} chromosomes, {sum(s[3] for s in spans)} sites", file=sys.stderr) - - -# --- report (step 43) ------------------------------------------------------------ - -def read_segments(path, sample="SAMPLE"): - """IBDmix's output rows for SAMPLE: (chrom, start, end, lod).""" - segs = [] - with open(path) as f: - head = f.readline().rstrip("\n").split("\t") - if head[:5] != ["ID", "chrom", "start", "end", "slod"]: - die(f"{path} is not IBDmix output (header {' '.join(head[:5])})") - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 5 or c[0] != sample: - continue - segs.append((c[1], int(c[2]), int(c[3]), float(c[4]))) - return segs - - -def keep(segs, min_lod=MIN_LOD, min_length=MIN_LENGTH): - return [s for s in segs if s[3] >= min_lod and s[2] - s[1] >= min_length] - - -def merge(segs): - """Overlapping or touching segments of one class as one: (chrom, start, end, lod, members).""" - out = [] - for chrom, start, end, lod, who in sorted(segs, key=lambda s: (chrom_key(s[0]), s[1], s[2])): - if out and out[-1][0] == chrom and start <= out[-1][2]: - c, s0, e0, l0, w0 = out[-1] - out[-1] = (c, s0, max(e0, end), max(l0, lod), w0 | who) - else: - out.append((chrom, start, end, lod, set(who))) - return out - - -def chrom_key(c): - c = str(c) - c = c[3:] if c.startswith("chr") else c - return int(c) if c.isdigit() else 99 - - -def overlaps(seg, others): - return [o for o in others if o[0] == seg[0] and o[1] < seg[2] and seg[1] < o[2]] - - -def read_genes(path): - """Protein-coding genes of a GENCODE gene GTF: chrom -> [(start, end, name)].""" - genes = {} - if not path: - return genes - with opener(path) as f: - for line in f: - if line.startswith("#"): - continue - c = line.rstrip("\n").split("\t") - if len(c) < 9 or c[2] != "gene": - continue - if 'gene_type "protein_coding"' not in c[8]: - continue - m = re.search(r'gene_name "([^"]+)"', c[8]) - if m: - genes.setdefault(c[0], []).append((int(c[3]), int(c[4]), m.group(1))) - return genes - - -def genes_in(genes, chrom, start, end): - """The names of the genes overlapping [start, end), once each, in order. - IBDmix's start is the segment's first site and its end the next site, past - the segment (its README, --inclusive-end); a GTF gene is [s, e], 1-based.""" - name = chrom if str(chrom).startswith("chr") else f"chr{chrom}" - seen = [] - for s, e, n in sorted(genes.get(name, [])): - if s < end and e >= start and n not in seen: - seen.append(n) - return seen - - -def classify(per_archaic): - """Neanderthal: the union of the Altai and Vindija segments. Denisovan: the - Denisova segments that overlap no Neanderthal one (a stretch the Denisova - genome and a Neanderthal both match is counted as Neanderthal, the more - common source outside Oceania). Returns a list of - (chrom, start, end, ancestry, matched, lod).""" - nea = merge([(c, s, e, l, {n}) for n in NEANDERTHALS for c, s, e, l in per_archaic.get(n, [])]) - den = merge([(c, s, e, l, {"Denisova"}) for c, s, e, l in per_archaic.get("Denisova", [])]) - rows = [] - for c, s, e, l, who in nea: - also = {"Denisova"} if overlaps((c, s, e), den) else set() - rows.append((c, s, e, "Neanderthal", sorted(who) + sorted(also), l)) - for c, s, e, l, who in den: - if not overlaps((c, s, e), nea): - rows.append((c, s, e, "Denisovan", ["Denisova"], l)) - rows.sort(key=lambda r: (chrom_key(r[0]), r[1])) - return rows - - -def summarise(rows, spans_bp): - out = {} - for anc in ("Neanderthal", "Denisovan"): - r = [x for x in rows if x[3] == anc] - bp = sum(x[2] - x[1] for x in r) - key = anc.lower() - out[f"{key}_segments"] = len(r) - out[f"{key}_bp"] = bp - out[f"{key}_percent"] = f"{100.0 * bp / spans_bp:.2f}" if spans_bp else "0.00" - out[f"{key}_longest_bp"] = max((x[2] - x[1] for x in r), default=0) - return out - - -def cmd_report(a): - spans = [] - with open(a.spans) as f: - head = f.readline().rstrip("\n").split("\t") - if head[:4] != ["chrom", "first", "last", "sites"]: - die(f"{a.spans} is not the spans table of the genotypes step") - for line in f: - c = line.rstrip("\n").split("\t") - spans.append((c[0], int(c[1]), int(c[2]), int(c[3]))) - if not spans: - die(f"{a.spans} lists no chromosome") - spans_bp = sum(last - first + 1 for _, first, last, _ in spans) - per = {n: [] for n in ARCHAICS} - for chrom, *_ in spans: - for n in ARCHAICS: - p = os.path.join(a.ibdmix_dir, f"{chrom}.{n}.txt") - if not os.path.isfile(p): - die(f"no IBDmix output {p}") - per[n] += [(f"chr{c}" if not str(c).startswith("chr") else c, s, e, l) - for c, s, e, l in keep(read_segments(p))] - rows = classify(per) - genes = read_genes(a.genes) - summary = summarise(rows, spans_bp) - os.makedirs(a.out_dir, exist_ok=True) - seg_path = os.path.join(a.out_dir, f"{a.sample}_archaic_segments.tsv") - with open(seg_path, "w") as f: - f.write("\t".join(SEGMENT_HEADER) + "\n") - for c, s, e, anc, who, lod in rows: - g = genes_in(genes, c, s, e) if a.genes else [] - f.write(f"{c}\t{s}\t{e}\t{e - s}\t{anc}\t{','.join(who)}\t{lod:.1f}\t{','.join(g) if g else '.'}\n") - longest = sorted(rows, key=lambda r: (-(r[2] - r[1]), chrom_key(r[0]), r[1]))[:TOP] - meta = { - "sample": a.sample, - "method": "IBDmix", - "archaic_genomes": "Altai Neanderthal, Vindija 33.19 Neanderthal, Altai Denisovan", - "population": a.pop, - "min_lod": f"{MIN_LOD:g}", - "min_length_bp": MIN_LENGTH, - "analysed_bp": spans_bp, - "analysed_chromosomes": len(spans), - "analysed_sites": sum(s[3] for s in spans), - "genes": "GENCODE protein-coding" if a.genes else "none (no gene annotation installed)", - **summary, - } - with open(os.path.join(a.out_dir, f"{a.sample}_archaic.tsv"), "w") as f: - f.write("key\tvalue\n") - for k, v in meta.items(): - f.write(f"{k}\t{v}\n") - for i, (c, s, e, anc, who, lod) in enumerate(longest, 1): - g = genes_in(genes, c, s, e) if a.genes else [] - f.write(f"longest_{i}\t{c}:{s}-{e} {anc} {(e - s) / 1e6:.2f} Mb " - f"({'/'.join(who)}); genes: {', '.join(g) if g else 'none'}\n") - lines = [f"Archaic ancestry of {a.sample} (IBDmix against the Altai and Vindija Neanderthals and the Altai Denisovan)", - f" Analysed: {len(spans)} autosomes, {spans_bp / 1e6:.1f} Mb, {meta['analysed_sites']} sites; " - f"modern frequencies: 1000 Genomes {a.pop}", - f" Neanderthal: {summary['neanderthal_percent']}% ({summary['neanderthal_bp'] / 1e6:.1f} Mb in " - f"{summary['neanderthal_segments']} segments)", - f" Denisovan: {summary['denisovan_percent']}% ({summary['denisovan_bp'] / 1e6:.1f} Mb in " - f"{summary['denisovan_segments']} segments)"] - if longest: - lines.append(f" The {len(longest)} longest segments:") - for c, s, e, anc, who, lod in longest: - g = genes_in(genes, c, s, e) if a.genes else [] - lines.append(f" {c}:{s}-{e} {(e - s) / 1e6:.2f} Mb {anc} ({'/'.join(who)}, LOD {lod:.0f}) " - f"genes: {', '.join(g) if g else 'none'}") - lines.append(f" {CAVEAT}") - lines.append(f" Every segment: {os.path.basename(seg_path)}") - with open(os.path.join(a.out_dir, f"{a.sample}_archaic.txt"), "w") as f: - f.write("\n".join(lines) + "\n") - print("\n".join(lines)) - - -def main(argv=None): - p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - sub = p.add_subparsers(dest="cmd", required=True) - t = sub.add_parser("table", help="one GRCh37 chromosome of the archaic table (setup.sh --archaic)") - t.add_argument("--chrom", required=True) - t.add_argument("--modern", required=True) - t.add_argument("--archaic", action="append", default=[], help="NAME=FILE, once each for Altai, Vindija, Denisova") - t.add_argument("--mask", action="append", default=[], - help="NAME=BED: read that archaic genome only inside this mask (its MPI-EVA FilterBed)") - t.add_argument("--exclude", action="append", default=[], - help="BED: leave out every site inside these regions (segmental duplications, indel neighbourhoods)") - t.add_argument("--out", required=True) - m = sub.add_parser("strict", help="mark the rows outside the GRCh38 strict mask masked (setup.sh --archaic)") - m.add_argument("--chrom", required=True) - m.add_argument("--strict", required=True, help="BED: the 1000 Genomes GRCh38 strict mask") - m.add_argument("--rows", required=True, help="the lifted rows, tab-separated, no header") - m.add_argument("--out", required=True) - g = sub.add_parser("genotypes", help="IBDmix's genotype files for one sample (step 43)") - g.add_argument("--table-dir", required=True) - g.add_argument("--vcf", required=True) - g.add_argument("--pop", default="ALL") - g.add_argument("--pseudo", type=int, default=PSEUDO) - g.add_argument("--out-dir", required=True) - r = sub.add_parser("report", help="segments, shares and the ten longest (step 43)") - r.add_argument("--sample", required=True) - r.add_argument("--ibdmix-dir", required=True) - r.add_argument("--spans", required=True) - r.add_argument("--genes", default="") - r.add_argument("--pop", default="ALL") - r.add_argument("--out-dir", required=True) - a = p.parse_args(argv) - if a.cmd == "genotypes" and a.pseudo < 1: - die("--pseudo must be 1 or more", 2) - {"table": cmd_table, "strict": cmd_strict, "genotypes": cmd_genotypes, "report": cmd_report}[a.cmd](a) - - -if __name__ == "__main__": - main() diff --git a/bin/collect_summary.py b/bin/collect_summary.py index 054d9a684..5aa4fd6eb 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -2169,7 +2169,7 @@ def sec_bloodtype(d, s): def sec_archaic(d, s): - """Step 43's summary (archaic/_archaic.tsv, bin/archaic_report.py): the + """Step 43's summary (archaic/_archaic.tsv, og archaic report): the Neanderthal and Denisovan shares and segment counts, the analysed span and the ten longest segments.""" p = first_existing(d, [f"archaic/{s}_archaic.tsv"]) diff --git a/rust/crates/og-ancestry/src/ancient.rs b/rust/crates/og-ancestry/src/ancient.rs index 612a71212..6a3b6fc52 100644 --- a/rust/crates/og-ancestry/src/ancient.rs +++ b/rust/crates/og-ancestry/src/ancient.rs @@ -1529,6 +1529,9 @@ mod tests { assert_eq!(year_label(1950.0), "1 BCE"); assert_eq!(year_label(1949.0), "1 CE"); assert_eq!(year_label(1750.5), "200 CE"); + assert_eq!(year_label(8025.0), "6075 BCE"); + assert_eq!(year_label(450.0), "1500 CE"); + assert_eq!(range_label(3000.0, 3000.0), "1050 BCE"); assert_eq!(range_label(7250.0, 6850.0), "5300-4900 BCE"); assert_eq!(range_label(1750.0, 1500.0), "200-450 CE"); assert_eq!(range_label(2250.0, 1750.0), "300 BCE-200 CE"); diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 37bdf7624..3d1b4e811 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -260,7 +260,6 @@ impl Stream { /// SNVs are skipped. fn next(&mut self) -> Out> { loop { - let path = self.text.path.display().to_string(); let Some(raw) = self.text.raw()? else { return Ok(None); }; @@ -268,10 +267,10 @@ impl Stream { let c: Vec<&str> = line.split('\t').collect(); if self.src == 0 { if c.len() < 9 { + let (n, head) = (c.len(), head80(raw)); return die(format!( - "{path}: a line has {} columns, want 9: {}", - c.len(), - head80(raw) + "{}: a line has {n} columns, want 9: {head}", + self.text.path.display() )); } if c[1].chars().count() != 1 || c[2].chars().count() != 1 { @@ -288,10 +287,10 @@ impl Stream { ))); } if c.len() < 4 { + let (n, head) = (c.len(), head80(raw)); return die(format!( - "{path}: a line has {} columns, want 4: {}", - c.len(), - head80(raw) + "{}: a line has {n} columns, want 4: {head}", + self.text.path.display() )); } let pos = need_int(c[0])?; @@ -732,7 +731,6 @@ struct Vcf { impl Vcf { fn next(&mut self) -> Out> { loop { - let path = self.text.path.display().to_string(); let Some(raw) = self.text.raw()? else { return Ok(None); }; @@ -743,10 +741,11 @@ impl Vcf { let c: Vec<&str> = line.split('\t').collect(); if raw.starts_with('#') { if c.len() != 10 { + let n = c.len().saturating_sub(9); return die(format!( - "{path} holds {} samples; step 43 reads a one-sample VCF \ + "{} holds {n} samples; step 43 reads a one-sample VCF \ (keep one with bcftools view -s ).", - c.len().saturating_sub(9) + self.text.path.display() )); } continue; diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index f76068dcf..e2f0a3f24 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -47,11 +47,11 @@ alignment-markdup-gvcf-sex-2-ploidy scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-4-bwamem2 scripts/02a-alignment-bwamem2.sh scripts/ci/fake-docker/lib.sh ancestry-fine-grained-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgs_scores.tsv assets/stub/sample.vcf.gz bin/ancestry_fine_report.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancestry_fine/* modules/local/html_report/* modules/local/prs/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/26b-ancestry-fine.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/ancient_report.py bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/og bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh annotsv-delly-depth-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh tests/fixtures/annotsv/class45.sh annotsv-report-section-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh tests/fixtures/annotsv/class45.sh annotsv-sv-inputs-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -archaic-ancestry-1 assets/pgs_scores.tsv bin/archaic_report.py bin/collect_summary.py bin/traits_report.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh +archaic-ancestry-1 assets/pgs_scores.tsv bin/og bin/collect_summary.py bin/traits_report.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh bash-step-08-hla - bash-step-09-str assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh bash-step-10-telomere scripts/10-telomere-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index 49bf9e2a5..645b5a3e4 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -1198,42 +1198,53 @@ def archaic(d, rng): def ancient(d, rng): - """Step 44: bin/ancient_report.py's nearest-group table, info table and - PC plot (its own svg_plot), made-up distances and PCs over a few - AADR-style group labels.""" - import ancient_report - groups = [("England_BellBeaker", "2450-2000 BCE", "United Kingdom", "Amesbury Down"), - ("Germany_CordedWare", "2900-2350 BCE", "Germany", "Esperstedt"), - ("Spain_BellBeaker", "2500-2000 BCE", "Spain", "Camino de las Yeseras"), - ("Czechia_EBA_Unetice", "2200-1700 BCE", "Czech Republic", "Moravia"), - ("Hungary_LBA", "1500-1100 BCE", "Hungary", "Tiszafüred"), - ("Italy_Imperial_Rome", "1-300 CE", "Italy", "Rome"), - ("Sweden_Viking", "800-1050 CE", "Sweden", "Sigtuna"), - ("France_Neolithic", "4500-3000 BCE", "France", "Gurgy"), - ("Turkey_Neolithic", "6500-6000 BCE", "Turkey", "Barcın"), - ("Russia_Yamnaya_Samara", "3300-2700 BCE", "Russia", "Samara")] + """Step 44: what og ancient report writes (the nearest-group table, the + info table, the text and the PC plot) for a made-up panel: present-day + people around the origin, ancient people of a few AADR-style group labels + each around its own point, and the person among them.""" + groups = [("England_BellBeaker", 4200, "United Kingdom", "Amesbury Down"), + ("Germany_CordedWare", 4600, "Germany", "Esperstedt"), + ("Spain_BellBeaker", 4200, "Spain", "Camino de las Yeseras"), + ("Czechia_EBA_Unetice", 3900, "Czech Republic", "Moravia"), + ("Hungary_LBA", 3250, "Hungary", "Tiszafüred"), + ("Italy_Imperial_Rome", 1800, "Italy", "Rome"), + ("Sweden_Viking", 1030, "Sweden", "Sigtuna"), + ("France_Neolithic", 5700, "France", "Gurgy"), + ("Turkey_Neolithic", 8200, "Turkey", "Barcın"), + ("Russia_Yamnaya_Samara", 4950, "Russia", "Samara")] person = (0.01234, -0.00456) - dist, nearest = 0.0, [] - lines = ["rank\tgroup\tn\tdistance\tpc1\tpc2\tdate_bp\tdates\tcountry\tlocality\n"] - for i, (g, dates, country, loc) in enumerate(groups, 1): - dist += rng.uniform(0.002, 0.012) - pc1, pc2 = person[0] + rng.uniform(-dist, dist), person[1] + rng.uniform(-dist, dist) - nearest.append({"group": g, "pc1": pc1, "pc2": pc2}) - lines.append(f"{i}\t{g}\t{rng.randint(3, 40)}\t{dist:.5f}\t{pc1:.5f}\t" - f"{pc2:.5f}\t{rng.randint(1000, 8500)}\t{dates}\t{country}\t{loc}\n") - put(os.path.join(d, "ancient", f"{SAMPLE}_ancient.tsv"), "".join(lines)) - panel = {f"m{i}": ("present", (rng.gauss(0.0, 0.03), rng.gauss(0.0, 0.025))) for i in range(300)} - ancients = {} - for i in range(200): - panel[f"a{i}"] = (ancient_report.ANCIENT_LABEL, (rng.gauss(0.005, 0.035), rng.gauss(0.0, 0.03))) - ancients[f"a{i}"] = {"date_bp": str(rng.randint(500, 9000))} - put(os.path.join(d, "ancient", f"{SAMPLE}_ancient_pca.svg"), - ancient_report.svg_plot(person, panel, ancients, SAMPLE, nearest)) - info = [("release", "v62.0"), ("sites_used", rng.randint(520_000, 560_000)), ("panel_sites", 597_573), - ("present_day_people", 819), ("ancient_people", 4_210), ("ancient_groups", 612), - ("pcs_in_distance", 4), ("pc1", f"{person[0]:.5f}"), ("pc2", f"{person[1]:.5f}")] - put(os.path.join(d, "ancient", f"{SAMPLE}_ancient_info.tsv"), - "key\tvalue\n" + "".join(f"{k}\t{v}\n" for k, v in info)) + evals = "#eigvals: 9.000 4.000 2.000 1.000\n" + moderns = [f"m{i} " + " ".join(f"{rng.gauss(0.0, sd):.6f}" for sd in (0.03, 0.025, 0.02, 0.015)) + + f" {('French', 'Sardinian', 'Basque')[i % 3]}\n" for i in range(300)] + ancients, anc_rows = [], ["id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n"] + for i, (g, bp, country, loc) in enumerate(groups): + spread = 0.004 + 0.006 * i + cx, cy = person[0] + rng.uniform(-spread, spread), person[1] + rng.uniform(-spread, spread) + for j in range(rng.randint(3, 12)): + aid = f"a{i}_{j}" + ancients.append(f"{aid} {cx + rng.gauss(0, 0.004):.6f} {cy + rng.gauss(0, 0.004):.6f} 0.000000 0.000000 Ancient\n") + date = bp + rng.randint(-150, 150) + anc_rows.append(f"{aid}\t{g}\t{date}\t50\t.\t{loc}\t{country}\t50\t10\t500000\n") + with tempfile.TemporaryDirectory() as work: + paths = {n: os.path.join(work, n) for n in ("panel.evec", "run.evec", "ancients.tsv", "stats.tsv")} + put(paths["panel.evec"], evals + "".join(moderns + ancients)) + put(paths["run.evec"], evals + "".join(moderns) + + f"{SAMPLE} {person[0]:.6f} {person[1]:.6f} 0.000000 0.000000 Person\n") + put(paths["ancients.tsv"], "".join(anc_rows)) + put(paths["stats.tsv"], f"panel_sites\t597573\nsites_genotyped\t560000\nother_allele\t120\n" + f"sites_used\t{rng.randint(520_000, 560_000)}\n") + out = os.path.join(d, "ancient") + os.makedirs(out, exist_ok=True) + r = subprocess.run([og_binary(), "ancient", "report", "--panel-evec", paths["panel.evec"], + "--run-evec", paths["run.evec"], "--ancients", paths["ancients.tsv"], + "--sample", SAMPLE, "--release", "v62.0", "--stats", paths["stats.tsv"], + "--out-tsv", os.path.join(out, f"{SAMPLE}_ancient.tsv"), + "--out-info", os.path.join(out, f"{SAMPLE}_ancient_info.tsv"), + "--out-txt", os.path.join(out, f"{SAMPLE}_ancient.txt"), + "--out-svg", os.path.join(out, f"{SAMPLE}_ancient_pca.svg")], + stdout=subprocess.DEVNULL) + if r.returncode: + sys.exit("ERROR: og ancient report failed on the made-up panel") def exomiser(d): diff --git a/tests/e2e/ancient-similarity-1.sh b/tests/e2e/ancient-similarity-1.sh index c1b0e8e7f..21941d932 100755 --- a/tests/e2e/ancient-similarity-1.sh +++ b/tests/e2e/ancient-similarity-1.sh @@ -46,7 +46,7 @@ for e in ANNO:anno IND:ind SNP:snp; do curl -fsSL --retry 3 -o "${REAL}/aadr.${e#*:}" "https://dataverse.harvard.edu/api/access/datafile/${!id_var}" check_eq "AADR ${AADR_VERSION} .${e#*:}: the md5 versions.env pins" "$(md5sum "${REAL}/aadr.${e#*:}" | cut -d' ' -f1)" "${!md5_var}" done -python3 "${REPO}/bin/ancient_report.py" select --anno "${REAL}/aadr.anno" --ind "${REAL}/aadr.ind" \ +"${REPO}/bin/og" ancient select --anno "${REAL}/aadr.anno" --ind "${REAL}/aadr.ind" \ --out-ind "${REAL}/panel.ind" --poplist "${REAL}/poplist.txt" --ancients "${REAL}/ancients.tsv" | tee "${CASE_TMP}/select.log" check_ge "real AADR: present-day reference people" "$(awk '$3 != "Ignore" && $3 != "Ancient"' "${REAL}/panel.ind" | wc -l | tr -d ' ')" 800 check_ge "real AADR: present-day groups" "$(wc -l < "${REAL}/poplist.txt" | tr -d ' ')" 35 diff --git a/tests/test_ancient_report.py b/tests/test_ancient_report.py index f91218a6d..b0e7efee9 100644 --- a/tests/test_ancient_report.py +++ b/tests/test_ancient_report.py @@ -1,34 +1,35 @@ #!/usr/bin/env python3 -"""bin/ancient_report.py (step 44) on synthetic files. +"""og ancient (step 44) on synthetic files, run as the pipeline runs it. Checks: - 1. dates: years before 1950 become BCE and CE the way the AADR writes - them (4250 BP is the middle of its 2600-2000 BCE; 1950 BP is 1 BCE, - 1949 BP is 1 CE), and a population's span is written oldest first, with - one era when both ends share it; - 2. ranking: on a synthetic .evec the nearest ancient group is the one whose + 1. ranking: on a synthetic .evec the nearest ancient group is the one whose mean position is closest on the weighted PCs, a group far on PC1 but with a small eigenvalue on PC3 is ranked by the weighted distance, and - each row carries its dates, country and member count; - 3. axes: a run whose PC2 has the opposite sign is turned back before the + each row carries its dates (oldest first, AADR style BCE), country and + member count; + 2. axes: a run whose PC2 has the opposite sign is turned back before the person is placed, and a run whose present-day coordinates differ stops with the reason instead of placing the person on other axes; - 4. liftover: a site inside a block maps with the block's offset, a site on - a minus-strand chain maps to the mirrored position with its alleles - complemented, a site in a gap or covered by two chains is dropped; - 5. genotypes: a bcftools genotype gives the copies of the .snp file's first - allele, phased or not; a half call, an indel and an allele that is - neither of the two are missing; person-geno appends one column, 9 where - the person has no genotype; - 6. select: the present-day reference people come from MODERN_GROUPS, the + 3. liftover: lift keeps a site inside a block, a site on a minus-strand + chain with its alleles complemented, and drops a site in a gap, one + whose alleles miss the GRCh38 base, chrX and a monomorphic marker; + 4. genotypes: person-geno appends one column, 9 where the person has no + usable genotype, and stops on a genotype file shorter than the .snp file; + 5. select: the present-day reference people come from MODERN_GROUPS, the ancient ones need a date, a passing assessment, enough sites and a place in West Eurasia, one genetic id per individual; - 7. the report command writes the table, the summary, the text and an SVG + 6. the report command writes the table, the summary, the text and an SVG with the person marked; - 8. bin/render_report.py shows them: an HTML card with the table and the + 7. bin/render_report.py shows them: an HTML card with the table and the plot, and a section of the text report. -Run: python3 tests/test_ancient_report.py +The Rust unit tests (rust/crates/og-ancestry, og-intervals chain) check the +date labels, the dosages and the single-position lift in detail, and +tests/parity checks each command against the Python it replaced. Without an +og binary the test is skipped with a note; with OG_REQUIRED set the skip is a +failure. + +Run: OG=bin/og python3 tests/test_ancient_report.py """ import os import shutil @@ -37,11 +38,13 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import ancient_report as ar # noqa: E402 - -SCRIPT = os.path.join(REPO, "bin", "ancient_report.py") +OG = os.environ.get("OG", "") FAILS = 0 +CAVEAT = "Closeness on a PCA is genetic similarity, not descent" + +if not OG: + print("skip: OG is not set (OG=bin/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) def check(desc, ok, detail=""): @@ -57,17 +60,11 @@ def write(path, text): return path -# --- 1. dates --------------------------------------------------------------------- -check("4250 BP is 2300 BCE, the AADR's midpoint of 2600-2000 BCE", ar.year_label(4250) == "2300 BCE", ar.year_label(4250)) -check("8025 BP is 6075 BCE", ar.year_label(8025) == "6075 BCE", ar.year_label(8025)) -check("1950 BP is 1 BCE", ar.year_label(1950) == "1 BCE", ar.year_label(1950)) -check("1949 BP is 1 CE", ar.year_label(1949) == "1 CE", ar.year_label(1949)) -check("450 BP is 1500 CE", ar.year_label(450) == "1500 CE", ar.year_label(450)) -check("a span in one era is written once", ar.range_label(7250, 6850) == "5300-4900 BCE", ar.range_label(7250, 6850)) -check("a span across the eras names both", ar.range_label(2250, 1750) == "300 BCE-200 CE", ar.range_label(2250, 1750)) -check("a span of one date is that date", ar.range_label(3000, 3000) == "1050 BCE", ar.range_label(3000, 3000)) +def og(*args): + return subprocess.run([OG, "ancient", *args], capture_output=True, text=True) + -# --- 2. ranking ------------------------------------------------------------------- +# --- 1. ranking ------------------------------------------------------------------- # Present-day people M0..M19 build the axes; three ancient groups. EVALS = [9.0, 4.0, 0.01, 0.01] @@ -94,36 +91,62 @@ def evec(rows, evals=EVALS): "B1": {"id": "B1", "group": "Anatolia_N", "date_bp": "8200", "country": "Turkey", "locality": "Barcin"}, "C1": {"id": "C1", "group": "Odd_C", "date_bp": "3000", "country": "Italy", "locality": "Somewhere"}, } -person = [0.08, 0.09, 0.0, 0.0] -rows, n = ar.nearest_populations(person, PANEL, ANCIENTS, EVALS, 4) -check("three ancient groups ranked", n == 3 and len(rows) == 3, (n, rows)) -order = [r["group"] for r in rows] -check("the nearest group is first (Steppe_EBA)", order[0] == "Steppe_EBA", order) -check("a group far only on a PC with a tiny eigenvalue is second, not last (Odd_C)", order[1] == "Odd_C", order) -check("the farthest group is last (Anatolia_N)", order[-1] == "Anatolia_N", order) -st = rows[0] -check("the row has the member count, dates and place", st["n"] == 2 and st["dates"] == "2850-2650 BCE" - and st["country"] == "Russia" and st["locality"] == "Samara", st) -check("ranks are 1..n", [r["rank"] for r in rows] == [1, 2, 3], rows) -unweighted = ar.nearest_populations(person, PANEL, ANCIENTS, [1.0, 1.0, 1.0, 1.0], 4)[0] -check("control: without the eigenvalue weights Odd_C falls behind Anatolia_N", - [r["group"] for r in unweighted] == ["Steppe_EBA", "Anatolia_N", "Odd_C"], [r["group"] for r in unweighted]) +# The person's run: the present-day people with PC2 flipped, as smartpca may +# write it, and the person. +RUN = {k: (lab, [pcs[0], -pcs[1], pcs[2], pcs[3]]) for k, (lab, pcs) in MODERNS.items()} +RUN["S"] = ("Person", [0.08, -0.09, 0.0, 0.0]) -# --- 3. axes ---------------------------------------------------------------------- -run = {k: (lab, [pcs[0], -pcs[1], pcs[2], pcs[3]]) for k, (lab, pcs) in MODERNS.items()} -run["S"] = ("Person", [0.08, -0.09, 0.0, 0.0]) -signs = ar.axis_signs(PANEL, run, 2) -check("a flipped PC2 is found and turned back", signs == [1.0, -1.0], signs) -bad = {k: (lab, [pcs[0] * (1 if int(k[1:]) % 3 else -1)] + pcs[1:]) for k, (lab, pcs) in MODERNS.items()} -try: - ar.axis_signs(PANEL, bad, 2) - check("other axes stop the report", False, "no error") -except ar.Fail as e: - check("other axes stop the report, naming the PC", "PC1" in str(e), str(e)) - -# --- 4. liftover -------------------------------------------------------------------- tmp = tempfile.mkdtemp() try: + stats = write(os.path.join(tmp, "stats.tsv"), "panel_sites\t4\nsites_genotyped\t3\nother_allele\t1\nsites_used\t2\n") + at = os.path.join(tmp, "anc.tsv") + with open(at, "w") as f: + f.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") + for r in ANCIENTS.values(): + f.write(f"{r['id']}\t{r['group']}\t{r['date_bp']}\t50\tx\t{r['locality']}\t{r['country']}\t50\t40\t500000\n") + outs = {k: os.path.join(tmp, f"S_{k}") for k in ("ancient.tsv", "info.tsv", "ancient.txt", "pca.svg")} + + def report(panel_evec, run_evec, with_stats=True): + return og("report", "--panel-evec", panel_evec, "--run-evec", run_evec, "--ancients", at, + "--sample", "S", "--release", "test", *(["--stats", stats] if with_stats else []), + "--out-tsv", outs["ancient.tsv"], "--out-info", outs["info.tsv"], + "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]) + + def table(): + with open(outs["ancient.tsv"]) as f: + head, *rows = [line.split("\t") for line in f.read().splitlines()] + return [dict(zip(head, r)) for r in rows] + + pe = write(os.path.join(tmp, "panel.evec"), evec(PANEL)) + re_ = write(os.path.join(tmp, "run.evec"), evec(RUN)) + p = report(pe, re_) + check("report exits 0", p.returncode == 0, p.stderr) + rows = table() if p.returncode == 0 else [] + order = [r["group"] for r in rows] + check("three ancient groups ranked", len(rows) == 3, rows) + check("the nearest group is first (Steppe_EBA), from the flipped run turned back", + order[:1] == ["Steppe_EBA"], order) + check("a group far only on a PC with a tiny eigenvalue is second, not last (Odd_C)", order[1:2] == ["Odd_C"], order) + check("the farthest group is last (Anatolia_N)", order[-1:] == ["Anatolia_N"], order) + st = rows[0] if rows else {} + check("the row has the member count, dates and place", st.get("n") == "2" and st.get("dates") == "2850-2650 BCE" + and st.get("country") == "Russia" and st.get("locality") == "Samara", st) + check("ranks are 1..n", [r["rank"] for r in rows] == ["1", "2", "3"], rows) + flat_pe = write(os.path.join(tmp, "flat_panel.evec"), evec(PANEL, [1.0, 1.0, 1.0, 1.0])) + flat_re = write(os.path.join(tmp, "flat_run.evec"), evec(RUN, [1.0, 1.0, 1.0, 1.0])) + p = report(flat_pe, flat_re) + check("control: without the eigenvalue weights Odd_C falls behind Anatolia_N", + p.returncode == 0 and [r["group"] for r in table()] == ["Steppe_EBA", "Anatolia_N", "Odd_C"], p.stderr) + + # --- 2. axes ---------------------------------------------------------------------- + p = og("align", "--panel-evec", pe, "--run-evec", re_, "--pcs", "2") + check("align: a flipped PC2 is found", p.returncode == 0 and "PC1+ PC2-" in p.stdout, (p.stdout, p.stderr)) + bad = {k: (lab, [pcs[0] * (1 if int(k[1:]) % 3 else -1)] + pcs[1:]) for k, (lab, pcs) in MODERNS.items()} + bad_re = write(os.path.join(tmp, "bad.evec"), evec(bad)) + p = og("align", "--panel-evec", pe, "--run-evec", bad_re, "--pcs", "2") + check("align: other axes stop, naming the PC", p.returncode == 1 and "PC1" in p.stderr, (p.returncode, p.stderr)) + + # --- 3. liftover -------------------------------------------------------------------- chain = write(os.path.join(tmp, "test.chain"), "\n".join([ # chr1 source 1000-1100 -> target 2000-2100 (+), gap of 50, then 1150-1200 -> 2150-2200 "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1", @@ -134,19 +157,6 @@ def evec(rows, evals=EVALS): # chr3: two chains over 100-200 "chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3", "100", "", "chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4", "100", ""]) + "\n") - ix = ar.index_blocks(ar.read_chain(chain)) - check("a site in the first block moves by its offset", ar.lift_point(ix, "chr1", 1051) == ("chr1", 2051, "+"), - ar.lift_point(ix, "chr1", 1051)) - check("a site in the second block moves by its offset", ar.lift_point(ix, "chr1", 1160) == ("chr1", 2160, "+"), - ar.lift_point(ix, "chr1", 1160)) - check("a site in the gap is dropped", ar.lift_point(ix, "chr1", 1120) is None, ar.lift_point(ix, "chr1", 1120)) - # 0-based 510 -> reverse 310 -> forward 10000 - 1 - 310 = 9689 (0-based) = 9690 1-based - check("a minus-strand site is mirrored", ar.lift_point(ix, "chr2", 511) == ("chr2", 9690, "-"), - ar.lift_point(ix, "chr2", 511)) - check("a site two chains cover is dropped", ar.lift_point(ix, "chr3", 171) is None, ar.lift_point(ix, "chr3", 171)) - check("a site one of them covers maps", ar.lift_point(ix, "chr3", 121) == ("chr3", 121, "+"), - ar.lift_point(ix, "chr3", 121)) - # The lift command against a small FASTA. seq1 = ["N"] * 3000 seq1[2050] = "G" # 1-based 2051 seq1[2159] = "T" # 1-based 2160 @@ -174,38 +184,32 @@ def evec(rows, evals=EVALS): "rs7 1 0.0 1051 G X", # monomorphic marker: dropped ]) + "\n") out = os.path.join(tmp, "sites.tsv") - rc = ar.main(["lift", "--snp", snp, "--chain", chain, "--fasta", fa, "--out", out]) + p = og("lift", "--snp", snp, "--chain", chain, "--fasta", fa, "--out", out) got = open(out).read().splitlines() if os.path.exists(out) else [] check("lift keeps rs1, rs2 and the complemented rs4, in GRCh38 order", - rc == 0 and got == ["snp\tchrom\tpos\tallele1\tallele2", "rs1\tchr1\t2051\tG\tA", - "rs2\tchr1\t2160\tC\tT", "rs4\tchr2\t9690\tC\tA"], (rc, got)) + p.returncode == 0 and got == ["snp\tchrom\tpos\tallele1\tallele2", "rs1\tchr1\t2051\tG\tA", + "rs2\tchr1\t2160\tC\tT", "rs4\tchr2\t9690\tC\tA"], (p.returncode, got, p.stderr)) - # --- 5. genotypes ----------------------------------------------------------------- - check("A/A is two copies of allele1 A", ar.dosage("A/A", "A", "G") == 2) - check("A|G is one copy", ar.dosage("A|G", "A", "G") == 1) - check("G/G is none", ar.dosage("G/G", "A", "G") == 0) - check("a half call is missing", ar.dosage("A", "A", "G") is None) - check("an allele outside the pair is missing", ar.dosage("A/T", "A", "G") is None) - check("an indel is missing", ar.dosage("A/AT", "A", "G") is None) + # --- 4. genotypes ----------------------------------------------------------------- msnp = write(os.path.join(tmp, "moderns.snp"), "rs1 1 0.0 1051 G A\nrs2 1 0.0 1160 C T\nrsX 23 0.0 5 A G\nrs4 2 0.0 511 G T\n") mgeno = write(os.path.join(tmp, "moderns.geno"), "012\n220\n999\n101\n") gts = write(os.path.join(tmp, "gts.tsv"), "chr1\t2051\tG\tA\tG/A\nchr1\t2160\tT\t.\tT/T\nchr2\t9690\tC\tG\tC/G\n") merged = os.path.join(tmp, "merged.geno") - stats = os.path.join(tmp, "stats.tsv") - rc = ar.main(["person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", mgeno, - "--out", merged, "--stats", stats]) + pstats = os.path.join(tmp, "pstats.tsv") + p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", mgeno, + "--out", merged, "--stats", pstats) got = open(merged).read().splitlines() if os.path.exists(merged) else [] check("person-geno appends one column: 1, 0, 9 for X, 9 for an allele outside the pair", - rc == 0 and got == ["0121", "2200", "9999", "1019"], (rc, got)) - st = dict(l.split("\t") for l in open(stats).read().splitlines()) if os.path.exists(stats) else {} + p.returncode == 0 and got == ["0121", "2200", "9999", "1019"], (p.returncode, got, p.stderr)) + st = dict(line.split("\t") for line in open(pstats).read().splitlines()) if os.path.exists(pstats) else {} check("its counts", st == {"panel_sites": "4", "sites_genotyped": "3", "other_allele": "1", "sites_used": "2"}, st) short = write(os.path.join(tmp, "short.geno"), "012\n220\n") - rc = ar.main(["person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", short, - "--out", merged, "--stats", stats]) - check("a genotype file shorter than the .snp file stops", rc == 1, rc) + p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", short, + "--out", merged, "--stats", pstats) + check("a genotype file shorter than the .snp file stops", p.returncode == 1, (p.returncode, p.stderr)) - # --- 6. select ------------------------------------------------------------------------ + # --- 5. select ------------------------------------------------------------------------ head = ["Genetic ID (long text)", "Persistent Genetic ID", "Individual ID", '"Date mean in BP in years before 1950"', "Date standard deviation in BP [x]", "Full Date One of two formats", "Group ID", "Locality", "Political Entity", "Latitude", "Longitude", "Data type", @@ -231,43 +235,32 @@ def arow(gid, ind, bp, group, lat, lon, dtype, snps, ass): anno = write(os.path.join(tmp, "x.anno"), "\n".join("\t".join(r) for r in [head] + rows_) + "\n") ind = write(os.path.join(tmp, "x.ind"), "".join(f"{r[0]} U {r[6]}\n" for r in rows_)) oind, plist, anc = (os.path.join(tmp, n) for n in ("panel.ind", "poplist.txt", "ancients.tsv")) - rc = ar.main(["select", "--anno", anno, "--ind", ind, "--out-ind", oind, "--poplist", plist, "--ancients", anc]) - labels = dict(l.split("\t")[0::2] for l in open(oind).read().splitlines()) if rc == 0 else {} + p = og("select", "--anno", anno, "--ind", ind, "--out-ind", oind, "--poplist", plist, "--ancients", anc) + rc = p.returncode + labels = dict(line.split("\t")[0::2] for line in open(oind).read().splitlines()) if rc == 0 else {} check("select: the 60 French shotgun genomes are the reference, one id per person (F0.DG)", - rc == 0 and sum(1 for v in labels.values() if v == "French") == 60 and labels.get("F0.SG") == "Ignore", labels) + rc == 0 and sum(1 for v in labels.values() if v == "French") == 60 and labels.get("F0.SG") == "Ignore", + (labels, p.stderr)) check("select: Yakut and a Human Origins genome are ignored", labels.get("Y1.DG") == "Ignore" and labels.get("H1.HO") == "Ignore", labels) check("select: only A1 is projected", [k for k, v in labels.items() if v == "Ancient"] == ["A1.AG"], labels) check("select: the population list", open(plist).read() == "French\n" if rc == 0 else False) check("select: ancients.tsv holds A1 with its group", - rc == 0 and open(anc).read().splitlines()[1].startswith("A1.AG\tRussia_Steppe\t4800"), open(anc).read() if rc == 0 else "") + rc == 0 and open(anc).read().splitlines()[1].startswith("A1.AG\tRussia_Steppe\t4800"), + open(anc).read() if rc == 0 else "") - # --- 7. the report command ---------------------------------------------------------- - pe = write(os.path.join(tmp, "panel.evec"), evec(PANEL)) - re_ = write(os.path.join(tmp, "run.evec"), evec(run)) - at = os.path.join(tmp, "anc.tsv") - with open(at, "w") as f: - f.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") - for r in ANCIENTS.values(): - f.write(f"{r['id']}\t{r['group']}\t{r['date_bp']}\t50\tx\t{r['locality']}\t{r['country']}\t50\t40\t500000\n") - outs = {k: os.path.join(tmp, f"S_{k}") for k in ("ancient.tsv", "info.tsv", "ancient.txt", "pca.svg")} - p = subprocess.run([sys.executable, SCRIPT, "report", "--panel-evec", pe, "--run-evec", re_, "--ancients", at, - "--sample", "S", "--release", "test", "--stats", stats, - "--out-tsv", outs["ancient.tsv"], "--out-info", outs["info.tsv"], - "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]], capture_output=True, text=True) + # --- 6. the report command ---------------------------------------------------------- + p = report(pe, re_) check("report exits 0", p.returncode == 0, p.stderr) - tsv = open(outs["ancient.tsv"]).read().splitlines() if p.returncode == 0 else [] - check("report: the table's first row is Steppe_EBA, from the flipped run turned back", - len(tsv) == 4 and tsv[1].split("\t")[1] == "Steppe_EBA", tsv) - info = dict(l.split("\t") for l in open(outs["info.tsv"]).read().splitlines()[1:]) if p.returncode == 0 else {} + info = dict(line.split("\t") for line in open(outs["info.tsv"]).read().splitlines()[1:]) if p.returncode == 0 else {} check("report: the person's PC2 in the panel's sign", info.get("pc2") == "0.09000", info) check("report: the sites used", info.get("sites_used") == "2", info) svg = open(outs["pca.svg"]).read() if p.returncode == 0 else "" check("report: the SVG is an SVG with a dark background and the person's label", svg.startswith("S" in svg and svg.rstrip().endswith(""), svg[:200]) txt = open(outs["ancient.txt"]).read() if p.returncode == 0 else "" - check("report: the text carries the caveat and says qpAdm is not run", ar.CAVEAT in txt and "qpAdm" in txt, txt) - # --- 8. both reports read the outputs ---------------------------------------------- + check("report: the text carries the caveat and says qpAdm is not run", CAVEAT in txt and "qpAdm" in txt, txt) + # --- 7. both reports read the outputs ---------------------------------------------- sd = os.path.join(tmp, "S") os.makedirs(os.path.join(sd, "ancient")) for k, f in outs.items(): @@ -284,10 +277,7 @@ def arow(gid, ind, bp, group, lat, lon, dtype, snps, ass): check("the text report has the section with the nearest group and the caveat", "Ancient Populations (AADR PCA, step 44)" in text and "1. Steppe_EBA (" in text and "not descent" in text, text[-600:]) - p = subprocess.run([sys.executable, SCRIPT, "report", "--panel-evec", pe, "--run-evec", pe, "--ancients", at, - "--sample", "S", "--release", "test", "--out-tsv", outs["ancient.tsv"], "--out-info", - outs["info.tsv"], "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]], - capture_output=True, text=True) + p = report(pe, pe, with_stats=False) check("report: a run without the person stops", p.returncode == 1 and "no row for S" in p.stderr, p.stderr) finally: shutil.rmtree(tmp) diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index a54d508e9..844658cd4 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""bin/archaic_report.py (step 43) on a synthetic archaic panel, with the real IBDmix image. +"""og archaic (step 43) on a synthetic archaic panel, with the real IBDmix image. Checks: 1. table: a 1000 Genomes site keeps its frequencies and gets each archaic @@ -37,7 +37,12 @@ Set ARCHAIC_TEST_NO_DOCKER=1 to skip part 4 on a machine without docker; CI never sets it. -Run: python3 tests/test_archaic_report.py +The Rust unit tests (rust/crates/og-ancestry) check pick_call's symmetry and +which genes touch a segment's ends, and tests/parity checks each command +against the Python it replaced. Without an og binary the test is skipped with +a note; with OG_REQUIRED set the skip is a failure. + +Run: OG=bin/og python3 tests/test_archaic_report.py """ import gzip import os @@ -49,10 +54,17 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import archaic_report as ar # noqa: E402 - -SCRIPT = os.path.join(REPO, "bin", "archaic_report.py") +OG = os.environ.get("OG", "") FAILS = 0 +TABLE_HEADER = ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", + "AF_ALL", "AF_AFR", "AF_AMR", "AF_EAS", "AF_EUR", "AF_SAS"] +SEGMENT_HEADER = ["chrom", "start", "end", "length_bp", "ancestry", "matched", "lod", "genes"] +MASKED = 10 +ARCHAICS = ("Altai", "Vindija", "Denisova") + +if not OG: + print("skip: OG is not set (OG=bin/og)") + sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) def check(desc, ok, detail=""): @@ -68,7 +80,7 @@ def write(path, text): def run(*args): - return subprocess.run([sys.executable, SCRIPT, *args], capture_output=True, text=True) + return subprocess.run([OG, "archaic", *args], capture_output=True, text=True) def image(var): @@ -166,25 +178,30 @@ def table_info(out, extra=()): r.returncode == 1 and "no interval" in r.stderr, r.stderr) # --- 1b. strict: rows outside the GRCh38 strict mask are marked, not dropped ------- -# Lifted rows (13 columns, no header). Strict mask 1-based 101-300 on chr22: -# 100 is outside, 150 and 300 inside, 301 outside. A duplicate position keeps -# the first row. Codes 0, 1, 2 outside become 10, 11, 12; 9 stays 9; a code -# already masked is not marked twice. +# Lifted rows (13 columns, no header), sorted by position as Picard writes +# them. Strict mask 1-based 101-300 on chr22: 100 is outside, 150 and 300 +# inside, 301 outside. A duplicate position keeps the first row. Codes 0, 1, 2 +# outside become 10, 11, 12; 9 stays 9; a code already masked is not marked +# twice. rows = ["chr22\t100\tA\tG\t2\t0\t9\t0.5\t0\t0\t0\t0\t0", "chr22\t150\tA\tG\t1\t1\t1\t0.5\t0\t0\t0\t0\t0", + "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0", "chr22\t301\tC\tT\t0\t10\t12\t0.5\t0\t0\t0\t0\t0", - "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0", - "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0"] + "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0"] write(f"{TMP}/lifted.tsv", "\n".join(rows) + "\n") write(f"{TMP}/strict.bed", "chr22\t100\t300\n") r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked.tsv") check("strict exits 0", r.returncode == 0, r.stderr) got = [l.rstrip("\n").split("\t") for l in open(f"{TMP}/marked.tsv")] by = {int(c[1]): c[4:7] for c in got} -check("every position is kept, once", [int(c[1]) for c in got] == [100, 150, 301, 300], [c[1] for c in got]) +check("every position is kept, once", [int(c[1]) for c in got] == [100, 150, 300, 301], [c[1] for c in got]) check("outside the strict mask 2, 0 and unknown become 12, 10 and 9", by[100] == ["12", "10", "9"], by.get(100)) check("inside the strict mask the calls are untouched (last base 300 too)", by[150] == ["1", "1", "1"] and by[300] == ["0", "0", "0"], (by.get(150), by.get(300))) check("a masked call stays as it is, and the first of a duplicate wins", by[301] == ["10", "10", "12"], by.get(301)) +write(f"{TMP}/unsorted.tsv", "\n".join([rows[0], rows[3], rows[2]]) + "\n") +r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/unsorted.tsv", "--out", f"{TMP}/marked3.tsv") +check("rows out of position order stop the step (a duplicate is found only next to its twin)", + r.returncode == 2 and "sorted" in r.stderr and not os.path.exists(f"{TMP}/marked3.tsv"), (r.returncode, r.stderr)) write(f"{TMP}/strict_other.bed", "chr21\t0\t10\n") r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict_other.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked2.tsv") check("a strict mask with no interval on the chromosome stops the step", r.returncode != 0 and "no interval" in r.stderr, r.stderr) @@ -192,7 +209,7 @@ def table_info(out, extra=()): # --- 2. genotypes ------------------------------------------------------------------ tdir = f"{TMP}/table" os.makedirs(tdir) -hdr = "\t".join(ar.TABLE_HEADER) + "\n" +hdr = "\t".join(TABLE_HEADER) + "\n" rows = ["chr22\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5", "chr22\t2000\tC\tT\t0\t0\t0\t0.01\t0\t0\t0\t0.02\t0", "chr22\t3000\tG\tA\t1\t1\t2\t0\t0\t0\t0\t0\t0", @@ -267,12 +284,6 @@ def table_info(out, extra=()): check("a 0/0 half after the carried SNV keeps the SNV call", code3(6000) == "1", g3.get(6000)) check("a usable record of the same SNV wins over a filtered one", code3(7000) == "2", g3.get(7000)) check("two 0/0 halves are the reference (0)", code3(8000) == "0", g3.get(8000)) -pc_code = lambda r: 9 if r is ar.CLASH else r[2] -check("pick_call gives the same code in either order", - all(pc_code(ar.pick_call(x, y)) == pc_code(ar.pick_call(y, x)) - for x, y in [(("A", "G", 1), ("A", None, 9)), (("A", "G", 1), ("A", "T", 0)), (("A", "G", 9), ("A", "G", 2)), - (("A", "G", 0), ("A", "T", 0)), (("A", "G", 1), ("A", "T", 2)), (("A", "G", 9), ("A", "T", 0)), - (("A", "G", 1), ("A", "G", 2))])) # Masked calls: IBDmix gets the call itself and a mask per genome of the # positions where it is masked, the runs of written rows closed by the next @@ -291,7 +302,7 @@ def table_info(out, extra=()): check("IBDmix gets each masked call as its ALT count (12, 10, 11 are 2, 0, 1)", gm.get(100) == ["2", "0", "9"] and gm.get(200) == ["0", "1", "0"] and gm.get(600) == ["1", "0", "0"], gm) masks = {g: open(f"{TMP}/gtm/chr22.{g}.mask.bed").read() if os.path.exists(f"{TMP}/gtm/chr22.{g}.mask.bed") else None - for g in ar.ARCHAICS} + for g in ARCHAICS} check("Altai's mask: 100-200, then 400-600 (the left-out row 500 does not close it)", masks["Altai"] == "22\t99\t200\n22\t399\t600\n", masks["Altai"]) check("Vindija's mask: 200 alone", masks["Vindija"] == "22\t199\t200\n", masks["Vindija"]) @@ -321,7 +332,7 @@ def table_info(out, extra=()): "--out-dir", f"{TMP}/out") check("report exits 0", r.returncode == 0, r.stderr) segs = [l.split("\t") for l in open(f"{TMP}/out/S1_archaic_segments.tsv").read().splitlines()] -check("segments header", segs[0] == ar.SEGMENT_HEADER, segs[0]) +check("segments header", segs[0] == SEGMENT_HEADER, segs[0]) body = segs[1:] check("two segments: the merged Neanderthal one and the Denisovan one", len(body) == 2, body) check("Altai 1.0-1.3 and Vindija 1.2-1.5 Mb merge into Neanderthal 1.0-1.5 Mb", @@ -367,9 +378,6 @@ def table_info(out, extra=()): check("a row cut after its key is unreadable, not a crash", False) except cs.Unreadable: check("a row cut after its key is unreadable, not a crash", True) -genes = {"chr22": [(90, 100, "ENDS_AT_START"), (100, 120, "INSIDE"), (200, 210, "STARTS_AT_END"), (50, 99, "BEFORE")]} -check("genes: a segment [100, 200) takes a gene ending at its first site, not one starting at its end", - ar.genes_in(genes, "22", 100, 200) == ["ENDS_AT_START", "INSIDE"], ar.genes_in(genes, "22", 100, 200)) check("no file: the section is missing", cs.sec_archaic(f"{TMP}/nowhere", "S1") == (None, {})) # --- 4. the real IBDmix ------------------------------------------------------------- @@ -411,14 +419,14 @@ def synthetic(d, plant, mid=None): if mid and MID[0] <= pos < MID[1]: if kind == 0: g = [False, False] - ta, tv = (al + ar.MASKED, vi + ar.MASKED) if mid == "masked" else (9, 9) + ta, tv = (al + MASKED, vi + MASKED) if mid == "masked" else (9, 9) rows.append(f"chr22\t{pos}\tA\tG\t{ta}\t{tv}\t{de}\t" + "\t".join([f"{f:.4f}"] * 6)) if plant and DEN[0] <= pos < DEN[1]: g[0] = de == 2 if any(g): calls.append(f"chr22\t{pos}\t.\tA\tG\t50\tPASS\t.\tGT\t{'1/1' if all(g) else '0/1'}") with gzip.open(f"{d}/table/chr22.tsv.gz", "wt") as f: - f.write("\t".join(ar.TABLE_HEADER) + "\n" + "\n".join(rows) + "\n") + f.write("\t".join(TABLE_HEADER) + "\n" + "\n".join(rows) + "\n") write(f"{d}/s.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + "\n".join(calls) + "\n") From 796f4cd6681d4a72aa0a6c4393002e95352a576d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:24:21 +0100 Subject: [PATCH 038/255] docs(ancestry): steps 43 and 44 run og archaic and og ancient Bead: pgp-bs9u.21 --- CHANGELOG.md | 8 ++++++++ docs/43-archaic.md | 6 +++--- docs/44-ancient.md | 2 +- docs/rust.md | 6 +++--- rust/README.md | 2 +- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f5f4625..7dfccbfb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 43's genotype files for 3 million table rows take 6 s and 159 MB instead of 12 s and 213 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: + - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes, so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. + - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. + - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. + - A gzip input is told by its first bytes, not by a `.gz` name, and a `.fai` that does not match its FASTA stops `og ancient lift`. + - Malformed input stops with one `ERROR:` line, not a Python traceback. + - `sites_genotyped` in step 44's stats still counts a position twice when the AADR holds two SNPs there, as the Python did; the genotypes themselves are right. + - Step 37's own Y tools now run in the Rust `og` binary: `og y tree-positions` and `og y tree-place` (the placement on the current YFull YTree), `og y private-snps` (the private SNP list) and `og y yleaf-positions` (Yleaf's marker positions). They replace `bin/y_tree_place.py` and `bin/y_private_snps.py`, which are gone, and the `positions` command of `bin/yleaf_run.py`, which keeps only `predict`. og is static, so each part still runs in the Yleaf image. The three tables are the same, byte for byte, on the 43 parity cases, except where a line below says otherwise. On the pinned YTree 14.06.0 with a synthetic pileup of 362,055 positions, `og y tree-place` takes 0.8 s and 99 MB where the Python took 3.6 s and 389 MB; on a synthetic chrY VCF of 20,000 records, `og y private-snps` takes 0.3 s and 24 MB where the Python took 1.1 s and 330 MB. What changes on purpose: - `og y tree-place` reads the pileup by contig and position, and only its `chrY` (or `Y`) lines. The Python keyed it by position only, so a line on another contig could overwrite a chrY count. Lines on another contig are now counted at the end of the step line, and a pileup with both `chrY` and `Y` lines stops with exit 1. Step 37 makes a chrY-only pileup, so its results do not change. - `og y private-snps` refuses a callable mask with more than one FASTA record (exit 1). The Python joined the records into one sequence, which shifts every position after the first. The pinned mask has one record. diff --git a/docs/43-archaic.md b/docs/43-archaic.md index 25e33f93b..3e706e1ba 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -13,7 +13,7 @@ Most people outside Africa carry about 2% Neanderthal DNA, and people with ances ## Tool - [IBDmix](https://github.com/PrincetonUniversity/IBDmix) 1.0.1 (Chen et al. 2020, Cell 180:677, doi:10.1016/j.cell.2020.01.012) -- `bin/archaic_report.py` (standard library Python): IBDmix's genotype files, then the segments and the summary +- `og archaic` (OwnGenome's Rust binary, `docs/rust.md`): the table at setup, IBDmix's genotype files, then the segments and the summary IBDmix needs no unadmixed reference population. It scores each site from the modern allele frequency and the two genotypes, so it finds archaic segments in Africans too, where methods that use Africans as the unadmixed reference cannot. @@ -57,13 +57,13 @@ Chen et al. do not score every site. Each archaic genome comes with its own mask - a genome's call outside that genome's FilterBed is kept and marked masked in the table (10, 11 or 12: 10 plus its ALT count); - every call at a site inside a segmental duplication (UCSC's GRCh37 track), or inside the window of a 1000 Genomes indel, is marked masked the same way. The window is 5 bp on each side of the indel, reaching past the longer of its REF and ALT, as the published script writes it; - step 43 writes each masked call to IBDmix as its ALT count, and the masked positions to a mask file per chromosome and archaic genome, which it passes to IBDmix with `-r`; -- every call at a site outside the 1000 Genomes GRCh38 strict mask is marked masked the same way, in all three archaic columns. `setup.sh --archaic` does that after the liftover (`archaic_report.py strict`), because the mask is on GRCh38 and the table is built on GRCh37. +- every call at a site outside the 1000 Genomes GRCh38 strict mask is marked masked the same way, in all three archaic columns. `setup.sh --archaic` does that after the liftover (`og archaic strict`), because the mask is on GRCh38 and the table is built on GRCh37. The first version of the masks dropped the masked calls instead (code 9, or no row), and a second one still dropped every row outside the strict mask. IBDmix then had nothing to score at those sites, the mismatches included, so a segment could run through masked sequence that contradicts it. That is why HG002's share stayed above the published figure after the masks; see Runtime below. One of their exclusions is not applied here: the CpG mask, which needs a GRCh37 reference and the chimpanzee, orangutan and macaque SNP sets, none of which this pipeline installs. What that leaves out is written down in the Notes below. -The filters cut hard: on chr22 the strict mask covers 25.2 Mb of 50.8, the Vindija FilterBed 19.6 Mb, and segmental duplications take out 4.3 Mb. Without them the step called sequence from positions the archaic genomes cannot be read at, and HG002's Neanderthal total was higher still: the filters took it from 115.2 Mb to 107.5 Mb, which is still 2.1 times the published median (see the same-person section under Runtime). The filters are part of the data version: a change to a pin or to a mask means `snpad2016-1kgp3-v3` and a rebuild. `setup.sh --archaic` also rebuilds a table whose build code changed (its stamp holds the sha256 of the table half of `bin/archaic_report.py`), so a table built before the masked codes, or before the strict-mask rows were kept, is rebuilt on the next `setup.sh --archaic`. Until then step 43 reads it as before: a table with no masked code gives empty masks. +The filters cut hard: on chr22 the strict mask covers 25.2 Mb of 50.8, the Vindija FilterBed 19.6 Mb, and segmental duplications take out 4.3 Mb. Without them the step called sequence from positions the archaic genomes cannot be read at, and HG002's Neanderthal total was higher still: the filters took it from 115.2 Mb to 107.5 Mb, which is still 2.1 times the published median (see the same-person section under Runtime). The filters are part of the data version: a change to a pin or to a mask means `snpad2016-1kgp3-v3` and a rebuild. `setup.sh --archaic` also rebuilds a table whose format changed (its stamp holds the table format id of `og archaic`, `ARCHAIC_TABLE_FORMAT` in `setup.sh`), so a table built before the masked codes, or before the strict-mask rows were kept, is rebuilt on the next `setup.sh --archaic`. A table the last Python build code wrote has the same bytes, so its stamp is taken as current once and rewritten, and no install downloads the archaic genomes again for the move to Rust. Until then step 43 reads it as before: a table with no masked code gives empty masks. The table is built on GRCh37 and lifted once to GRCh38 with Picard `LiftoverVcf`. A site that does not lift, or lifts to another chromosome, is dropped. The archaic files are downloaded one chromosome at a time (150 GB in all, at most 13.8 GB at once, for chromosome 2) and deleted once the chromosome is built, so an interrupted install resumes at the chromosome it stopped in. `ARCHAIC_REGIONS` (a GRCh37 BED, chromosomes named 1 to 22) builds only those regions. diff --git a/docs/44-ancient.md b/docs/44-ancient.md index 785913ca9..ec0c31253 100644 --- a/docs/44-ancient.md +++ b/docs/44-ancient.md @@ -14,7 +14,7 @@ Most people who look at their ancestry want to know where they fall among ancien - **smartpca** and **convertf** (EIGENSOFT 9.0.0): the PCA, and the conversion of the present-day people to EIGENSTRAT text. 9.0.0 is the first release that reads the AADR's TGENO genotype format. - **bcftools** (`convert --gvcf2vcf`, `view`, `query`) reads your genotypes at the panel's sites from the gVCF. -- **`bin/ancient_report.py`** (Python standard library): chooses the panel's people, lifts the AADR sites to GRCh38, adds your column to the present-day genotypes, aligns the axes and writes the report and the plot. +- **`og ancient`** (OwnGenome's Rust binary, `docs/rust.md`): chooses the panel's people, lifts the AADR sites to GRCh38, adds your column to the present-day genotypes, aligns the axes and writes the report and the plot. ## Docker Image diff --git a/docs/rust.md b/docs/rust.md index 241f92311..5c60ea02c 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -2,7 +2,7 @@ This page is for contributors. OwnGenome's own tools are moving from Python to Rust, one tool per pull request, into a single binary, `og`. This page says how the Rust code is laid out, how the binary reaches the pipeline, how it stays byte-for-byte compatible with the Python it replaces, and what CI checks. -Steps 04, 05, 07 and 25 call `og` (the filters of `og sv` and `og pgx`, and `og prs format`), step 06 and its Nextflow module `CLINVAR_SCREEN` call `og clinvar`, step 33 calls it (`og qc sex-loss`, the mosaic loss of Y or X), steps 23 and 31 and their Nextflow modules call it (`og vcf csq-rank` and `og table constraint-join`), step 27 and `CPIC_LOOKUP` call it (`og pgx cpic-report`), and so do steps 09b, 21 and 32: the CYP2D6 depth check (`og pgx cyp2d6-bed`, `cyp2d6-measure`, `cyp2d6-check`) and the RFC1 motif count (`og str motif`) replaced their Python. Steps 06, 17 and 31 and their Nextflow modules run the reads at the variant in `og`: the cDNA read check and scan (`og cdna check`, `og cdna scan`) and the phase of the compound-het pairs (`og comphet phase`); step 32 and its module read the BAM's contig names with `og bam contigs`. `scripts/setup.sh` calls it for the STRchive catalog, Exomiser's zips, the genetic map and the masked reference. Step 26b calls `og ancestry-fine`. Step 40 calls `og consumer`. Step 38 and the Nextflow family workflow call `og family`. So a run that reaches those steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull. The pieces below are the base every port builds on. +Steps 04, 05, 07 and 25 call `og` (the filters of `og sv` and `og pgx`, and `og prs format`), step 06 and its Nextflow module `CLINVAR_SCREEN` call `og clinvar`, step 33 calls it (`og qc sex-loss`, the mosaic loss of Y or X), steps 23 and 31 and their Nextflow modules call it (`og vcf csq-rank` and `og table constraint-join`), step 27 and `CPIC_LOOKUP` call it (`og pgx cpic-report`), and so do steps 09b, 21 and 32: the CYP2D6 depth check (`og pgx cyp2d6-bed`, `cyp2d6-measure`, `cyp2d6-check`) and the RFC1 motif count (`og str motif`) replaced their Python. Steps 06, 17 and 31 and their Nextflow modules run the reads at the variant in `og`: the cDNA read check and scan (`og cdna check`, `og cdna scan`) and the phase of the compound-het pairs (`og comphet phase`); step 32 and its module read the BAM's contig names with `og bam contigs`. `scripts/setup.sh` calls it for the STRchive catalog, Exomiser's zips, the genetic map and the masked reference. Step 26b calls `og ancestry-fine`, step 43 and `setup.sh --archaic` call `og archaic`, and step 44 and `setup.sh --ancient-panel` call `og ancient`. Step 40 calls `og consumer`. Step 38 and the Nextflow family workflow call `og family`. So a run that reaches those steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull. The pieces below are the base every port builds on. ## Layout @@ -26,7 +26,7 @@ rust/ crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read crates/og-bloodtype/ step 41, blood groups: og bloodtype report, check-markers, systems - crates/og-ancestry/ step 26b, fine-grained ancestry: og ancestry-fine + crates/og-ancestry/ steps 26b, 43 and 44: og ancestry-fine, og archaic, og ancient crates/og-reports/ the opt-in step reports of steps 45, 47, 48 and 50 to 53, and step 37's Y tree tools crates/og-lookup/ the genotype lookup of scripts/owngenome-lookup.sh: og lookup query, resolve, report crates/og-fresh/ og dev freshness: the monthly freshness issue's upstream lookups, typed answers @@ -36,7 +36,7 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, the commands of `copy_number.py` (which keeps the catalog reader the catalog tests import) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. diff --git a/rust/README.md b/rust/README.md index 5071aad12..a6a412a2a 100644 --- a/rust/README.md +++ b/rust/README.md @@ -18,7 +18,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-lookup` | the genotype lookup of `scripts/owngenome-lookup.sh`: `og lookup query`, `og lookup resolve`, `og lookup report` | | `crates/og-filters` | the VCF filters of steps 04, 05 and 07: `og sv svlen-fix`, `og sv inv-filter`, `og pgx refblock-fill`, `og pgx mito-merge` | | `crates/og-setup` | what `scripts/setup.sh` runs once: `og str catalog` (the STRchive catalog of steps 09 and 9b), `og setup unzip` (Exomiser's zips), `og setup genetic-map` (the genetic map of steps 26b and 51) | -| `crates/og-ancestry` | step 26b, fine-grained ancestry: `og ancestry-fine panel-map`, `check-map`, `order`, `sample-bed`, `chroms`, `report` | +| `crates/og-ancestry` | step 26b, fine-grained ancestry: `og ancestry-fine panel-map`, `check-map`, `order`, `sample-bed`, `chroms`, `report`; step 43's `og archaic table`, `strict`, `genotypes`, `report`; step 44's `og ancient select`, `lift`, `person-geno`, `align`, `report` | | `crates/og-steps` | step helpers, among them `og pgx outside-calls`, `og pgx hla-concordance` (steps 08, 36), `og pgx pypgx-summary` (step 32) and `og paraphase check-bam`, `og paraphase table` (step 35b) | | `crates/og-fresh` | `og dev freshness`: the upstream lookups of the monthly freshness issue, each answer typed, HTTP through the host's `curl` | | `crates/og-recheck` | `og recheck`: which of a finished sample's results a newer ClinVar or catalog would change, `--rerun`, `--affected`, `--launch-dir` | From 7cb54dc8ef4641e4d472ea7142a1c85cac92484f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:28:18 +0100 Subject: [PATCH 039/255] test(ancestry): a mask holds the 1-based bases of its BED intervals only Bead: pgp-bs9u.21 --- rust/crates/og-ancestry/src/archaic.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 3d1b4e811..5e6a01b60 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -1609,6 +1609,20 @@ mod tests { } } + #[test] + fn a_mask_holds_its_bed_bases_only() { + // BED 99-100 and 200-300 are the 1-based bases 100 and 201 to 300. + let mut m = Mask { + iv: vec![(99, 100), (200, 300)], + i: 0, + }; + let got: Vec = [99, 100, 101, 200, 201, 300, 301] + .into_iter() + .filter(|&p| m.holds(p)) + .collect(); + assert_eq!(got, [100, 201, 300]); + } + #[test] fn gt_codes() { assert_eq!(gt_code("0/1"), Some(1)); From 3b492b9a30db6e823bae87486040d9c0462f94e6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:28:18 +0100 Subject: [PATCH 040/255] test(ancestry): RED CONTROL, plant a 1-based mask and a chain block that holds its end Reverted in the next commit. Guard must go red: cargo test in og-ancestry and og-intervals, the archaic-table and ancient-lift parity cases, and tests/test_archaic_report.py. Bead: pgp-bs9u.21 --- rust/crates/og-ancestry/src/archaic.rs | 2 +- rust/crates/og-intervals/src/chain.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 5e6a01b60..ff03f7791 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -224,7 +224,7 @@ struct Mask { impl Mask { fn holds(&mut self, pos: i64) -> bool { - let p = pos - 1; + let p = pos; while self.i < self.iv.len() && self.iv[self.i].1 <= p { self.i += 1; } diff --git a/rust/crates/og-intervals/src/chain.rs b/rust/crates/og-intervals/src/chain.rs index 559a4d71e..160aaf502 100644 --- a/rust/crates/og-intervals/src/chain.rs +++ b/rust/crates/og-intervals/src/chain.rs @@ -240,10 +240,10 @@ impl Chain { // may still reach past p. let mut j = src.blocks.partition_point(|b| i64::from(b.start) <= p); let mut first: Option<&Block> = None; - while j > 0 && i64::from(src.reach[j - 1]) > p { + while j > 0 && i64::from(src.reach[j - 1]) >= p { j -= 1; let b = &src.blocks[j]; - if i64::from(b.end) > p { + if i64::from(b.end) >= p { match first { None => first = Some(b), Some(f) if f.chain != b.chain => return None, From bb71d5692e77523235e27243e6453b9d4f3e4cf2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:38:15 +0100 Subject: [PATCH 041/255] test(ancestry): revert the red control of 3b492b9a Bead: pgp-bs9u.21 --- rust/crates/og-ancestry/src/archaic.rs | 2 +- rust/crates/og-intervals/src/chain.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index ff03f7791..5e6a01b60 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -224,7 +224,7 @@ struct Mask { impl Mask { fn holds(&mut self, pos: i64) -> bool { - let p = pos; + let p = pos - 1; while self.i < self.iv.len() && self.iv[self.i].1 <= p { self.i += 1; } diff --git a/rust/crates/og-intervals/src/chain.rs b/rust/crates/og-intervals/src/chain.rs index 160aaf502..559a4d71e 100644 --- a/rust/crates/og-intervals/src/chain.rs +++ b/rust/crates/og-intervals/src/chain.rs @@ -240,10 +240,10 @@ impl Chain { // may still reach past p. let mut j = src.blocks.partition_point(|b| i64::from(b.start) <= p); let mut first: Option<&Block> = None; - while j > 0 && i64::from(src.reach[j - 1]) >= p { + while j > 0 && i64::from(src.reach[j - 1]) > p { j -= 1; let b = &src.blocks[j]; - if i64::from(b.end) >= p { + if i64::from(b.end) > p { match first { None => first = Some(b), Some(f) if f.chain != b.chain => return None, From 4ab9d0754bc8c11eff60d6230b4798544a6ac6d1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:41:17 +0100 Subject: [PATCH 042/255] fix(ancestry): join genotype calls in one position window --- .github/workflows/guard.yml | 18 ++ rust/crates/og-ancestry/src/archaic.rs | 263 ++++++++++++++++--------- tests/parity/cases.tsv | 2 +- tests/test_archaic_report.py | 31 +++ 4 files changed, 222 insertions(+), 92 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index e22eba988..0f2d5f70a 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -266,6 +266,24 @@ jobs: grep -E 'planted_failure.*FAILED' "$log" || { cat "$log"; echo "ERROR: the run failed, but not on the planted test"; exit 1; } echo "Control: the planted test failed, as it must." + - name: Control (archaic duplicate window must fail when equal positions are split) + run: | + source=crates/og-ancestry/src/archaic.rs + saved=$(mktemp "${RUNNER_TEMP}/archaic-window.XXXXXX") + cp "$source" "$saved" + trap 'cp -f "$saved" "$source"; rm -f "$saved"' EXIT + sed 's/r.pos == kept.pos/r.pos < kept.pos/' "$saved" > "$source" + grep -q 'r.pos < kept.pos' "$source" + log="${RUNNER_TEMP}/archaic-window-red.log" + if cargo test --locked --no-fail-fast -p og-ancestry duplicate_window_keeps_only_one_record_of_lookahead > "$log" 2>&1; then + cat "$log" + echo "ERROR: splitting a duplicate-position window passed" + exit 1 + fi + cat "$log" + grep -E 'duplicate_window_keeps_only_one_record_of_lookahead.*FAILED' "$log" + echo "Control: splitting the window failed its regression test." + - name: cargo test run: cargo test --workspace --locked --no-fail-fast diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 5e6a01b60..4468fc815 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -25,7 +25,7 @@ //! Each command writes what `bin/archaic_report.py` wrote (checked by //! `tests/parity`). Deliberate changes: every output is written under a //! temporary name and renamed into place; a gzip input is told by its first -//! bytes, not its name; `strict` reads its rows in position order and stops +//! bytes, not its name; `strict` and `genotypes` read in position order and stop //! (exit 2) on a row that goes back, instead of keeping every position of a //! chromosome in memory; and malformed input is an `ERROR:` line, never a //! traceback. @@ -726,9 +726,17 @@ struct VcfRec { /// the reference. struct Vcf { text: Text, + previous: Option<(String, i64)>, } impl Vcf { + fn new(text: Text) -> Self { + Self { + text, + previous: None, + } + } + fn next(&mut self) -> Out> { loop { let Some(raw) = self.text.raw()? else { @@ -750,13 +758,28 @@ impl Vcf { } continue; } - if c.len() < 10 || c[3].chars().count() != 1 || c[4] == "." { + if c.len() < 10 { + continue; + } + let pos = need_int(c[1])?; + if let Some((chrom, prev)) = &self.previous + && chrom == c[0] + && pos < *prev + { + let message = format!("{}:{pos} comes after {prev}", c[0]); + return usage(format!( + "{} is not sorted: {message}", + self.text.path.display() + )); + } + self.previous = Some((c[0].to_owned(), pos)); + if c[3].chars().count() != 1 || c[4] == "." { continue; } if c[4].contains(',') || c[4].chars().count() != 1 { return Ok(Some(VcfRec { chrom: c[0].to_owned(), - pos: need_int(c[1])?, + pos, call: Sample { r#ref: c[3].to_owned(), alt: None, @@ -775,7 +798,7 @@ impl Vcf { } return Ok(Some(VcfRec { chrom: c[0].to_owned(), - pos: need_int(c[1])?, + pos, call: Sample { r#ref: c[3].to_owned(), alt: Some(c[4].to_owned()), @@ -786,39 +809,48 @@ impl Vcf { } } -/// The sample's calls on one chromosome, one per position after -/// [`pick_call`], sorted by position: a compact array, not a map. -fn settle(mut recs: Vec<(i64, Sample)>) -> Vec<(i64, Sample)> { - // A stable sort keeps the records of one position in file order. - recs.sort_by_key(|(p, _)| *p); - let mut out: Vec<(i64, Sample)> = Vec::new(); - let mut clashed = false; - for (pos, call) in recs { - match out.last_mut() { - Some((p, kept)) if *p == pos => { - if clashed { - continue; - } - let r#ref = call.r#ref.clone(); - match pick_call(Some(kept.clone()), call) { - Picked::Call(c) => *kept = c, - Picked::Clash => { - *kept = Sample { - r#ref, - alt: None, - code: 9, +/// Reduce one duplicate-position window in file order. Only the reduced +/// call and one record of lookahead survive, even for very long windows. +struct Calls { + vcf: Vcf, + pending: Option, +} + +impl Calls { + fn next(&mut self) -> Out> { + let first = match self.pending.take() { + Some(r) => Some(r), + None => self.vcf.next()?, + }; + let Some(mut kept) = first else { + return Ok(None); + }; + let mut clashed = false; + loop { + match self.vcf.next()? { + Some(r) if r.chrom == kept.chrom && r.pos == kept.pos => { + if !clashed { + let r#ref = r.call.r#ref.clone(); + kept.call = match pick_call(Some(kept.call), r.call) { + Picked::Call(c) => c, + Picked::Clash => { + clashed = true; + Sample { + r#ref, + alt: None, + code: 9, + } + } }; - clashed = true; } } - } - _ => { - clashed = false; - out.push((pos, call)); + other => { + self.pending = other; + return Ok(Some(kept)); + } } } } - out } /// `chrN.tsv.gz` with N from 1 to 22: the chromosome's name. @@ -893,13 +925,14 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { put(&Path::new(out_dir).join("samples.txt"), samples.as_bytes())?; let vcf_path = a.req("--vcf"); - let mut vcf = Vcf { - text: Text::open(vcf_path)?, + let mut calls = Calls { + vcf: Vcf::new(Text::open(vcf_path)?), + pending: None, }; let mut spans: Vec = Vec::new(); let mut done: Vec = Vec::new(); - let mut pending = vcf.next()?; - while let Some(first) = pending.take() { + let mut pending = calls.next()?; + while let Some(first) = &pending { let chrom = first.chrom.clone(); if done.contains(&chrom) { return die(format!( @@ -908,33 +941,24 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { } done.push(chrom.clone()); let table = tables.iter().find(|(c, _)| *c == chrom).map(|(_, t)| t); - // The sample's calls on this chromosome; on a chromosome the table - // does not hold they are read past. - let mut recs = Vec::new(); - if table.is_some() { - recs.push((first.pos, first.call)); - } - loop { - match vcf.next()? { - Some(r) if r.chrom == chrom => { - if table.is_some() { - recs.push((r.pos, r.call)); - } - } - other => { - pending = other; - break; - } + if let Some(table) = table { + if let Some(span) = write_chrom( + &chrom, + table, + &mut calls, + &mut pending, + &head, + &pseudo, + af_col, + two_n, + out_dir, + )? { + spans.push(span); + } + } else { + while pending.as_ref().is_some_and(|r| r.chrom == chrom) { + pending = calls.next()?; } - } - let Some(table) = table else { - continue; - }; - let calls = settle(recs); - if let Some(span) = write_chrom( - &chrom, table, &calls, &head, &pseudo, af_col, two_n, out_dir, - )? { - spans.push(span); } } let mut text = @@ -967,7 +991,8 @@ type Span = (String, i64, i64, u64, u64, u64); fn write_chrom( chrom: &str, table: &str, - calls: &[(i64, Sample)], + calls: &mut Calls, + pending: &mut Option, head: &str, pseudo: &[String], af_col: usize, @@ -991,8 +1016,14 @@ fn write_chrom( let (mut n, mut first, mut last, mut dropped, mut carried) = (0u64, 0i64, 0i64, 0u64, 0u64); // Per archaic genome, the runs of written rows whose call is masked: // [first, last] positions, closed at the next unmasked row. - let mut runs: [Vec<(i64, i64)>; 3] = [Vec::new(), Vec::new(), Vec::new()]; + let mask_paths = + ARCHAICS.map(|name| Path::new(out_dir).join(format!("{chrom}.{name}.mask.bed"))); + let mut masks: Vec = mask_paths + .iter() + .map(|p| AtomicFile::create(p).map_err(|e| cannot_write(p, &e))) + .collect::>()?; let mut open_run: [Option<(i64, i64)>; 3] = [None; 3]; + let mut previous = None; let mut row = String::new(); while let Some(line) = t.next()? { let c: Vec<&str> = line.split('\t').collect(); @@ -1004,6 +1035,13 @@ fn write_chrom( )); } let pos = need_int(c[1])?; + if previous.is_some_and(|p| pos < p) { + return usage(format!( + "{table} is not sorted: {chrom}:{pos} comes after {}", + previous.unwrap() + )); + } + previous = Some(pos); let (r, alt) = (c[2], c[3]); let mut arch = [0u8; 3]; for k in 0..3 { @@ -1020,10 +1058,19 @@ fn write_chrom( } }; } - let code = match calls.binary_search_by_key(&pos, |(p, _)| *p) { - Err(_) => 0, - Ok(i) => { - let call = &calls[i].1; + while pending + .as_ref() + .is_some_and(|r| r.chrom == chrom && r.pos < pos) + { + *pending = calls.next()?; + } + let code = match pending + .as_ref() + .filter(|r| r.chrom == chrom && r.pos == pos) + { + None => 0, + Some(record) => { + let call = &record.call; if call.r#ref != r || call.alt.as_ref().is_some_and(|x| x != alt) { // Another allele than the row's: IBDmix's generate_gt // drops the line. @@ -1052,7 +1099,8 @@ fn write_chrom( Some(run) => run.1 = pos, } } else if let Some(run) = open_run[k].take() { - runs[k].push(run); + writeln!(masks[k], "{num}\t{}\t{}", run.0 - 1, run.1) + .map_err(|e| cannot_write(&mask_paths[k], &e))?; } } row.clear(); @@ -1069,6 +1117,11 @@ fn write_chrom( last = pos; n += 1; } + // Read the tail too: an unsorted or unreadable record past the last + // table site must fail before committing this chromosome's outputs. + while pending.as_ref().is_some_and(|r| r.chrom == chrom) { + *pending = calls.next()?; + } if n == 0 { drop(gz); match std::fs::remove_file(&path) { @@ -1082,18 +1135,13 @@ fn write_chrom( gz.finish().and_then(AtomicFile::commit).map_err(werr)?; // IBDmix's mask (-r): the chromosome as in the genotype file, and a // line covers the positions start + 1 to end, as in a BED. - for k in 0..3 { - if let Some(run) = open_run[k] { - runs[k].push(run); + for (k, mut mask) in masks.into_iter().enumerate() { + if let Some((s, e)) = open_run[k] { + writeln!(mask, "{num}\t{}\t{e}", s - 1) + .map_err(|e| cannot_write(&mask_paths[k], &e))?; } - let text: String = runs[k] - .iter() - .map(|(s, e)| format!("{num}\t{}\t{e}\n", s - 1)) - .collect(); - put( - &Path::new(out_dir).join(format!("{chrom}.{}.mask.bed", ARCHAICS[k])), - text.as_bytes(), - )?; + mask.commit() + .map_err(|e| cannot_write(&mask_paths[k], &e))?; } Ok(Some((chrom.to_owned(), first, last, n, carried, dropped))) } @@ -1155,7 +1203,7 @@ type Merged = (String, i64, i64, f64, Vec<&'static str>); /// Overlapping or touching segments of one class as one. fn merge(mut segs: Vec<(Seg, &'static str)>) -> Vec { - segs.sort_by(|(a, _), (b, _)| (chrom_key(&a.0), a.1, a.2).cmp(&(chrom_key(&b.0), b.1, b.2))); + segs.sort_by_key(|(a, _)| (chrom_key(&a.0), a.1, a.2)); let mut out: Vec = Vec::new(); for ((chrom, start, end, lod), who) in segs { match out.last_mut() { @@ -1205,7 +1253,7 @@ fn classify(per: &[Vec; 3]) -> Vec { rows.push((c, s, e, "Denisovan", vec!["Denisova"], l)); } } - rows.sort_by(|a, b| (chrom_key(&a.0), a.1).cmp(&(chrom_key(&b.0), b.1))); + rows.sort_by_key(|a| (chrom_key(&a.0), a.1)); rows } @@ -1659,16 +1707,49 @@ mod tests { #[test] fn a_clash_ends_its_position() { - let recs = vec![ - (4000, s("T", Some("C"), 1)), - (4000, s("T", Some("G"), 1)), - (4000, s("T", Some("C"), 1)), - (10, s("A", Some("G"), 2)), - ]; - assert_eq!( - settle(recs), - vec![(10, s("A", Some("G"), 2)), (4000, s("T", None, 9))] - ); + let mut calls = sample_calls(&[ + (10, "A", "G", "1/1"), + (4000, "T", "C", "0/1"), + (4000, "T", "G", "0/1"), + (4000, "T", "C", "0/1"), + ]); + assert_eq!(calls.next().unwrap().unwrap().call, s("A", Some("G"), 2)); + assert_eq!(calls.next().unwrap().unwrap().call, s("T", None, 9)); + assert!(calls.next().unwrap().is_none()); + } + + fn sample_calls(rows: &[(i64, &str, &str, &str)]) -> Calls { + let mut text = String::new(); + for (p, r, a, gt) in rows { + text += &format!("chr22\t{p}\t.\t{r}\t{a}\t.\tPASS\t.\tGT\t{gt}\n"); + } + Calls { + vcf: Vcf::new(Text { + path: "sample.vcf".into(), + lines: Lines::new(Box::new(io::Cursor::new(text.into_bytes())), Utf8::Strict), + line: String::new(), + }), + pending: None, + } + } + + #[test] + fn duplicate_window_keeps_only_one_record_of_lookahead() { + let mut rows = vec![(100, "A", "AT", "0/1"); 10_000]; + rows.push((100, "A", "G", "1/1")); + rows.push((200, "C", "T", "0/1")); + rows.push((150, "G", "A", "0/1")); // must not be read in the first window + let mut calls = sample_calls(&rows); + let first = calls.next().unwrap().unwrap(); + assert_eq!(first.pos, 100); + assert_eq!(first.call, s("A", Some("G"), 2)); + assert_eq!(calls.pending.as_ref().unwrap().pos, 200); + let error = calls + .next() + .err() + .expect("the second window sees the unsorted tail"); + assert_eq!(error.0, 2); + assert!(error.1.contains("not sorted")); } #[test] diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index dd63836b6..aae29404c 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -963,7 +963,7 @@ archaic-table empty-exclude gen:archaic-inputs.sh --chrom 22 --modern modern.tsv archaic-strict plain gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows.tsv --out s.tsv s.tsv bytes PYTHON_IMAGE archaic-strict unsorted gen:archaic-inputs.sh --chrom chr22 --strict strict.bed --rows rows-unsorted.tsv --out s.tsv - bytes PYTHON_IMAGE archaic-strict empty-mask gen:archaic-inputs.sh --chrom 22 --strict empty.bed --rows rows.tsv --out s.tsv - bytes PYTHON_IMAGE -archaic-genotypes plain gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop EUR --pseudo 3 --out-dir gt gt/samples.txt,gt/spans.tsv,gt/chr21.gt.gz,gt/chr22.gt.gz,gt/chr22.Altai.mask.bed *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes plain gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop EUR --pseudo 3 --out-dir gt gt/samples.txt,gt/spans.tsv,gt/chr21.gt.gz,gt/chr22.gt.gz,gt/chr21.Altai.mask.bed,gt/chr21.Vindija.mask.bed,gt/chr21.Denisova.mask.bed,gt/chr22.Altai.mask.bed,gt/chr22.Vindija.mask.bed,gt/chr22.Denisova.mask.bed *.gz=gz-content bytes PYTHON_IMAGE archaic-genotypes default gen:archaic-inputs.sh --table-dir . --vcf s.vcf.gz --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE archaic-genotypes afr-one gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop AFR --pseudo 1 --out-dir gt gt/spans.tsv,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE archaic-genotypes two-samples gen:archaic-inputs.sh --table-dir . --vcf two.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index 844658cd4..cffe87e77 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -315,6 +315,37 @@ def table_info(out, extra=()): r = run("genotypes", "--table-dir", bdir, "--vcf", f"{TMP}/m.vcf", "--out-dir", f"{TMP}/gtb") check("a code the table never holds (7) stops the step", r.returncode == 1 and "code '7'" in r.stderr, r.stderr) +# A streaming join must refuse backwards positions, including records past +# the table's end and records whose genotype it would otherwise ignore. +vcf_header = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" +for label, records in ( + ("within-table", [(3000, "G", "A", "GT"), (1000, "A", "G", "GT")]), + ("past-table", [(9000, "A", "G", "GT"), (8000, "A", "G", "GT")]), + ("ignored-call", [(3000, "G", "A", "GT"), (1000, "A", ".", "GQ")])): + bad_vcf = f"{TMP}/{label}.vcf" + write(bad_vcf, vcf_header + "".join( + f"chr22\t{p}\t.\t{ref}\t{alt}\t.\tPASS\t.\t{fmt}\t0/1\n" + for p, ref, alt, fmt in records)) + out = f"{TMP}/order-{label}" + os.makedirs(out) + protected = ["chr22.gt.gz", *(f"chr22.{g}.mask.bed" for g in ARCHAICS)] + for name in protected: + write(f"{out}/{name}", "previous output\n") + r = run("genotypes", "--table-dir", tdir, "--vcf", bad_vcf, "--out-dir", out) + check(f"unsorted VCF {label} exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) + check(f"unsorted VCF {label} keeps prior genotype and masks", + all(open(f"{out}/{name}", "rb").read() == b"previous output\n" for name in protected)) + +unordered_table = f"{TMP}/table_unordered" +os.makedirs(unordered_table) +with gzip.open(f"{unordered_table}/chr22.tsv.gz", "wt") as f: + f.write(hdr + "\n".join([rows[1], rows[0], *rows[2:]]) + "\n") +r = run("genotypes", "--table-dir", unordered_table, "--vcf", f"{TMP}/s.vcf", + "--out-dir", f"{TMP}/gt_unordered") +check("an unsorted table exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) +check("an unsorted table leaves no partial genotype or mask", + not any(name.endswith((".gt.gz", ".mask.bed")) for name in os.listdir(f"{TMP}/gt_unordered"))) + # --- 3. report --------------------------------------------------------------------- idir = f"{TMP}/ibd" os.makedirs(idir) From b101631c6db40d1fe2318d33815b72a70dc0f33c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:44:17 +0100 Subject: [PATCH 043/255] docs(ancestry): record sorted-input contract and bounded memory --- CHANGELOG.md | 3 ++- docs/43-archaic.md | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08831d0a4..bfa29e96b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,9 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 43's genotype files for 3 million table rows take 6 s and 159 MB instead of 12 s and 213 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: +- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes, so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. + - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. - A gzip input is told by its first bytes, not by a `.gz` name, and a `.fai` that does not match its FASTA stops `og ancient lift`. diff --git a/docs/43-archaic.md b/docs/43-archaic.md index 3e706e1ba..d9689a32c 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -72,6 +72,12 @@ The table is built on GRCh37 and lifted once to GRCh38 with Picard `LiftoverVcf` - `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample. A site with no record is read as the reference allele, a record whose FILTER is not PASS as unknown, and a site where you carry another ALT than the table's is left out. - The GENCODE gene lines `setup.sh` installs, for the genes of each segment (without them the segments are listed without genes) +`og archaic genotypes` reads the VCF and table together, keeping one +duplicate-position window. Positions within each chromosome must be in +ascending order in both inputs. A position that goes backwards stops the +command with exit 2. Mask runs are written as they close. Duplicate calls, +clashes, masked sites and the output columns keep their previous meaning. + ## Command ```bash From 76be60b73ce0d1ce40600d3b454a1ecd5560ace0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:07:45 +0100 Subject: [PATCH 044/255] fix(live): keep the map's APOE warning independent of closed results --- CHANGELOG.md | 2 + assets/live/map.js | 5 +- docs/live-site.md | 1 + tests/test_live_demo.py | 181 ++++++++++++++++++++++++++++++++++++- tests/test_live_results.py | 47 ++++++++++ 5 files changed, 231 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f5bc06db..2c250902d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Opening APOE on a child's genome map no longer reveals whether another closed result overlaps APOE. Its map row follows its own warning switch. + - The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. - A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; the address never holds a closable finding's id, closed (`#f/closed-`) or opened (`#f/opened-`), so no history entry keeps it, and any other `#f/` on such a page, a closed result's id or nothing, lands on the first closed finding, so a guessed id tells nothing. The home page links a child's closed result to the child's report, not to the finding's own page. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and so do their own findings (a medicine gene, a CHIP call) and the lead's words on them, the ClinVar card's lines for the known deep intronic sites, the carrier list's two-hit line and the callability card's rows, all together; a table row, a finding or a chart mark closes with a result only by its gene and place fields, never by a code such as the inheritance `AR` or the structural-variant type `INS`, and prose is never read word by word against gene symbols: a card's headline, its lines and facts, a finding with no gene and no place, and the lead's own sentences close whole while any result is closed, unless they are structured (the carrier list's two-hit lines close by the gene each is built from) or on a list of prose that names no gene (the QC, ClinVar, mosaic, ROH and telomere lines, most cards' facts, the headlines built from counts), so a child's page with a result closed shows less prose, and no search hit, until the results are opened; a place that names two closed results (a segment of the lead, a duplication over two genes) stays closed until both are opened; the polygenic score card and its chart are never tagged, and the genome map no longer counts a finding a closed result ties among its findings with no place; a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels` and `veil.headline`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; each tag in `veil.rows`, `veil.lines`, `veil.facts`, `veil.findings`, `veil.lead`, `veil.marks`, `veil.viewer` and `veil.widget` is now a list of closed-result ids (or `*panel*`) instead of one id. The genome map's findings layer gains a `veils` column, every closed result a row overlaps; `veil` stays. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. diff --git a/assets/live/map.js b/assets/live/map.js index a4901128a..82b4606cc 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -169,12 +169,13 @@ // A row that is a finding of the page (opts.known) follows the page's own veil only: the page already // hides every finding it keeps closed (opts.now), and its card shows the rest. Hiding such a row for // another result's tag would drop a finding whose card the page shows, and so place that closed result - // inside it. A row the page has no finding for (the map's APOE row) follows its tag too. + // inside it. A map-owned gated row (APOE) follows its own gate for the same reason: opening it must not + // tell whether another, still-closed result overlaps it. Other rows follow every overlapping tag. function shownRow(r) { var own = M.closable.indexOf(r.id) < 0 || stateOf(r.id) !== 'closed'; // Closed while ANY result the row lies over is closed: a row over two closed results never comes back with // the first one opened. - var tag = (opts.known && opts.known(r.id)) || r.veils.every(function (id) { return stateOf(id) !== 'closed'; }); + var tag = !!r.gate || (opts.known && opts.known(r.id)) || r.veils.every(function (id) { return stateOf(id) !== 'closed'; }); return own && tag && (!opts.now || opts.now(r.id)); } // The runs of homozygosity, as this page shows them now: a replay before their step ended has none yet. diff --git a/docs/live-site.md b/docs/live-site.md index ee3005c9a..19a5948c3 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -59,6 +59,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. Two pages whose open rows are the same differ only in how many results are closed, not in which; the section that holds a closed result still says it holds one: - A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call, a repeat) close with them, whatever gene they name, and so does a segment of the lead that names one (`veil.findings` and `veil.lead` tag them `*panel*`): a medicine finding that closed only when the closed result is in its gene would tell the gene. A card can hold a fixed panel among its lines or rows: the ClinVar card prints a line for each known deep intronic site, hit or not; the carrier list prints its two-hit line, which names the copy-number callers it read; and the callability card lists the genes this reference does not hold and the hard genes the reads did not cover, nearly the same for everyone. Those lines and rows close all together while any result to act on is closed and come back once every result is opened (`FIXED_LINES` in `bin/live_results.py`; `veil.lines` and `veil.rows` tag each of them `*panel*`). A table row, a finding or a chart mark closes with a result only by the fields that name a gene or a place: a row by its gene and position columns, a finding by its gene, its genes and its loci, a chart mark by the gene and position of the summary row it draws. It never closes by a vocabulary cell (an inheritance code, a structural-variant type, a genotype, a class, a significance, a unit) or by a structural variant's title, because any code the report prints in capitals may also be a gene symbol: a closed result in AR would else close every recessive carrier row, and one in INS every insertion. `TABLE_COLUMNS` in `bin/live_results.py` says for each column of each card's table whether it names a gene or a place. A place that names two closed results closes with both: its tag is the list of every closed result it names, never the first one only, and it stays closed while any of them is closed. A segment of the lead that names two closed results, or a duplication over two closed genes, comes back only once both are opened, so opening one never tells the other. The page never matches the words of prose against gene symbols: any word the report prints in capitals may be a gene symbol (`AR` in "Cystic fibrosis; AR"), and a line that closes by such a word tells the gene. Prose with no field that names a gene closes whole (`*panel*`) while any result to act on is closed: a card's headline, its lines and facts, a finding with no gene and no place, and the lead's own sentences. The exceptions are structured or declared free of genes, each list pinned in `tests/test_live_results.py`. Structured: the carrier list's two-hit lines close by the gene and variant each line is built from, and its two-hit count by the two-hit genes (`FACT_NAMES`); a segment of the lead closes with the findings it prints (`veil.lead`); a line that gives a position closes by that position. Declared free of genes: the lines of the QC, ClinVar, mosaic, MoChA, ROH, telomere and consumer-export cards (`LINES_NO_GENE`), the facts of the cards in `FACTS_NO_GENE`, the headlines of the cards in `HEADLINES_NO_GENE` (built from counts and fixed words), a data-quality finding, and a card's own finding when its facts and headline are free. The cost: a child's page with any result closed shows less prose (the headline and lines of a card that can name a gene, the search's hits) until the results are opened. The polygenic score card, which names scores and traits only, is never tagged (`NAMES_NO_GENE`), and neither are the marks of a chart whose labels come from a list known before the run (the scores, the ancestry groups, the ClinVar star bars' labels). The genome map leaves a finding the veil ties to a closed result out of its count of findings with no place. +- The map's APOE row follows its own warning switch. Opening it shows the same mark even when another result in APOE stays closed. Otherwise its absence would tell which result is closed. Other map rows keep every tag they follow. - A closed finding carries no id in the page or the address, since its id names its gene and variant. It is known by its place among the page's closed findings (`#f/closed-2`). On a page that keeps a result closed, the address never holds the id of a finding the page can close, open or closed: once opened on purpose it is `#f/opened-`, its place among the opened ones, so no history entry keeps the id. Any other `#f/` on such a page, the id of a closed result or no id at all, lands the same way, on the first closed finding (`#f/closed-1`): the page never picks a closed card by its id, so a guessed id tells nothing. On the home page, a child's closed result links to the child's report, never to the finding's own page, whose address names it. Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: its index is prose, so while any result to act on is closed it shows no hit and suggests no gene, and its hits come back once every result is opened. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. While the traits card is closed whole on a child's page, the chip is the switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with the ids of every closed result it overlaps (the `veils` column; `veil` keeps the first). diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 27c3a53af..f67cd7fdd 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1510,11 +1510,11 @@ def page_now(b, base, hash_="", typed=False): def plant_summary(out, og, change): """The son's results.json built again by live_results from his summary.json with CHANGE(sections) applied, and - his map.json built again from it by og map build. Returns what to write back afterwards.""" + his map.json and rendered charts built again from that same summary. Returns what to write back afterwards.""" son = OTHERS[1] - rj, mj = (os.path.join(out, "runs", son, n) for n in ("results.json", "map.json")) + rj, mj, cj = (os.path.join(out, "runs", son, n) for n in ("results.json", "map.json", f"{son}_charts.json")) keep = {} - for p in (rj, mj): + for p in (rj, mj, cj): with open(p, "rb") as fh: keep[p] = fh.read() root = os.path.join(os.path.dirname(out), "root") @@ -1524,6 +1524,8 @@ def plant_summary(out, og, change): res = live_results.build(son, summary) with open(rj, "w") as fh: json.dump(res, fh) + with open(cj, "w") as fh: + json.dump(live_results.rr.charts_json(summary), fh) doc = build_live_demo.og_map(og, ["build", "--root", root, "--run", son, "--assets", os.path.join(REPO, "assets"), "--cache-dir", os.path.join(root, ".owngenome-live", "map")], json.dumps({"results": res}).encode()) @@ -1906,6 +1908,178 @@ def two_closed_problems(b, base, out, og, cases=TWO_CLOSED): return bad +def apoe_pair_problems(b, base, out, og): + """Opening the map's own gate cannot say which other result is still closed.""" + got, bad = [], [] + for gene in ("APOE", "BRCA2"): + keep = plant_summary(out, og, cpsr(gene)) + try: + stages = [page_now(b, base)] + if not b.eval("(function () { var c = document.querySelector('.gm-chip[data-closed=gate]'); " + "if (!c) { return false; } c.querySelector('button').click(); " + "document.querySelector('.gm-chip[data-closed=gate] .gm-yes').click(); return true; }())"): + return [f"{gene}: no APOE gate"] + time.sleep(0.2) + stages.append(page_snapshot(b)) + if not b.eval("!!document.querySelector('.gm-marks [data-id=apoe]')"): + bad.append(f"{gene}: opening APOE did not draw its mark") + if not b.eval("(function () { var c = Array.from(document.querySelectorAll('.gm-chip.opened')).find(" + "function (e) { return e.textContent.indexOf('Opened: APOE') === 0; }); " + "if (!c) { return false; } c.querySelector('button').click(); return true; }())"): + return bad + [f"{gene}: no close button for APOE"] + time.sleep(0.2) + stages.append(page_snapshot(b)) + if b.eval("!!document.querySelector('.gm-marks [data-id=apoe]')"): + bad.append(f"{gene}: closing APOE kept its mark") + got.append(stages) + finally: + unplant(keep) + for stage, a, z in zip(("before", "opened", "closed again"), *got): + if a != z: + bad.append(f"APOE against BRCA2 {stage}: HTML {a[0] == z[0]}, address {a[1] == z[1]}, canvas {a[2] == z[2]}") + return bad + + +def page_snapshot(b): + """The same complete document, address and canvas comparison used by page_now, after a click.""" + got = json.loads(b.eval(PAGE_JS) or "{}") + return (got.get("html") or "").replace("><", ">\n<").split("\n"), got.get("href") or "", \ + hashlib.sha256((got.get("canvas") or "").encode()).hexdigest()[:12] + + +def related_set_problems(b, base, out, og): + """On each finding's own page, opening one of two results cannot open a related finding naming both.""" + change = dup_over(cpsrs("BRCA1", "BRCA2"), "chr5:1000000-1300000", ["BRCA1", "BRCA2"]) + keep = plant_summary(out, og, change) + made, bad = [], [] + folder = os.path.join(out, "runs", SON, "finding") + # The demo server is static: give the planted findings the same page template as its original findings. + original = "sf-kcnh2-chr7-150960000-c-t" + with open(os.path.join(folder, original, "index.html")) as fh: + template = fh.read() + try: + for gene in ("BRCA1", "BRCA2"): + fid = cid(gene) + dest = os.path.join(folder, fid) + os.makedirs(dest) + made.append(dest) + with open(os.path.join(dest, "index.html"), "w") as fh: + fh.write(template.replace(original, fid)) + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{SON}/finding/{fid}/") + if not b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.querySelector('.veil button') " + "&& 'ready'", 10): + bad.append(f"{gene}: finding never closed") + continue + b.eval("document.querySelector('.veil button').click(); true") + if not b.wait("!document.querySelector('.veil') && document.querySelector('#related') && 'opened'", 5): + bad.append(f"{gene}: finding never opened") + related = b.eval("document.querySelector('#related').outerHTML") or "" + if "sv-chr5-1000000-1300000" in related or "Large DUP" in related: + bad.append(f"{gene} opened alone: the duplication names the other closed result in Related results") + if not b.eval("document.querySelector('#what').textContent.indexOf(" + json.dumps(gene) + ") >= 0"): + bad.append(f"{gene}: opening its finding did not show it") + finally: + unplant(keep) + for dest in made: + shutil.rmtree(dest) + return bad + + +def chart_position_problems(b, base, out, og): + """Gene-only chart tags and coordinate-only mosaic tags, including reopening with two overlapping acts.""" + def change(S): + dup_over(cpsr("KCNQ1"), "chr5:1000000-1300000", ["NKD2", "KCNQ1"])(S) + second_hit(S) # KCNH2, chr7:150970000: outside both closed results' positions, tied by gene alone. + S["mosaic"] = {"state": "ok", "data": {"status": "assessed", "floor": 0.05, "depth": 30, + "floor_read": 0.05, "floor_min": 0.05, "het_sites": 1000, "chromosomes": 22, "calls_reported": 1, + "calls": [{"chrom": "chr7", "start": 150900000, "end": 151000000, "length_mb": 0.1, + "type": "Loss", "arms": "7q", "cell_fraction": 0.2}]}} + keep = plant_summary(out, og, change) + bad, states = [], [] + try: + with open(os.path.join(out, "runs", SON, f"{SON}_charts.json")) as fh: + marks = next(sp["marks"] for sp in json.load(fh)["charts"] if sp["id"] == "genome") + hit = next(m["id"] for m in marks if m.get("kind") == "hit" and m.get("pos") == 150970000) + sv = next(m["id"] for m in marks if m.get("kind") == "sv" and m.get("pos") == 1000000) + # Inspect each surface separately. A link/title leak counts even when its visible row is absent. + expression = """JSON.stringify((function () { + var c = document.querySelector('#card-mosaic-mca'); + return [!!document.querySelector('figure[data-chart=genome] [data-mark=%s]'), + !!document.querySelector('figure[data-chart=genome] [data-mark=%s]'), + !!document.querySelector('[data-finding=mca-chr7-150900000]'), + !!(c && Array.from(c.querySelectorAll('tbody tr')).some(function (r) { return r.textContent.includes('150900000'); })), + !!document.querySelector('a[href*="150900000"]'), + !!(c && Array.from(c.querySelectorAll('li')).some(function (e) { return e.textContent.includes('150900000'); }))]; + }()))""" % (hit, sv) + page_now(b, base) + states.append(json.loads(b.eval(expression))) + for fid in ("sf-kcnh2-chr7-150960000-c-t", "sv-chr7-150955000-150965000", cid("KCNQ1")): + if not open_closed(b, fid): + return [f"no closed card for {fid}"] + states.append(json.loads(b.eval(expression))) + # Positive first: every planted mark and surface must reach the browser when opened. Otherwise a missing + # rendered charts file could make the hidden-state assertions vacuous. + if states[-1] != [True] * 6: + bad.append(f"all opened: planted hit, SV, mosaic finding, row, viewer link and line must exist: {states[-1]}") + for stage, got, want in zip(("all closed", "SNV alone opened", "SNV and deletion opened"), states, + ([False] * 6, [False] * 6, [True, False, True, True, True, True])): + if got != want: + bad.append(f"{stage}: hit, SV, mosaic finding/row/link/line {got}, want {want}") + # Reverse the overlapping results' opening order too. The SNV has no chip of its own on this demo map. + page_now(b, base) + if not open_closed(b, "sv-chr7-150955000-150965000"): + bad.append("no closed deletion after reloading") + elif json.loads(b.eval(expression)) != [False] * 6: + bad.append(f"deletion alone opened: {b.eval(expression)}") + finally: + unplant(keep) + return bad + + +def gate_regression_checks(b, base, out, og): + """Each guard has a browser control that removes exactly the reviewed rule.""" + t0 = time.time() + for desc, probe in (("APOE's gate before opening, opened and closed again gives the same HTML, address and canvas " + "with APOE or BRCA2 closed", apoe_pair_problems), + ("a finding page opened for either BRCA1 or BRCA2 keeps their duplication out of Related results", + related_set_problems), + ("planted charts reach the browser and close by gene alone, and a gene-free mosaic finding, row, " + "viewer link and line close until both overlapping results open", chart_position_problems)): + bad = probe(b, base, out, og) + check("browser: " + desc, not bad, bad) + for file, a, z, desc, probe in ( + ("map.js", "var tag = !!r.gate ||", "var tag =", "map-owned APOE row following overlapping act tags", apoe_pair_problems), + ("finding.js", "return t.length ? t : null;", "return t.length ? t.slice(0, 1) : null;", + "finding.js tagOf keeping only the first id", related_set_problems)): + p = os.path.join(out, "assets", file) + with open(p) as fh: + source = fh.read() + try: + if source.count(a) != 1: + check("control: " + desc, False, "missing mutation anchor") + continue + with open(p, "w") as fh: + fh.write(source.replace(a, z)) + control(desc, not probe(b, base, out, og)) + finally: + with open(p, "w") as fh: + fh.write(source) + names, spans = live_results.mark_names, live_results.closable + try: + for kind, desc in (("hit", "ClinVar chart mark without its gene"), ("sv", "SV chart mark with only its first gene")): + live_results.mark_names = lambda S, sp, m: (([], "") if kind == "hit" else (names(S, sp, m)[0][:1], "")) \ + if m.get("kind") == kind else names(S, sp, m) + control(desc, not chart_position_problems(b, base, out, og)) + live_results.mark_names = names + live_results.closable = lambda found: [(fid, g, None, None, None) for fid, g, *_ in spans(found)] + control("closable without coordinates", not chart_position_problems(b, base, out, og)) + finally: + live_results.mark_names, live_results.closable = names, spans + print(f" gate regression checks: {time.time() - t0:.2f} s") + + def infer_checks(b, base, out): """The son's page is a function of his open results and the count of his closed ones: two pages that differ only in which result is closed are the same before any click (INFER), and a closed deletion's second gene is named @@ -1916,6 +2090,7 @@ def infer_checks(b, base, out): print("[SKIP] infer checks: the demo has no map.json") return t0 = time.time() + gate_regression_checks(b, base, out, og) bad = infer_problems(b, base, out, og) check("browser: two pages of the son that differ only in which result is closed are the same before any click, " "HTML, attributes, address and map canvas (" + "; ".join(x[0] for x in INFER) + ")", not bad, bad) diff --git a/tests/test_live_results.py b/tests/test_live_results.py index bc0d0596d..60313726a 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -550,6 +550,39 @@ def sv_genes_problems(): return out +def gene_and_position_problems(): + """Chart genes without position overlap, and a gene-free call with position overlap, need independent tags.""" + summary = {"sections": { + "roh": {"state": "ok", "data": {}}, + "annotsv": {"state": "ok", "data": {"rows": [ + {"sv": "chr17:43044295-43170245", "type": "DEL", "length": -125950, "acmg_class": "5", + "genes": ["BRCA1", "NBR2"], "gt": "0/1", "support": 1, "caller": "Manta"}, + {"sv": "chr5:1000000-1300000", "type": "DUP", "length": 300000, "acmg_class": "4", + "genes": ["NKD2", "NBR2"], "gt": "0/1", "support": 1, "caller": "Manta"}]}}, + "clinvar": {"state": "ok", "data": {"hits": [ + {"gene": "NBR2", "chrom": "chr1", "pos": 5, "ref": "A", "alt": "G", "significance": "Pathogenic"}]}}, + "mosaic": {"state": "ok", "data": {"status": "assessed", "calls": [ + {"chrom": "chr17", "start": 43050000, "end": 43100000, "length_mb": 0.05, + "type": "Loss", "arms": "17q", "cell_fraction": 0.2}]}}}} + res = live_results.build("T", summary) + v, out = res["veil"], [] + closed = "sv-chr17-43044295-43170245" + for mark in ("hit-0", "sv-1"): + got = v.get("marks", {}).get("genome", {}).get(mark) + if got != [closed]: + out.append(f"gene-only chart {mark}: {got}, want {[closed]}") + if v.get("findings", {}).get("mca-chr17-43050000") != [closed]: + out.append("coordinate-only mosaic finding not tagged with the deletion") + if v.get("rows", {}).get("mosaic-mca") != [[closed]]: + out.append("coordinate-only mosaic table row not tagged with the deletion") + card = next(c for ch in res["chapters"] for c in ch["sections"] if c["id"] == "mosaic-mca") + lines = [(ln, tag) for ln, tag in zip(card["lines"], v.get("lines", {}).get("mosaic-mca", [])) + if "chr17:43050000-43100000" in ln] + if not lines or any(tag != [closed] for _, tag in lines): + out.append(f"coordinate-only mosaic lines: {lines}") + return out + + # The veil's tables, pinned here so that an entry dropped from one turns this file red (each is checked by what # veil() does with it, not by reading the table back): the fixed-panel cards, the fixed lines of a card (a sample # line or row naming a gene, its card, and whether it is a row), the cards that name no gene, the cards whose lines, @@ -936,6 +969,20 @@ def tagged(sample_dir): check("8. a closed structural variant over two genes closes the rows and the finding that name only its second " "gene, veil.spans lists it per gene, veil.panels lists the fixed-panel cards, and a duplication over two " "closed results carries both ids", not bad, bad) + bad = gene_and_position_problems() + check("8. gene-only ClinVar and second-gene SV chart marks, and coordinate-only mosaic findings, rows and lines " + "carry the closed result's tag", not bad, bad) + names, spans = live_results.mark_names, live_results.closable + try: + for kind in ("hit", "sv"): + live_results.mark_names = lambda S, sp, m: (([], "") if kind == "hit" else (names(S, sp, m)[0][:1], "")) \ + if m.get("kind") == kind else names(S, sp, m) + control(f"{kind} chart mark without every gene", not gene_and_position_problems()) + live_results.mark_names = names + live_results.closable = lambda found: [(fid, g, None, None, None) for fid, g, *_ in spans(found)] + control("closable without positions", not gene_and_position_problems()) + finally: + live_results.mark_names, live_results.closable = names, spans first = live_results.closable try: live_results.closable = lambda found: first([{k: v for k, v in f.items() if k != "genes"} for f in found]) From 2b21e9f0de2ae122af045ba4cecefc303d9aa5fb Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:30:09 +0100 Subject: [PATCH 045/255] test(live): run the family veil checks in a separate Guard shard --- docs/live-site.md | 2 +- tests/test_live_demo.py | 5 ++++- tests/test_owngenome_live_family.py | 34 ++++++++++++++++++++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/docs/live-site.md b/docs/live-site.md index 19a5948c3..9210ac150 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -192,4 +192,4 @@ The core files (`bin/owngenome_live.py`, `assets/live/index.html`, `home.html`, The page has a replay mode for a demo with no server: `` reads `frames.json` (`{"interval_ms": 1000, "frames": [{"t": ..., "state": }]}`) and plays it with a play button and a slider, through the same drawing code. A replay whose last frame is a finished run opens at that frame, on the finished report, until you press Play, which starts again from the first frame; with `"autoplay": true` it plays from the start at once. The bar shows each frame's moment as day, month and time in UTC ("2 Jan 10:05"), the full time on hover. -`tests/test_owngenome_live.py` checks the server, the login, the stored messages, the plugin interface and the page in a headless Chrome; `tests/test_live_results.py` checks `results.json` (every card's caveat word for word, no wording that reads as an all-clear, `results.json` behind the login); `tests/test_live_journal.py` checks the levels, the lead paragraph and the finding page's route; `tests/test_live_demo.py` builds the demo and checks every page, phone width included; `tests/test_live_post.sh` and `tests/fake-docker/run-all-live.sh` check what `run-all.sh` posts. +`tests/test_owngenome_live.py` checks the server, the login, the stored messages, the plugin interface and the page in a headless Chrome; `tests/test_live_results.py` checks `results.json` (every card's caveat word for word, no wording that reads as an all-clear, `results.json` behind the login); `tests/test_live_journal.py` checks the levels, the lead paragraph and the finding page's route; `tests/test_live_demo.py` builds the demo and checks every page, phone width included; the family veil browser checks run from `tests/test_owngenome_live_family.py` in a separate CI shard, with every control kept; `tests/test_live_post.sh` and `tests/fake-docker/run-all-live.sh` check what `run-all.sh` posts. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 5e3d98d9d..58e59179b 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -116,6 +116,9 @@ every line, a step timeline left open) also run against a planted mistake and must fail there (2: a build with one plugin's state hook dropped): a check that cannot fail is no check. +The family veil group described above runs from tests/test_owngenome_live_family.py in another Guard shard. +Run both files for every browser check. + Run: python3 tests/test_live_demo.py """ import base64 @@ -373,7 +376,7 @@ def browser_checks(out): veil_checks(b, base, out) planted_checks(b, base, out) map_checks(b, base, out) - family_veil_checks(b, base, out) + # The family veil group runs from test_owngenome_live_family.py in a different Guard shard. widget_veil_checks(b, base, out) infer_checks(b, base, out) finally: diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index 42d4253fa..6ec41d174 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -26,7 +26,9 @@ row draws the four children and a 1-in-2 row gives its chance in words with no drawing, a missing trio file and one linking out of the folder are worded apart (the link's target never shown), and the side by side lists every - blood group system with a stale value marked in its cell. + blood group system with a stale value marked in its cell; + 7. the demo family's veil checks and every mutation control from test_live_demo.py, + run here in Guard shard 0 so the person-page suite in shard 2 stays below its time limit. Checks 2 and 4 are run again on a copy of the fixture with two parents swapped and must fail there: a check that cannot fail is no check. @@ -161,6 +163,35 @@ def route_checks(app): ok and descents(body) == DESCENT, (code, ctype, auth, ok and descents(body)))] +def family_browser_checks(work): + """Run the demo's family veil group here so its browser time is in another Guard shard.""" + import test_live_demo as demo + chrome = demo.find_chrome() + if not chrome: + if os.environ.get("CI") == "true": + check("family browser group has Chrome in CI", False) + else: + print("[SKIP] no Chrome or Chromium for the family browser group") + return + og = demo.og_binary() or demo.build_live_demo.OG + out = os.path.join(work, "family-browser", "demo") + demo.build_live_demo.build(out, work=os.path.join(work, "family-browser", "root"), og=og) + handler = type("H", (demo.Static,), {"__init__": lambda self, *a, **k: demo.SimpleHTTPRequestHandler.__init__( + self, *a, directory=out, **k)}) + srv = demo.ThreadingHTTPServer(("127.0.0.1", 0), handler) + demo.threading.Thread(target=srv.serve_forever, daemon=True).start() + b = demo.Chrome(chrome, os.path.join(work, "family-browser", "chrome")) + before, t0 = len(demo.FAILS), demo.time.time() + try: + demo.family_veil_checks(b, f"http://127.0.0.1:{srv.server_address[1]}", out) + FAILS.extend(demo.FAILS[before:]) + finally: + b.close() + srv.shutdown() + srv.server_close() + print(f" family browser group: {demo.time.time() - t0:.2f} s") + + def main(): work = tempfile.mkdtemp(prefix="pgp-live-family-", dir=os.environ.get("TMPDIR")) try: @@ -277,6 +308,7 @@ def main(): # 6. the real server real_server(root) + family_browser_checks(work) finally: shutil.rmtree(work, ignore_errors=True) if FAILS: From 3a423006b0fa612864274b216159d0a00764db73 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:47:42 +0100 Subject: [PATCH 046/255] test(live): exercise both related-list opening orders --- tests/test_live_demo.py | 57 +++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 58e59179b..86286a9be 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1952,8 +1952,6 @@ def page_snapshot(b): def related_set_problems(b, base, out, og): """On each finding's own page, opening one of two results cannot open a related finding naming both.""" - change = dup_over(cpsrs("BRCA1", "BRCA2"), "chr5:1000000-1300000", ["BRCA1", "BRCA2"]) - keep = plant_summary(out, og, change) made, bad = [], [] folder = os.path.join(out, "runs", SON, "finding") # The demo server is static: give the planted findings the same page template as its original findings. @@ -1961,30 +1959,39 @@ def related_set_problems(b, base, out, og): with open(os.path.join(folder, original, "index.html")) as fh: template = fh.read() try: - for gene in ("BRCA1", "BRCA2"): - fid = cid(gene) - dest = os.path.join(folder, fid) - os.makedirs(dest) - made.append(dest) - with open(os.path.join(dest, "index.html"), "w") as fh: - fh.write(template.replace(original, fid)) - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{SON}/finding/{fid}/") - if not b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.querySelector('.veil button') " - "&& 'ready'", 10): - bad.append(f"{gene}: finding never closed") - continue - b.eval("document.querySelector('.veil button').click(); true") - if not b.wait("!document.querySelector('.veil') && document.querySelector('#related') && 'opened'", 5): - bad.append(f"{gene}: finding never opened") - related = b.eval("document.querySelector('#related').outerHTML") or "" - if "sv-chr5-1000000-1300000" in related or "Large DUP" in related: - bad.append(f"{gene} opened alone: the duplication names the other closed result in Related results") - if not b.eval("document.querySelector('#what').textContent.indexOf(" + json.dumps(gene) + ") >= 0"): - bad.append(f"{gene}: opening its finding did not show it") + for genes in (["BRCA1", "BRCA2"], ["BRCA2", "BRCA1"]): + # Related links use the finding's first gene. Reverse that as well as the CPSR order, so both + # directions can actually list the duplication and expose a first-id reader. + gene, fid = genes[0], cid(genes[0]) + keep = plant_summary(out, og, dup_over(cpsrs(*genes), "chr5:1000000-1300000", genes)) + try: + with open(os.path.join(out, "runs", SON, "results.json")) as fh: + finding = next(f for f in json.load(fh)["findings"] if f["id"] == fid) + if "sv-chr5-1000000-1300000" not in finding.get("related", []): + bad.append(f"{gene}: fixture never offers the duplication as a related result") + dest = os.path.join(folder, fid) + os.makedirs(dest) + made.append(dest) + with open(os.path.join(dest, "index.html"), "w") as fh: + fh.write(template.replace(original, fid)) + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{SON}/finding/{fid}/") + if not b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.querySelector('.veil button') " + "&& 'ready'", 10): + bad.append(f"{gene}: finding never closed") + continue + b.eval("document.querySelector('.veil button').click(); true") + if not b.wait("!document.querySelector('.veil') && document.querySelector('#related') && 'opened'", 5): + bad.append(f"{gene}: finding never opened") + related = b.eval("document.querySelector('#related').outerHTML") or "" + if "sv-chr5-1000000-1300000" in related or "Large DUP" in related: + bad.append(f"{gene} opened alone: the duplication names the other closed result in Related results") + if not b.eval("document.querySelector('#what').textContent.indexOf(" + json.dumps(gene) + ") >= 0"): + bad.append(f"{gene}: opening its finding did not show it") + finally: + unplant(keep) finally: - unplant(keep) for dest in made: shutil.rmtree(dest) return bad From 98213613df8768ad601eddeaabcd47bfa0308413 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:09:30 +0100 Subject: [PATCH 047/255] fix(live): keep panel titles closed on a child's home view --- CHANGELOG.md | 2 +- assets/live/home.js | 8 ++++-- bin/live_results.py | 8 ++++-- docs/live-site.md | 2 +- tests/test_live_demo.py | 59 ++++++++++++++++++++++++++++++++++++-- tests/test_live_results.py | 23 +++++++++++++-- 6 files changed, 90 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f61c9b656..308f16e2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,7 +105,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. - A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; the address never holds a closable finding's id, closed (`#f/closed-`) or opened (`#f/opened-`), so no history entry keeps it, and any other `#f/` on such a page, a closed result's id or nothing, lands on the first closed finding, so a guessed id tells nothing. The home page links a child's closed result to the child's report, not to the finding's own page. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and so do their own findings (a medicine gene, a CHIP call) and the lead's words on them, the ClinVar card's lines for the known deep intronic sites, the carrier list's two-hit line and the callability card's rows, all together; a table row, a finding or a chart mark closes with a result only by its gene and place fields, never by a code such as the inheritance `AR` or the structural-variant type `INS`, and prose is never read word by word against gene symbols: a card's headline, its lines and facts, a finding with no gene and no place, and the lead's own sentences close whole while any result is closed, unless they are structured (the carrier list's two-hit lines close by the gene each is built from) or on a list of prose that names no gene (the QC, ClinVar, mosaic, ROH and telomere lines, most cards' facts, the headlines built from counts), so a child's page with a result closed shows less prose, and no search hit, until the results are opened; a place that names two closed results (a segment of the lead, a duplication over two genes) stays closed until both are opened; the polygenic score card and its chart are never tagged, and the genome map no longer counts a finding a closed result ties among its findings with no place; a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels` and `veil.headline`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; each tag in `veil.rows`, `veil.lines`, `veil.facts`, `veil.findings`, `veil.lead`, `veil.marks`, `veil.viewer` and `veil.widget` is now a list of closed-result ids (or `*panel*`) instead of one id. The genome map's findings layer gains a `veils` column, every closed result a row overlaps; `veil` stays. -- Opening APOE on a child's genome map no longer reveals whether another closed result overlaps APOE. Its map row follows its own warning switch. +- Opening APOE on a child's genome map no longer reveals whether another closed result overlaps APOE. Its map row follows its own warning switch. A child's home view also keeps panel finding titles and links closed, through the same veil tags. Adults and runs with no result to act on keep their titles and destinations. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. - Steps 21, 32 and 9b (and the CYRIUS, PYPGX and STRANGER_RFC1 modules) now run the CYP2D6 depth check and the RFC1 motif count with `og`, and `bin/cyp2d6_depth_check.py` and `bin/rfc1_motif.py` are gone. A run that reaches these steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull; without it the step stops with `og is not built`. The output files, their names and formats do not change; in CI, every e2e case that runs these steps in bash checks `og` against the deleted Python, byte for byte. diff --git a/assets/live/home.js b/assets/live/home.js index 2afb54fa9..9c5e54c55 100644 --- a/assets/live/home.js +++ b/assets/live/home.js @@ -147,7 +147,11 @@ }); }); if (rolesUnknown) { runs.forEach(function (r) { if (!listed[r.name]) { closedRuns[r.name] = true; } }); } - function closedTitle(r, f) { return closedRuns[r.name] && f.level === 'act' ? 'A result to act on, closed until opened on the report' : f.title; } + function homeShut(r, f) { return !!closedRuns[r.name] && (f.level === 'act' || (f.veil || []).indexOf('*panel*') >= 0); } + function closedTitle(r, f) { + return homeShut(r, f) ? (f.level === 'act' ? 'A result to act on, closed until opened on the report' + : 'A finding in a panel, closed until opened on the report') : f.title; + } // The dateline and the headline sentence. var dl = el('div', { 'class': 'dateline' }); var today = new Date(); @@ -193,7 +197,7 @@ items.sort(function (a, b) { return order[a.f.level] - order[b.f.level]; }); items.slice(0, 6).forEach(function (x) { // A child's closed result links to the run's page, never to its own page: its id names its gene and variant. - var a = el('a', { href: x.r.url + (closedRuns[x.r.name] && x.f.level === 'act' ? '' : 'finding/' + encodeURIComponent(x.f.id) + '/') }); + var a = el('a', { href: x.r.url + (homeShut(x.r, x.f) ? '' : 'finding/' + encodeURIComponent(x.f.id) + '/') }); var who = el('span', { 'class': 'who' }); var pc = el('span', { 'class': 'pc' }, x.r.name); if (colourOf[x.r.name]) { pc.style.setProperty('--pcol', colourOf[x.r.name]); } diff --git a/bin/live_results.py b/bin/live_results.py index 544832f15..6527beac1 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -1638,13 +1638,15 @@ def overview(summary): order["trust"] = -1 # a problem with the data comes before any finding read from it # A finding that names a result to act on (veil: a symptom ranking of its gene) is not listed: a child's tile # closes only the results to act on themselves, and the result it names already comes first. - # A finding tagged PANEL (a fixed-panel card's own, veil()) names no result: it stays, the same for everyone. - named = {k: v for k, v in ((r.get("veil") or {}).get("findings") or {}).items() if any(x != PANEL for x in v)} + # A PANEL-only finding stays in top with its tag: the home page closes its title and link for a child, + # while adults and runs without a result to act on keep the same title and destination. + finding_tags = (r.get("veil") or {}).get("findings") or {} + named = {k: v for k, v in finding_tags.items() if any(x != PANEL for x in v)} top = sorted((f for f in r["findings"] if f["level"] != "info" and f["id"] not in named), key=lambda f: order[f["level"]])[:4] tags = (r.get("veil") or {}).get("lead") or [] return {"attention": r["attention"], "levels": r["levels"], "screened": screened(summary), - "top": [{"id": f["id"], "level": f["level"], "title": f["title"]} for f in top], + "top": [{"id": f["id"], "level": f["level"], "title": f["title"], "veil": finding_tags.get(f["id"])} for f in top], "lead": "".join(x[0] for x in r["lead"]), # The same lead with the names of the findings to act on left out, and every other segment that names # one of them: a child's tile shows this one. diff --git a/docs/live-site.md b/docs/live-site.md index 9210ac150..b97e4c174 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -94,7 +94,7 @@ Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with it The reads cross the network to the browser that logged in: only the byte ranges igv.js asks for, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS); without it the password and the reads cross the network in clear. Opening the viewer writes one access-log row; the byte ranges igv.js reads write none. -`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. +`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title and veil tag; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). A child's home view keeps a panel-tagged finding's title and link closed too, at every level. Adults and runs with no result to act on keep the title and link. While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the places outside a result's own card that close with it, each tagged with the list of closed results it names, or `*panel*`; `spans`, the closed results with their position, one row per gene they name, from which the family page knows whether a child has a result still closed; and `panels`, the cards of a fixed panel). A structural variant's finding also lists every gene it covers in `genes`. It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 86286a9be..b12130105 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -2048,6 +2048,46 @@ def change(S): return bad +def home_panel_problems(b, base, out, og): + """A panel finding's title and id stay off a closed child's home view, but remain for adults and no-act runs.""" + p = os.path.join(out, "runs.json") + with open(p, "rb") as fh: + original = fh.read() + idx = json.loads(original) + idx["runs"] = [dict(r, state="running") for r in idx["runs"] if r["name"] == SON] + bad = [] + + def change(S): + S.clear() + cpsr("TP53")(S) + S["chip"] = {"state": "ok", "data": {"calls": [{"gene": "TP53", "protein_change": "p.Demo", + "vaf": 0.1, "alt_reads": 10, "depth": 100, "status": "reported"}]}} + + try: + for case in ("child", "adult", "no act"): + view = copy.deepcopy(idx) + if case == "adult": + view["families"] = [] + with open(p, "w") as fh: + json.dump(view, fh) + + def plant(S): + change(S) + if case == "no act": + S["cpsr"]["data"]["plp"] = [] + + html = "\n".join(home_now(b, base, out, plant)) + if case == "child": + if "TP53" in html or "chip-tp53-p-demo" in html: + bad.append("closed child's home view exposes a panel finding's title or id") + elif "A blood clone in TP53" not in html or "finding/chip-tp53-p-demo/" not in html: + bad.append(f"{case}: the open panel title or finding link is missing") + finally: + with open(p, "wb") as fh: + fh.write(original) + return bad + + def gate_regression_checks(b, base, out, og): """Each guard has a browser control that removes exactly the reviewed rule.""" t0 = time.time() @@ -2055,6 +2095,8 @@ def gate_regression_checks(b, base, out, og): "with APOE or BRCA2 closed", apoe_pair_problems), ("a finding page opened for either BRCA1 or BRCA2 keeps their duplication out of Related results", related_set_problems), + ("a child's home view hides panel finding titles and links, while adults and no-act runs keep them", + home_panel_problems), ("planted charts reach the browser and close by gene alone, and a gene-free mosaic finding, row, " "viewer link and line close until both overlapping results open", chart_position_problems)): bad = probe(b, base, out, og) @@ -2062,7 +2104,9 @@ def gate_regression_checks(b, base, out, og): for file, a, z, desc, probe in ( ("map.js", "var tag = !!r.gate ||", "var tag =", "map-owned APOE row following overlapping act tags", apoe_pair_problems), ("finding.js", "return t.length ? t : null;", "return t.length ? t.slice(0, 1) : null;", - "finding.js tagOf keeping only the first id", related_set_problems)): + "finding.js tagOf keeping only the first id", related_set_problems), + ("home.js", "(f.level === 'act' || (f.veil || []).indexOf('*panel*') >= 0)", "f.level === 'act'", + "home.js ignoring panel tags", home_panel_problems)): p = os.path.join(out, "assets", file) with open(p) as fh: source = fh.read() @@ -2087,6 +2131,17 @@ def gate_regression_checks(b, base, out, og): control("closable without coordinates", not chart_position_problems(b, base, out, og)) finally: live_results.mark_names, live_results.closable = names, spans + overview = live_results.overview + try: + def no_panel(summary): + r = overview(summary) + for f in r["top"]: + f.pop("veil", None) + return r + live_results.overview = no_panel + control("overview dropping panel tags", not home_panel_problems(b, base, out, og)) + finally: + live_results.overview = overview print(f" gate regression checks: {time.time() - t0:.2f} s") @@ -2127,7 +2182,7 @@ def infer_checks(b, base, out): home = os.path.join(out, "assets", "home.js") with open(home) as fh: keep_home = fh.read() - a = "x.r.url + (closedRuns[x.r.name] && x.f.level === 'act' ? '' : 'finding/' + encodeURIComponent(x.f.id) + '/')" + a = "x.r.url + (homeShut(x.r, x.f) ? '' : 'finding/' + encodeURIComponent(x.f.id) + '/')" try: if a not in keep_home: check("control: home.js that links a closed result's own page (the planted line is in home.js)", False, a) diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 60313726a..1c0456d44 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -832,10 +832,13 @@ def overview_problems(): (a symptom ranking of its gene) is listed among the first findings, or the closed lead still names it.""" r = {"attention": 0, "levels": {}, "chapters": [], "findings": [{"id": "sf-gla-test", "level": "act", "title": "Secondary finding in GLA"}, - {"id": "exomiser-gla", "level": "review", "title": "Symptom ranking: GLA ranks 1"}], + {"id": "exomiser-gla", "level": "review", "title": "Symptom ranking: GLA ranks 1"}, + {"id": "pgx-demo", "level": "medicine", "title": "Medicine panel finding"}, + {"id": "chip-tp53", "level": "review", "title": "A blood clone in TP53"}], "lead": [["Of everything, ", False], [": Secondary finding in GLA. ", False, "act"], ["One result is worth a look", True], [": Symptom ranking: GLA ranks 1.", False]], - "veil": {"findings": {"exomiser-gla": ["sf-gla-test"]}, "lead": [None, None, None, ["sf-gla-test"]]}} + "veil": {"findings": {"exomiser-gla": ["sf-gla-test"], "pgx-demo": [live_results.PANEL], + "chip-tp53": [live_results.PANEL]}, "lead": [None, None, None, ["sf-gla-test"]]}} keep = live_results.build try: live_results.build = lambda *a, **k: r @@ -843,8 +846,11 @@ def overview_problems(): finally: live_results.build = keep out = [] - if [f["id"] for f in o["top"]] != ["sf-gla-test"]: + if [f["id"] for f in o["top"]] != ["sf-gla-test", "pgx-demo", "chip-tp53"]: out.append(f"the home page's first findings: {o['top']}") + for f in o["top"]: + if f["id"] in ("pgx-demo", "chip-tp53") and f.get("veil") != [live_results.PANEL]: + out.append(f"home panel finding has no panel tag: {f}") if "GLA" in o["lead_closed"] or "worth a look" not in o["lead_closed"]: out.append(f"the closed lead: {o['lead_closed']}") return out @@ -961,6 +967,17 @@ def tagged(sample_dir): bad = overview_problems() check("8. a child's home tile lists no finding that names a closed result, and its closed lead names none", not bad, bad) + overview = live_results.overview + try: + def no_panel(summary): + r = overview(summary) + for f in r["top"]: + f.pop("veil", None) + return r + live_results.overview = no_panel + control("overview drops panel tags from medicine and review findings", not overview_problems()) + finally: + live_results.overview = overview bad = widget_problems() check("8. a CHIP call, a copy-number dot and the two-hit fact and line carry the id of a closed result in their " "gene field, never by their words (AR), the CHIP chart's gene panel none, a fixed-panel card's facts the " From cf7159a7e62530e866197672f25946c68000ee9b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:25:18 +0100 Subject: [PATCH 048/255] fix(live): close unclassified child content by default --- CHANGELOG.md | 2 + assets/live/finding.js | 2 + assets/live/home.js | 16 +- assets/live/person.js | 63 +++++++- bin/live_results.py | 144 +++++++---------- bin/report_charts.py | 13 +- docs/live-site.md | 14 +- tests/test_live_demo.py | 240 +++++++++++++++++++++++++--- tests/test_live_results.py | 40 +++-- tests/test_owngenome_live_family.py | 1 + 10 files changed, 400 insertions(+), 135 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 308f16e2a..52fe67323 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- A child's live page now closes unclassified data text and chart notes by default. Its home summary uses level counts while results stay closed. HPO labels and mitochondrial HGVS rows follow the same veil, and chart geometry no longer identifies an off-contig closed result. + - The container helpers check (`scripts/ci/check-container-helpers.sh`, Guard job `helper-binaries`) now derives its table with the Rust `og` binary, `og dev helpers`. It replaces the Python inside the script, which is gone; the script keeps the docker probe, the orad download and its self-test. On this tree the table and the rule 1 output are the same, byte for byte. The static run takes 0.01 s and 6 MB where the Python took 0.2 s and 15 MB. What changes: - A helper called in a process's `shell:` block is checked too. The Python read only `script:` blocks, so such a call went unchecked. No module has one today. diff --git a/assets/live/finding.js b/assets/live/finding.js index 253957d8c..cb1f6fe79 100644 --- a/assets/live/finding.js +++ b/assets/live/finding.js @@ -83,7 +83,9 @@ // A tag is the list of every result it names: the finding stays closed while any of them is. var tags = (r.veil && r.veil.findings) || {}; function tagOf(x) { + if (x.level === 'act') { return null; } var t = tags[x.id]; + if (t === undefined) { t = ['*panel*']; } t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (y) { return y !== x.id; }); return t.length ? t : null; } diff --git a/assets/live/home.js b/assets/live/home.js index 9c5e54c55..42da76af5 100644 --- a/assets/live/home.js +++ b/assets/live/home.js @@ -46,6 +46,13 @@ // The runs whose results to act on stay closed on this page: a child in any family, and, while a family's // members cannot be read, every run that no readable family lists (it may be that family's child). var closedRuns = {}; + function closedSummary(r) { return !!(r && closedRuns[r.name] && r.levels && r.levels.act); } + function homeTop(r) { + if (!closedSummary(r)) { return r.top || []; } + return ['trust', 'act', 'medicine', 'carrier', 'review'].filter(function (level) { + return r.levels[level] > 0; + }).map(function (level) { return { level: level, title: 'Results closed until opened on the report' }; }); + } function tile(m, fam, r, colour) { var div = el('div', { 'class': 'ptile person-card' + (r ? ' has-run' : ' norun') + (r && r.state === 'running' ? ' running' : ''), 'data-sample': m.sample }); @@ -58,7 +65,8 @@ add(meta, (fam ? role(m, fam) + ' · ' : '') + progressText(r)); if (r && r.failed) { add(meta, ' · ', el('span', { 'class': 'warn' }, r.failed + ' step' + (r.failed > 1 ? 's' : '') + ' not assessed')); } add(div, meta); - var lead = r && (closedRuns[m.sample] ? r.lead_closed || '' : r.lead); + var lead = r && (closedSummary(r) ? 'Results to act on stay closed until opened on the report. Read the report for the other findings and their caveats.' + : closedRuns[m.sample] ? r.lead_closed || '' : r.lead); if (lead) { // Two sentences: the first often says only that nothing needs a doctor, the second names what differs. var one = U.firstSentence(lead), rest = lead.slice(one.length).trim(); @@ -147,7 +155,7 @@ }); }); if (rolesUnknown) { runs.forEach(function (r) { if (!listed[r.name]) { closedRuns[r.name] = true; } }); } - function homeShut(r, f) { return !!closedRuns[r.name] && (f.level === 'act' || (f.veil || []).indexOf('*panel*') >= 0); } + function homeShut(r, f) { return closedSummary(r) || (!!closedRuns[r.name] && (f.level === 'act' || (f.veil || []).indexOf('*panel*') >= 0)); } function closedTitle(r, f) { return homeShut(r, f) ? (f.level === 'act' ? 'A result to act on, closed until opened on the report' : 'A finding in a panel, closed until opened on the report') : f.title; @@ -186,7 +194,7 @@ // One line per person and level: the first finding names the line, the rest are counted on it. runs.forEach(function (r) { var seen = {}; - (r.top || []).forEach(function (f) { + homeTop(r).forEach(function (f) { if (order[f.level] === undefined) { return; } if (seen[f.level]) { return; } // The rest of the level from its count, not from the first four findings the index names. @@ -234,7 +242,7 @@ var c2 = el('div', {}); add(c2, el('div', { 'class': 'fig-label' }, 'Found so far')); var fs = el('ul', { 'class': 'fs' }); - (r.top || []).forEach(function (f) { add(fs, add(el('li', {}), U.findingLevel(f.level, ''), ' ', closedTitle(r, f))); }); + homeTop(r).forEach(function (f) { add(fs, add(el('li', {}), U.findingLevel(f.level, ''), ' ', closedTitle(r, f))); }); if (!fs.childNodes.length) { add(fs, el('li', {}, r.lead ? U.firstSentence(closedRuns[r.name] ? r.lead_closed || '' : r.lead) : 'No section is written yet: the first ones appear as their steps end.')); } add(c2, fs); var c3 = el('div', {}); diff --git a/assets/live/person.js b/assets/live/person.js index c29db59f8..5506c3fbc 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -335,22 +335,66 @@ var f = findingById(id); return closedPage() && (!f || veiled(f)); } + // Explicit null is a structured place with no matching act. Missing metadata is unclassified: PANEL. + function placeClosed(tag) { return tag === undefined ? tagClosed(PANEL) : !!tag && tagClosed(tag); } function veilTags(kind, key) { return ((results && results.veil && results.veil[kind]) || {})[key] || []; } // A card's headline is prose: tagged PANEL unless the summary key it comes from prints no gene (veil.headline). - function headlineShut(c) { var t = ((results && results.veil && results.veil.headline) || {})[c.id]; return !!(t && tagClosed(t)); } + function headlineShut(c) { var t = ((results && results.veil && results.veil.headline) || {})[c.id]; return placeClosed(t); } // The charts (plugins/charts.js) ask for each mark. A closed page with no results.json yet cannot tell which // marks those are, so it leaves every mark out until it can. window.PGP.markClosed = function (chart, mark) { if (!closedPage()) { return false; } if (!results) { return true; } var id = ((results.veil && results.veil.marks && results.veil.marks[chart]) || {})[mark]; - return !!(id && tagClosed(id)); + return placeClosed(id); }; + // Chart prose and dimensions have no structured result tag. The chart reader asks the same PANEL rule. + function chartPanelClosed() { return closedPage() && (!results || tagClosed(PANEL)); } + // The chart plugin owns mark links and their click handlers. Keep those nodes, but give the child's + // closed drawing only the producer's fixed geometry. The complete SVG lives off the document until + // PANEL opens. Older chart files without fixed geometry stay closed in full. + function veilCharts() { + Array.prototype.forEach.call(document.querySelectorAll('figure[data-chart]'), function (fig) { + var sp = fig.pgpSpec, shut = chartPanelClosed(); + if (!sp || fig.pgpPanelState === shut) { return; } + fig.pgpPanelState = shut; + if (!fig.pgpFullSVG) { fig.pgpFullSVG = fig.querySelector('svg'); } + var current = fig.querySelector('svg,.chart-panel-closed'); + if (!shut) { + if (current && current !== fig.pgpFullSVG) { current.remove(); } + if (fig.pgpFullSVG && fig.pgpFullSVG.parentNode !== fig) { fig.appendChild(fig.pgpFullSVG); } + return; + } + var replacement = el('p', { 'class': 'dim chart-panel-closed' }, + 'This chart stays closed until every result to act on is opened.'); + if (sp.svg_closed && fig.pgpFullSVG) { + var doc = new DOMParser().parseFromString(sp.svg_closed, 'image/svg+xml'); + if (doc.documentElement.nodeName === 'svg') { + replacement = document.importNode(doc.documentElement, true); + Array.prototype.forEach.call(replacement.querySelectorAll('a[data-mark]'), function (a) { + var id = a.getAttribute('data-mark'); + var live = Array.prototype.find.call(fig.pgpFullSVG.querySelectorAll('a[data-mark]'), function (n) { + return n.getAttribute('data-mark') === id; + }); + // Use a clone in the fixed geometry and delegate its click to the plugin's original node. + if (!live) { a.remove(); } + else { a.addEventListener('click', function (e) { e.preventDefault(); live.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); }); } + }); + } + } + if (current) { current.replaceWith(replacement); } else { fig.appendChild(replacement); } + }); + } + // The plugin places figures after person.js draws the slots. Mutation callbacks run before paint, + // including on the first async charts response; the card event also handles PANEL changing without + // any mark changing. No chart filename or data value is used to decide whether PANEL is closed. + new MutationObserver(veilCharts).observe(document.body, { childList: true, subtree: true }); + document.addEventListener('pgp:cards', function () { queueMicrotask(veilCharts); }); // A card's table without its closed rows: [table or null, how many rows it left out]. function openTable(c) { var tags = veilTags('rows', c.id), t = c.table; - if (!t || !t.rows || !tags.length) { return [t, 0]; } - var rows = t.rows.filter(function (r, i) { return !(tags[i] && tagClosed(tags[i])); }); + if (!t || !t.rows) { return [t, 0]; } + var rows = t.rows.filter(function (r, i) { return !placeClosed(tags[i]); }); var out = {}; Object.keys(t).forEach(function (k) { out[k] = t[k]; }); out.rows = rows; @@ -359,7 +403,7 @@ // A card's facts and drawing without their closed items (a fact, a CHIP call or a copy-number dot that names a // closed result's gene): [facts, widget, how many items they left out]. function openFacts(c) { - var ft = veilTags('facts', c.id), facts = (c.facts || []).filter(function (f, i) { return !(ft[i] && tagClosed(ft[i])); }); + var ft = veilTags('facts', c.id), facts = (c.facts || []).filter(function (f, i) { return !placeClosed(ft[i]); }); var w = c.widget, key = w && (w.kind === 'vaf' ? 'calls' : w.kind === 'copies' ? 'genes' : null); var wt = veilTags('widget', c.id), shut = (c.facts || []).length - facts.length; if (key && wt.length && w[key]) { @@ -370,6 +414,7 @@ copy[key] = items; w = copy; } + if (w && placeClosed(((results.veil || {}).widget_panel || {})[c.id])) { w = null; shut += 1; } return [facts, w, shut]; } function veiledOf(c) { return (c.findings || []).map(findingById).filter(function (f) { return f && veiled(f); }); } @@ -461,7 +506,7 @@ ' to act on in adult life. Its rows stay closed until ' + (closedFs.length === 1 ? 'it is' : 'each is') + ' opened.', null, function () { openFinding(closedFs[0]); redraw(true); var x = document.getElementById('f-' + closedFs[0].id) || document.getElementById('card-' + c.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); } else { - (c.notes || []).forEach(function (n) { add(text, el('p', { 'class': 'note-stale' }, n)); }); + (c.notes || []).forEach(function (n, i) { if (!placeClosed(veilTags('notes', c.id)[i])) { add(text, el('p', { 'class': 'note-stale' }, n)); } }); if (c.headline && !headlineShut(c)) { add(text, el('p', { 'class': 'headline' }, c.headline)); } var of = openFacts(c); add(margin, U.facts(of[0])); @@ -472,7 +517,7 @@ if (c.image) { add(text, el('img', { 'class': 'plot', src: c.image.src, alt: c.image.alt })); } var ot = openTable(c), tb = U.table(ot[0]); var lt = veilTags('lines', c.id); - var lines = (c.lines || []).filter(function (l, i) { return !(lt[i] && tagClosed(lt[i])); }); + var lines = (c.lines || []).filter(function (l, i) { return !placeClosed(lt[i]); }); var shut = ot[1] + (c.lines || []).length - lines.length + (c.headline && headlineShut(c) ? 1 : 0), items = of[2]; if (tb) { add(text, tb); } else if (c.empty_text && !ot[1]) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } if (shut || items) { @@ -502,7 +547,7 @@ var foot = el('p', { 'class': 'card-links' }); add(foot, el('a', { href: 'report.html#' + encodeURIComponent(c.anchor), 'class': 'report' }, 'Open in the report')); var vtags = veilTags('viewer', c.id); - var reads = (c.viewer || []).filter(function (x, i) { return !(vtags[i] && tagClosed(vtags[i])); }); + var reads = (c.viewer || []).filter(function (x, i) { return !placeClosed(vtags[i]); }); if (g === 'show' && reads.length && !panelShut(c)) { add(foot, el('a', { href: 'viewer/#' + reads[0], 'class': 'viewer' }, 'See the reads at ' + reads[0])); } @@ -577,7 +622,9 @@ // A finding that is not itself a result to act on but names one (a symptom ranking can rank the gene: results.veil.findings) // stays closed while that result is, and opening it opens that result. function findingTag(f) { + if (f.level === 'act') { return null; } // its own switch; never an unclassified secondary place var t = ((results && results.veil && results.veil.findings) || {})[f.id]; + if (t === undefined) { t = [PANEL]; } t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (x) { return x !== f.id; }); return t.length ? t : null; } diff --git a/bin/live_results.py b/bin/live_results.py index 6527beac1..a0e0a7bcf 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -179,7 +179,7 @@ def a_clinvar(S, s): lines = [x for x in (rr.cdna_scan_lines(d) or [])] for x in d.get("deep_intronic") or []: # Always reported by the report, hit or not: each site's call stays visible on the card. - lines.append(f"{FIXED_LINES['clinvar'][0]} {x.get('gene')} {x.get('hgvs')} " + lines.append(f"Known deep intronic site (always reported): {x.get('gene')} {x.get('hgvs')} " f"{x.get('chrom')}:{x.get('pos')} {x.get('ref')}>{x.get('alt')}: {rr.deep_intronic_call(x)}") freq = rr.clinvar_freq_line(d) if freq: @@ -1187,66 +1187,24 @@ def seg(text, bold, shows=(), act=False): "traits-panel": "the trait panel's genes", "bloodtype": "the blood group systems", } -# The lines and table rows of a card that are a fixed panel inside it (card -> the words such a line, or a cell of -# such a row, starts with): the ClinVar card prints one line per known deep intronic site of -# assets/deep_intronic_sites.tsv, hit or not; the callability card lists the catalog genes this reference does -# not hold, the same for every genome on it, and the hard genes the reads did not cover, which are nearly the same -# for every genome (segmental duplications, collapsed regions); the runs-of-homozygosity card names the imprinted -# regions of assets/imprinted_regions.tsv (their genes) on a chromosome a long run covers. (The carrier list's line of -# the two-hit callers, which names SMN1, HBA1 and HBA2, is prose of a card whose lines are not declared gene-free: -# PANEL as well.) veil() tags each of them -# PANEL, never by a gene, so they close all together while any result to act on is closed and open all together -# once the last one is opened (assets/live/person.js tagClosed()). Every other line or row says something about -# this genome alone. -FIXED_LINES = {"clinvar": ("Known deep intronic site (always reported):",), - "callability": ("not on this reference", "not read"), - "roh": ("Imprinted ",)} +# These catalog rows have structured gene cells but must never close one gene at a time. +# The other fixed-panel sentences (ClinVar's deep-intronic sites and ROH's imprinted regions) +# already take the default PANEL rule, so need no exemption or prefix guard. +FIXED_LINES = {"callability": ("not on this reference", "not read")} PANEL = "*panel*" -# The cards whose rows, lines and facts name no gene, only things known before the run: veil() never tags them. The -# score card names its scores and their traits (assets/pgs_scores.tsv) and its words "OR 1.57", "HR 1.4", "(TG)" -# are odds and hazard ratios and a trait, though HR and TG are gene symbols too. -NAMES_NO_GENE = {"prs": "the polygenic scores of assets/pgs_scores.tsv"} -# The cards whose lines never name a gene (card -> what they print), FIXED_LINES aside: veil() tags their lines by -# the places they print only. Every other card's line is prose that can name one, and is tagged PANEL. -LINES_NO_GENE = { - "qc": "the sex chromosomes' karyotype and copy numbers (render_report.karyotype_lines)", - "clinvar": "where the frequencies came from, and the places of the cDNA scan", - "mosaic": "the loss of an X or the Y, from depth and from allele fractions", - "mosaic-mca": "the cell-fraction floor, and each call's type, arm and place", - "roh": "F_ROH, its band, and the chromosome a long run covers", - "telomere": "the telomeric read counts", - "consumer-export": "the genotype counts and the file names", -} -# The cards whose facts never name a gene (card -> what they print): counts, states, versions, haplogroups, a -# partner's sample name. FACT_NAMES overrides one fact of such a card; every fact of a card on neither list is -# tagged PANEL (the fixed-panel cards' facts, closed with their card anyway). -FACTS_NO_GENE = { - "qc": "depth, read counts, sex, contamination, a matching sample", - "callability": "counts", "variants": "counts", "clinvar": "counts and the ClinVar file date", - "clinical-filter": "counts", "cpsr": "CPSR's counts", "slivar": "counts", - "exomiser": "the symptoms searched (HPO terms)", "structural-variants": "the callers' counts", - "mito": "counts", "mosaic": "the call", "mosaic-mca": "the status and counts", - "carrier-list": "counts", "couple": "the partner's sample name and counts", "pharmcat": "the report's version", - "ancestry-fine": "the groups' shares", "mt-haplogroup": "the haplogroup", "y-chromosome": "the haplogroups", - "archaic": "shares", "ancient": "counts", "roh": "lengths and F_ROH", - "telomere": "the telomere content", "viral": "the HHV-6 call and counts", "consumer-export": "counts", -} -# The facts of a FACTS_NO_GENE card that name genes: (card, label) -> the summary field that lists them, or None -# for prose (PANEL). -FACT_NAMES = {("carrier-list", "Genes with two hits"): "two_hits", - ("clinical-filter", "High-impact rare variants in its genes"): None} -# The summary keys whose headline (bin/owngenome_live.py HEADLINES) never names a gene (key -> what it prints). A -# card whose headline comes from another key (the couple screen's, which lists the shared genes) gets PANEL. -HEADLINES_NO_GENE = { - "clinvar": "the P/LP count", "carrier": "the carrier list's size and count", "roh": "F_ROH or the chromosome a run covers", - "pypgx": "CYP2D6's depth check", "annotsv": "the class 4 and 5 counts", "viral": "the HHV-6 call", - "archaic": "the Neanderthal share", "lpa": "the KIV-2 count", "cpsr": "the P/LP count and the ClinVar releases", - "mito": "the poly-C or ClinVar count", -} +# No card-wide prose exemption survives the fail-closed audit. Even counts often arrive with +# sample notes, ontology labels or data-dependent record presence. Structured rows and lines below +# retain their individual tags; all other data text takes PANEL. Keep these registries explicit so +# adding an exemption requires a producer-based invariance test. +NAMES_NO_GENE = {} +LINES_NO_GENE = {} +FACTS_NO_GENE = {} +FACT_NAMES = {("carrier-list", "Genes with two hits"): "two_hits"} +HEADLINES_NO_GENE = {} # Every column of every card's table (the adapters' table() calls), and what it holds: a gene ("gene": a gene # symbol, or a list of them), a place ("place": its loci are read, never its words) or the report's own vocabulary # (None: an inheritance code such as AR, a structural-variant type such as INS, a genotype, a class, a -# significance, a unit, a caller, a condition), never read. A column missing here counts as a gene column; +# significance, a unit, a caller, a condition), never read. An unknown column defaults to PANEL; # tests/test_live_results.py fails until every column of every table() call is listed. G, P = "gene", "place" TABLE_COLUMNS = { @@ -1388,13 +1346,16 @@ def row_tag(spans, columns, row, genes=()): cols = list(columns or []) names, where, at = list(genes or ()), [], {} for i, x in enumerate(cells): - kind = TABLE_COLUMNS.get(cols[i], G) if i < len(cols) else G + kind = TABLE_COLUMNS.get(cols[i]) if i < len(cols) else None if kind == G: names += gene_cell(x) elif kind == P: where.append(str(x)) if i < len(cols) and cols[i] in ("Chromosome", "Start", "End"): at[cols[i]] = x + coordinate = f"{at.get('Chromosome')}:{at.get('Start')}-{at.get('End')}" + if not names and not _loci(" ".join(where)) and not _loci(coordinate): + return [PANEL] if spans else None return veil_tags(spans, names, " ".join(where), at.get("Chromosome"), at.get("Start"), at.get("End")) @@ -1429,13 +1390,8 @@ def two_hit_names(S): def headline_free(card): - """Whether CARD's headline names no gene: it comes from a key of HEADLINES_NO_GENE, or from no headline helper.""" - try: - import owngenome_live - helpers = owngenome_live.HEADLINES - except Exception: # without the helpers no key is known to name no gene - helpers = {} - return all(k in HEADLINES_NO_GENE or k not in helpers for k in card.get("keys") or []) + """Only an explicitly audited producer can exempt a headline from PANEL.""" + return bool(card.get("keys")) and all(k in HEADLINES_NO_GENE for k in card["keys"]) def mark_names(S, sp, m): @@ -1477,19 +1433,24 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): if not spans: return {} out = {"rows": {}, "viewer": {}, "lines": {}, "facts": {}, "headline": {}, "widget": {}, "marks": {}, - "findings": {}, "lead": [], "spans": [list(s) for s in spans], + "findings": {}, "notes": {}, "lead": [], "spans": [list(s) for s in spans], "panels": [c["id"] for ch in chapters for c in ch["sections"] if c["id"] in FIXED_PANELS and c.get("state") != "missing"]} S = summary.get("sections") or {} hits = [h for h in _items(_data(S, "clinvar").get("hits"))[:50]] tags = [veil_tags(spans, [h.get("gene")], "", h.get("chrom"), h.get("pos")) if isinstance(h, dict) else None for h in hits] - if any(tags): + if tags: out["rows"]["clinvar"] = tags sv_rows = [x for x in _items(_data(S, "annotsv").get("rows"))] two = two_hit_names(S) th = _data(S, "carrier").get("two_hits") two_genes = [x.get("gene") if isinstance(x, dict) else x for x in _items((th or {}).get("genes"))] \ if isinstance(th, dict) else [] + mca_lines = {} + for x in _items(_data(S, "mosaic").get("calls")): + if isinstance(x, dict) and all(k in x for k in ("type", "arms", "chrom", "start", "end", "length_mb", "cell_fraction")): + for ln in rr.mosaic_mca_lines(dict(_data(S, "mosaic"), calls=[x]))[1:2]: + mca_lines[plain(ln)] = x cards = {} for ch in chapters: for c in ch["sections"]: @@ -1497,7 +1458,7 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): if c["id"] in NAMES_NO_GENE: continue rows = (c.get("table") or {}).get("rows") or [] - if c["id"] != "clinvar" and rows: + if rows and (c["id"] != "clinvar" or not hits or (c.get("table") or {}).get("columns", [None])[0] != "Gene"): cols = (c.get("table") or {}).get("columns") tags = [] for i, r in enumerate(rows): @@ -1507,14 +1468,20 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): more = [] if c["id"] == "structural-variants" and i < len(sv_rows) and isinstance(sv_rows[i], dict): more = [g for g in sv_rows[i].get("genes") or [] if g] # the whole list the cell shortens - tags.append(row_tag(spans, cols, r, more)) - if any(tags): + if c["id"] == "mito" and cols == ["Site", "Allele fraction"]: + sites = _items(_data(S, "mito").get("heteroplasmic_sites")) + site = sites[i] if i < len(sites) and isinstance(sites[i], dict) else {} + tags.append(veil_tags(spans, (), chrom="chrM", pos=site["pos"]) + if site.get("pos") is not None else [PANEL]) + else: + tags.append(row_tag(spans, cols, r, more)) + if tags: out["rows"][c["id"]] = tags tags = [veil_tags(spans, (), v) for v in c.get("viewer") or []] if c["id"] == "clinvar": # one link per row: a row closed by its gene alone closes its link too rt = out["rows"].get("clinvar") or [] tags = [union(rt[i] if i < len(rt) else None, t) for i, t in enumerate(tags)] - if any(tags): + if tags: out["viewer"][c["id"]] = tags tags = [] for ln in c.get("lines") or []: @@ -1522,11 +1489,14 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): tags.append([PANEL]) elif c["id"] == "carrier-list" and ln in two: tags.append(veil_tags(spans, [two[ln][0]], two[ln][1])) + elif c["id"] == "mosaic-mca" and ln in mca_lines: + x = mca_lines[ln] + tags.append(veil_tags(spans, (), chrom=x.get("chrom"), pos=x.get("start"), end=x.get("end"))) elif c["id"] in LINES_NO_GENE: tags.append(veil_tags(spans, (), ln)) else: tags.append(prose_tag(spans, ln)) - if any(tags): + if tags: out["lines"][c["id"]] = tags tags = [] for f in c.get("facts") or []: @@ -1538,8 +1508,10 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): tags.append(prose_tag(spans, " ".join(map(str, f)) if isinstance(f, (list, tuple)) else f)) else: tags.append(None) - if any(tags): + if tags: out["facts"][c["id"]] = tags + if c.get("notes"): + out["notes"][c["id"]] = [prose_tag(spans, n) for n in c["notes"]] if c.get("headline") and not headline_free(c): out["headline"][c["id"]] = prose_tag(spans, c["headline"]) # A drawing's items: the CHIP calls (gene) and the copy-number dots (gene), by their gene field. The CHIP @@ -1547,7 +1519,9 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): w = c.get("widget") or {} items = w.get("calls") if w.get("kind") == "vaf" else w.get("genes") if w.get("kind") == "copies" else None tags = [veil_tags(spans, gene_cell(x.get("gene"))) for x in items or []] - if any(tags): + if w: + out.setdefault("widget_panel", {})[c["id"]] = [PANEL] + if tags: out["widget"][c["id"]] = tags try: specs = rr.chart_specs(summary) @@ -1555,12 +1529,13 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): specs = [] for sp in specs: for m in sp.get("marks") or []: - if m.get("kind") == "roh": # a run of homozygosity is no result's source + if m.get("kind") == "roh": # structured coordinates; a run of homozygosity is no result's source + out["marks"].setdefault(sp["id"], {})[m["id"]] = None continue genes, where = mark_names(S, sp, m) - t = veil_tags(spans, genes, where, m.get("chrom"), m.get("pos"), m.get("end")) - if t: - out["marks"].setdefault(sp["id"], {})[m["id"]] = t + structured = genes or _loci(where) or (m.get("chrom") and m.get("pos") is not None) + t = veil_tags(spans, genes, where, m.get("chrom"), m.get("pos"), m.get("end")) if structured else [PANEL] + out["marks"].setdefault(sp["id"], {})[m["id"]] = t # A finding of a fixed-panel card (a medicine gene, a CHIP call, a repeat) is tagged PANEL whatever it names: it # closes with its card while any result to act on is closed, so the findings left open never tell which gene the # closed one is in. A finding with no gene and no place is prose (prose_finding()). @@ -1572,8 +1547,7 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): else: got = finding_names(f) t = veil_tags(spans, got[0], got[1]) if got else prose_finding(spans, f, cards.get(f.get("card")) or {}) - if t: - out["findings"][f["id"]] = t + out["findings"][f["id"]] = t # A lead segment is tagged by the findings it prints (lead()): every closed result one of them names. The # lead's own sentences print no gene. tags = [] @@ -1585,12 +1559,10 @@ def veil(summary, found, chapters, lead_parts=(), lead_names=()): def prose_finding(spans, f, card): - """The tag of finding F, which has no gene and no place: None when its words are known to name no gene (a - data-quality finding of its own sentences and a sample's name, a row that could not be read, or a card's - headline finding whose headline and facts name none), else PANEL.""" + """An unstructured finding takes PANEL unless an audited fact/headline producer exempts it.""" cid = f.get("card") - if cid == "qc" or str(f.get("id") or "").startswith("unreadable-"): - return None + # Quality findings and unreadable records can include sample names and parser messages. + # They have no structured gene/position proof either. if f.get("id") == slug("card", cid) and cid in FACTS_NO_GENE and headline_free(card) \ and not any((cid, a) in FACT_NAMES for a, _b in card.get("facts") or []): return None @@ -1641,7 +1613,7 @@ def overview(summary): # A PANEL-only finding stays in top with its tag: the home page closes its title and link for a child, # while adults and runs without a result to act on keep the same title and destination. finding_tags = (r.get("veil") or {}).get("findings") or {} - named = {k: v for k, v in finding_tags.items() if any(x != PANEL for x in v)} + named = {k: v for k, v in finding_tags.items() if any(x != PANEL for x in v or ())} top = sorted((f for f in r["findings"] if f["level"] != "info" and f["id"] not in named), key=lambda f: order[f["level"]])[:4] tags = (r.get("veil") or {}).get("lead") or [] diff --git a/bin/report_charts.py b/bin/report_charts.py index bd463e730..7ed86620b 100755 --- a/bin/report_charts.py +++ b/bin/report_charts.py @@ -324,6 +324,13 @@ def build(summary, notes, helpers): for x in out: if x is not None: x["svg"] = svg(x) + # The live child's drawing keeps structured marks, but no data-derived note, empty-state + # sentence, legend choice or variable dimension. Render its geometry from a stable spec. + # The complete SVG above remains the report's and an opened/adult page's drawing. + if x["kind"] in ("ideogram", "scatter"): + closed = dict(x, not_drawn=[], axis=dict(x["axis"], sites_missing=0), veil_geometry=True) + closed["marks"] = list(x["marks"]) + x["svg_closed"] = svg(closed) return [x for x in out if x is not None] @@ -501,7 +508,7 @@ def inside(a, w): + _text(x0 + 258, y + 8, "ClinVar hit", 11) + f'' + _text(x0 + 348, y + 8, "low-confidence ClinVar call", 11)) - kinds = {m.get("kind") for m in sp["marks"]} + kinds = {"sv", "mca"} if sp.get("veil_geometry") else {m.get("kind") for m in sp["marks"]} if kinds & {"sv", "mca"}: # A second legend row, drawn only when the chart has these marks. y += 18 @@ -531,7 +538,9 @@ def svg_scatter(sp): for lo, hi in ((303, 315), (16184, 16193)): body.append(f'') - if sp["axis"].get("sites_missing"): + if sp.get("veil_geometry"): + pass # empty-state prose is PANEL too + elif sp["axis"].get("sites_missing"): body.append(_text((x0 + x1) / 2, (y0 + y1) / 2, "The positions of the heteroplasmic sites are not in this " "summary", 12, "middle")) elif not sp["marks"]: diff --git a/docs/live-site.md b/docs/live-site.md index b97e4c174..329f91d77 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -58,7 +58,17 @@ Every result carries one of six levels, with a word, a shape and a colour, so th A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. Two pages whose open rows are the same differ only in how many results are closed, not in which; the section that holds a closed result still says it holds one: -- A card whose rows are a fixed panel, the same genes or loci for everyone, would tell the closed result's gene by the row it leaves out. So these cards close whole while any result to act on is closed, behind one box with the same words whatever the result: the known mobile element insertions, the repeat expansions, VNtyper, Lp(a), CHIP, the paralog copy numbers (SMN1, SMN2, HBA1, HBA2, the RCCX module), CPIC, CYP2D6, HLA, Locityper, the traits panel and the blood groups. The one list is `FIXED_PANELS` in `bin/live_results.py`, and `results.json` gives it as `veil.panels`. Their "Show the result" opens the next closed result, and the cards come back once every result is opened. The cost: a child's page with a closed result shows none of these cards until the results are opened. Their own findings (a medicine gene, a CHIP call, a repeat) close with them, whatever gene they name, and so does a segment of the lead that names one (`veil.findings` and `veil.lead` tag them `*panel*`): a medicine finding that closed only when the closed result is in its gene would tell the gene. A card can hold a fixed panel among its lines or rows: the ClinVar card prints a line for each known deep intronic site, hit or not; the carrier list prints its two-hit line, which names the copy-number callers it read; and the callability card lists the genes this reference does not hold and the hard genes the reads did not cover, nearly the same for everyone. Those lines and rows close all together while any result to act on is closed and come back once every result is opened (`FIXED_LINES` in `bin/live_results.py`; `veil.lines` and `veil.rows` tag each of them `*panel*`). A table row, a finding or a chart mark closes with a result only by the fields that name a gene or a place: a row by its gene and position columns, a finding by its gene, its genes and its loci, a chart mark by the gene and position of the summary row it draws. It never closes by a vocabulary cell (an inheritance code, a structural-variant type, a genotype, a class, a significance, a unit) or by a structural variant's title, because any code the report prints in capitals may also be a gene symbol: a closed result in AR would else close every recessive carrier row, and one in INS every insertion. `TABLE_COLUMNS` in `bin/live_results.py` says for each column of each card's table whether it names a gene or a place. A place that names two closed results closes with both: its tag is the list of every closed result it names, never the first one only, and it stays closed while any of them is closed. A segment of the lead that names two closed results, or a duplication over two closed genes, comes back only once both are opened, so opening one never tells the other. The page never matches the words of prose against gene symbols: any word the report prints in capitals may be a gene symbol (`AR` in "Cystic fibrosis; AR"), and a line that closes by such a word tells the gene. Prose with no field that names a gene closes whole (`*panel*`) while any result to act on is closed: a card's headline, its lines and facts, a finding with no gene and no place, and the lead's own sentences. The exceptions are structured or declared free of genes, each list pinned in `tests/test_live_results.py`. Structured: the carrier list's two-hit lines close by the gene and variant each line is built from, and its two-hit count by the two-hit genes (`FACT_NAMES`); a segment of the lead closes with the findings it prints (`veil.lead`); a line that gives a position closes by that position. Declared free of genes: the lines of the QC, ClinVar, mosaic, MoChA, ROH, telomere and consumer-export cards (`LINES_NO_GENE`), the facts of the cards in `FACTS_NO_GENE`, the headlines of the cards in `HEADLINES_NO_GENE` (built from counts and fixed words), a data-quality finding, and a card's own finding when its facts and headline are free. The cost: a child's page with any result closed shows less prose (the headline and lines of a card that can name a gene, the search's hits) until the results are opened. The polygenic score card, which names scores and traits only, is never tagged (`NAMES_NO_GENE`), and neither are the marks of a chart whose labels come from a list known before the run (the scores, the ancestry groups, the ClinVar star bars' labels). The genome map leaves a finding the veil ties to a closed result out of its count of findings with no place. +- A fixed panel lists genes or loci known before the run. Hiding only one row would identify the closed result's gene. These cards close whole while any result to act on stays closed. They include known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP and paralog copy numbers. CPIC, CYP2D6, HLA, Locityper, traits and blood groups follow the same rule. The paralog panel includes SMN1, SMN2, HBA1, HBA2 and RCCX. The single list is `FIXED_PANELS` in `bin/live_results.py`, exported as `veil.panels`. Each box uses the same words and opens the next closed result. The cards return after every act opens. + + Their own findings and the lead segments naming them also take `*panel*`, whatever gene they name. Fixed panels inside other cards follow this rule too. These include ClinVar's known deep intronic sites, the carrier list's copy-number caller sentence, and callability's catalog genes missing from the reference or unread by the reads. Callability's row prefixes remain in `FIXED_LINES`. The other sentences take the default prose rule. + + Structured tags use gene and coordinate fields. `TABLE_COLUMNS` classifies every table column. A row uses its gene and position cells, a finding its genes and loci, and a chart mark its source record. An SV uses every gene it covers. Vocabulary cells never count as genes: inheritance, SV type, genotype, class, significance and units. The page never matches prose words against gene symbols. Otherwise a closed result in AR could hide every recessive carrier row, and INS could hide every insertion. + + Each tag lists every matching closed result. A place naming two results stays closed while either remains closed. This includes a duplication over two closed genes and a lead segment naming two findings. The carrier list's two-hit lines use their source gene and variant. Its two-hit count uses the two-hit genes (`FACT_NAMES`). MoChA call lines use their source coordinates. Mitochondrial rows use each record's structured chrM position, so HGVS text and chart marks stay closed until every matching result opens. Lead segments follow the findings they print. The genome map excludes findings tied to closed results from its open findings without a place. + + Unclassified rows, facts, lines, headlines, stale notes and findings default to `*panel*`. No card-wide gene-free exemption remains. HPO labels, sample names, haplogroups, trait labels and record counts cannot prove that their producer is safe. The genome and mitochondrial charts retain coordinate-tagged marks on fixed geometry. Data-dependent notes, empty-state sentences and legend choices stay absent. Other charts close whole. The complete chart returns after the last act opens and closes again if any act closes. Unstructured card drawings, including the Y path, also stay closed until every act opens. + + The accepted cost is less of a child's report while any result to act on stays closed. More prose, tables and charts remain closed until every act opens. This is a browser veil: the result and chart files still hold their complete data. - The map's APOE row follows its own warning switch. Opening it shows the same mark even when another result in APOE stays closed. Otherwise its absence would tell which result is closed. Other map rows keep every tag they follow. - A closed finding carries no id in the page or the address, since its id names its gene and variant. It is known by its place among the page's closed findings (`#f/closed-2`). On a page that keeps a result closed, the address never holds the id of a finding the page can close, open or closed: once opened on purpose it is `#f/opened-`, its place among the opened ones, so no history entry keeps the id. Any other `#f/` on such a page, the id of a closed result or no id at all, lands the same way, on the first closed finding (`#f/closed-1`): the page never picks a closed card by its id, so a guessed id tells nothing. On the home page, a child's closed result links to the child's report, never to the finding's own page, whose address names it. @@ -94,7 +104,7 @@ Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with it The reads cross the network to the browser that logged in: only the byte ranges igv.js asks for, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS); without it the password and the reads cross the network in clear. Opening the viewer writes one access-log row; the byte ranges igv.js reads write none. -`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title and veil tag; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). A child's home view keeps a panel-tagged finding's title and link closed too, at every level. Adults and runs with no result to act on keep the title and link. While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. +`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title and veil tag; a finding tied to a result to act on is omitted from this legacy summary, which the closed child view does not use), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a child has any result to act on closed, the home page builds attention rows and the running tile's findings from level counts alone. Each row links to the report and uses generic text. The tile's lead is gene-free. This also hides whether a symptom ranking names a closed gene. Adults and runs with no result to act on keep their existing titles, prose and destinations. While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the places outside a result's own card that close with it, each tagged with the list of closed results it names, or `*panel*`; `spans`, the closed results with their position, one row per gene they name, from which the family page knows whether a child has a result still closed; and `panels`, the cards of a fixed panel). A structural variant's finding also lists every gene it covers in `genes`. It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index b12130105..7913eb5a8 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -439,6 +439,10 @@ def widget_counts(b, base): b.go(f"{base}/runs/{person}/#s/{cid}") b.wait("document.documentElement.dataset.pgpResults && !!document.getElementById(%s) && 'drawn'" % json.dumps("card-" + cid)) + if kind == "ypath": + acts = json.loads(b.eval("JSON.stringify((function () { var x = new XMLHttpRequest(); x.open('GET', 'results.json', false); x.send(); return JSON.parse(x.responseText).findings.filter(function (f) { return f.level === 'act'; }).map(function (f) { return f.id; }); }()))")) + for fid in acts: + open_closed(b, fid) if kind in ("MoChA bar", "AnnotSV span"): b.wait("document.documentElement.dataset.pgpCharts") time.sleep(0.5) @@ -670,6 +674,7 @@ def planted_closed(b, base): function shut(f) { if (f.level === 'act') { return !open(f.id); } var t = tags[f.id]; + if (t === undefined) { t = ['*panel*']; } t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (x) { return x !== f.id; }); return t.some(function (x) { return x === '*panel*' ? anyShut : !byId[x] || !open(x); }); } @@ -774,7 +779,7 @@ def planted_pages(b, base, out): b.wait("document.documentElement.dataset.pgpStatus === 'ready' && 'drawn'", 10) shown = b.eval(shown_js) or "" out_ += [f"{w} on the finding page {fid}" for w in [gene] + hit + sv if w in shown] - named = [k for k in ((res.get("veil") or {}).get("findings") or {}) if k.startswith("exomiser-")] + named = [k for k, t in ((res.get("veil") or {}).get("findings") or {}).items() if t and k.startswith("exomiser-")] if len(ids) < 10 or not named: return out_ + [f"the son's findings are not the planted ones: {ids}, ranking {named}"] b.go("about:blank") @@ -813,10 +818,23 @@ def write(p, text): res = json.loads(keep[rj]) res["veil"] = {} write(rj, json.dumps(res)) - control("results.json without its veil tags", not planted_closed(b, base)) + bad = planted_closed(b, base) + check("browser: results.json without veil metadata still keeps secondary places closed", not bad, bad) + res = json.loads(keep[rj]) + def unsafe_tags(value): + if isinstance(value, dict): + return {k: unsafe_tags(v) for k, v in value.items()} + if isinstance(value, list): + return None if all(isinstance(v, str) for v in value) else [unsafe_tags(v) for v in value] + return value + for kind in ("rows", "viewer", "lines", "facts", "headline", "marks", "findings"): + if kind in res["veil"]: + res["veil"][kind] = unsafe_tags(res["veil"][kind]) + write(rj, json.dumps(res)) + control("results.json falsely exempts tagged secondary places", not planted_closed(b, base)) write(rj, keep[rj]) for p, a, z, desc in ( - (pj, "return !(tags[i] && tagClosed(tags[i]));", "return true;", "person.js whose ClinVar rows ignore their tags"), + (pj, "return !placeClosed(tags[i]);", "return true;", "person.js whose ClinVar rows ignore their tags"), (pj, "return !!(t && tagClosed(t));", "return false;", "person.js whose findings ignore their tags"), (cj, "return ask ? (sp.marks", "return false ? (sp.marks", "charts.js that keeps every mark"), (sj, "it.veil || null", "null", "search.js that ignores a hit's tag")): @@ -1791,6 +1809,9 @@ def home_problems(b, base, out, cases=HOME): bad = [] for what, one, two in cases: h1, h2 = home_now(b, base, out, one), home_now(b, base, out, two) + links = json.loads(b.eval("JSON.stringify(Array.from(document.querySelectorAll('#attention a')).filter(function (a) { var who = a.querySelector('.who .pc'); return who && who.textContent === " + json.dumps(SON) + "; }).map(function (a) { return a.getAttribute('href'); }))") or "[]") + if any("finding/" in link for link in links): + bad.append(f"home, {what}: a closed summary links to a finding page") diff = [x for x in difflib.unified_diff(h1, h2, lineterm="", n=0) if x[:1] in "+-" and x[:3] not in ("+++", "---")] if diff: bad.append(f"home, {what}: {len(diff)} lines differ: {diff[:4]}") @@ -2105,8 +2126,8 @@ def gate_regression_checks(b, base, out, og): ("map.js", "var tag = !!r.gate ||", "var tag =", "map-owned APOE row following overlapping act tags", apoe_pair_problems), ("finding.js", "return t.length ? t : null;", "return t.length ? t.slice(0, 1) : null;", "finding.js tagOf keeping only the first id", related_set_problems), - ("home.js", "(f.level === 'act' || (f.veil || []).indexOf('*panel*') >= 0)", "f.level === 'act'", - "home.js ignoring panel tags", home_panel_problems)): + ("home.js", "if (!closedSummary(r)) { return r.top || []; }", "if (true) { return r.top || []; }", + "home.js ignoring count-based child summaries", lambda *args: fail_closed_problems(*args, cases=["home"]))): p = os.path.join(out, "assets", file) with open(p) as fh: source = fh.read() @@ -2131,18 +2152,196 @@ def gate_regression_checks(b, base, out, og): control("closable without coordinates", not chart_position_problems(b, base, out, og)) finally: live_results.mark_names, live_results.closable = names, spans - overview = live_results.overview + print(f" gate regression checks: {time.time() - t0:.2f} s") + + +def fail_closed_problems(b, base, out, og, cases=None, producers=None): + """Producer plantings: home rankings, ontology facts, chart geometry and structured mitochondrial sites.""" + bad = [] + cases = cases or ("home", "hpo", "contig", "mito", "producers", "metadata") + if "home" in cases: + for n in (1, 2, 3): + def ranked(g): + def change(S): + S.clear() + cpsrs(g, *["MLH1", "MSH2"][:n - 1])(S) + S["exomiser"] = {"state": "ok", "data": {"terms": [], "genes": [ + {"gene": "BRCA1", "rank": 1, "moi": "AD", "exomiser_score": 0.9, "variants": []}]}} + return change + a, z = (home_now(b, base, out, ranked(g)) for g in ("BRCA1", "BRCA2")) + if a != z: + bad.append(f"home: {n} closed results change the whole document with the same BRCA1 ranking") + if "hpo" in cases: + import collect_summary + with tempfile.TemporaryDirectory() as td: + folder = os.path.join(td, "exomiser") + os.mkdir(folder) + with open(os.path.join(folder, f"{SON}_exomiser_top.tsv"), "w") as fh: + fh.write("rank\tgene\tmoi\texomiser_score\tphenotype_score\tvariant\tgenotype\tconsequence\thgvs\tacmg\tdisease\n" + "1\tCFTR\tAR\t0.9\t0.8\tchr7:117559590\thet\tmissense\tc.1521_1523del\tPathogenic\tDemo condition\n") + with open(os.path.join(folder, f"{SON}_exomiser_terms.tsv"), "w") as fh: + fh.write("hpo_id\tlabel\nHP:0020055\tDiminished MLH1 expression\n") + _, data = collect_summary.sec_exomiser(td, SON) + def hpo(S): + cpsrs("MLH1", "MSH2")(S) + S["exomiser"] = {"state": "ok", "data": data} + keep = plant_summary(out, og, hpo) + try: + page_now(b, base) + for stage in ("closed", "partly opened", "all opened"): + if stage == "partly opened": + assert open_closed(b, cid("MLH1")) + if stage == "all opened": + with open(os.path.join(out, "runs", SON, "results.json")) as fh: + acts = [f["id"] for f in json.load(fh)["findings"] if f["level"] == "act"] + for fid in acts: + open_closed(b, fid) + visible = b.eval("document.querySelector('#card-exomiser').textContent.includes('Diminished MLH1 expression')") + if visible != (stage == "all opened"): + bad.append(f"hpo: label visibility {visible} while {stage}") + finally: + unplant(keep) + if "contig" in cases: + views = [] + for locus in ("chrUn_KI270442v1:100-200", "chr13:32300000-32300100"): + keep = plant_summary(out, og, sv_over(locus, ["BRCA1"], False)) + try: + views.append(page_now(b, base)) + with open(os.path.join(out, "runs", SON, "results.json")) as fh: + acts = [f["id"] for f in json.load(fh)["findings"] if f["level"] == "act"] + for fid in acts: + assert open_closed(b, fid), fid + note = b.eval("document.querySelector('figure[data-chart=genome]').textContent.includes('SVs on other contigs')") + if note != locus.startswith("chrUn"): + bad.append("contig: the full chart note did not return after every act opened") + assert b.eval("(function () { var b = document.querySelector('.gm-chip.opened button'); if (!b) return false; b.click(); return true; }())") + if b.eval("document.querySelector('figure[data-chart=genome]').textContent.includes('SVs on other contigs')"): + bad.append("contig: re-closing an act leaves the chart note visible") + finally: + unplant(keep) + if views[0] != views[1]: + bad.append("contig: closed SV changes text, attributes, address, canvas or SVG geometry") + if "mito" in cases: + def mito(S): + cpsrs("MT-TL1", "MT-TL2")(S) + for row in S["cpsr"]["data"]["plp"]: + row["variant"] = "chrM:3243 A>G" + S["mito"] = {"state": "ok", "data": {"pass": 1, "heteroplasmic": 1, + "heteroplasmic_sites": [{"pos": 3243, "ref": "A", "alt": "G", "af": 0.25}]}} + keep = plant_summary(out, og, mito) + try: + page_now(b, base) + expr = "JSON.stringify([!!document.querySelector('#card-mito tbody tr'), !!document.querySelector('figure[data-chart=mito] [data-mark=m3243-G]')])" + for stage, fid in enumerate((None, "cpsr-mt-tl1-chrm-3243-a-g", "cpsr-mt-tl2-chrm-3243-a-g")): + if fid and not open_closed(b, fid): + bad.append(f"mito: no closed result {fid}") + break + got = json.loads(b.eval(expr)) + if got != [stage == 2] * 2: + bad.append(f"mito: row/mark after {stage} matching openings {got}") + finally: + unplant(keep) + if "metadata" in cases: + keep = plant_summary(out, og, cpsr("MLH1")) + try: + path = os.path.join(out, "runs", SON, "results.json") + with open(path) as fh: + result = json.load(fh) + result["veil"] = {k: v for k, v in result["veil"].items() if k in ("spans", "panels")} + card = next(c for ch in result["chapters"] for c in ch["sections"] if c["id"] == "qc") + token = "MLH1_UNCLASSIFIED_NOTE" + card.update(headline=token, facts=[["Unknown", token]], lines=[token], notes=[token], + table={"columns": ["Detail"], "rows": [[token]]}) + with open(path, "w") as fh: + json.dump(result, fh) + if token in "\n".join(page_now(b, base)[0]): + bad.append("metadata: a missing tag exposes unclassified content") + for finding in result["findings"]: + if finding["level"] == "act": + assert open_closed(b, finding["id"]), finding["id"] + if token not in (b.eval("document.documentElement.outerHTML") or ""): + bad.append("metadata: content does not return after every act opens") + finally: + unplant(keep) + if "producers" in cases: + # Each adapter receives an unknown free-text field and an extra unstructured row, in turn. Neither + # the token's spelling nor its record count earns an exemption from the default PANEL rule. + for producer, adapter in list(live_results.ADAPTERS.items()): + if producers and producer not in producers: + continue + def injected(S, summary): + got = adapter(S, summary) + got.update(facts=[["Unclassified", "MLH1"]], lines=["MLH1 sample note"], + table={"columns": ["Detail"], "rows": [["MLH1"], ["extra MLH1 record"]]}) + return got + live_results.ADAPTERS[producer] = injected + try: + # Supply each adapter's real demo section. Missing sections are tested by the unit counterpart. + views = [] + for gene in ("MLH1", "BRCA2"): + keep = plant_summary(out, og, cpsr(gene)) + try: + views.append(page_now(b, base)) + if any(token in "\n".join(views[-1][0]) for token in ("MLH1 sample note", "extra MLH1 record")): + bad.append(f"producer {producer}: unclassified token reached the closed DOM") + finally: + unplant(keep) + if views[0] != views[1]: + bad.append(f"producer {producer}: unclassified planting changes the closed document") + finally: + live_results.ADAPTERS[producer] = adapter + return bad + + +def fail_closed_checks(b, base, out, og): + started = time.monotonic() + for case in ("home", "hpo", "contig", "mito", "producers", "metadata"): + bad = fail_closed_problems(b, base, out, og, [case]) + check("browser: fail-closed producers " + case, not bad, bad) + prose, row, veil = live_results.prose_tag, live_results.row_tag, live_results.veil try: - def no_panel(summary): - r = overview(summary) - for f in r["top"]: - f.pop("veil", None) - return r - live_results.overview = no_panel - control("overview dropping panel tags", not home_panel_problems(b, base, out, og)) + live_results.FACTS_NO_GENE["exomiser"] = "unsafe HPO labels" + control("an HPO fact declared gene-free", not fail_closed_problems(b, base, out, og, ["hpo"])) + live_results.FACTS_NO_GENE.pop("exomiser", None) + def no_mito(*args, **kwargs): + got = veil(*args, **kwargs) + got.get("rows", {}).pop("mito", None) + return got + live_results.veil = no_mito + control("mitochondrial rows without structured positions", not fail_closed_problems(b, base, out, og, ["mito"])) + live_results.veil = veil + live_results.prose_tag = lambda spans, text: None + control("unclassified producer prose left open", not fail_closed_problems(b, base, out, og, ["hpo"])) + live_results.prose_tag = prose + live_results.row_tag = lambda spans, columns, values, genes=(): \ + live_results.veil_tags(spans, live_results.gene_cell(values[0])) if columns == ["Detail"] \ + else row(spans, columns, values, genes) + control("unclassified rows read as gene cells", not fail_closed_problems(b, base, out, og, ["producers"], ["qc"])) finally: - live_results.overview = overview - print(f" gate regression checks: {time.time() - t0:.2f} s") + live_results.FACTS_NO_GENE.pop("exomiser", None) + live_results.prose_tag, live_results.row_tag, live_results.veil = prose, row, veil + path = os.path.join(out, "assets", "person.js") + with open(path) as fh: + reader = fh.read() + try: + old = "parseFromString(sp.svg_closed," + assert reader.count(old) == 1 + with open(path, "w") as fh: + fh.write(reader.replace(old, "parseFromString(sp.svg,")) + control("chart notes and geometry outside PANEL", not fail_closed_problems(b, base, out, og, ["contig"])) + finally: + with open(path, "w") as fh: + fh.write(reader) + try: + old = "tag === undefined ? tagClosed(PANEL)" + assert reader.count(old) == 1 + with open(path, "w") as fh: + fh.write(reader.replace(old, "tag === undefined ? false")) + control("missing tag metadata treated as open", not fail_closed_problems(b, base, out, og, ["metadata"])) + finally: + with open(path, "w") as fh: + fh.write(reader) + print(f" fail-closed producer checks: {time.monotonic() - started:.2f} s") def infer_checks(b, base, out): @@ -2281,7 +2480,7 @@ def infer_checks(b, base, out): live_results.finding_names = names live_results.mark_names = lambda S, sp, m: (re.findall(r"[A-Z0-9]+", f"{m.get('label') or ''} {m.get('detail') or ''}"), "") control("a veil that reads a chart mark's label word by word", - not infer_problems(b, base, out, og, [x for x in INFER if "(TG)" in x[0]])) + not infer_problems(b, base, out, og, [x for x in INFER if "(INS)" in x[0]])) finally: live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names = cols, names, marks og_map = build_live_demo.og_map @@ -2356,17 +2555,18 @@ def phone_problems(): def shown(): return b.eval("JSON.stringify([" + SHOWN_JS + ", (document.querySelector('[data-widget=copies]') || {}).textContent || '', " - "document.querySelectorAll('.finding[data-level=act] .veil').length])") + "document.querySelectorAll('.finding[data-level=act] .veil').length, " + "document.querySelectorAll('#card-paralogs tbody tr,#card-chip tbody tr').length])") def problems(text, opened): - page, cops, _ = json.loads(text) if text else ["", "", 0] + page, cops, _, rows = json.loads(text) if text else ["", "", 0, 0] out_ = [] - deep = page.count(live_results.FIXED_LINES["clinvar"][0]) + deep = page.count("Known deep intronic site (always reported):") if opened and not ("SMN1" in cops and "SMN2" in cops and "p.R175H" in page and "SMN1 copies" in page and deep == sites[0]): out_.append(f"every result opened, yet a dot, the call, the fact or a deep intronic line ({deep} of {sites[0]}) is " f"missing: {cops[:200]}") - if not opened and (cops or "p.R175H" in page or "SMN1 copies" in page or deep): + if not opened and (rows or cops or "p.R175H" in page or "SMN1 copies" in page or deep): out_.append(f"a closed result, yet the copy-number or CHIP card shows its rows or the ClinVar card " f"{deep} deep intronic lines: {cops[:200]}") return out_ diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 1c0456d44..da0d3f94e 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -383,7 +383,8 @@ def veil_problems(res, idx): view = (v.get("viewer") or {}).get("clinvar") or [] if [sorted(t) if t else None for t in view] != [both if x == f"{c}:{p}" else None for x in card.get("viewer") or []]: out.append(f"ClinVar viewer links tagged {view}") - marks = {k: {m: sorted(t) for m, t in x.items()} for k, x in (v.get("marks") or {}).items()} + marks = {k: {m: sorted(t) for m, t in x.items() if t and t != [live_results.PANEL]} for k, x in (v.get("marks") or {}).items()} + marks = {k: x for k, x in marks.items() if x} specs = {sp["id"]: sp for sp in rr.chart_specs(res["_summary"])} hit = [m["id"] for m in specs["genome"]["marks"] if m.get("kind") == "hit" and m["pos"] == p and m["chrom"] == c] span = [m["id"] for m in specs["genome"]["marks"] if m.get("kind") == "sv" and (m["chrom"], m["pos"], m["end"]) == (sv_c, a, b)] @@ -403,7 +404,7 @@ def veil_problems(res, idx): exo = (v.get("rows") or {}).get("exomiser") or [] ranked = [f["id"] for f in res["findings"] if f["level"] != "act" and gene in f["title"]] # A fixed-panel card's own findings (a medicine gene) carry the panel tag, whatever the closed result. - named = {k: t for k, t in (v.get("findings") or {}).items() if t != [live_results.PANEL]} + named = {k: t for k, t in (v.get("findings") or {}).items() if t and t != [live_results.PANEL]} if not any(sf in (t or ()) for t in exo) or not ranked or set(named) != set(ranked) \ or any(sf not in named[i] for i in ranked): out.append(f"the symptom ranking of {gene}: rows tagged {exo}, findings tagged {v.get('findings')}, want {ranked} -> {sf}") @@ -427,7 +428,7 @@ def veil_lines_problems(): found = [{"id": "sf-gla-test", "level": "act", "gene": "GLA", "detail": "GLA"}] v = live_results.veil({"sections": S}, found, [{"sections": [{"id": "clinvar", "lines": lines}]}]) panel = [live_results.PANEL] - want = [panel if ln.startswith(live_results.FIXED_LINES["clinvar"]) else None for ln in lines] + want = [panel for ln in lines] got = (v.get("lines") or {}).get("clinvar") out = [] if want.count(panel) == 2 and got == want else [f"lines {lines} tagged {got}, want {want}"] # A segment of the lead is tagged by the findings it prints (lead()'s names), beside the act names the page @@ -594,14 +595,11 @@ def gene_and_position_problems(): ("callability", ["NBPF1", "not read", "69%", "segdup, collapsed"], True), ("roh", "Imprinted 6q24 (PLAGL1, HYMAI): from the mother: -; from the father: Transient neonatal diabetes " "mellitus (6q24-related) (PMID 20301706)", False)) -WANT_NO_GENE = (("prs", ["Asthma", "PGS002727", "68.3 (EUR)", "2605/2872", "HR 1.57 (EUR, PPM014751)"], "HR"),) -WANT_LINES_NO_GENE = ("qc", "clinvar", "mosaic", "mosaic-mca", "roh", "telomere", "consumer-export") -WANT_FACTS_NO_GENE = ("qc", "callability", "variants", "clinvar", "clinical-filter", "cpsr", "slivar", "exomiser", - "structural-variants", "mito", "mosaic", "mosaic-mca", "carrier-list", "couple", "pharmcat", - "ancestry-fine", "mt-haplogroup", "y-chromosome", "archaic", "ancient", "roh", "telomere", - "viral", "consumer-export") +WANT_NO_GENE = () +WANT_LINES_NO_GENE = () +WANT_FACTS_NO_GENE = () WANT_FACT_PROSE = (("clinical-filter", "High-impact rare variants in its genes"),) -WANT_HEADLINES_NO_GENE = ("clinvar", "carrier", "roh", "pypgx", "annotsv", "viral", "archaic", "lpa", "cpsr", "mito") +WANT_HEADLINES_NO_GENE = () # Prose, each with the gene it names or the gene its word is spelled like: the tag must be the panel tag either way. # The carrier list's summary of the two-hit callers (it names SMN1, HBA1 and HBA2 as the callers it read) is prose of # a card with no gene-free lines: the panel tag, as every other line of that card but a two-hit gene's own. @@ -627,6 +625,9 @@ def table_problems(): of a card declared gene-free there tagged by the panel tag (or one of a card not declared not), prose read word by word, or a gene cell not read as its list of symbols.""" out = [] + for name in ("NAMES_NO_GENE", "LINES_NO_GENE", "FACTS_NO_GENE", "HEADLINES_NO_GENE"): + if getattr(live_results, name): + out.append(f"{name}: no card-wide free-text exemption passed the producer audit") panel = [live_results.PANEL] found = lambda g: [{"id": "sf-" + g.lower(), "level": "act", "gene": g, "detail": g}] chapters = [{"sections": [{"id": c, "state": "ok"} for c in WANT_PANELS]}] @@ -634,7 +635,7 @@ def table_problems(): out += [f"card {c} is not in veil.panels" for c in WANT_PANELS if c not in got] for card, item, is_row in WANT_LINES: g = gene_of(item) - sec = {"id": card, "table": {"head": ["a", "b", "c", "d"], "rows": [item]}} if is_row else {"id": card, "lines": [item]} + sec = {"id": card, "table": {"columns": ["Gene", "Status", "Bases read", "Why"], "rows": [item]}} if is_row else {"id": card, "lines": [item]} v = live_results.veil({"sections": {}}, found(g), [{"sections": [sec]}]) tag = ((v.get("rows" if is_row else "lines") or {}).get(card) or [None])[0] if tag != panel: @@ -669,6 +670,16 @@ def table_problems(): v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [sec]}]) if ((v.get("headline") or {}).get("c")) != (None if key in WANT_HEADLINES_NO_GENE else panel): out.append(f"the headline from {key} tagged {(v.get('headline') or {}).get('c')}") + for card in live_results.ADAPTERS: + sec = {"id": card, "keys": ["unknown-producer"], "headline": "MLH1 sample note", + "facts": [["Unclassified", "MLH1"]], "lines": ["MLH1 sample note"], + "table": {"columns": ["Future field"], "rows": [["MLH1"], ["extra MLH1 record"]]}} + v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [sec]}]) + for kind in ("facts", "lines", "rows"): + if not (v.get(kind) or {}).get(card) or any(t != panel for t in v[kind][card]): + out.append(f"{card}: unclassified {kind} did not default to PANEL") + if (v.get("headline") or {}).get(card) != panel: + out.append(f"{card}: unknown headline did not default to PANEL") # Prose read with no gene field: the panel tag whatever it names, so the tag never tells the words apart. for g, text in WANT_PROSE: for spans in (live_results.closable(found(g)), live_results.closable(found("BRCA2"))): @@ -696,8 +707,8 @@ def table_controls(): live_results.FIXED_LINES = dict(keep["FIXED_LINES"], **{card: tuple(x for x in words if x != w)}) control(f"FIXED_LINES[{card}] without {w!r}", not table_problems()) live_results.FIXED_LINES = keep["FIXED_LINES"] - live_results.NAMES_NO_GENE = {} - control("NAMES_NO_GENE empty", not table_problems()) + live_results.NAMES_NO_GENE = {"prs": "an unsafe blanket exemption"} + control("NAMES_NO_GENE with an unsafe exemption", not table_problems()) live_results.NAMES_NO_GENE = keep["NAMES_NO_GENE"] for n in ("LINES_NO_GENE", "FACTS_NO_GENE", "FACT_NAMES", "HEADLINES_NO_GENE"): for k in keep[n]: @@ -948,11 +959,14 @@ def tagged(sample_dir): "row closed by its gene closes its reads link, and a lead segment carries the ids of every closed result " "its findings name", not bad, bad) fixed = live_results.FIXED_LINES + prose = live_results.prose_tag try: # the card still prints its lines; veil() no longer knows they are a panel live_results.FIXED_LINES = type("Unknown", (dict,), {"get": lambda self, k, d=None: None})(fixed) + live_results.prose_tag = lambda spans, text: live_results.veil_tags(spans, re.findall(r"[A-Z][A-Z0-9]+", text), text) control("a veil that tags a known deep intronic site's line by its gene", not veil_lines_problems()) finally: live_results.FIXED_LINES = fixed + live_results.prose_tag = prose bad = table_problems() check("8. every fixed-panel card, fixed line and gene-free card of the veil's tables closes as a panel or never, " "prose is the panel tag whatever it names unless declared gene-free, and a gene cell is its symbols", not bad, bad) diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index 6ec41d174..3fbf4477f 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -184,6 +184,7 @@ def family_browser_checks(work): before, t0 = len(demo.FAILS), demo.time.time() try: demo.family_veil_checks(b, f"http://127.0.0.1:{srv.server_address[1]}", out) + demo.fail_closed_checks(b, f"http://127.0.0.1:{srv.server_address[1]}", out, og) FAILS.extend(demo.FAILS[before:]) finally: b.close() From 30214ca340f5463f5d6ffe83778fc8f103a5a55b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:31:32 +0100 Subject: [PATCH 049/255] fix(live): preserve open addresses when no result needs a veil --- assets/live/finding.js | 2 +- assets/live/person.js | 2 +- tests/test_live_demo.py | 23 ++++++++++++++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/assets/live/finding.js b/assets/live/finding.js index cb1f6fe79..901d015eb 100644 --- a/assets/live/finding.js +++ b/assets/live/finding.js @@ -85,7 +85,7 @@ function tagOf(x) { if (x.level === 'act') { return null; } var t = tags[x.id]; - if (t === undefined) { t = ['*panel*']; } + if (t === undefined) { t = (r.findings || []).some(function (item) { return item.level === 'act'; }) ? ['*panel*'] : null; } t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (y) { return y !== x.id; }); return t.length ? t : null; } diff --git a/assets/live/person.js b/assets/live/person.js index 5506c3fbc..3a85cd487 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -624,7 +624,7 @@ function findingTag(f) { if (f.level === 'act') { return null; } // its own switch; never an unclassified secondary place var t = ((results && results.veil && results.veil.findings) || {})[f.id]; - if (t === undefined) { t = [PANEL]; } + if (t === undefined) { t = ((results && results.findings) || []).some(function (x) { return x.level === 'act'; }) ? [PANEL] : null; } t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (x) { return x !== f.id; }); return t.length ? t : null; } diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 7913eb5a8..82c492d2f 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -2241,6 +2241,18 @@ def mito(S): bad.append(f"mito: row/mark after {stage} matching openings {got}") finally: unplant(keep) + if "no-act" in cases: + def no_act(S): + S.clear() + S["exomiser"] = {"state": "ok", "data": {"terms": [], "genes": [ + {"gene": "BRCA1", "rank": 1, "moi": "AD", "exomiser_score": 0.9, "variants": []}]}} + keep = plant_summary(out, og, no_act) + try: + page_now(b, base, "#f/exomiser-brca1") + if b.eval("location.hash") != "#f/exomiser-brca1": + bad.append("no-act: missing metadata changes an open finding's address") + finally: + unplant(keep) if "metadata" in cases: keep = plant_summary(out, og, cpsr("MLH1")) try: @@ -2295,7 +2307,7 @@ def injected(S, summary): def fail_closed_checks(b, base, out, og): started = time.monotonic() - for case in ("home", "hpo", "contig", "mito", "producers", "metadata"): + for case in ("home", "hpo", "contig", "mito", "producers", "metadata", "no-act"): bad = fail_closed_problems(b, base, out, og, [case]) check("browser: fail-closed producers " + case, not bad, bad) prose, row, veil = live_results.prose_tag, live_results.row_tag, live_results.veil @@ -2341,6 +2353,15 @@ def no_mito(*args, **kwargs): finally: with open(path, "w") as fh: fh.write(reader) + try: + old = "if (t === undefined) { t = ((results && results.findings) || []).some(function (x) { return x.level === 'act'; }) ? [PANEL] : null; }" + assert reader.count(old) == 1 + with open(path, "w") as fh: + fh.write(reader.replace(old, "if (t === undefined) { t = [PANEL]; }")) + control("a no-act page treating every unclassified finding as closable", not fail_closed_problems(b, base, out, og, ["no-act"])) + finally: + with open(path, "w") as fh: + fh.write(reader) print(f" fail-closed producer checks: {time.monotonic() - started:.2f} s") From 071681c13e7fb38a9287efb55194df859b8361e5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:13:56 +0100 Subject: [PATCH 050/255] fix(live): apply the veil to copied QC facts and report metadata --- CHANGELOG.md | 12 +++++-- assets/live/person.js | 8 ++++- docs/live-site.md | 2 +- tests/test_live_demo.py | 80 +++++++++++++++++++++++++++++++++++++++-- 4 files changed, 96 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52fe67323..0bc5a509b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,8 +106,16 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Step 38, step 39 with a families file, `run-all.sh` with `FAMILIES` and the family workflow now need `og` (`scripts/setup.sh --og`). - The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. -- A child's page no longer tells which result to act on is closed. A closed finding's card, its place in the page and the address carry a handle (`#f/closed-2`) instead of its id, which names the gene and variant; the address never holds a closable finding's id, closed (`#f/closed-`) or opened (`#f/opened-`), so no history entry keeps it, and any other `#f/` on such a page, a closed result's id or nothing, lands on the first closed finding, so a guessed id tells nothing. The home page links a child's closed result to the child's report, not to the finding's own page. A closed structural variant now closes the rows and findings that name any of its genes, not only the first. The cards whose rows are the same genes for everyone (known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, the copy numbers, CPIC, CYP2D6, HLA, Locityper, the traits panel, the blood groups) close whole while any result is closed, behind one box, and so do their own findings (a medicine gene, a CHIP call) and the lead's words on them, the ClinVar card's lines for the known deep intronic sites, the carrier list's two-hit line and the callability card's rows, all together; a table row, a finding or a chart mark closes with a result only by its gene and place fields, never by a code such as the inheritance `AR` or the structural-variant type `INS`, and prose is never read word by word against gene symbols: a card's headline, its lines and facts, a finding with no gene and no place, and the lead's own sentences close whole while any result is closed, unless they are structured (the carrier list's two-hit lines close by the gene each is built from) or on a list of prose that names no gene (the QC, ClinVar, mosaic, ROH and telomere lines, most cards' facts, the headlines built from counts), so a child's page with a result closed shows less prose, and no search hit, until the results are opened; a place that names two closed results (a segment of the lead, a duplication over two genes) stays closed until both are opened; the polygenic score card and its chart are never tagged, and the genome map no longer counts a finding a closed result ties among its findings with no place; a card's note on its closed rows no longer counts them. `results.json` gains `veil.panels` and `veil.headline`, a `genes` list on each structural-variant finding, and one `veil.spans` row per gene; each tag in `veil.rows`, `veil.lines`, `veil.facts`, `veil.findings`, `veil.lead`, `veil.marks`, `veil.viewer` and `veil.widget` is now a list of closed-result ids (or `*panel*`) instead of one id. The genome map's findings layer gains a `veils` column, every closed result a row overlaps; `veil` stays. -- Opening APOE on a child's genome map no longer reveals whether another closed result overlaps APOE. Its map row follows its own warning switch. A child's home view also keeps panel finding titles and links closed, through the same veil tags. Adults and runs with no result to act on keep their titles and destinations. +- A child's page no longer tells which result to act on is closed. Closed findings use ordinal handles such as `#f/closed-2`. Opened findings use `#f/opened-`, so history retains no closable result id. A guessed `#f/` lands on the first closed finding. The home page links closed results to the report and uses level counts and fixed prose while any act stays closed. + + Structural variants retain every gene they cover. Rows, findings and chart marks use structured gene and coordinate fields. Vocabulary such as inheritance `AR` or variant type `INS` never counts as a gene. A place naming two closed results stays closed until both open. Hidden-row notes use fixed words without counts. + + Fixed panels close whole: known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, copy numbers, CPIC, CYP2D6, HLA, Locityper, traits and blood groups. Their findings and lead segments close together. ClinVar's deep-intronic lines, the carrier caller sentence and callability's catalog rows also take the uniform PANEL rule. + + Unclassified facts, lines, headlines, notes, rows and findings close while any act stays closed. This includes QC facts copied into the trust line, their title attributes, and the polygenic score rows and chart. No card-wide prose exemption remains. Unclassified limitation lists, missing-step lists, version metadata and their counts also close until every act opens. Carrier two-hit lines and counts use their source genes. MoChA lines and mitochondrial rows use their source coordinates. The genome and mitochondrial charts retain structured marks on fixed geometry, without data-derived notes or legend choices. Other charts and unstructured drawings close whole until every act opens. The cost is less of a child's report and no search hit while results stay closed. + + `results.json` adds `veil.panels`, `veil.headline`, `veil.notes` and `veil.widget_panel`. SV findings retain `genes`, and `veil.spans` has one row per gene. Tags in `veil.rows`, `veil.lines`, `veil.facts`, `veil.findings`, `veil.lead`, `veil.marks`, `veil.viewer` and `veil.widget` hold every matching closed id or `*panel*`. Explicit null tags mark structured places with no matching act. Chart specifications add `svg_closed`. The original SVG remains. The genome map adds `veils` and retains the first-id `veil` column. Its count of findings without a place excludes findings tied to a closed result. No output file is renamed or removed. +- Opening APOE on a child's genome map no longer reveals whether another closed result overlaps APOE. Its map row follows its own warning switch. A child's home view also keeps panel finding titles and links closed while any result to act on stays closed. Adults and runs with no result to act on keep their titles and destinations. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. - Steps 21, 32 and 9b (and the CYRIUS, PYPGX and STRANGER_RFC1 modules) now run the CYP2D6 depth check and the RFC1 motif count with `og`, and `bin/cyp2d6_depth_check.py` and `bin/rfc1_motif.py` are gone. A run that reaches these steps needs the binary: run `./scripts/setup.sh --og` once after setup and after each pull; without it the step stops with `og is not built`. The output files, their names and formats do not change; in CI, every e2e case that runs these steps in bash checks `og` against the deleted Python, byte for byte. diff --git a/assets/live/person.js b/assets/live/person.js index 3a85cd487..23463db9c 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -863,6 +863,11 @@ add(body, nf); return body; } + if ((sec.n === 9 || sec.n === 10) && tagClosed(PANEL)) { + add(body, U.veil('This part can contain data text, so it stays closed until every result to act on is opened.', + null, function () { var f = nextClosed(); if (f) { openFinding(f); } redraw(true); })); + return body; + } if (sec.n === 9) { var lists = [['No step looks at these', (results && results.not_assessed) || []], ['Steps that did not run', (results && results.not_run) || []]]; lists.forEach(function (x) { @@ -942,6 +947,7 @@ sec.cards.map(cardById).some(function (c) { return c && flaggedAlone(c); }))); } function sectionLine(sec) { + if (sec.n === 9 && tagClosed(PANEL)) { return 'Closed until every result to act on is opened.'; } if (sec.n === 9) { var na = ((results && results.not_assessed) || []).length, nr = ((results && results.not_run) || []).length; return na + (na === 1 ? ' thing' : ' things') + ' no step looks at, ' + nr + (nr === 1 ? ' step' : ' steps') + ' that did not run'; @@ -1032,7 +1038,7 @@ add(trust, U.level('trust', 'Data quality', problems.length ? 'problem' : '')); var qc = cardById('qc'); if (qc && gate(qc) === 'show') { - (qc.facts || []).filter(function (f) { return /depth|contamination|sex from/i.test(f[0]); }).slice(0, 3).forEach(function (f, i) { + openFacts(qc)[0].filter(function (f) { return /depth|contamination|sex from/i.test(f[0]); }).slice(0, 3).forEach(function (f, i) { if (i) { add(trust, el('span', { 'class': 'sep' }, '·')); } add(trust, el('span', { title: f[0] + ': ' + f[1] }, String(f[0]).replace(/\s*\([^)]*\)/g, '') + ': ' + String(f[1]).split(/[.;(]\s/)[0].replace(/[.;]$/, '').trim())); }); diff --git a/docs/live-site.md b/docs/live-site.md index 329f91d77..426ca5c5f 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -66,7 +66,7 @@ A card has its rows (the first ones, then a button for all), its own sentences a Each tag lists every matching closed result. A place naming two results stays closed while either remains closed. This includes a duplication over two closed genes and a lead segment naming two findings. The carrier list's two-hit lines use their source gene and variant. Its two-hit count uses the two-hit genes (`FACT_NAMES`). MoChA call lines use their source coordinates. Mitochondrial rows use each record's structured chrM position, so HGVS text and chart marks stay closed until every matching result opens. Lead segments follow the findings they print. The genome map excludes findings tied to closed results from its open findings without a place. - Unclassified rows, facts, lines, headlines, stale notes and findings default to `*panel*`. No card-wide gene-free exemption remains. HPO labels, sample names, haplogroups, trait labels and record counts cannot prove that their producer is safe. The genome and mitochondrial charts retain coordinate-tagged marks on fixed geometry. Data-dependent notes, empty-state sentences and legend choices stay absent. Other charts close whole. The complete chart returns after the last act opens and closes again if any act closes. Unstructured card drawings, including the Y path, also stay closed until every act opens. + Unclassified rows, facts, lines, headlines, stale notes and findings default to `*panel*`. No card-wide gene-free exemption remains. HPO labels, sample names, haplogroups, trait labels and record counts cannot prove that their producer is safe. The genome and mitochondrial charts retain coordinate-tagged marks on fixed geometry. Data-dependent notes, empty-state sentences and legend choices stay absent. Other charts close whole. The complete chart returns after the last act opens and closes again if any act closes. Unstructured card drawings, including the Y path, also stay closed until every act opens. The trust line reads QC facts through the same veil. Unclassified limitation lists, missing-step lists, database versions and update notes in sections 9 and 10 close together, including their record counts. The accepted cost is less of a child's report while any result to act on stays closed. More prose, tables and charts remain closed until every act opens. This is a browser veil: the result and chart files still hold their complete data. - The map's APOE row follows its own warning switch. Opening it shows the same mark even when another result in APOE stays closed. Otherwise its absence would tell which result is closed. Other map rows keep every tag they follow. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 82c492d2f..26626d8c2 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -2241,6 +2241,60 @@ def mito(S): bad.append(f"mito: row/mark after {stage} matching openings {got}") finally: unplant(keep) + if "report-metadata" in cases: + phone_at(b, 390) + try: + views = [] + for gene in ("MLH1", "BRCA2"): + build = live_results.build + def metadata(sample, summary, files=()): + token = "MLH1_METADATA_NOTE" + records = [token] * (2 if gene == "MLH1" else 1) + planted = dict(summary, not_assessed=records, not_run=records, + databases=dict(summary.get("databases") or {}, test_metadata=token)) + return build(sample, planted, files) + live_results.build = metadata + try: + keep = plant_summary(out, og, cpsr(gene)) + finally: + live_results.build = build + try: + views.append(page_now(b, base)) + if "MLH1_METADATA_NOTE" in "\n".join(views[-1][0]): + bad.append("report metadata: unclassified limitation, missing-step or version text reaches the DOM") + with open(os.path.join(out, "runs", SON, "results.json")) as fh: + acts = [f["id"] for f in json.load(fh)["findings"] if f["level"] == "act"] + for fid in acts: + assert open_closed(b, fid), fid + if "MLH1_METADATA_NOTE" not in (b.eval("document.documentElement.outerHTML") or ""): + bad.append("report metadata: text does not return after every act opens") + finally: + unplant(keep) + if views[0] != views[1]: + bad.append("report metadata: extra records change closed text, attributes, address, canvas or SVG sizes") + finally: + b.call("Emulation.clearDeviceMetricsOverride", {}, b.session) + if "trust" in cases: + def qc_note(S): + cpsr("MLH1")(S) + S["sample_qc"] = {"state": "ok", "data": {"sex_check": "not_checked", "inferred_sex": "male", + "sex_check_reason": "MLH1_TRUST_NOTE", "contamination": "not_checked"}} + keep = plant_summary(out, og, qc_note) + try: + page_now(b, base) + if "MLH1_TRUST_NOTE" in (b.eval("document.documentElement.outerHTML") or ""): + bad.append("trust: a PANEL QC fact survives in the trust-line text or title") + assert open_closed(b, cid("MLH1")) + if "MLH1_TRUST_NOTE" in (b.eval("document.documentElement.outerHTML") or ""): + bad.append("trust: opening one act exposes a QC fact while other acts remain closed") + with open(os.path.join(out, "runs", SON, "results.json")) as fh: + acts = [f["id"] for f in json.load(fh)["findings"] if f["level"] == "act"] + for fid in acts: + open_closed(b, fid) + if "MLH1_TRUST_NOTE" not in (b.eval("document.querySelector('.trustline').outerHTML") or ""): + bad.append("trust: the QC fact does not return after every act opens") + finally: + unplant(keep) if "no-act" in cases: def no_act(S): S.clear() @@ -2283,7 +2337,7 @@ def no_act(S): continue def injected(S, summary): got = adapter(S, summary) - got.update(facts=[["Unclassified", "MLH1"]], lines=["MLH1 sample note"], + got.update(facts=[["Mean depth", "MLH1 sample note"]], lines=["MLH1 sample note"], table={"columns": ["Detail"], "rows": [["MLH1"], ["extra MLH1 record"]]}) return got live_results.ADAPTERS[producer] = injected @@ -2307,7 +2361,7 @@ def injected(S, summary): def fail_closed_checks(b, base, out, og): started = time.monotonic() - for case in ("home", "hpo", "contig", "mito", "producers", "metadata", "no-act"): + for case in ("home", "hpo", "contig", "mito", "producers", "metadata", "no-act", "trust", "report-metadata"): bad = fail_closed_problems(b, base, out, og, [case]) check("browser: fail-closed producers " + case, not bad, bad) prose, row, veil = live_results.prose_tag, live_results.row_tag, live_results.veil @@ -2362,6 +2416,28 @@ def no_mito(*args, **kwargs): finally: with open(path, "w") as fh: fh.write(reader) + try: + old = "openFacts(qc)[0].filter(function (f) { return /depth|contamination|sex from/i.test(f[0]); })" + assert reader.count(old) == 1 + with open(path, "w") as fh: + fh.write(reader.replace(old, "(qc.facts || []).filter(function (f) { return /depth|contamination|sex from/i.test(f[0]); })")) + control("trust line reading QC facts outside their veil", not fail_closed_problems(b, base, out, og, ["trust"])) + finally: + with open(path, "w") as fh: + fh.write(reader) + for old, new, desc in ( + ("(sec.n === 9 || sec.n === 10) && tagClosed(PANEL)", "false", + "report metadata read outside PANEL"), + ("sec.n === 9 && tagClosed(PANEL)", "false", + "unclassified report metadata counted in the contents")): + try: + assert reader.count(old) == 1 + with open(path, "w") as fh: + fh.write(reader.replace(old, new)) + control(desc, not fail_closed_problems(b, base, out, og, ["report-metadata"])) + finally: + with open(path, "w") as fh: + fh.write(reader) print(f" fail-closed producer checks: {time.monotonic() - started:.2f} s") From e4855c85c11e7f5962a8c7921a60d97819779810 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:56:59 +0100 Subject: [PATCH 051/255] fix(live): keep panel findings closed through partial opening --- CHANGELOG.md | 2 +- assets/live/finding.js | 16 +++++-- docs/live-site.md | 2 +- tests/test_live_demo.py | 92 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce266e3f7..0ce57d89f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,7 +115,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n Structural variants retain every gene they cover. Rows, findings and chart marks use structured gene and coordinate fields. Vocabulary such as inheritance `AR` or variant type `INS` never counts as a gene. A place naming two closed results stays closed until both open. Hidden-row notes use fixed words without counts. - Fixed panels close whole: known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, copy numbers, CPIC, CYP2D6, HLA, Locityper, traits and blood groups. Their findings and lead segments close together. ClinVar's deep-intronic lines, the carrier caller sentence and callability's catalog rows also take the uniform PANEL rule. + Fixed panels close whole: known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, copy numbers, CPIC, CYP2D6, HLA, Locityper, traits and blood groups. Their findings and lead segments close together. A PANEL finding's own page opens one act per click and reveals the finding only after every act opens. ClinVar's deep-intronic lines, the carrier caller sentence and callability's catalog rows also take the uniform PANEL rule. Unclassified facts, lines, headlines, notes, rows and findings close while any act stays closed. This includes QC facts copied into the trust line, their title attributes, and the polygenic score rows and chart. No card-wide prose exemption remains. Unclassified limitation lists, missing-step lists, version metadata and their counts also close until every act opens. Carrier two-hit lines and counts use their source genes. MoChA lines and mitochondrial rows use their source coordinates. The genome and mitochondrial charts retain structured marks on fixed geometry, without data-derived notes or legend choices. Other charts and unstructured drawings close whole until every act opens. The cost is less of a child's report and no search hit while results stay closed. diff --git a/assets/live/finding.js b/assets/live/finding.js index 901d015eb..754512812 100644 --- a/assets/live/finding.js +++ b/assets/live/finding.js @@ -15,6 +15,7 @@ var replay = root.getAttribute('data-mode') === 'replay'; var main = document.getElementById('finding'); var ROMAN = ['i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii']; + var opened = {}; function abs(u) { return !u ? u : (u.charAt(0) === '/' ? u : BASE + u); } function note(head, body, cls) { @@ -93,13 +94,11 @@ // A finding of a fixed-panel card (tag PANEL, bin/live_results.py FIXED_PANELS) names no result: it stays closed // while any result to act on is, whichever, as on the person page. var PANEL = '*panel*'; - var opened = {}; - if (draw.open) { opened[f.id] = true; (tagOf(f) || []).forEach(function (t) { if (t !== PANEL) { opened[t] = true; } }); } function actShut() { return (r.findings || []).some(function (x) { return x.level === 'act' && !opened[x.id]; }); } function shut(x) { if (!child) { return false; } if (x.level === 'act') { return !opened[x.id]; } - return (tagOf(x) || []).some(function (t) { return t === PANEL ? !opened[x.id] && actShut() : !opened[t]; }); + return (tagOf(x) || []).some(function (t) { return t === PANEL ? actShut() : !opened[t]; }); } // Closed: no title, gene or variant anywhere on the page (its title, crumb, cover, rail) until opened. var closed = shut(f); @@ -163,7 +162,16 @@ (f.level === 'act' ? '' : 'This finding names a result to act on. ') + 'A result to act on in adult life is usually not looked at before the child can choose. It stays closed until you open it.')); var b = el('button', { type: 'button', 'class': 'btn' }, 'Show the result'); - b.addEventListener('click', function () { draw.open = true; draw(r, family); }); + b.addEventListener('click', function () { + var ids = tagOf(f) || []; + // A panel names no particular act. Its button opens the next closed act, one per click. + var nextAct = ids.indexOf(PANEL) >= 0 ? (r.findings || []).filter(function (x) { + return x.level === 'act' && !opened[x.id]; + })[0] : null; + if (nextAct) { opened[nextAct.id] = true; } + else { opened[f.id] = true; ids.forEach(function (id) { if (id !== PANEL) { opened[id] = true; } }); } + draw(r, family); + }); add(v, b); add(col, v); add(body, rail, col); diff --git a/docs/live-site.md b/docs/live-site.md index 426ca5c5f..3ad7c96d8 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -58,7 +58,7 @@ Every result carries one of six levels, with a word, a shape and a colour, so th A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. Two pages whose open rows are the same differ only in how many results are closed, not in which; the section that holds a closed result still says it holds one: -- A fixed panel lists genes or loci known before the run. Hiding only one row would identify the closed result's gene. These cards close whole while any result to act on stays closed. They include known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP and paralog copy numbers. CPIC, CYP2D6, HLA, Locityper, traits and blood groups follow the same rule. The paralog panel includes SMN1, SMN2, HBA1, HBA2 and RCCX. The single list is `FIXED_PANELS` in `bin/live_results.py`, exported as `veil.panels`. Each box uses the same words and opens the next closed result. The cards return after every act opens. +- A fixed panel lists genes or loci known before the run. Hiding only one row would identify the closed result's gene. These cards close whole while any result to act on stays closed. They include known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP and paralog copy numbers. CPIC, CYP2D6, HLA, Locityper, traits and blood groups follow the same rule. The paralog panel includes SMN1, SMN2, HBA1, HBA2 and RCCX. The single list is `FIXED_PANELS` in `bin/live_results.py`, exported as `veil.panels`. Each box uses the same words and opens the next closed result. The cards return after every act opens. A PANEL finding's own page follows this rule too: its button opens one act per click, and the finding appears after the last act opens. Their own findings and the lead segments naming them also take `*panel*`, whatever gene they name. Fixed panels inside other cards follow this rule too. These include ClinVar's known deep intronic sites, the carrier list's copy-number caller sentence, and callability's catalog genes missing from the reference or unread by the reads. Callability's row prefixes remain in `FIXED_LINES`. The other sentences take the default prose rule. diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 26626d8c2..cf2023b41 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -2359,8 +2359,100 @@ def injected(S, summary): return bad +def panel_finding_problems(b, base, out, og, cases=("child", "missing tags", "adult", "no acts")): + """A PANEL finding opens one act per click and reveals its content only after the last act opens.""" + folder = os.path.join(out, "runs", SON, "finding") + original, fid = "sf-kcnh2-chr7-150960000-c-t", "chip-tp53-p-demo" + dest = os.path.join(folder, fid) + with open(os.path.join(folder, original, "index.html")) as fh: + template = fh.read() + fj = os.path.join(out, "runs", FAM, "family.json") + with open(fj) as fh: + family = fh.read() + os.makedirs(dest) + bad = [] + try: + with open(os.path.join(dest, "index.html"), "w") as fh: + fh.write(template.replace(original, fid)) + for case in cases: + def change(S): + S.clear() + cpsrs("TP53", "BRCA2")(S) + if case == "no acts": + S["cpsr"]["data"]["plp"] = [] + S["chip"] = {"state": "ok", "data": {"calls": [{"gene": "TP53", "protein_change": "p.Demo", + "vaf": 0.1, "alt_reads": 10, "depth": 100, "status": "reported"}]}} + keep = plant_summary(out, og, change) + try: + rj = os.path.join(out, "runs", SON, "results.json") + with open(rj) as fh: + res = json.load(fh) + tags = (res.get("veil", {}).get("findings") or {}).get(fid) + if case != "no acts" and tags != ["*panel*"]: + bad.append(f"{case}: producer did not classify the CHIP finding as PANEL: {tags}") + if case == "missing tags": + del res["veil"]["findings"][fid] + with open(rj, "w") as fh: + json.dump(res, fh) + fam = json.loads(family) + if case == "adult": + for member in fam["state"]["members"]: + if member["sample"] == SON: + member["father"] = member["mother"] = None + with open(fj, "w") as fh: + json.dump(fam, fh) + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{SON}/finding/{fid}/") + if not b.wait("document.documentElement.dataset.pgpStatus === 'ready' && 'ready'", 10): + bad.append(f"{case}: finding never ready") + continue + for clicks in range(3 if case in ("child", "missing tags") else 1): + should_open = case in ("adult", "no acts") or clicks == 2 + visible = b.eval("document.querySelector('#what').textContent.includes('A blood clone in TP53') " + "&& document.title.includes('TP53') && !!document.querySelector('#evidence')") + shut = b.eval("!!document.querySelector('.veil button')") + if bool(visible) != should_open or bool(shut) == should_open: + bad.append(f"{case}: after {clicks} clicks PANEL finding visible={visible}, closed={shut}; " + f"expected visible={should_open}") + if not should_open: + b.eval("var button = document.querySelector('.veil button'); button && button.click(); true") + if case == "child": + b.go(f"{base}/runs/{SON}/finding/{fid}/") + if not b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.querySelector('.veil button') " + "&& !document.querySelector('#evidence') && 'closed'", 10): + bad.append("child: a reload did not close the PANEL finding again") + finally: + unplant(keep) + finally: + with open(fj, "w") as fh: + fh.write(family) + shutil.rmtree(dest) + return bad + + def fail_closed_checks(b, base, out, og): started = time.monotonic() + bad = panel_finding_problems(b, base, out, og) + check("browser: a PANEL finding stays closed until one click per act, with missing tags, adult and no-act controls", + not bad, bad) + finding_path = os.path.join(out, "assets", "finding.js") + with open(finding_path) as fh: + finding_reader = fh.read() + for old, new, desc in ( + ("return t === PANEL ? actShut() : !opened[t];", "return t === PANEL ? false : !opened[t];", + "a finding page that ignores PANEL"), + ("if (nextAct) { opened[nextAct.id] = true; }", + "if (nextAct) { (r.findings || []).forEach(function (x) { if (x.level === 'act') { opened[x.id] = true; } }); }", + "a PANEL finding button that opens every act in one click")): + try: + assert finding_reader.count(old) == 1 + with open(finding_path, "w") as fh: + fh.write(finding_reader.replace(old, new)) + control(desc, not panel_finding_problems(b, base, out, og, ["child"])) + finally: + with open(finding_path, "w") as fh: + fh.write(finding_reader) for case in ("home", "hpo", "contig", "mito", "producers", "metadata", "no-act", "trust", "report-metadata"): bad = fail_closed_problems(b, base, out, og, [case]) check("browser: fail-closed producers " + case, not bad, bad) From d1581b9e3b8e6ed4dea59436b4cb4f76ad0da15f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:04:28 +0100 Subject: [PATCH 052/255] test(live): retain the finding-tag mutation control --- tests/test_live_demo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index cf2023b41..bc2162c3a 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -848,7 +848,7 @@ def unsafe_tags(value): check("browser: the home page and every finding page of the son name the planted gene and positions nowhere " "(the rail's next link and the related list included) until opened, and the symptom ranking's page " "opened on purpose shows it", not bad, bad) - a = "return (tagOf(x) || []).some(function (t) { return t === PANEL ? !opened[x.id] && actShut() : !opened[t]; });" + a = "return (tagOf(x) || []).some(function (t) { return t === PANEL ? actShut() : !opened[t]; });" if a not in keep[fj]: check("control: finding.js whose findings ignore their tags (the planted line is in finding.js)", False, a) else: From e76767f6f35a59813c57adc9a6ac8cc8e8c34ba4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:49:21 +0100 Subject: [PATCH 053/255] fix(live): gate child report data before rendering --- CHANGELOG.md | 14 +- assets/live/core.js | 332 ++- assets/live/family-page.js | 168 +- assets/live/finding.js | 58 +- assets/live/home.js | 38 +- assets/live/map.js | 9 +- assets/live/person.js | 281 +- bin/live_results.py | 493 +--- docs/live-site.md | 34 +- rust/crates/og-map/README.md | 7 +- rust/crates/og-map/src/doc.rs | 13 +- rust/crates/og-map/src/facts.rs | 51 - rust/crates/og-map/src/findings.rs | 6 +- rust/crates/og-map/src/veil.rs | 53 +- rust/crates/og-map/tests/findings.rs | 3 - tests/demo/gate-property.js | 37 + tests/test_live_demo.py | 3594 ++++---------------------- tests/test_live_results.py | 641 +---- tests/test_owngenome_live_family.py | 62 +- 19 files changed, 978 insertions(+), 4916 deletions(-) create mode 100644 tests/demo/gate-property.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 842b36c54..fd642a3bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,19 +125,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Each output is written whole or not at all: a run that stops leaves no part-written table behind. A read-only `family_plan.tsv` is replaced; a folder that cannot be written still stops the plan. - Step 38, step 39 with a families file, `run-all.sh` with `FAMILIES` and the family workflow now need `og` (`scripts/setup.sh --og`). -- The family page now keeps a child's results to act on closed, as the child's own page does, and what it shows of a child no longer depends on which result is closed or where it lies. While a child has a result still closed, or the page cannot read the child's results, these close for that child, one box each: the painting's grandparents (every stretch drawn from its parent alone), the chromosome map image, all de novo and compound-het rows, the child's medicines and HLA cells in the side by side, and the carrier screen of a couple the child is part of. Each "Show the result" opens one result; the places come back once the last one is opened, and a reload closes them again. While the results cannot be read, each box opens only its own place. On a child's page, a copy-number dot and a card's fact that name a closed result's gene now close with it too, on a phone in the card's line as well. `results.json` adds `veil.spans`, `veil.facts` and `veil.widget`; nothing is renamed or removed. The veil is still a browser veil: `results.json` and `family.json` hold every row. -- A child's page no longer tells which result to act on is closed. Closed findings use ordinal handles such as `#f/closed-2`. Opened findings use `#f/opened-`, so history retains no closable result id. A guessed `#f/` lands on the first closed finding. The home page links closed results to the report and uses level counts and fixed prose while any act stays closed. - - A child's live page now closes unclassified data text and chart notes by default. Its home summary uses level counts while results stay closed. HPO labels and mitochondrial HGVS rows follow the same veil, and chart geometry no longer identifies an off-contig closed result. - - Structural variants retain every gene they cover. Rows, findings and chart marks use structured gene and coordinate fields. Vocabulary such as inheritance `AR` or variant type `INS` never counts as a gene. A place naming two closed results stays closed until both open. Hidden-row notes use fixed words without counts. - - Fixed panels close whole: known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP, copy numbers, CPIC, CYP2D6, HLA, Locityper, traits and blood groups. Their findings and lead segments close together. A PANEL finding's own page opens one act per click and reveals the finding only after every act opens. ClinVar's deep-intronic lines, the carrier caller sentence and callability's catalog rows also take the uniform PANEL rule. - - Unclassified facts, lines, headlines, notes, rows and findings close while any act stays closed. This includes QC facts copied into the trust line, their title attributes, and the polygenic score rows and chart. No card-wide prose exemption remains. Unclassified limitation lists, missing-step lists, version metadata and their counts also close until every act opens. Carrier two-hit lines and counts use their source genes. MoChA lines and mitochondrial rows use their source coordinates. The genome and mitochondrial charts retain structured marks on fixed geometry, without data-derived notes or legend choices. Other charts and unstructured drawings close whole until every act opens. The cost is less of a child's report and no search hit while results stay closed. - - `results.json` adds `veil.panels`, `veil.headline`, `veil.notes` and `veil.widget_panel`. SV findings retain `genes`, and `veil.spans` has one row per gene. Tags in `veil.rows`, `veil.lines`, `veil.facts`, `veil.findings`, `veil.lead`, `veil.marks`, `veil.viewer` and `veil.widget` hold every matching closed id or `*panel*`. Explicit null tags mark structured places with no matching act. Chart specifications add `svg_closed`. The original SVG remains. The genome map adds `veils` and retains the first-id `veil` column. Its count of findings without a place excludes findings tied to a closed result. No output file is renamed or removed. -- Opening APOE on a child's genome map no longer reveals whether another closed result overlaps APOE. Its map row follows its own warning switch. A child's home view also keeps panel finding titles and links closed while any result to act on stays closed. Adults and runs with no result to act on keep their titles and destinations. +- A child's live report now has one gate at the data loader. While any act result is unopened, the page shows fixed text, opaque opening controls and the own record of each act opened on purpose. It shows none of the rest of the report until every act opens. This replaces per-element tags, fixed-panel lists and exemption registries, which repeatedly missed copied or new fields. Home summaries use fixed text. Family cells close for that child while parents retain their values. Closing an act replaces the document and clears view caches and search results. Reload closes the acts again. Map and section addresses return to the default view after the final opening. Adults and children with no act result keep the full report. APOE retains its separate warning. The boundary remains browser-only: static JSON files remain downloadable. `results.veil.acts` replaces the element metadata introduced during this unreleased work. The map retains its first-tag `veil` column and drops the unreleased multi-tag `veils` column. No output file is renamed. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. diff --git a/assets/live/core.js b/assets/live/core.js index 1e2f225c1..7546874dd 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -146,19 +146,327 @@ return isFinite(x) && isFinite(y) && y > x ? (y - x) / 3600e3 : null; } - // --- the veil: a child's results to act on stay closed until someone opens them on purpose --------- - // The closed box, the same on a child's page (person.js) and on the family page (family-page.js): what stays - // closed, and the button that opens it. - function veil(text, button, onOpen) { - var v = el('div', { 'class': 'veil' }); - add(v, add(el('div', { 'class': 'vt' }), el('b', {}, 'A result about a child. '), text)); - var b = el('button', { type: 'button', 'class': 'btn' }, button || 'Show the result'); - b.addEventListener('click', onOpen); - add(v, b); - return v; - } window.PGPUI = { el: el, svg: svg, add: add, getJSON: getJSON, avatar: avatar, day: day, caveat: caveat, facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, - hm: hm, dm: dm, hours: hours, veil: veil }; + hm: hm, dm: dm, hours: hours }; + // The report gate owns raw payloads. Renderers only see its projection until the final act opens. + var originalFetch = window.fetch.bind(window); + var records = {}, current = null, ownsPage = false, eventSources = []; + var CLOSED_TEXT = "This child's report stays closed until every result to act on in adult life is opened on purpose."; + var TRANSFER = 'og-report-opening'; + function jsonAt(url) { + return originalFetch(url, { credentials: 'same-origin', cache: 'no-store' }).then(function (r) { + if (r.status === 401) { window.location.href = '/login?next=' + encodeURIComponent(location.pathname); } + return r.ok ? r.json() : null; + }).catch(function () { return null; }); + } + function copy(x) { return x === undefined ? null : JSON.parse(JSON.stringify(x)); } + function absolute(url, base) { return new URL(url, base || location.href).href; } + function actsOf(r) { + if (!r || !Array.isArray(r.findings)) { return null; } + // The producer's act manifest is independent of every other record. Ignore unrelated malformed rows. + // Level reconciliation supports older documents and refuses to lose an act omitted from a manifest. + var ids = r.veil ? r.veil.acts : undefined; + if (ids !== undefined && (!Array.isArray(ids) || ids.some(function (id) { return typeof id !== 'string'; }))) { return null; } + ids = (ids || []).slice(); + r.findings.forEach(function (f) { if (f && f.level === 'act' && typeof f.id === 'string' && ids.indexOf(f.id) < 0) { ids.push(f.id); } }); + var acts = ids.map(function (id) { return r.findings.filter(function (f) { return f && f.id === id; })[0]; }); + return acts.every(Boolean) ? acts : null; + } + function shut(rec) { return rec.child && (!rec.acts || rec.acts.some(function (f) { return !rec.opened[f.id]; })); } + function record(url, child) { + url = absolute(url); + var old = records[url]; + return jsonAt(url + 'results.json').then(function (r) { + var nextActs = actsOf(r); + var opened = {}; + (nextActs || []).forEach(function (f) { + if (old && old.opened[f.id] && (old.acts || []).some(function (prev) { return JSON.stringify(prev) === JSON.stringify(f); })) { opened[f.id] = true; } + }); + var rec = { url: url, child: child !== undefined ? child : old ? old.child : true, acts: nextActs, opened: opened }; + records[url] = rec; + return rec; + }); + } + function projection(rec) { + // No data-derived default: even missing/unreadable results have only the fixed explanation. + return { gated: true, acts: (rec.acts || []).map(function (f) { + var closed = rec.acts.filter(function (a) { return !rec.opened[a.id]; }); + var opened = rec.acts.filter(function (a) { return rec.opened[a.id]; }); + return rec.opened[f.id] ? { handle: 'opened-' + (opened.indexOf(f) + 1), open: true, finding: copy(f) } + : { handle: 'closed-' + (closed.indexOf(f) + 1), open: false }; + }) }; + } + function normalizeHash(p) { + if (!location.hash && !location.search) { return; } + var h = location.hash.slice(3); + var found = /^#f\/(closed|opened)-[1-9][0-9]*$/.test(location.hash) && p.acts.some(function (a) { return a.handle === h; }); + var to = found ? location.hash : p.acts.length ? '#f/' + (p.acts.filter(function (a) { return !a.open; })[0] || p.acts[0]).handle : ''; + if (location.hash !== to || location.search) { history.replaceState(null, '', location.pathname + to); } + } + function actRecord(f) { + var box = el('div', { 'class': 'opened-act' }); + add(box, el('h2', {}, f.title), el('p', {}, f.detail), el('p', {}, f.text)); + var plan = el('ol', {}); + (f.plan || []).forEach(function (t) { add(plan, el('li', {}, t)); }); + add(box, plan, facts(f.evidence), caveat(f.caveat)); + return box; + } + function controls(rec, changed, only) { + var p = projection(rec), box = el('section', { 'class': 'report-gate' }); + add(box, el('p', { 'class': 'lede' }, CLOSED_TEXT)); + add(box, el('p', {}, rec.acts ? p.acts.length + ' results to act on' : 'The results cannot be read yet. Reload to try again.')); + p.acts.forEach(function (a, i) { + var row = el('article', { id: 'f-' + a.handle, 'data-finding': a.handle, 'class': 'finding' }); + if (a.open && (!only || only === a.finding.id)) { add(row, actRecord(a.finding)); } + var b = el('button', { type: 'button', 'class': 'btn', 'data-opening': a.handle }, a.open ? 'Close it again' : 'Show the result'); + b.addEventListener('click', function () { + var active = records[rec.url] || rec, id = rec.acts[i].id; + var f = (active.acts || []).filter(function (item) { return item.id === id; })[0]; + if (!f || JSON.stringify(f) !== JSON.stringify(rec.acts[i])) { changed(); return; } // stale controls cannot open replacement records + var at = p.acts.map(function (item) { return '#f/' + item.handle; }).indexOf(location.hash); + var addressed = at >= 0 ? rec.acts[at].id : null; + if (a.open) { delete active.opened[f.id]; } else { active.opened[f.id] = true; } + if (addressed) { + var next = (active.acts || []).map(function (item) { return item.id; }).indexOf(addressed); + history.replaceState(null, '', next >= 0 ? '#f/' + projection(active).acts[next].handle : location.pathname); + } + changed(); + }); + add(row, b); add(box, row); + }); + return box; + } + function cleanPage() { + // A gated page has no renderer, search widget, replay, canvas, progress view or external report link. + document.body.textContent = ''; + Array.prototype.slice.call(root.attributes).forEach(function (a) { + if (a.name.indexOf('data-pgp-') === 0 || a.name === 'data-finding') { root.removeAttribute(a.name); } + }); + document.title = 'Results closed: OwnGenome'; + root.removeAttribute('data-pgp-search'); + } + function renderGate() { + cleanPage(); + var p = projection(current); + normalizeHash(p); + var wrap = el('main', { 'class': 'wrap', id: 'report-gate' }); + add(wrap, el('h1', {}, 'Results about a child')); + var requested = /\/finding\//.test(location.pathname) ? decodeURIComponent(location.pathname.split('/finding/')[1].split('/')[0]) : null; + add(wrap, controls(current, function () { if (shut(current)) { renderGate(); } else { transfer(); } }, requested)); + add(document.body, wrap); + root.setAttribute('data-pgp-status', 'ready'); + root.setAttribute('data-pgp-gated', 'true'); + } + function proof(rec) { + var bytes = new TextEncoder().encode(JSON.stringify(rec.acts)); + return crypto.subtle.digest('SHA-256', bytes).then(function (h) { return Array.from(new Uint8Array(h)).map(function (v) { return v.toString(16).padStart(2, '0'); }).join(''); }); + } + function transfer() { + // Only opaque booleans and a digest cross the document replacement. The receiver consumes them once. + // Replacing the document discards every view closure, image, chart, search cache and in-flight callback. + proof(current).then(function (stamp) { + sessionStorage.setItem(TRANSFER, JSON.stringify({ path: location.pathname, stamp: stamp, time: Date.now(), + opened: current.acts.map(function (f) { return !!current.opened[f.id]; }) })); + location.replace(location.pathname); // requested map/section returns to the default view + }); + } + function consume(rec) { + var text = sessionStorage.getItem(TRANSFER); sessionStorage.removeItem(TRANSFER); + if (!text) { return Promise.resolve(); } + return proof(rec).then(function (stamp) { + try { + var t = JSON.parse(text); + if (t.path === location.pathname && t.stamp === stamp && Date.now() - t.time < 5000 && t.opened.length === rec.acts.length) { + rec.acts.forEach(function (f, i) { if (t.opened[i] === true) { rec.opened[f.id] = true; } }); + } + } catch (e) { /* An invalid or stale ticket opens nothing. */ } + }); + } + function preparePerson() { + var url = absolute(/\/finding\//.test(location.pathname) ? '../../' : './'); + var replaying = root.getAttribute('data-mode') === 'replay'; + return Promise.all([record(url), jsonAt(url + (replaying ? 'frames.json' : 'state.json'))]).then(function (x) { + current = x[0]; + var state = replaying ? x[1] && x[1].frames && x[1].frames.slice(-1)[0] : null; + state = replaying ? state && state.state : x[1]; + var fam = state && state.plugins && state.plugins.family; + var entry = fam && fam.kind === 'member' && (fam.families || [])[0]; + if (!entry) { current.child = !state || !state.plugins || !!(fam && fam.unknown); return; } + return jsonAt(absolute(entry.url, url) + 'family.json').then(function (f) { + var member = f && f.state && (f.state.members || []).filter(function (m) { return m.sample === root.getAttribute('data-run'); })[0]; + current.child = !member || !!(member.father || member.mother); + }); + }).then(function () { return consume(current); }).then(function () { + ownsPage = shut(current); + if (ownsPage) { renderGate(); } + }).catch(function () { + current = { child: true, acts: null, opened: {} }; ownsPage = true; renderGate(); + }); + } + function familyRecords(f, base) { + return Promise.all(((f.state && f.state.members) || []).filter(function (m) { return m.father || m.mother; }).map(function (m) { + var run = f.runs && f.runs[m.sample]; + return run && run.url ? record(absolute(run.url, base), true).then(function (r) { return [m.sample, r]; }) : Promise.resolve([m.sample, { child: true, acts: null, opened: {} }]); + })).then(function (entries) { var out = {}; entries.forEach(function (e) { out[e[0]] = e[1]; }); return out; }); + } + function projectFamily(raw, recs) { + var closed = Object.keys(recs).filter(function (k) { return shut(recs[k]); }); + if (!closed.length) { return raw; } + // An ownership allowlist, not a classifier of genes, prose or individual report fields. + // Anything a new producer adds stays out until its owner is explicit here. + var f = { state: { kind: raw.state.kind, name: raw.state.name, pedigree: raw.state.pedigree, + members: (raw.state.members || []).map(function (m) { return { sample: m.sample, sex: m.sex, father: m.father, mother: m.mother, partner: m.partner, sequenced: m.sequenced }; }), children: [], couples: copy(raw.state.couples || []), plan: [], reports: [] }, + layout: raw.layout ? { width: raw.layout.width, height: raw.layout.height, size: raw.layout.size, lines: copy(raw.layout.lines), + people: (raw.layout.people || []).map(function (m) { return { sample: m.sample, sex: m.sex, sequenced: m.sequenced, x: m.x, y: m.y }; }) } : null, relationships: copy(raw.relationships || []), ibd_pairs: copy(raw.ibd_pairs || []), + couples: copy(raw.couples || {}), couple_note: raw.couple_note, + segments: copy(raw.segments || {}), de_novo: copy(raw.de_novo || []), compound_het: copy(raw.compound_het || []), + trio_status: copy(raw.trio_status || {}), side_by_side: copy(raw.side_by_side), chrom_lengths: copy(raw.chrom_lengths), + runs: copy(raw.runs || {}) }; + // Ownership cut: discard child report objects wholesale. Unknown child fields are discarded too. + f.gated = closed; + f.openings = {}; closed.forEach(function (k) { f.openings[k] = projection(recs[k]); }); + f.state.children = (raw.state.members || []).filter(function (m) { return m.father || m.mother; }).map(function (m) { + return closed.indexOf(m.sample) >= 0 ? { child: m.sample, gated: true } : (raw.state.children || []).filter(function (c) { return c.child === m.sample; })[0]; + }).filter(Boolean); + ['segments', 'trio_status'].forEach(function (key) { closed.forEach(function (k) { if (f[key]) { delete f[key][k]; } }); }); + ['de_novo', 'compound_het'].forEach(function (key) { f[key] = (f[key] || []).filter(function (r) { return closed.indexOf(r.child) < 0; }); }); + f.relationships = (f.relationships || []).filter(function (r) { return !closed.some(function (c) { return JSON.stringify(r).indexOf(c) >= 0; }); }); + f.state.plan = []; f.state.reports = []; + closed.forEach(function (k) { if (f.runs && f.runs[k]) { f.runs[k] = { url: f.runs[k].url, gated: true }; } }); + f.state.couples = (f.state.couples || []).filter(function (c) { return closed.indexOf(c.a) < 0 && closed.indexOf(c.b) < 0; }); + Object.keys(f.couples || {}).forEach(function (key) { if (key.split(' ').some(function (k) { return closed.indexOf(k) >= 0; })) { delete f.couples[key]; } }); + f.ibd_pairs = (f.ibd_pairs || []).filter(function (r) { return closed.indexOf(r.sample_a) < 0 && closed.indexOf(r.sample_b) < 0; }); + if (f.side_by_side) { + f.side_by_side = { members: f.side_by_side.members, rows: f.side_by_side.rows }; + var sb = f.side_by_side, seenRows = Object.create(null); + // Presence comes only from actual open-member values, never an API absence marker. + sb.rows = sb.rows.filter(function (row) { return row.values.some(function (v, i) { return closed.indexOf(sb.members[i]) < 0 && v !== null && v !== undefined && typeof v !== 'object' && ['', '.', '—', 'no summary yet', 'summary not readable'].indexOf(v) < 0; }); }).map(function (row) { + row.values = row.values.map(function (v, i) { return closed.indexOf(sb.members[i]) >= 0 ? 'closed' : v; }); + row.stale = (row.stale || []).map(function (v, i) { return closed.indexOf(sb.members[i]) >= 0 ? false : v; }); + return { group: row.group, label: row.label, values: row.values, stale: row.stale }; + }).filter(function (row) { + var key = JSON.stringify(row); + if (seenRows[key]) { return false; } seenRows[key] = true; return true; + }).sort(function (a, b) { + var x = JSON.stringify([a.group, a.label]), y = JSON.stringify([b.group, b.label]); + return x < y ? -1 : x > y ? 1 : 0; + }); + } + return f; + } + var familyGates = {}; + function loadedJSON(url, data) { + var u = new URL(url, location.href); + if (u.pathname.endsWith('/family.json') && document.body.classList.contains('page-family')) { + return familyRecords(data, u.href).then(function (recs) { familyGates = recs; return projectFamily(data, recs); }); + } + if (u.pathname.endsWith('/runs.json')) { + var idx = copy(data); + var families = idx.families || [], listed = {}; + families.forEach(function (f) { (f.members || []).forEach(function (m) { listed[m.sample] = true; }); }); + var unknown = families.some(function (f) { return !(f.members || []).length; }); + var unknownRuns = unknown ? (idx.runs || []).filter(function (r) { return !listed[r.name]; }).map(function (r) { + return record(absolute(r.url), true).then(function (rec) { var out = {}; out[r.name] = rec; return out; }); + }) : []; + return Promise.all(unknownRuns.concat(families.map(function (entry) { + return jsonAt(absolute(entry.url) + 'family.json').then(function (f) { + if (!f || !f.state) { + var runs = {}; + (idx.runs || []).forEach(function (r) { runs[r.name] = { url: absolute(r.url) }; }); + f = { state: { members: entry.members || [] }, runs: runs }; + } + return familyRecords(f, absolute(entry.url)); + }); + }))).then(function (all) { + all.forEach(function (recs) { + (idx.runs || []).forEach(function (r, i) { + if (recs[r.name] && shut(recs[r.name])) { + idx.runs[i] = { name: r.name, url: r.url, gated: true, state: 'closed', levels: { act: recs[r.name].acts ? recs[r.name].acts.length : 0 }, + top: [{ level: 'act', title: CLOSED_TEXT }], lead: CLOSED_TEXT, lead_closed: CLOSED_TEXT }; + } + }); + }); + return idx; + }); + } + return Promise.resolve(data); + } + var ready = Promise.resolve(); + if (document.body.classList.contains('page-person') || document.body.classList.contains('page-finding')) { ready = preparePerson(); } + // Permission is checked after parsing too: a response started while open may finish after closing. + function viewResponse(response) { + var read = response.json.bind(response); + response.json = function () { return read().then(function (data) { return ownsPage ? new Promise(function () {}) : data; }); }; + return response; + } + // All fetch-based readers (including plugins) share this boundary. An unrouted view receives nothing. + window.fetch = function (url, options) { + return ready.then(function () { + var requestURL = absolute(url && url.url || url); + var closedRun = Object.keys(records).some(function (key) { return requestURL.indexOf(key) === 0 && shut(records[key]); }); + if (ownsPage || closedRun) { return new Promise(function () {}); } + return originalFetch(url, options).then(function (response) { + if (ownsPage) { return new Promise(function () {}); } + var path = new URL(url && url.url || url, location.href).pathname; + if (response.ok && current && current.child && path === new URL(current.url + 'results.json').pathname) { + return response.clone().json().then(function (r) { + var nextActs = actsOf(r); + if (JSON.stringify(nextActs) !== JSON.stringify(current.acts)) { + var opened = {}; + (nextActs || []).forEach(function (f) { + if (current.opened[f.id] && (current.acts || []).some(function (p) { return JSON.stringify(p) === JSON.stringify(f); })) { opened[f.id] = true; } + }); + current.acts = nextActs; current.opened = opened; + if (shut(current)) { ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return new Promise(function () {}); } + } + return viewResponse(response); + }); + } + if (response.ok && (path.endsWith('/family.json') || path.endsWith('/runs.json'))) { + return response.json().then(function (data) { return loadedJSON(url, data); }).then(function (data) { + return viewResponse(new Response(JSON.stringify(data), { status: response.status, headers: response.headers })); + }); + } + return viewResponse(response); + }); + }); + }; + window.addEventListener('hashchange', function () { if (ownsPage) { renderGate(); } }); + window.addEventListener('pagehide', function () { if (current && current.child && current.acts && current.acts.length) { ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); } }); + window.addEventListener('pageshow', function (e) { if (e.persisted) { location.reload(); } }); + // live.js also has an event reader. Delay it at the same gate; it never sees child events while closed. + var NativeEvents = window.EventSource; + window.EventSource = function (url, options) { + var proxy = this, source = null; + proxy.readyState = 0; + proxy.close = function () { if (source) { source.close(); } proxy.readyState = 2; }; + ready.then(function () { + var requestURL = absolute(url); + if (ownsPage || Object.keys(records).some(function (key) { return requestURL.indexOf(key) === 0 && shut(records[key]); }) || proxy.readyState === 2) { return; } + source = new NativeEvents(url, options); eventSources.push(source); + source.onmessage = function (e) { if (!ownsPage && proxy.onmessage) { proxy.onmessage(e); } }; + source.onerror = function (e) { proxy.readyState = source.readyState; if (proxy.onerror) { proxy.onerror(e); } }; + source.onopen = function (e) { proxy.readyState = source.readyState; if (proxy.onopen) { proxy.onopen(e); } }; + }); + }; + window.EventSource.CLOSED = NativeEvents.CLOSED; + window.PGPUI.ready = ready; + window.PGPUI.gateOwnsPage = function () { return ownsPage; }; + window.PGPUI.closedText = CLOSED_TEXT; + window.PGPUI.openedActs = function () { return current && !shut(current) ? copy(current.opened) : {}; }; + window.PGPUI.closeAct = function (id) { + if (!current || !current.child || !current.acts || !current.acts.some(function (f) { return f.id === id; })) { return false; } + delete current.opened[id]; ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return true; + }; + window.PGPUI.familyStatus = function () { return Object.keys(familyGates).sort().map(function (k) { return k + ':' + (familyGates[k].acts ? 'known' : 'failed'); }).join(' '); }; + window.PGPUI.familyControls = function (child, redraw) { + function build() { + var box = controls(familyGates[child], function () { box.replaceWith(build()); redraw(); }); + box.setAttribute('data-gate-child', child); return box; + } + return build(); + }; + }()); diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 575d1ee72..1d10edfe4 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -7,10 +7,11 @@ // members side by side. It draws what the steps wrote and computes nothing about genetics. Text through textContent, // the drawing through createElementNS; images from this server only. // A child's results to act on stay closed here as on the child's own page (person.js): each child's results.json -// gives the closed results (veil.spans), and while one is closed every place of this page that could carry one -// closes for that child, the same whichever result it is (the veil below). +// is read by the shared gate, which passes no report data of a child with an unopened act to this view. (function () { 'use strict'; + window.PGPUI.ready.then(function () { + if (window.PGPUI.gateOwnsPage()) { return; } var U = window.PGPUI; if (!U) { return; } var el = U.el, add = U.add, root = document.documentElement; @@ -54,7 +55,6 @@ out.push([seq + ' genome' + (seq === 1 ? '' : 's') + ' of this family are read together here. ', false]); (st.couples || []).forEach(function (c) { if (!c.screened) { out.push([c.a + ' and ' + c.b + ' are not screened as a couple yet. ', false]); return; } - if (coupleShut(f, c).length) { out.push([c.a + ' and ' + c.b + '’s carrier screen stays closed in chapter III. ', false]); return; } if (c.shared) { out.push([c.a + ' and ' + c.b + ' both carry a change in ' + c.shared + ' gene' + (c.shared > 1 ? 's' : '') + ': chapter III gives the chance per pregnancy for each. ', true]); } else { out.push([c.a + ' and ' + c.b + ' share no carrier gene in what the screen sees; ', false], ['it does not see everything. ', false]); } if (c.x_linked) { out.push([c.x_linked + ' X-linked finding' + (c.x_linked > 1 ? 's' : '') + ' bear on sons. ', true]); } @@ -74,94 +74,6 @@ } function memberUrl(f, s) { return f.runs && f.runs[s] ? f.runs[s].url : null; } - // --- the veil --------------------------------------------------------------------------------------- - // What this page knows of each child's results: { status: 'pending', 'known' or 'failed', spans, acts }. A child - // with no run here, or whose results.json has not answered or could not be read, cannot say whether a result to - // act on is closed (UNKNOWN). - // What the page shows of a child never depends on which result is closed or where it lies: while a child has any - // result to act on still closed here, or while the page cannot tell, every place that could carry one closes for - // that child in the same way, with one box each and the same words whatever the result. The places: the - // painting's grandparents, the map as an image, the de novo and compound-het rows, the child's medicines and HLA - // cells of the side by side, and the carrier screen of a couple the child is part of. A known result opens with - // its own click, from any of those boxes; the places open once the last one is. While the page cannot tell, each - // box opens only its own place. Opened results are kept for the life of this page, as on the child's page: a - // reload closes them again. They are kept per child and per run (two children can carry the same result id). - var UNKNOWN = '?'; - var GENE_GROUPS = { Medicines: true, HLA: true }; // the side-by-side groups whose rows name a gene - var kids = {}, opened = {}, last = null; - function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; } - function isChild(f, s) { return (f.state.children || []).some(function (c) { return c.child === s; }); } - function loadKids(f) { - (f.state.children || []).forEach(function (c) { - var url = memberUrl(f, c.child); - var k = kids[c.child]; - if (!url) { kids[c.child] = { status: 'failed' }; return; } - if (k && k.url === url && k.busy) { return; } - kids[c.child] = k && k.url === url ? k : { status: 'pending', url: url }; - kids[c.child].busy = true; - U.getJSON(url + 'results.json').then(function (r) { - var cur = kids[c.child]; - if (!cur || cur.url !== url) { return; } - cur.busy = false; - var v = (r && r.veil) || {}; - // Results the server could not build come back without their findings: as unreadable as no answer. - var usable = !!(r && Array.isArray(r.findings)); - var acts = usable ? r.findings.filter(function (x) { return x.level === 'act'; }).map(function (x) { return x.id; }) : []; - // Spans that are not [id, ...] entries naming every result to act on cannot say which are closed: read - // as none given (UNKNOWN while there are results to act on). - var spansOk = Array.isArray(v.spans) && v.spans.every(function (s) { return Array.isArray(s) && typeof s[0] === 'string'; }) && - acts.every(function (id) { return v.spans.some(function (s) { return s[0] === id; }); }); - var next = usable ? { status: 'known', url: url, spans: spansOk ? v.spans : null, acts: acts } - : { status: 'failed', url: url }; - // An answer that cannot be read keeps what was known: the page never opens on a failed request. - if (!usable && cur.status === 'known') { return; } - var key = JSON.stringify([next.status, next.spans, next.acts]); - if (key === cur.key) { return; } - next.key = key; - kids[c.child] = next; - if (last) { redraw(); } - }).catch(function () { - var cur = kids[c.child]; - if (!cur || cur.url !== url) { return; } - cur.busy = false; - if (cur.status !== 'known' && cur.status !== 'failed') { cur.status = 'failed'; cur.key = 'failed'; if (last) { redraw(); } } - }); - }); - } - // The results of CHILD still closed, as the place BOX sees them: their ids, [UNKNOWN] while this page cannot read - // them (until BOX's own button opens BOX), or [] once every one is opened. Results with act findings but no - // veil.spans cannot say which they are: UNKNOWN too. - function shutOf(child, box) { - var k = kids[child]; - if (!k || k.status !== 'known' || (!k.spans && k.acts.length)) { - return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN]; - } - var out = []; - (k.spans || []).forEach(function (s) { if (!opened[openKey(child, s[0])] && out.indexOf(s[0]) < 0) { out.push(s[0]); } }); - return out; - } - // The closed box of the place BOX of CHILD: WHAT says what stays closed; one button opens one result (BUTTON - // while the page cannot read the child's results, and then it opens this place only). - function veilBox(child, shut, box, what, button) { - if (!shut.length) { return null; } - var v = shut[0] === UNKNOWN - ? U.veil(what + ': closed, as this page cannot read ' + child + '’s results yet, so it cannot tell whether a ' + - 'result to act on in adult life is among them.', button, function () { opened[openKey(child, 'box:' + box)] = true; redraw(); }) - : U.veil(what + ': closed while ' + child + ' has a result to act on in adult life that is closed here.', null, - function () { opened[openKey(child, shut[0])] = true; redraw(); }); - v.setAttribute('data-veil-child', child); - v.setAttribute('data-veil-box', box); - return v; - } - - // A couple's carrier screen names the partners' carrier genes and their positions, so it closes while a partner - // who is a child of this family has a result still closed: [[that partner, their closed results]]. - function coupleShut(f, c) { - return [c.a, c.b].filter(function (m) { return isChild(f, m); }) - .map(function (m) { return [m, shutOf(m, 'couple:' + c.a + ' ' + c.b)]; }) - .filter(function (x) { return x[1].length; }); - } - // The pedigree drawn from the layout the server gives: a sequenced person with a run here is a link. function pedigree(f) { var L = f.layout, st = f.state; @@ -225,7 +137,7 @@ if (m.father || m.mother) { rel.push('child of ' + [m.father, m.mother].filter(Boolean).join(' and ')); } if (m.partner) { rel.push('partner of ' + m.partner); } if (rel.length) { add(txt, el('span', { 'class': 'person-place' }, rel.join('; '))); } - var state = !m.sequenced ? 'Not sequenced' : !r ? 'No run on this server' + var state = r && r.gated ? U.closedText : !m.sequenced ? 'Not sequenced' : !r ? 'No run on this server' : r.state === 'finished' ? 'Finished' : (r.state === 'stopped' ? 'Stopped, ' : '') + r.done + ' of ' + r.planned + ' steps'; add(txt, el('span', { 'class': 'person-state' }, state)); if (r && r.results) { @@ -264,16 +176,6 @@ add(s, p); return; } - var cs = coupleShut(f, c); - if (cs.length) { - // The whole screen closes: its counts, its boxes, its table and its report. - p.textContent = c.a + ' and ' + c.b + ': the carrier screen stays closed here.'; - add(s, p); - cs.forEach(function (x) { - add(s, veilBox(x[0], x[1], 'couple:' + c.a + ' ' + c.b, 'The carrier screen of ' + c.a + ' and ' + c.b, 'Show the screen')); - }); - return; - } var rows = (f.couples || {})[c.a + ' ' + c.b] || []; var one = rows.filter(function (r) { return r.result === 'one_partner'; }).length; // A count of zero is not a clean result: the screen sees only what it lists as assessed. @@ -389,17 +291,7 @@ // coloured by the grandparent it came from (or by the parent when no grandparent was sequenced). var SIDE = { paternal: 'father', maternal: 'mother' }; function painting(f, child, all) { - // A stretch drawn from a grandparent says which grandparent's copy the child got there, so one over a - // closed result says whether the child got it. A stretch from a parent alone says nothing. Closing only the - // stretches over a result, or the painting only when one lies over it, would point at where the result lies, so - // while the child has any result still closed every stretch is drawn and listed from its parent alone, with no - // grandparent, basis or closed mark, behind one box. - var anc = all.some(function (r) { return r.ancestor && r.ancestor !== '.'; }); - var shut = anc ? shutOf(child, 'paint') : []; - var rows = !shut.length ? all : all.map(function (r) { - return { side: r.side, parent: r.parent, chrom: r.chrom, start: r.start, end: r.end, sites: r.sites, - discordant: r.discordant, ancestor: '.', basis: 'parent' }; - }); + var rows = all; var lengths = f.chrom_lengths || []; var longest = Math.max.apply(null, lengths.map(function (x) { return x[1]; }).concat([1])); var colours = Object.create(null), next = 1; @@ -444,8 +336,6 @@ }); add(leg, el('span', { 'class': 'dim' }, ' Grey: no stretch called (no informative site, a gap of the reference).')); add(fig, leg); - // One box for the whole painting: each stretch shows its parent only. - add(fig, veilBox(child, shut, 'paint', 'The grandparents of ' + child + '’s stretches', 'Show the grandparents')); var d = el('details', { 'class': 'w-fallback' }); add(d, el('summary', {}, 'Every stretch (' + rows.length + ')')); add(d, U.table({ columns: ['Copy', 'Chromosome', 'Start', 'End', 'From', 'Sites', 'Discordant', 'Basis'], @@ -468,6 +358,10 @@ return 'Not assessed for this child: ' + why + '.'; } f.state.children.forEach(function (c, i) { + if (c.gated) { + add(s, el('h3', {}, c.child), U.familyControls(c.child, fetchFamily)); + return; + } var d = el('details', { 'data-child': c.child }); if (i === 0) { d.open = true; } add(d, el('summary', {}, c.child + ': father ' + (c.father || 'not named') + ', mother ' + (c.mother || 'not named'))); @@ -478,12 +372,7 @@ } else if (c.segments) { // The drawing as an image is coloured by grandparent too, and cannot close one stretch: it stays closed // whole, as the painting does. - var shut = shutOf(c.child, 'map'); - if (shut.length) { - add(d, veilBox(c.child, shut, 'map', 'The chromosome map of ' + c.child + ' and its colours', 'Show the map')); - } else { - add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); - } + add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); } else { add(d, el('p', { 'class': 'dim' }, 'No chromosome map yet: step 38 draws one for a child with a sequenced parent.')); } @@ -493,14 +382,9 @@ add(d, el('p', { 'class': 'dim' }, 'In the child and in neither parent. Most candidates in a genome are genotype errors: ' + 'check the reads before reading anything into one.')); // Every row closes while a result is: a row closed on its own would say where the result lies. - var dshut = shutOf(c.child, 'denovo'); - if (dshut.length) { - add(d, veilBox(c.child, dshut, 'denovo', 'The de novo candidates of ' + c.child, 'Show the rows')); - } else { add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); - } } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'de_novo' }, empty(c.child, 'de_novo', 'de_novo.tsv'))); } @@ -508,13 +392,8 @@ add(d, el('h3', {}, 'Compound-het pairs')); if (ch.length) { add(d, el('p', { 'class': 'dim' }, 'Two candidate variants in one gene, in trans first: one from each parent.')); - var hshut = shutOf(c.child, 'het'); - if (hshut.length) { - add(d, veilBox(c.child, hshut, 'het', 'The compound-het pairs of ' + c.child, 'Show the rows')); - } else { add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); - } } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'compound_het' }, empty(c.child, 'compound_het', 'compound_het.tsv'))); } @@ -530,18 +409,11 @@ if (!sb || !sb.rows.length) { add(s, el('p', { 'class': 'dim' }, 'No member has a summary yet, or no summary lists these results.')); return s; } add(s, el('p', { 'class': 'dim' }, 'Each member’s own report, one column per person: CYP2D6, the CPIC genes ' + '(phenotypes in CPIC’s words), HLA, the haplogroups and the blood groups.')); - // A child's medicines and HLA cells name genes (a CPIC gene can be the gene of a result to act on, RYR1 or - // CACNA1S). Closing only the row of a closed result's gene would leave a gap in a list everyone knows, so - // while a child has any result still closed every medicines and HLA cell of the child's column reads "closed": - // every row stays, with the parents' values. The haplogroups and blood groups name no gene. - var shutBy = {}; - sb.members.forEach(function (m) { if (isChild(f, m)) { shutBy[m] = shutOf(m, 'side'); } }); // A value from a section older than its input (state "stale") says so in its own cell. - var group = null, rows = [], old = 0, closedCells = {}; + var group = null, rows = [], old = 0; sb.rows.forEach(function (r) { rows.push([r.group === group ? '' : r.group, r.label].concat(r.values.map(function (v, i) { var m = sb.members[i]; - if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length) { closedCells[m] = true; return 'closed'; } if (!(r.stale || [])[i]) { return v; } old++; return v + STALE; @@ -549,9 +421,6 @@ group = r.group; }); add(s, U.table({ columns: ['', 'Gene or system'].concat(sb.members), rows: rows }, 14)); - sb.members.forEach(function (m) { - if (closedCells[m]) { add(s, veilBox(m, shutBy[m], 'side', 'The medicines and HLA cells of ' + m, 'Show the cells')); } - }); if (old) { add(s, el('p', { 'class': 'dim', 'data-stale': String(old) }, 'Marked' + STALE + ': the member’s result file is from an ' + 'earlier run, and the latest run did not write it again; that member’s page says why.')); @@ -568,19 +437,14 @@ } if (f.state.reports.length) { var ul = el('ul', { 'class': 'lines' }); - // A couple report of a screen that stays closed is left out with it. - var shutReports = (f.state.couples || []).filter(function (c) { return c.report && coupleShut(f, c).length; }) - .map(function (c) { return c.report; }); - f.state.reports.forEach(function (r) { - if (shutReports.indexOf(r.url) < 0) { add(ul, add(el('li', {}), el('a', { href: r.url }, r.name))); } - }); + f.state.reports.forEach(function (r) { add(ul, add(el('li', {}), el('a', { href: r.url }, r.name))); }); add(s, ul); } return s; } function draw(f) { - last = f; + var main = document.getElementById('family'); // A redraw (a result opened, a child's results answered) keeps each child's fold as the reader left it. var folds = {}; @@ -622,10 +486,10 @@ }); window.scrollTo(0, y); // What the page knows of each child's results (for the tests): "DEMO-003:known". - root.setAttribute('data-pgp-kids', Object.keys(kids).sort().map(function (c) { return c + ':' + kids[c].status; }).join(' ')); + root.setAttribute('data-pgp-kids', U.familyStatus()); root.setAttribute('data-pgp-status', 'ready'); } - function redraw() { if (last) { draw(last); } } + // The page asks again every 30 s and redraws only when family.json changed (a member's run moved on, // step 39 finished). A failed first request says so, with a retry; a later one keeps what is shown. @@ -644,7 +508,6 @@ if (!f) { failed(); return; } var k = JSON.stringify(f); // Each child's results are asked again with the family: a result to act on can appear while a run goes. - if (!f.state.error) { loadKids(f); } if (k !== drawnKey) { drawnKey = k; draw(f); } }).catch(failed); } @@ -653,4 +516,5 @@ window.setInterval(function () { if (!document.hidden) { fetchFamily(); } }, 30000); } if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', start); } else { start(); } + }); }()); diff --git a/assets/live/finding.js b/assets/live/finding.js index 754512812..9b0041875 100644 --- a/assets/live/finding.js +++ b/assets/live/finding.js @@ -6,6 +6,8 @@ // genetics. A result to act on about a child of the family stays closed until it is opened on purpose. (function () { 'use strict'; + window.PGPUI.ready.then(function () { + if (window.PGPUI.gateOwnsPage()) { return; } var U = window.PGPUI; if (!U) { return; } var el = U.el, add = U.add; @@ -15,7 +17,6 @@ var replay = root.getAttribute('data-mode') === 'replay'; var main = document.getElementById('finding'); var ROMAN = ['i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii']; - var opened = {}; function abs(u) { return !u ? u : (u.charAt(0) === '/' ? u : BASE + u); } function note(head, body, cls) { @@ -78,38 +79,13 @@ var known = unknown ? null : family; var me = known && (known.state.members || []).filter(function (m) { return m.sample === r.sample; })[0]; // A family that does not list this person cannot say they are not a child: closed as well. - var child = unknown || (!!known && !me) || !!(me && (me.father || me.mother)); - // A finding that is not itself a result to act on but names one (a symptom ranking can rank the gene: - // results.veil.findings, bin/live_results.py veil()) stays closed with it, as on the person page. - // A tag is the list of every result it names: the finding stays closed while any of them is. - var tags = (r.veil && r.veil.findings) || {}; - function tagOf(x) { - if (x.level === 'act') { return null; } - var t = tags[x.id]; - if (t === undefined) { t = (r.findings || []).some(function (item) { return item.level === 'act'; }) ? ['*panel*'] : null; } - t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (y) { return y !== x.id; }); - return t.length ? t : null; - } - // Opening this finding opens it and the results it names, nothing else: another result keeps its own veil. - // A finding of a fixed-panel card (tag PANEL, bin/live_results.py FIXED_PANELS) names no result: it stays closed - // while any result to act on is, whichever, as on the person page. - var PANEL = '*panel*'; - function actShut() { return (r.findings || []).some(function (x) { return x.level === 'act' && !opened[x.id]; }); } - function shut(x) { - if (!child) { return false; } - if (x.level === 'act') { return !opened[x.id]; } - return (tagOf(x) || []).some(function (t) { return t === PANEL ? actShut() : !opened[t]; }); - } - // Closed: no title, gene or variant anywhere on the page (its title, crumb, cover, rail) until opened. - var closed = shut(f); - var title = !closed ? f.title : f.level === 'act' ? 'A result to act on' : 'A finding about a result to act on'; + var title = f.title; document.title = title + ': ' + r.sample + ': OwnGenome'; document.getElementById('finding-crumb').textContent = title; var sec = (r.journal || []).filter(function (s) { return s.n === f.section; })[0]; var same = (r.findings || []).filter(function (x) { return x.level === f.level; }); var order = (r.findings || []).filter(function (x) { return x.level !== 'info'; }); var next = order[order.indexOf(f) + 1]; - if (next && shut(next)) { next = null; } // The rail: back to the report, this page's parts, the next finding of the report. var rail = el('aside', { 'class': 'toc frail', 'aria-label': 'This finding' }); var back = el('a', { 'class': 'back', href: BASE + '#f/' + f.id }); @@ -145,7 +121,7 @@ if (f.detail) { cs.push(f.detail); } if (f.viewer && cs.join(' ').indexOf(f.viewer) < 0) { cs.push(f.viewer); } cs.forEach(function (c) { add(chips, el('span', { 'class': 'chip mono' }, c)); }); - if (cs.length && !closed) { add(cover, chips); } + if (cs.length) { add(cover, chips); } var main0 = el('div', {}); add(main0, cover); var acts = el('aside', { 'class': 'actions' }); @@ -156,29 +132,6 @@ var top = el('div', { 'class': 'row' }); add(top, main0, acts); add(col, top); - if (closed) { - var v = el('div', { 'class': 'veil' }); - add(v, add(el('div', { 'class': 'vt' }), el('b', {}, 'A result about a child. '), - (f.level === 'act' ? '' : 'This finding names a result to act on. ') + - 'A result to act on in adult life is usually not looked at before the child can choose. It stays closed until you open it.')); - var b = el('button', { type: 'button', 'class': 'btn' }, 'Show the result'); - b.addEventListener('click', function () { - var ids = tagOf(f) || []; - // A panel names no particular act. Its button opens the next closed act, one per click. - var nextAct = ids.indexOf(PANEL) >= 0 ? (r.findings || []).filter(function (x) { - return x.level === 'act' && !opened[x.id]; - })[0] : null; - if (nextAct) { opened[nextAct.id] = true; } - else { opened[f.id] = true; ids.forEach(function (id) { if (id !== PANEL) { opened[id] = true; } }); } - draw(r, family); - }); - add(v, b); - add(col, v); - add(body, rail, col); - add(main, body); - root.setAttribute('data-pgp-status', 'ready'); - return; - } var row1 = el('div', { 'class': 'row' }); var m1 = el('div', {}); add(m1, el('p', { 'class': 'lede' }, f.text)); @@ -220,7 +173,7 @@ } // Related findings and the family. var rel = (f.related || []).map(function (id) { return (r.findings || []).filter(function (x) { return x.id === id; })[0]; }) - .filter(function (x) { return x && !shut(x); }); + .filter(Boolean); if (known) { var sf = el('section', { 'class': 'sec', id: 'family' }); add(sf, add(el('div', { 'class': 'sechead' }), el('span', { 'class': 'secnum' }, ROMAN[parts.length - 3]), el('h2', { 'class': 'h2' }, 'In the family'))); @@ -289,4 +242,5 @@ }).catch(failed); } if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', start); } else { start(); } + }); }()); diff --git a/assets/live/home.js b/assets/live/home.js index 42da76af5..803fc0522 100644 --- a/assets/live/home.js +++ b/assets/live/home.js @@ -6,6 +6,8 @@ // member of a family keeps one colour on this page. The plain list of runs is what shows without scripts. (function () { 'use strict'; + window.PGPUI.ready.then(function () { + if (window.PGPUI.gateOwnsPage()) { return; } var U = window.PGPUI; if (!U) { return; } var el = U.el, add = U.add; @@ -43,16 +45,8 @@ if (!row.childNodes.length) { add(row, el('span', { 'class': 'dim' }, 'No finding in what was assessed; read the caveats')); } return row; } - // The runs whose results to act on stay closed on this page: a child in any family, and, while a family's - // members cannot be read, every run that no readable family lists (it may be that family's child). - var closedRuns = {}; - function closedSummary(r) { return !!(r && closedRuns[r.name] && r.levels && r.levels.act); } - function homeTop(r) { - if (!closedSummary(r)) { return r.top || []; } - return ['trust', 'act', 'medicine', 'carrier', 'review'].filter(function (level) { - return r.levels[level] > 0; - }).map(function (level) { return { level: level, title: 'Results closed until opened on the report' }; }); - } + // The loader has already replaced each gated child's summary. + function homeTop(r) { return r.top || []; } function tile(m, fam, r, colour) { var div = el('div', { 'class': 'ptile person-card' + (r ? ' has-run' : ' norun') + (r && r.state === 'running' ? ' running' : ''), 'data-sample': m.sample }); @@ -61,12 +55,12 @@ var h = el('h3', {}); add(h, r ? el('a', { href: r.url, 'class': 'person-link' }, m.sample) : el('span', {}, m.sample)); add(div, h); + if (r && r.gated) { add(div, el('p', { 'class': 'lead' }, U.closedText)); return div; } var meta = el('div', { 'class': 'meta' }); add(meta, (fam ? role(m, fam) + ' · ' : '') + progressText(r)); if (r && r.failed) { add(meta, ' · ', el('span', { 'class': 'warn' }, r.failed + ' step' + (r.failed > 1 ? 's' : '') + ' not assessed')); } add(div, meta); - var lead = r && (closedSummary(r) ? 'Results to act on stay closed until opened on the report. Read the report for the other findings and their caveats.' - : closedRuns[m.sample] ? r.lead_closed || '' : r.lead); + var lead = r && (r.lead); if (lead) { // Two sentences: the first often says only that nothing needs a doctor, the second names what differs. var one = U.firstSentence(lead), rest = lead.slice(one.length).trim(); @@ -145,21 +139,8 @@ // nothing needs a doctor. var screened = runs.filter(function (r) { return r.levels && r.screened; }).length; // A child of a family: results to act on stay closed on this page too (the report opens them). - closedRuns = {}; - var listed = {}; - var rolesUnknown = fams.some(function (f) { return !f.members.length; }); - fams.forEach(function (f) { - f.members.forEach(function (m) { - listed[m.sample] = true; - if (f.members.some(function (x) { return x.sample === m.father || x.sample === m.mother; })) { closedRuns[m.sample] = true; } - }); - }); - if (rolesUnknown) { runs.forEach(function (r) { if (!listed[r.name]) { closedRuns[r.name] = true; } }); } - function homeShut(r, f) { return closedSummary(r) || (!!closedRuns[r.name] && (f.level === 'act' || (f.veil || []).indexOf('*panel*') >= 0)); } - function closedTitle(r, f) { - return homeShut(r, f) ? (f.level === 'act' ? 'A result to act on, closed until opened on the report' - : 'A finding in a panel, closed until opened on the report') : f.title; - } + function homeShut(r) { return !!r.gated; } + function closedTitle(r, f) { return r.gated ? U.closedText : f.title; } // The dateline and the headline sentence. var dl = el('div', { 'class': 'dateline' }); var today = new Date(); @@ -243,7 +224,7 @@ add(c2, el('div', { 'class': 'fig-label' }, 'Found so far')); var fs = el('ul', { 'class': 'fs' }); homeTop(r).forEach(function (f) { add(fs, add(el('li', {}), U.findingLevel(f.level, ''), ' ', closedTitle(r, f))); }); - if (!fs.childNodes.length) { add(fs, el('li', {}, r.lead ? U.firstSentence(closedRuns[r.name] ? r.lead_closed || '' : r.lead) : 'No section is written yet: the first ones appear as their steps end.')); } + if (!fs.childNodes.length) { add(fs, el('li', {}, r.lead ? U.firstSentence(r.lead) : 'No section is written yet: the first ones appear as their steps end.')); } add(c2, fs); var c3 = el('div', {}); add(c3, el('div', { 'class': 'fig-label' }, 'Timing')); @@ -324,4 +305,5 @@ }).catch(function () { root.setAttribute('data-pgp-status', 'offline'); setTimeout(start, 30000); }); } if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', start); } else { start(); } + }); }()); diff --git a/assets/live/map.js b/assets/live/map.js index 82b4606cc..5f0b2f3de 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -90,9 +90,8 @@ (f.id || []).forEach(function (fid, i) { if (!(f.chrom[i] in by)) { return; } rows.push({ id: fid, level: f.level[i], gate: f.gate[i], chrom: f.chrom[i], pos: f.pos[i], s: f.s[i], e: f.e[i], - gene: f.gene[i], title: f.title[i], href: f.href[i], phrase: f.phrase[i], veil: f.veil[i] >= 0 ? veilIds[f.veil[i]] : null, - // Every closable finding the row lies over (veils); a document from before the column has its first only. - veils: ((f.veils && f.veils[i]) || (f.veil[i] >= 0 ? [f.veil[i]] : [])).map(function (k) { return veilIds[k]; }) }); + gene: f.gene[i], title: f.title[i], href: f.href[i], phrase: f.phrase[i], veil: f.veil[i] >= 0 ? veilIds[f.veil[i]] : null }); + }); var gated = {}; rows.forEach(function (r) { if (r.gate) { gated[r.id] = r; } }); @@ -170,12 +169,12 @@ // hides every finding it keeps closed (opts.now), and its card shows the rest. Hiding such a row for // another result's tag would drop a finding whose card the page shows, and so place that closed result // inside it. A map-owned gated row (APOE) follows its own gate for the same reason: opening it must not - // tell whether another, still-closed result overlaps it. Other rows follow every overlapping tag. + // tell whether another, still-closed result overlaps it. Other rows retain the document's APOE warning tag. function shownRow(r) { var own = M.closable.indexOf(r.id) < 0 || stateOf(r.id) !== 'closed'; // Closed while ANY result the row lies over is closed: a row over two closed results never comes back with // the first one opened. - var tag = !!r.gate || (opts.known && opts.known(r.id)) || r.veils.every(function (id) { return stateOf(id) !== 'closed'; }); + var tag = !!r.gate || (opts.known && opts.known(r.id)) || !r.veil || stateOf(r.veil) !== 'closed'; return own && tag && (!opts.now || opts.now(r.id)); } // The runs of homozygosity, as this page shows them now: a replay before their step ended has none yet. diff --git a/assets/live/person.js b/assets/live/person.js index 23463db9c..3ea338406 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -20,7 +20,8 @@ var results = null, state = null, family = null, familyFor = null, fetchedFor = null, drawnKey = null, folded = false; // 'none': in no family; 'pending' or 'failed': in one whose roles are not known (yet); 'known'. var familyStatus = 'none'; - var shown = {}; // veiled findings the reader opened, by id + var openingSeeded = false; + var shown = {}; // act openings transferred from the gate, plus the independent APOE switch var genomeMap = null, resultsKey = null; // Figure 1 (map.js), and the results it was built for var panel = document.getElementById('results'); var rail = document.getElementById('rail'); @@ -320,147 +321,15 @@ add(b, text); return b; } - // The findings of a card that a child's page keeps closed: while one is closed the card's own rows, - // numbers and drawing stay closed too, or its table would name the gene and variant under the veil. - // A place outside its result's card (a ClinVar row, a card's line, fact or headline, a viewer link, a chart's mark, - // a search hit) carries a tag (results.veil, bin/live_results.py veil()): the list of every result to act on it - // names. It stays closed while ANY of them is, so a place naming two closed results never comes back with the - // first one opened. A tag naming no finding of these results stays closed. PANEL (a fixed panel's line, or prose - // that cannot be shown to name no gene) names no result: it stays closed while any result to act on is, - // whichever, so what is missing never tells the gene. The ids of a tag never reach the page. - var PANEL = '*panel*'; - function tagClosed(id) { - if (Array.isArray(id)) { return id.some(tagClosed); } - if (id === PANEL) { return closedPage() && closedActs().length > 0; } - var f = findingById(id); - return closedPage() && (!f || veiled(f)); - } - // Explicit null is a structured place with no matching act. Missing metadata is unclassified: PANEL. - function placeClosed(tag) { return tag === undefined ? tagClosed(PANEL) : !!tag && tagClosed(tag); } - function veilTags(kind, key) { return ((results && results.veil && results.veil[kind]) || {})[key] || []; } - // A card's headline is prose: tagged PANEL unless the summary key it comes from prints no gene (veil.headline). - function headlineShut(c) { var t = ((results && results.veil && results.veil.headline) || {})[c.id]; return placeClosed(t); } - // The charts (plugins/charts.js) ask for each mark. A closed page with no results.json yet cannot tell which - // marks those are, so it leaves every mark out until it can. - window.PGP.markClosed = function (chart, mark) { - if (!closedPage()) { return false; } - if (!results) { return true; } - var id = ((results.veil && results.veil.marks && results.veil.marks[chart]) || {})[mark]; - return placeClosed(id); - }; - // Chart prose and dimensions have no structured result tag. The chart reader asks the same PANEL rule. - function chartPanelClosed() { return closedPage() && (!results || tagClosed(PANEL)); } - // The chart plugin owns mark links and their click handlers. Keep those nodes, but give the child's - // closed drawing only the producer's fixed geometry. The complete SVG lives off the document until - // PANEL opens. Older chart files without fixed geometry stay closed in full. - function veilCharts() { - Array.prototype.forEach.call(document.querySelectorAll('figure[data-chart]'), function (fig) { - var sp = fig.pgpSpec, shut = chartPanelClosed(); - if (!sp || fig.pgpPanelState === shut) { return; } - fig.pgpPanelState = shut; - if (!fig.pgpFullSVG) { fig.pgpFullSVG = fig.querySelector('svg'); } - var current = fig.querySelector('svg,.chart-panel-closed'); - if (!shut) { - if (current && current !== fig.pgpFullSVG) { current.remove(); } - if (fig.pgpFullSVG && fig.pgpFullSVG.parentNode !== fig) { fig.appendChild(fig.pgpFullSVG); } - return; - } - var replacement = el('p', { 'class': 'dim chart-panel-closed' }, - 'This chart stays closed until every result to act on is opened.'); - if (sp.svg_closed && fig.pgpFullSVG) { - var doc = new DOMParser().parseFromString(sp.svg_closed, 'image/svg+xml'); - if (doc.documentElement.nodeName === 'svg') { - replacement = document.importNode(doc.documentElement, true); - Array.prototype.forEach.call(replacement.querySelectorAll('a[data-mark]'), function (a) { - var id = a.getAttribute('data-mark'); - var live = Array.prototype.find.call(fig.pgpFullSVG.querySelectorAll('a[data-mark]'), function (n) { - return n.getAttribute('data-mark') === id; - }); - // Use a clone in the fixed geometry and delegate its click to the plugin's original node. - if (!live) { a.remove(); } - else { a.addEventListener('click', function (e) { e.preventDefault(); live.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); }); } - }); - } - } - if (current) { current.replaceWith(replacement); } else { fig.appendChild(replacement); } - }); - } - // The plugin places figures after person.js draws the slots. Mutation callbacks run before paint, - // including on the first async charts response; the card event also handles PANEL changing without - // any mark changing. No chart filename or data value is used to decide whether PANEL is closed. - new MutationObserver(veilCharts).observe(document.body, { childList: true, subtree: true }); - document.addEventListener('pgp:cards', function () { queueMicrotask(veilCharts); }); - // A card's table without its closed rows: [table or null, how many rows it left out]. - function openTable(c) { - var tags = veilTags('rows', c.id), t = c.table; - if (!t || !t.rows) { return [t, 0]; } - var rows = t.rows.filter(function (r, i) { return !placeClosed(tags[i]); }); - var out = {}; - Object.keys(t).forEach(function (k) { out[k] = t[k]; }); - out.rows = rows; - return [rows.length ? out : null, t.rows.length - rows.length]; - } - // A card's facts and drawing without their closed items (a fact, a CHIP call or a copy-number dot that names a - // closed result's gene): [facts, widget, how many items they left out]. - function openFacts(c) { - var ft = veilTags('facts', c.id), facts = (c.facts || []).filter(function (f, i) { return !placeClosed(ft[i]); }); - var w = c.widget, key = w && (w.kind === 'vaf' ? 'calls' : w.kind === 'copies' ? 'genes' : null); - var wt = veilTags('widget', c.id), shut = (c.facts || []).length - facts.length; - if (key && wt.length && w[key]) { - var items = w[key].filter(function (x, i) { return !(wt[i] && tagClosed(wt[i])); }); - shut += w[key].length - items.length; - var copy = {}; - Object.keys(w).forEach(function (k) { copy[k] = w[k]; }); - copy[key] = items; - w = copy; - } - if (w && placeClosed(((results.veil || {}).widget_panel || {})[c.id])) { w = null; shut += 1; } - return [facts, w, shut]; - } - function veiledOf(c) { return (c.findings || []).map(findingById).filter(function (f) { return f && veiled(f); }); } - // The results to act on this page keeps closed, drawn yet or not. - function closedActs() { return ((results && results.findings) || []).filter(function (x) { return x.level === 'act' && veiled(x); }); } - // A card of a fixed panel (results.veil.panels, bin/live_results.py FIXED_PANELS: the same genes or loci for - // everyone, known in advance): a row or dot of it closed with a result would tell the result's gene by what is - // missing. So it closes whole while this page keeps any result to act on closed, behind one box with the same - // words whatever the result, and comes back once the last one is opened. - function panelShut(c) { - var p = (results && results.veil && results.veil.panels) || []; - return closedPage() && p.indexOf(c.id) >= 0 && closedActs().length > 0; - } - var PANEL_WORDS = 'This section lists the same genes for everyone, so it stays closed while a result to act on is closed ' + - 'on this page: what it left out would tell which gene. It opens once every such result is opened.'; - var PANEL_LINE = 'The same genes for everyone, closed while a result about a child is closed.'; - // The next closed result in the page's order (the map's Open opens the same one). - function nextClosed() { var ids = mapClosed(); return ids.length ? findingById(ids[0]) : closedActs()[0]; } - // A closed finding's handle in the page: its place in the page's list of closed findings, never its id, which - // names its gene and variant. The id comes back once the finding is opened on purpose (#f/ too). - function closedOrder() { - var out = []; - sectionsList().forEach(function (sec) { - (sec.findings || []).forEach(function (id) { var f = findingById(id); if (f && veiled(f) && out.indexOf(id) < 0) { out.push(id); } }); - }); - ((results && results.findings) || []).forEach(function (f) { if (veiled(f) && out.indexOf(f.id) < 0) { out.push(f.id); } }); - return out; - } - function handle(f) { return veiled(f) ? 'closed-' + (closedOrder().indexOf(f.id) + 1) : f.id; } - // The search box (plugins/search.js) asks the same question for each hit, by the summary key and the anchor - // of the section it comes from: a hit from a card this page keeps closed is left out. A closed page with - // no results.json yet cannot tell which cards those are, so it leaves every hit out until it can. - // Asked with no key, it answers whether this page keeps anything closed at all. - // A hit tagged with a result to act on (veil, bin/results_index.py) is left out while that result is closed. - window.PGP.searchHidden = function (key, anchor, tag) { - if (!closedPage()) { return false; } - if (!results || key === null) { return true; } - if (tag && tagClosed(tag)) { return true; } - return cards().some(function (c) { - return (c.id === anchor || (c.keys || []).indexOf(key) >= 0) && (veiledOf(c).length > 0 || panelShut(c)); - }); - }; + // This renderer receives the full report only after the shared data gate releases. + function openTable(c) { return [c.table, 0]; } + function openFacts(c) { return [c.facts || [], c.widget, 0]; } + function handle(f) { return f.id; } + window.PGP.markClosed = function () { return false; }; + window.PGP.searchHidden = function () { return false; }; function card(c) { var g = gate(c); - var closedFs = g === 'show' ? veiledOf(c) : []; var art = el('article', { 'class': 'card result g-' + g + (attentionNow(c) ? ' attn' : ''), id: 'card-' + c.id, 'data-section': c.id, 'data-step': (c.steps || []).join(' '), 'data-state': c.state, 'data-level': c.level || '', tabindex: '-1', 'aria-labelledby': 'h-' + c.id }); @@ -496,18 +365,9 @@ 'Step ' + (c.steps || []).join(', ') + ' failed: the step timeline says why, and a rerun of the step fills this section.')); } else if (g === 'empty') { add(text, el('p', { 'class': 'notassessed' }, 'Not assessed: its step ended without a result for this section.')); - } else if (panelShut(c)) { - // A fixed-panel card says the same words whichever result is closed, its own findings' too: so it comes first. - // One button opens one result, the next closed one in the page's order, as the map's Open does. - add(text, U.veil(PANEL_WORDS, null, function () { var f = nextClosed(); if (f) { openFinding(f); } redraw(true); })); - } else if (closedFs.length) { - // One button opens one result: a card with two results to act on asks twice. - add(text, U.veil((closedFs.length === 1 ? 'This section holds a result' : 'This section holds ' + closedFs.length + ' results') + - ' to act on in adult life. Its rows stay closed until ' + (closedFs.length === 1 ? 'it is' : 'each is') + ' opened.', null, - function () { openFinding(closedFs[0]); redraw(true); var x = document.getElementById('f-' + closedFs[0].id) || document.getElementById('card-' + c.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); } else { - (c.notes || []).forEach(function (n, i) { if (!placeClosed(veilTags('notes', c.id)[i])) { add(text, el('p', { 'class': 'note-stale' }, n)); } }); - if (c.headline && !headlineShut(c)) { add(text, el('p', { 'class': 'headline' }, c.headline)); } + (c.notes || []).forEach(function (n) { add(text, el('p', { 'class': 'note-stale' }, n)); }); + if (c.headline) { add(text, el('p', { 'class': 'headline' }, c.headline)); } var of = openFacts(c); add(margin, U.facts(of[0])); add(text, widget(of[1])); @@ -516,15 +376,8 @@ }); if (c.image) { add(text, el('img', { 'class': 'plot', src: c.image.src, alt: c.image.alt })); } var ot = openTable(c), tb = U.table(ot[0]); - var lt = veilTags('lines', c.id); - var lines = (c.lines || []).filter(function (l, i) { return !placeClosed(lt[i]); }); - var shut = ot[1] + (c.lines || []).length - lines.length + (c.headline && headlineShut(c) ? 1 : 0), items = of[2]; - if (tb) { add(text, tb); } else if (c.empty_text && !ot[1]) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } - if (shut || items) { - // The same words however many rows or items close, and whichever result they belong to: a count would - // tell one closed result from another. - add(text, el('p', { 'class': 'dim veil-rows' }, 'Some of its rows or numbers stay closed with a result to act on, until that result is opened.')); - } + var lines = c.lines || []; + if (tb) { add(text, tb); } else if (c.empty_text) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } add(text, lineList(lines)); if (c.hidden) { // APOE and the like: behind their warning, opened on purpose. @@ -546,9 +399,8 @@ add(margin, U.caveat(c.caveat)); var foot = el('p', { 'class': 'card-links' }); add(foot, el('a', { href: 'report.html#' + encodeURIComponent(c.anchor), 'class': 'report' }, 'Open in the report')); - var vtags = veilTags('viewer', c.id); - var reads = (c.viewer || []).filter(function (x, i) { return !placeClosed(vtags[i]); }); - if (g === 'show' && reads.length && !panelShut(c)) { + var reads = c.viewer || []; + if (g === 'show' && reads.length) { add(foot, el('a', { href: 'viewer/#' + reads[0], 'class': 'viewer' }, 'See the reads at ' + reads[0])); } (g === 'show' ? c.links || [] : []).forEach(function (f) { @@ -619,32 +471,6 @@ add(box, d); return note('Caveat', box, 'caveat-note'); } - // A finding that is not itself a result to act on but names one (a symptom ranking can rank the gene: results.veil.findings) - // stays closed while that result is, and opening it opens that result. - function findingTag(f) { - if (f.level === 'act') { return null; } // its own switch; never an unclassified secondary place - var t = ((results && results.veil && results.veil.findings) || {})[f.id]; - if (t === undefined) { t = ((results && results.findings) || []).some(function (x) { return x.level === 'act'; }) ? [PANEL] : null; } - t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (x) { return x !== f.id; }); - return t.length ? t : null; - } - function veiled(f) { - if (!closedPage()) { return false; } - if (f.level === 'act') { return !shown[f.id]; } - var t = findingTag(f); - return !!(t && tagClosed(t)); - } - // Opening a finding opens every result its tag names. One tagged PANEL (a fixed-panel card's own) stays closed while - // any result to act on is: opening it opens the next closed result in the page's order too, as its card's box does. - function openFinding(f) { - var t = findingTag(f) || []; - keepHash(function () { - shown[f.id] = true; - t.forEach(function (x) { if (x !== PANEL) { shown[x] = true; } }); - var n = t.indexOf(PANEL) >= 0 ? nextClosed() : null; - if (n) { shown[n.id] = true; } - }); - } // The address never holds the id of a finding this page can close, closed or opened: #f/closed- while it is // closed (its handle) and #f/opened- once opened on purpose (its place among the opened ones), so no history // entry keeps the id. Any other #f/ on a page that keeps a result closed, a closed result's id or no id at all, @@ -653,25 +479,24 @@ // finding behind a handle in the address is kept here only (hashId); a new address from the reader forgets it. var hashId = null; window.addEventListener('hashchange', function () { hashId = null; }); - function closable(f) { return closedPage() && (f.level === 'act' || !!findingTag(f)); } + function needsHandle(f) { return closedPage() && f.level === 'act'; } function openedOrder() { var out = []; sectionsList().forEach(function (sec) { - (sec.findings || []).forEach(function (id) { var f = findingById(id); if (f && closable(f) && !veiled(f) && out.indexOf(id) < 0) { out.push(id); } }); + (sec.findings || []).forEach(function (id) { var f = findingById(id); if (f && needsHandle(f) && out.indexOf(id) < 0) { out.push(id); } }); }); - ((results && results.findings) || []).forEach(function (f) { if (closable(f) && !veiled(f) && out.indexOf(f.id) < 0) { out.push(f.id); } }); + ((results && results.findings) || []).forEach(function (f) { if (needsHandle(f) && out.indexOf(f.id) < 0) { out.push(f.id); } }); return out; } - function addr(f) { return !closable(f) ? f.id : veiled(f) ? handle(f) : 'opened-' + (openedOrder().indexOf(f.id) + 1); } + function addr(f) { return !needsHandle(f) ? f.id : 'opened-' + (openedOrder().indexOf(f.id) + 1); } function hashFinding() { var h; try { h = decodeURIComponent((window.location.hash || '').slice(1)); } catch (e) { return null; } if (h.indexOf('f/') !== 0) { return null; } var x = h.slice(2), m = /^(closed|opened)-(\d+)$/.exec(x); var f = (m && hashId && findingById(hashId)) || - (m ? findingById((m[1] === 'closed' ? closedOrder() : openedOrder())[Number(m[2]) - 1]) : findingById(x)); - if (f && !(veiled(f) && !m)) { return f; } - return closedOrder().length ? findingById(closedOrder()[0]) : null; + (m ? findingById(openedOrder()[Number(m[2]) - 1]) : findingById(x)); + return f || null; } function syncHash() { var f = hashFinding(); @@ -688,19 +513,15 @@ if (gated) { return shown[id] ? 'opened' : 'closed'; } if (!closedPage()) { return 'open'; } var f = findingById(id); - if (!f || veiled(f)) { return 'closed'; } - return f.level === 'act' || findingTag(f) ? 'opened' : 'open'; + if (!f) { return 'closed'; } + return f.level === 'act' ? 'opened' : 'open'; } // One switch for a result, from the map's chip or the card's "Show the result" (openFinding): opening it in // one place opens it in both. Closing it closes the findings that were opened with it. function setShown(id, on) { + if (!on && U.closeAct(id)) { return; } var f = findingById(id); - if (on) { if (f) { openFinding(f); } else { shown[id] = true; } } else { - keepHash(function () { - delete shown[id]; - ((results && results.findings) || []).forEach(function (x) { if ((findingTag(x) || []).indexOf(id) >= 0) { delete shown[x.id]; } }); - }); - } + shown[id] = !!on; redraw(true); } // Whether the page shows a finding of the map at this moment: its card's steps are done (a replay draws @@ -711,17 +532,17 @@ var f = findingById(id); if (!f) { var t = cards().filter(function (c) { return c.hidden; })[0]; return !!(t && gate(t) === 'show'); } var c = cardById(f.card); - return !!(c && gate(c) === 'show') && !veiled(f); + return !!(c && gate(c) === 'show'); } // The results to act on this page keeps closed at this moment, in the page's order: the map's chip counts // them, whether or not they have a place on the map. function mapClosed() { - return findingsNow().filter(function (f) { return f.level === 'act' && veiled(f); }).map(function (f) { return f.id; }); + return []; } // The title of a result once the page shows it, for the map's "Opened in its section" chip; never a closed one's. // The address of a result the map opened, for its "Opened in its section" link: never its id (addr()). function mapHref(id) { var f = findingById(id); return f ? addr(f) : id; } - function mapTitle(id) { var f = findingById(id); return f && !veiled(f) ? f.title : ''; } + function mapTitle(id) { var f = findingById(id); return f ? f.title : ''; } // Whether a row of the map is a finding of these results: then the page's own veil alone decides it (map.js). function mapKnown(id) { return !!findingById(id); } // Whether a step of the run has ended at this moment (a replay before it has not): the map's layers follow it. @@ -729,7 +550,7 @@ // Without the map (a browser with no DecompressionStream, or a map the server could not build): the findings // with a place, as a table, each through this page's veil first. function mapFallback() { - return findingsNow().filter(function (f) { return f.viewer && !veiled(f); }).map(function (f) { + return findingsNow().filter(function (f) { return f.viewer; }).map(function (f) { return [U.LEVEL_WORDS[f.level] || f.level, f.title, f.viewer]; }); } @@ -739,12 +560,6 @@ var at = newAt(f); if (at) { art.classList.add('new'); add(art, el('span', { 'class': 'newband' }, 'New at ' + U.hm(at))); add(art, el('br', {})); } add(art, U.findingLevel(f.level, f.level_words)); - if (veiled(f)) { - add(art, U.veil('A result to act on in adult life is usually not looked at before the child can choose. It stays closed until you open it.', - null, function () { openFinding(f); redraw(true); var x = document.getElementById('f-' + f.id); if (x) { x.scrollIntoView({ block: 'start' }); } })); - add(row, art, el('aside', {})); - return row; - } var h = el('h3', { 'class': big ? 'h1' : 'h3' }); add(h, el('a', { href: findingHref(f) }, f.title)); add(art, h); @@ -761,7 +576,7 @@ var ev = evidence(f.evidence); if (ev) { add(side, note('Evidence', ev)); } add(side, caveatNote(f.caveat, f.id)); - var rel = (f.related || []).map(findingById).filter(function (r) { return r && !veiled(r); }); + var rel = (f.related || []).map(findingById).filter(function (r) { return r; }); if (rel.length) { var box = el('div', {}); rel.slice(0, 4).forEach(function (r, i) { if (i) { add(box, el('br', {})); } add(box, el('a', { href: '#f/' + addr(r) }, r.title)); }); @@ -772,13 +587,6 @@ } function findingLine(f) { var li = el('li', { id: 'f-' + handle(f), 'data-finding': handle(f), 'data-level': f.level }); - if (veiled(f)) { - // A finding that names a closed result (a symptom ranking of its gene): closed with it, opened with it. - var b = el('button', { type: 'button', 'class': 'btn' }, 'Show the result'); - b.addEventListener('click', function () { openFinding(f); redraw(true); var x = document.getElementById('f-' + f.id); if (x) { x.scrollIntoView({ block: 'start' }); } }); - add(li, el('span', { 'class': 't' }, 'A result about a child, closed until opened.'), U.findingLevel(f.level, f.level_words), b); - return li; - } add(li, el('a', { 'class': 't', href: findingHref(f) }, f.title), U.findingLevel(f.level, f.level_words)); if (f.detail) { add(li, el('span', { 'class': 'gene' }, f.detail)); } add(li, el('span', { 'class': 'x' }, f.text)); @@ -863,11 +671,6 @@ add(body, nf); return body; } - if ((sec.n === 9 || sec.n === 10) && tagClosed(PANEL)) { - add(body, U.veil('This part can contain data text, so it stays closed until every result to act on is opened.', - null, function () { var f = nextClosed(); if (f) { openFinding(f); } redraw(true); })); - return body; - } if (sec.n === 9) { var lists = [['No step looks at these', (results && results.not_assessed) || []], ['Steps that did not run', (results && results.not_run) || []]]; lists.forEach(function (x) { @@ -909,9 +712,7 @@ // One line for a card: what it found, or why it has nothing to show yet. function cardLine(c) { var g = gate(c); - return g === 'show' && veiledOf(c).length ? 'A result about a child, closed until opened.' - : g === 'show' && panelShut(c) ? PANEL_LINE - : g === 'show' ? ((!headlineShut(c) && c.headline) || (openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)) + return g === 'show' ? ((c.headline) || (openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)) : pending(g) ? 'Waiting for ' + stepWords(c) + '.' : g === 'failed' ? 'Not assessed: its step failed.' : 'Not assessed in this run.'; } // A card the report flags that has no finding of its own: on a phone it is the only place that flag @@ -947,7 +748,6 @@ sec.cards.map(cardById).some(function (c) { return c && flaggedAlone(c); }))); } function sectionLine(sec) { - if (sec.n === 9 && tagClosed(PANEL)) { return 'Closed until every result to act on is opened.'; } if (sec.n === 9) { var na = ((results && results.not_assessed) || []).length, nr = ((results && results.not_run) || []).length; return na + (na === 1 ? ' thing' : ' things') + ' no step looks at, ' + nr + (nr === 1 ? ' step' : ' steps') + ' that did not run'; @@ -986,15 +786,6 @@ n.info = cards().filter(function (c) { return gate(c) === 'show' && c.level === 'info'; }).length; return n; } - // A segment of the lead that names a closed result, without the names: the list after a colon, the genes in - // brackets, or what follows "of". - function closedLead(t) { - var w = 'closed on this page until opened on purpose, in section 1'; - if (/^\s*:/.test(t)) { return ': ' + w + '. '; } - if (/\(.*\)/.test(t)) { return t.replace(/\(.*\)/, '(' + w + ')'); } - if (/ of /.test(t)) { return t.replace(/ of .*$/, ' result, ' + w); } - return w + '. '; - } function drawLead() { var slot = document.getElementById('lead-slot'); slot.textContent = ''; @@ -1002,12 +793,7 @@ var row = el('div', { 'class': 'row' }); var main = el('div', {}); var p = el('p', { 'class': 'lede dropcap', id: 'lead' }); - var closed = (results.findings || []).some(veiled); - var leadTags = (results.veil && results.veil.lead) || []; - var parts = (results.lead || []).map(function (x, i) { - if (x[2] === 'act' && closed) { return [': closed on this page until opened on purpose, in section 1. ', false]; } - return leadTags[i] && tagClosed(leadTags[i]) ? [closedLead(x[0]), x[1]] : x; - }); + var parts = results.lead || []; if (replay && running()) { // A replay holds the results of the finished run: before its end the lead would name findings the // sections do not show yet. @@ -1026,7 +812,7 @@ add(a, U.level(x[0], { act: 'Take to a doctor', medicine: 'Prescriber', review: 'Worth a look', carrier: 'Family planning', info: 'Information' }[x[0]])); add(a, el('span', { 'class': 'n' }, String(n[x[0]]))); var words = x[0] === 'info' ? 'Origins, traits, scores' : x[0] === 'act' && closed && n.act ? 'Closed until opened' : - fs.filter(function (f) { return f.level === x[0] && !veiled(f); }).slice(0, 2).map(function (f) { return f.gene || f.title; }).join(', ') || + fs.filter(function (f) { return f.level === x[0]; }).slice(0, 2).map(function (f) { return f.gene || f.title; }).join(', ') || (fs.some(function (f) { return f.level === x[0]; }) ? 'Closed until opened' : 'None in what was assessed'); add(a, el('span', { 'class': 'w' }, words)); add(led, a); @@ -1034,7 +820,7 @@ add(main, led); var trust = el('div', { 'class': 'trustline' }); var problems = fs.filter(function (f) { return f.level === 'trust'; }); - var openProblems = problems.filter(function (f) { return !veiled(f); }); + var openProblems = problems; add(trust, U.level('trust', 'Data quality', problems.length ? 'problem' : '')); var qc = cardById('qc'); if (qc && gate(qc) === 'show') { @@ -1372,6 +1158,7 @@ window.PGP.register({ name: 'person', onState: function (s) { + if (!openingSeeded) { shown = U.openedActs(); openingSeeded = true; } state = s; loadFamily(); if (!replay || !results) { load(); } diff --git a/bin/live_results.py b/bin/live_results.py index a0e0a7bcf..6d0911341 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -20,9 +20,8 @@ or a couple's shared or X-linked gene. Nothing says a section is fine: a card without attention is "for information". -Beside the cards, "veil" tags the rows that name a result to act on outside its own card (every card's -table rows and lines, the viewer links, the charts' marks, the other findings and the lead's names) with -that result's id, so a child's page closes them with it (veil() below). +The veil metadata lists act identities. The shared browser loader withholds a child's entire +report from views while any act remains unopened. No individual field is classified here. Nothing here reads a file or computes anything about genetics; the adapters pick fields the summary already holds. A section id with no adapter falls back to its @@ -793,8 +792,8 @@ class _Row: cannot read (a field of the wrong type, a stars value that is not a number) costs that row only: it becomes a finding on its card that says the row could not be read, and every other row and card still renders. Its words name no gene or variant: the row is unreadable. Its level is the highest the rule - can give (level "act" for a rule that can give one): a child's page veils a card that holds a finding - to act on, and the card's table may still draw the row the rule could not read.""" + can give (level "act" for a rule that can give one). Such an act keeps a child's report gate shut + until someone opens it. The full report may still draw the row the rule could not read.""" def __init__(self, out, card_id, level="review"): self.out, self.card_id, self.level = out, card_id, level @@ -888,7 +887,7 @@ def gene(x): gene=(x.get("genes") or [None])[0], detail=x.get("sv"), evidence=[["Class", x.get("acmg_class")], ["Callers", rr.annotsv_support_text(x)], ["Genotype", x.get("gt")]], viewer=str(x.get("sv") or "").replace(" ", "") or None)) - # Every gene it covers, not only the first: a closed one closes the rows that name any of them (closable()). + # Every gene covered by this structural variant belongs on its own finding record. gl = x.get("genes") out[-1]["genes"] = [str(g) for g in gl if g] if isinstance(gl, list) else [] # 6. Repeat expansions outside Stranger's "normal", or in an intermediate zone. @@ -1099,26 +1098,24 @@ def screened(summary): return any((S.get(k) or {}).get("state") in ("ok", "stale") for k in ("clinvar", "clinical", "cpsr")) -def lead(found, levels, summary, names=None): +def lead(found, levels, summary): """The report's lead paragraph: [[text, bold]] segments (a third item "act" marks the names of the - findings to act on), plain sentences that name the findings by level. It never says a section is fine: with no finding of a level it says what was assessed. - NAMES, when a list, gets one entry per segment: the findings whose titles or genes the segment prints ([] for - one of the lead's own sentences, which print no gene). veil() tags each segment by them.""" + findings to act on), plain sentences that name the findings by level. With no finding of a level, + it says what was assessed. The legacy act marker remains part of the document format.""" S = summary.get("sections") or {} by = {lv: [f for f in found if f["level"] == lv] for lv in LEVELS} - out, named = [], [] + out = [] - def seg(text, bold, shows=(), act=False): + def seg(text, bold, act=False): out.append([text, bold] + (["act"] if act else [])) - named.append(list(shows)) if by["trust"]: seg("Read the data-quality section first: ", True) - seg(_names(by["trust"]) + ". ", False, by["trust"][:3]) + seg(_names(by["trust"]) + ". ", False) if by["act"]: n = len(by["act"]) seg("Of everything this genome was screened for, ", False) seg(_count(n, "finding should", "findings should") + " go to a doctor", True) - seg(": " + _names(by["act"]) + ". ", False, by["act"][:3], act=True) # "act": the names a child's page keeps closed + seg(": " + _names(by["act"]) + ". ", False, act=True) # "act": the names a child's page keeps closed elif screened(summary): seg("No finding in what these steps assessed needs a doctor; ", True) seg("sections 8 and 9 say what they do not see. ", False) @@ -1129,13 +1126,13 @@ def seg(text, bold, shows=(), act=False): genes = sorted({f["gene"] for f in by["medicine"] if f["gene"]}) seg(_count(len(by["medicine"]), "medicine result", "medicine results").capitalize(), True) seg(f" bear{'s' if len(by['medicine']) == 1 else ''} on how drugs work ({', '.join(genes)}): tell whoever prescribes. ", - False, by["medicine"]) + False) if by["carrier"]: - seg(f"A carrier of {_names(by['carrier'], 2)}", True, by["carrier"][:2]) + seg(f"A carrier of {_names(by['carrier'], 2)}", True) seg(": carriers are not affected; it matters when a partner carries the same gene. ", False) if by["review"]: seg(_count(len(by["review"]), "result is", "results are").capitalize() + " worth a look", True) - seg(": " + _names(by["review"]) + ". ", False, by["review"][:3]) + seg(": " + _names(by["review"]) + ". ", False) cov = _data(S, "coverage").get("mean_depth") q = _data(S, "sample_qc") checked = q.get("contamination") == "ok" and q.get("sex_check") == "ok" # both ran and raised nothing @@ -1144,429 +1141,22 @@ def seg(text, bold, shows=(), act=False): else "; the identity and contamination checks have no result here: section 5 says why."), False) if out: out[-1][0] = out[-1][0].rstrip() - if isinstance(names, list): - names[:] = named - return out - - -# --- the veil's tags ----------------------------------------------------------------------------- -# A child's page keeps each result to act on closed (assets/live/person.js veiled()). The page closes a -# finding's own card with it, but other places can name the same gene or variant: any card's table, lines, facts -# and headline, the charts' marks, another finding (a symptom ranking), the lead, the search index. Each such place -# carries a tag: the SET of the closed results it names, as a list of their ids. The page keeps the place closed -# while ANY of them is closed, so a place that names two closed results never comes back with the first one -# opened (person.js tagClosed()). The veil lives in the browser: the files still hold every row. -# A place is tagged from its structured fields only, never by reading its words as gene symbols: a report prints -# codes in capitals (AR as an inheritance, INS as a structural-variant type, TG in a score's name) that are gene -# symbols too, so a word rule would close a place for a result it does not name, and what closes would tell the -# gene. The structured fields: a row's gene and place cells (TABLE_COLUMNS), a finding's gene, genes and loci -# (finding_names()), a chart mark's summary row (mark_names()), a two-hit line's gene (two_hit_names()), a -# drawing's gene, the findings a lead segment names (lead()). A place belongs to a closed result when it names one -# of the result's genes in such a field, or its position falls inside the result's position or span. -# Prose has no such field: a line, a fact, a headline or a finding that has none cannot be shown to be free of a -# closed result's name. So it is tagged PANEL (closed while any result is closed, the same whatever the result), -# unless it is on one of the lists below of prose that never names a gene (LINES_NO_GENE, FACTS_NO_GENE, -# HEADLINES_NO_GENE, NAMES_NO_GENE), each pinned in tests/test_live_results.py. The cost: a child's page with a -# result closed shows less prose until every result is opened. -# The cards whose rows are a fixed panel: the same genes or loci for everyone, known before the run. A row or -# dot of one that closes with a result would tell which gene the result is in by what is missing, so a child's -# page closes each of these cards whole while any result to act on is closed, behind one box with the same words -# whatever the result (results.json veil.panels; assets/live/person.js panelShut()). The one table: a card of a -# fixed panel is listed here and nowhere else. -FIXED_PANELS = { - "mei": "the known disease-causing insertions of assets/mei_sites.tsv", - "expansions": "the repeat loci of the catalog", - "vntyper": "MUC1", - "lpa": "LPA", - "chip": "the CHIP genes and their detection limits", - "paralogs": "SMN1, SMN2, HBA1, HBA2, the RCCX module and the other paralog genes", - "cpic": "the CPIC genes", - "cyp2d6": "CYP2D6", - "hla": "the HLA genes", - "locityper": "the Locityper loci", - "traits-panel": "the trait panel's genes", - "bloodtype": "the blood group systems", -} -# These catalog rows have structured gene cells but must never close one gene at a time. -# The other fixed-panel sentences (ClinVar's deep-intronic sites and ROH's imprinted regions) -# already take the default PANEL rule, so need no exemption or prefix guard. -FIXED_LINES = {"callability": ("not on this reference", "not read")} -PANEL = "*panel*" -# No card-wide prose exemption survives the fail-closed audit. Even counts often arrive with -# sample notes, ontology labels or data-dependent record presence. Structured rows and lines below -# retain their individual tags; all other data text takes PANEL. Keep these registries explicit so -# adding an exemption requires a producer-based invariance test. -NAMES_NO_GENE = {} -LINES_NO_GENE = {} -FACTS_NO_GENE = {} -FACT_NAMES = {("carrier-list", "Genes with two hits"): "two_hits"} -HEADLINES_NO_GENE = {} -# Every column of every card's table (the adapters' table() calls), and what it holds: a gene ("gene": a gene -# symbol, or a list of them), a place ("place": its loci are read, never its words) or the report's own vocabulary -# (None: an inheritance code such as AR, a structural-variant type such as INS, a genotype, a class, a -# significance, a unit, a caller, a condition), never read. An unknown column defaults to PANEL; -# tests/test_live_results.py fails until every column of every table() call is listed. -G, P = "gene", "place" -TABLE_COLUMNS = { - "Gene": G, "Genes": G, "Locus": G, "Variant": P, "Where": P, "Site": P, "Known site": P, "Change": P, - "Chromosome": P, "Start": P, "End": P, - "Status": None, "Bases read": None, "Why": None, "Genotype": None, "Significance": None, "Stars": None, - "Frequency": None, "Call": None, "Kind": None, "Rule": None, "Final class": None, "ClinVar": None, - "Rank": None, "Inheritance": None, "Score": None, "ACMG": None, "Disease": None, "Type": None, - "Length": None, "Class": None, "Support": None, "Checks": None, "Element": None, "Condition": None, - "Depth": None, "Repeats": None, "Stranger status": None, "Normal up to": None, "Pathologic from": None, - "95% interval": None, "Allele fraction": None, "VAF": None, "Alt reads": None, "Length (Mb)": None, - "Cell fraction": None, "Result": None, "Chance for a child": None, "This person": None, "Partner": None, - "Copies": None, "Detail": None, "Diplotype": None, "Phenotype (CPIC's words)": None, "Caller": None, - "CYP2D6": None, "Alleles (T1K quality)": None, "Panel haplotypes": None, "Quality": None, "Filter": None, - "Primary caller": None, "Level": None, "Group": None, "Share": None, "Bootstrap range": None, - "Branch": None, "Derived": None, "Ancestral": None, "No call": None, "Dates": None, "Country": None, - "Distance": None, "Trait": None, "Reading": None, "Percentile": None, "Variants matched": None, - "Effect": None, "System": None, "Predicted phenotype": None, "Confidence": None, "Virus": None, - "Reads": None, "Breadth": None, "Copies per cell": None, "Sizing": None, "Intermediate zone": None, -} - - -def panel_finding(f): - """Whether finding F is one of a fixed-panel card's own (a medicine gene, a CHIP call, a repeat), not a result - to act on: veil() tags it PANEL whatever it names.""" - return f.get("level") != "act" and f.get("card") in FIXED_PANELS - - -def fixed_line(card_id, text): - """Whether TEXT, a line of the card CARD_ID or a cell of one of its rows, is one of its fixed lines (FIXED_LINES).""" - return any(str(text).startswith(w) for w in FIXED_LINES.get(card_id) or ()) - - -LOCUS_RE = re.compile(r"(?= a for lc, ls, le in locs)): - out.append(fid) - return out or None - - -def prose_tag(spans, text): - """The tag of prose (TEXT: a line, a fact, a headline, a search hit's quote) that is on no list of prose naming no - gene: nothing proves it names no closed result, so PANEL while there is one to close, whatever its words. Its - words are never read: a code the report prints (AR, INS, TG) is a gene symbol too.""" - return [PANEL] if spans and str(text or "").strip() else None - - -def veil_id(spans, text="", chrom=None, pos=None, end=None, genes=None): - """veil_tags() for a caller that has the genes a place names (GENES, from its own gene field; TEXT then gives its - loci). A place given as TEXT alone is prose (prose_tag(); bin/results_index.py, whose search hits quote the - summary's rows).""" - if genes is None: - return prose_tag(spans, text) - return veil_tags(spans, genes, text, chrom, pos, end) - - -def union(*tags): - """One tag naming every id of TAGS (each a tag or None), in order; None for none.""" - out = [] - for t in tags: - for x in t or (): - if x not in out: - out.append(x) - return out or None - - -def row_tag(spans, columns, row, genes=()): - """A table row's tag, from its gene and place cells (TABLE_COLUMNS) and GENES its summary row names.""" - cells = list(row) if isinstance(row, (list, tuple)) else [row] - cols = list(columns or []) - names, where, at = list(genes or ()), [], {} - for i, x in enumerate(cells): - kind = TABLE_COLUMNS.get(cols[i]) if i < len(cols) else None - if kind == G: - names += gene_cell(x) - elif kind == P: - where.append(str(x)) - if i < len(cols) and cols[i] in ("Chromosome", "Start", "End"): - at[cols[i]] = x - coordinate = f"{at.get('Chromosome')}:{at.get('Start')}-{at.get('End')}" - if not names and not _loci(" ".join(where)) and not _loci(coordinate): - return [PANEL] if spans else None - return veil_tags(spans, names, " ".join(where), at.get("Chromosome"), at.get("Start"), at.get("End")) - - -def finding_names(f): - """(genes, loci text) a finding is tagged by: a finding with a gene field is known by its gene, its genes (a - structural variant's every gene) and the loci of its viewer, detail and evidence, never by its title's words (an - SV type such as INS, a condition); one with no gene but a place (a mosaic call) by that place alone; None for one - with neither (a card's own headline, a data-quality finding, a row that could not be read): prose_finding().""" - where = " ".join(str(x) for x in [f.get("viewer"), f.get("detail")] + [b for _a, b in f.get("evidence") or []] - if x not in (None, "")) - genes = [g for g in list(f.get("genes") or []) + [f.get("gene")] if g] - if genes or _loci(where): - return genes, where - return None - - -def two_hit_names(S): - """{line: (gene, loci text)} of the carrier list's two-hit lines, each from its row of carrier.two_hits.genes.""" - th = _data(S, "carrier").get("two_hits") - out = {} - if not isinstance(th, dict): - return out - for x in _items(th.get("genes")): - if not isinstance(x, dict): - continue - try: - ln = rr.two_hit_lines({"genes": [x], "deletions_read": [], "deletions_not_read": []})[0] - except (KeyError, TypeError, ValueError, AttributeError): - continue - out[plain(ln)] = (x.get("gene"), f"{x.get('deletion') or ''} {x.get('variant') or ''}") - return out - - -def headline_free(card): - """Only an explicitly audited producer can exempt a headline from PANEL.""" - return bool(card.get("keys")) and all(k in HEADLINES_NO_GENE for k in card["keys"]) - - -def mark_names(S, sp, m): - """(genes, loci text) of a chart mark, from the summary row it draws, never from its label: a ClinVar hit's gene - and position, a structural variant's genes, each star bar's hits (the ones its detail lists). A mark of a list - known before the run (a polygenic score, an ancestry group, a mitochondrial site) names no gene: its position - alone, if it has one.""" - hits = _items(_data(S, "clinvar").get("hits")) - mid = str(m.get("id") or "") - try: - if m.get("kind") == "hit": - h = hits[int(mid.split("-", 1)[1])] - return [h.get("gene")], "" - if m.get("kind") == "sv": - r = _items(_data(S, "annotsv").get("rows"))[int(mid.split("-", 1)[1])] - return list(r.get("genes") or []), "" - if sp.get("id") == "clinvar-stars" and mid.startswith("stars-"): - k = mid.split("-", 1)[1] - idx = [h for h in hits if isinstance(h, dict) and str(h.get("stars")) == k and not h.get("low_confidence")][:10] - return [h.get("gene") for h in idx], " ".join(f"{h.get('chrom')}:{h.get('pos')}" for h in idx) - except (IndexError, ValueError, AttributeError, TypeError): - pass - return [], "" - - -def veil(summary, found, chapters, lead_parts=(), lead_names=()): - """Each place a child's page closes with a result to act on, with its tag (veil_tags(): the ids of every closed - result it names, or PANEL): each card's table rows (rows: card -> one tag or None per row), viewer links - (viewer), lines (lines; a line of a fixed panel, such as a known deep intronic site, printed whether hit or not, - is PANEL: FIXED_LINES), facts (facts), headline (headline: card -> tag) and the items of its drawing (widget: card - -> one per CHIP call or copy-number dot), the marks of the report's charts (marks: chart -> {mark id: tag}), every - finding not to act on that names a closed result (findings: finding id -> tag; PANEL for every finding of a - FIXED_PANELS card), the lead's segments (lead: one per segment of LEAD_PARTS, from the findings LEAD_NAMES says - each prints; the "act" names left out: the page closes those itself), and the cards of a fixed panel (panels: - FIXED_PANELS, which close whole while any result is closed). spans is closable() itself: the family page - (assets/live/family-page.js) reads which results are closed from it, and closes every place of the child that - could carry one while one is. Empty when nothing can be closed.""" - spans = closable(found) - if not spans: - return {} - out = {"rows": {}, "viewer": {}, "lines": {}, "facts": {}, "headline": {}, "widget": {}, "marks": {}, - "findings": {}, "notes": {}, "lead": [], "spans": [list(s) for s in spans], - "panels": [c["id"] for ch in chapters for c in ch["sections"] if c["id"] in FIXED_PANELS and c.get("state") != "missing"]} - S = summary.get("sections") or {} - hits = [h for h in _items(_data(S, "clinvar").get("hits"))[:50]] - tags = [veil_tags(spans, [h.get("gene")], "", h.get("chrom"), h.get("pos")) if isinstance(h, dict) else None - for h in hits] - if tags: - out["rows"]["clinvar"] = tags - sv_rows = [x for x in _items(_data(S, "annotsv").get("rows"))] - two = two_hit_names(S) - th = _data(S, "carrier").get("two_hits") - two_genes = [x.get("gene") if isinstance(x, dict) else x for x in _items((th or {}).get("genes"))] \ - if isinstance(th, dict) else [] - mca_lines = {} - for x in _items(_data(S, "mosaic").get("calls")): - if isinstance(x, dict) and all(k in x for k in ("type", "arms", "chrom", "start", "end", "length_mb", "cell_fraction")): - for ln in rr.mosaic_mca_lines(dict(_data(S, "mosaic"), calls=[x]))[1:2]: - mca_lines[plain(ln)] = x - cards = {} - for ch in chapters: - for c in ch["sections"]: - cards[c["id"]] = c - if c["id"] in NAMES_NO_GENE: - continue - rows = (c.get("table") or {}).get("rows") or [] - if rows and (c["id"] != "clinvar" or not hits or (c.get("table") or {}).get("columns", [None])[0] != "Gene"): - cols = (c.get("table") or {}).get("columns") - tags = [] - for i, r in enumerate(rows): - if any(fixed_line(c["id"], x) for x in (r if isinstance(r, (list, tuple)) else [r])): - tags.append([PANEL]) - continue - more = [] - if c["id"] == "structural-variants" and i < len(sv_rows) and isinstance(sv_rows[i], dict): - more = [g for g in sv_rows[i].get("genes") or [] if g] # the whole list the cell shortens - if c["id"] == "mito" and cols == ["Site", "Allele fraction"]: - sites = _items(_data(S, "mito").get("heteroplasmic_sites")) - site = sites[i] if i < len(sites) and isinstance(sites[i], dict) else {} - tags.append(veil_tags(spans, (), chrom="chrM", pos=site["pos"]) - if site.get("pos") is not None else [PANEL]) - else: - tags.append(row_tag(spans, cols, r, more)) - if tags: - out["rows"][c["id"]] = tags - tags = [veil_tags(spans, (), v) for v in c.get("viewer") or []] - if c["id"] == "clinvar": # one link per row: a row closed by its gene alone closes its link too - rt = out["rows"].get("clinvar") or [] - tags = [union(rt[i] if i < len(rt) else None, t) for i, t in enumerate(tags)] - if tags: - out["viewer"][c["id"]] = tags - tags = [] - for ln in c.get("lines") or []: - if fixed_line(c["id"], ln): - tags.append([PANEL]) - elif c["id"] == "carrier-list" and ln in two: - tags.append(veil_tags(spans, [two[ln][0]], two[ln][1])) - elif c["id"] == "mosaic-mca" and ln in mca_lines: - x = mca_lines[ln] - tags.append(veil_tags(spans, (), chrom=x.get("chrom"), pos=x.get("start"), end=x.get("end"))) - elif c["id"] in LINES_NO_GENE: - tags.append(veil_tags(spans, (), ln)) - else: - tags.append(prose_tag(spans, ln)) - if tags: - out["lines"][c["id"]] = tags - tags = [] - for f in c.get("facts") or []: - label = f[0] if isinstance(f, (list, tuple)) and f else f - rule = FACT_NAMES.get((c["id"], label), "") - if rule == "two_hits": - tags.append(veil_tags(spans, two_genes)) - elif rule is None or c["id"] not in FACTS_NO_GENE: - tags.append(prose_tag(spans, " ".join(map(str, f)) if isinstance(f, (list, tuple)) else f)) - else: - tags.append(None) - if tags: - out["facts"][c["id"]] = tags - if c.get("notes"): - out["notes"][c["id"]] = [prose_tag(spans, n) for n in c["notes"]] - if c.get("headline") and not headline_free(c): - out["headline"][c["id"]] = prose_tag(spans, c["headline"]) - # A drawing's items: the CHIP calls (gene) and the copy-number dots (gene), by their gene field. The CHIP - # chart's list of genes with their detection limits is the same panel for everyone: never tagged. - w = c.get("widget") or {} - items = w.get("calls") if w.get("kind") == "vaf" else w.get("genes") if w.get("kind") == "copies" else None - tags = [veil_tags(spans, gene_cell(x.get("gene"))) for x in items or []] - if w: - out.setdefault("widget_panel", {})[c["id"]] = [PANEL] - if tags: - out["widget"][c["id"]] = tags - try: - specs = rr.chart_specs(summary) - except (KeyError, TypeError, ValueError, AttributeError, IndexError, ZeroDivisionError): - specs = [] - for sp in specs: - for m in sp.get("marks") or []: - if m.get("kind") == "roh": # structured coordinates; a run of homozygosity is no result's source - out["marks"].setdefault(sp["id"], {})[m["id"]] = None - continue - genes, where = mark_names(S, sp, m) - structured = genes or _loci(where) or (m.get("chrom") and m.get("pos") is not None) - t = veil_tags(spans, genes, where, m.get("chrom"), m.get("pos"), m.get("end")) if structured else [PANEL] - out["marks"].setdefault(sp["id"], {})[m["id"]] = t - # A finding of a fixed-panel card (a medicine gene, a CHIP call, a repeat) is tagged PANEL whatever it names: it - # closes with its card while any result to act on is closed, so the findings left open never tell which gene the - # closed one is in. A finding with no gene and no place is prose (prose_finding()). - for f in found: - if f.get("level") == "act": - continue - if panel_finding(f): - t = [PANEL] - else: - got = finding_names(f) - t = veil_tags(spans, got[0], got[1]) if got else prose_finding(spans, f, cards.get(f.get("card")) or {}) - out["findings"][f["id"]] = t - # A lead segment is tagged by the findings it prints (lead()): every closed result one of them names. The - # lead's own sentences print no gene. - tags = [] - for i, x in enumerate(lead_parts or ()): - shows = list(lead_names[i]) if i < len(lead_names or ()) else [] - tags.append(None if len(x) > 2 else union(*[out["findings"].get(f["id"]) for f in shows if f.get("level") != "act"])) - out["lead"] = tags if any(tags) else [] - return {k: v for k, v in out.items() if v} - - -def prose_finding(spans, f, card): - """An unstructured finding takes PANEL unless an audited fact/headline producer exempts it.""" - cid = f.get("card") - # Quality findings and unreadable records can include sample names and parser messages. - # They have no structured gene/position proof either. - if f.get("id") == slug("card", cid) and cid in FACTS_NO_GENE and headline_free(card) \ - and not any((cid, a) in FACT_NAMES for a, _b in card.get("facts") or []): - return None - return prose_tag(spans, f"{f.get('title') or ''} {f.get('text') or ''}") +def veil_id(acts, text="", **kwargs): + return ["*panel*"] if acts else None def build(sample, summary, files=()): @@ -1583,8 +1173,7 @@ def build(sample, summary, files=()): person = {"sample": sample, "declared_sex": (summary.get("run") or {}).get("declared_sex"), "inferred_sex": sexc.get("inferred_sex"), "karyotype": sexc.get("karyotype")} found, levels, sections = journal(chapters, summary) - lead_names = [] - lead_parts = lead(found, levels, summary, lead_names) + lead_parts = lead(found, levels, summary) return {"schema_version": SCHEMA, "sample": sample, "generated_utc": summary.get("generated_utc"), "person": person, "chapters": chapters, "findings": found, "levels": levels, "journal": sections, "lead": lead_parts, @@ -1594,7 +1183,7 @@ def build(sample, summary, files=()): "not_assessed": [plain(x) for x in summary.get("not_assessed") or []], "not_run": [plain(x) for x in summary.get("not_run") or []], "new_since": summary.get("new_since") or None, - "charts": CHART_SECTION, "veil": veil(summary, found, chapters, lead_parts, lead_names)} + "charts": CHART_SECTION, "veil": veil(found)} def attention_count(summary): @@ -1608,36 +1197,10 @@ def overview(summary): r = build("", summary) order = {lv: i for i, lv in enumerate(LEVELS)} order["trust"] = -1 # a problem with the data comes before any finding read from it - # A finding that names a result to act on (veil: a symptom ranking of its gene) is not listed: a child's tile - # closes only the results to act on themselves, and the result it names already comes first. - # A PANEL-only finding stays in top with its tag: the home page closes its title and link for a child, - # while adults and runs without a result to act on keep the same title and destination. - finding_tags = (r.get("veil") or {}).get("findings") or {} - named = {k: v for k, v in finding_tags.items() if any(x != PANEL for x in v or ())} - top = sorted((f for f in r["findings"] if f["level"] != "info" and f["id"] not in named), - key=lambda f: order[f["level"]])[:4] - tags = (r.get("veil") or {}).get("lead") or [] + top = sorted((f for f in r["findings"] if f["level"] != "info"), key=lambda f: order[f["level"]])[:4] return {"attention": r["attention"], "levels": r["levels"], "screened": screened(summary), - "top": [{"id": f["id"], "level": f["level"], "title": f["title"], "veil": finding_tags.get(f["id"])} for f in top], - "lead": "".join(x[0] for x in r["lead"]), - # The same lead with the names of the findings to act on left out, and every other segment that names - # one of them: a child's tile shows this one. - "lead_closed": "".join(": closed until opened on purpose. " if x[2:] == ["act"] - else closed_lead(x[0]) if i < len(tags) and tags[i] else x[0] - for i, x in enumerate(r["lead"]))} - - -def closed_lead(t): - """A segment of the lead that names a result to act on, without the names (person.js closedLead()): the - list after a colon, the genes in brackets, or what follows "of".""" - w = "closed until opened on purpose" - if t.lstrip().startswith(":"): - return f": {w}. " - if re.search(r"\(.*\)", t): - return re.sub(r"\(.*\)", f"({w})", t) - if " of " in t: - return re.sub(r" of .*$", f" result, {w}", t) - return f"{w}. " + "top": [{"id": f["id"], "level": f["level"], "title": f["title"]} for f in top], + "lead": "".join(x[0] for x in r["lead"]), "lead_closed": "Results closed until opened on purpose."} def strings(obj): diff --git a/docs/live-site.md b/docs/live-site.md index 3ad7c96d8..af13d80d7 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,27 +56,17 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. On a child's page in a family, a result to act on stays closed behind "Show the result" until someone opens it on purpose: its title, gene and variant stay out of the lead, the ledger, the genome map, the home page and its own page, and the card it comes from stays closed with it (its rows, numbers and drawing), in its section and in section 8. Anything else on the child's pages that names a closed result's gene or position stays closed with it: another card's rows, lines and facts, a viewer link, a chart's marks and their titles, another finding that names it (a symptom ranking of the gene) on the report, on its own page, in a finding page's next link and in its related list, and the part of the lead that names it. A structural variant names every gene it covers, so a row that names any of them closes with it. A card with such rows says that some of its rows stay closed, in the same words however many there are, and opening the result opens them with it. Two pages whose open rows are the same differ only in how many results are closed, not in which; the section that holds a closed result still says it holds one: +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. A child's report has one gate at the data loader. While any result to act on is unopened, the page shows fixed explanatory text, the number of act results and one opaque opening control for each. An act opened on purpose shows only its own explanation, plan, evidence and caveat. It has no related list, next finding, map mark or chart. Nothing else from that child's report reaches a view: no cards, tables, level counts, images, file links, search box, search status or progress headlines. -- A fixed panel lists genes or loci known before the run. Hiding only one row would identify the closed result's gene. These cards close whole while any result to act on stays closed. They include known mobile element insertions, repeat expansions, VNtyper, Lp(a), CHIP and paralog copy numbers. CPIC, CYP2D6, HLA, Locityper, traits and blood groups follow the same rule. The paralog panel includes SMN1, SMN2, HBA1, HBA2 and RCCX. The single list is `FIXED_PANELS` in `bin/live_results.py`, exported as `veil.panels`. Each box uses the same words and opens the next closed result. The cards return after every act opens. A PANEL finding's own page follows this rule too: its button opens one act per click, and the finding appears after the last act opens. +The cost is that a child with an unopened act result sees none of the rest of the report until every act result is opened. Opening the last act releases the full report. Closing an act from the map replaces the document with the closed view. That replacement discards search results, view caches and in-flight view callbacks. A short-lived, one-use opening ticket contains only opening flags and a digest, and is consumed immediately. Reloading closes the acts again. Map and section addresses use the closed controls while the gate is shut. After the last opening, the page returns to its default view. - Their own findings and the lead segments naming them also take `*panel*`, whatever gene they name. Fixed panels inside other cards follow this rule too. These include ClinVar's known deep intronic sites, the carrier list's copy-number caller sentence, and callability's catalog genes missing from the reference or unread by the reads. Callability's row prefixes remain in `FIXED_LINES`. The other sentences take the default prose rule. +The controls use ordinal handles such as `closed-1` and `opened-1`. Guessed finding hashes use the same first closed handle. Opening or closing a control preserves the addressed act as the ordinals change. A finding address for an opened act shows that act alone while the gate is shut. Other finding addresses show the fixed explanation and controls. Adults and children with no act result use the full report. Unknown family roles and a child's unreadable results stay closed. - Structured tags use gene and coordinate fields. `TABLE_COLUMNS` classifies every table column. A row uses its gene and position cells, a finding its genes and loci, and a chart mark its source record. An SV uses every gene it covers. Vocabulary cells never count as genes: inheritance, SV type, genotype, class, significance and units. The page never matches prose words against gene symbols. Otherwise a closed result in AR could hide every recessive carrier row, and INS could hide every insertion. +The home tile and attention row use fixed text for a child whose gate is shut. The family loader removes that child's report objects before rendering. Every retained side-by-side row has a fixed closed marker in that child's cell. Rows supplied only by that child are absent. Parents keep their values. Each child opens independently. The family tree remains visible because it comes from the declared pedigree, not the child's report. - Each tag lists every matching closed result. A place naming two results stays closed while either remains closed. This includes a duplication over two closed genes and a lead segment naming two findings. The carrier list's two-hit lines use their source gene and variant. Its two-hit count uses the two-hit genes (`FACT_NAMES`). MoChA call lines use their source coordinates. Mitochondrial rows use each record's structured chrM position, so HGVS text and chart marks stay closed until every matching result opens. Lead segments follow the findings they print. The genome map excludes findings tied to closed results from its open findings without a place. +This replaces the per-element veil. There is no fixed-panel list, gene-matching rule or prose exemption registry. `results.veil.acts` lists the act identities. The loader derives the controls from their own records. The map retains its separate APOE warning on every full report, including adults' reports. - Unclassified rows, facts, lines, headlines, stale notes and findings default to `*panel*`. No card-wide gene-free exemption remains. HPO labels, sample names, haplogroups, trait labels and record counts cannot prove that their producer is safe. The genome and mitochondrial charts retain coordinate-tagged marks on fixed geometry. Data-dependent notes, empty-state sentences and legend choices stay absent. Other charts close whole. The complete chart returns after the last act opens and closes again if any act closes. Unstructured card drawings, including the Y path, also stay closed until every act opens. The trust line reads QC facts through the same veil. Unclassified limitation lists, missing-step lists, database versions and update notes in sections 9 and 10 close together, including their record counts. - - The accepted cost is less of a child's report while any result to act on stays closed. More prose, tables and charts remain closed until every act opens. This is a browser veil: the result and chart files still hold their complete data. -- The map's APOE row follows its own warning switch. Opening it shows the same mark even when another result in APOE stays closed. Otherwise its absence would tell which result is closed. Other map rows keep every tag they follow. -- A closed finding carries no id in the page or the address, since its id names its gene and variant. It is known by its place among the page's closed findings (`#f/closed-2`). On a page that keeps a result closed, the address never holds the id of a finding the page can close, open or closed: once opened on purpose it is `#f/opened-`, its place among the opened ones, so no history entry keeps the id. Any other `#f/` on such a page, the id of a closed result or no id at all, lands the same way, on the first closed finding (`#f/closed-1`): the page never picks a closed card by its id, so a guessed id tells nothing. On the home page, a child's closed result links to the child's report, never to the finding's own page, whose address names it. - -Each result opens on its own: opening one leaves every other result to act on closed, on the report and on the finding's page. A person in a family whose family report has not answered yet, or answered without listing them, is treated the same way, since the page cannot tell whether they are a child. So is a page whose run state is unavailable or comes without its plugins, and a person no readable family lists while some family's pedigree cannot be read (the family card says so): an unknown role is never read as a standalone adult. The header search follows the same veil: its index is prose, so while any result to act on is closed it shows no hit and suggests no gene, and its hits come back once every result is opened. On the genome map a closed result has no mark, no tooltip, no line in the margin and no place in the address; a dashed chip above the map counts the closed results without naming a gene or a place, and its Open button asks once before it opens the next one. The chip counts every result to act on the page keeps closed, with a place on the map or without, so the count never tells whether a closed result has a place. Open opens one result per click, the next closed result in the page's order (the order of its closed cards), so the order does not tell either whether a closed result has a place. Opening a result there or with its card's "Show the result" is one switch: it opens on the map and in its card together, the chip then reads "Opened:" with the result's title and a button to close it again, and a reload closes it. A result with no place on the map opens in its section only: the map draws nothing new, and a chip reads "Opened in its section:" with the result's title, a link to its card and a button to close it again. That chip names only a result that is open, and it goes when the result closes. After Open the keyboard moves to "Show the result", and after "Show the result" or "Keep it closed" to the chips' first button, or to the map when no chip is left. A finding whose card the page shows stays on the map even when a closed result lies inside its gene's span: hiding it would show that the closed result sits in that gene. The page's own veil decides each of its findings, and a result opened with its card shows on the map even when it overlaps another result that is still closed. APOE is closed on the map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. While the traits card is closed whole on a child's page, the chip is the switch. The veil lives in the browser: the map document, like `results.json`, reaches the browser whole, with each row a closed result could reveal tagged with the ids of every closed result it overlaps (the `veils` column; `veil` keeps the first). - -A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. - -While the run goes, the page still opens on the report being written. A short band under the name says how far the run is ("23 of 27 are done, about 8–13 hours remain"), with the steps done as a number and a bar, the time elapsed, the time to go from the usual step times and the retries, and links to the step timeline. Each section and each line of the contents says whether it is written, being written or waiting. The run itself, below, closes the page after section 10, folded until opened (or reached through `#progress`, the band or the contents); a page with no results yet keeps it open. +This is a browser-only boundary. Static result, map, chart, search and family JSON files remain downloadable and contain their complete data. The gate protects the rendered document, attributes, accessibility text, title, address, canvas, SVG, browser storage and requests made by views. It is not an access-control boundary for those static files. - **How the run went.** One line per step the run planned, on a time axis: a bar from its start to its end, or to now while it runs, with an outline to its usual hours. The colour is the step's state from `bin/run_state.py` ([Where a Run Stands](run-state.md)): waiting, running, done, failed, skipped, not finished. Under it the tasks running now against their usual time, a step's detail, the wire of what was found so far and the plugins' cards (run health, the database re-check). - **A step's detail.** Click a step: its state and reason, start and end, the usual hours, each process with its tasks waiting, running, done, from the cache or failed, and the headline of its sections with a link into the report. @@ -84,15 +74,15 @@ While the run goes, the page still opens on the report being written. A short ba - **The report.** Once written, `report.html` opens the run's `_report.html`, and the files it links to beside it. - **The genome viewer.** `/runs//viewer/` shows the run's reads and variants in igv.js, as [`scripts/owngenome-view.sh`](viewer.md) does on one computer. The report's Viewer links, the charts' "See the reads" links and a finding page's reads open it at their position. -The header search finds a gene or rsID across the results and opens the card that names it; on a page that keeps results closed it leaves out every hit that names one (above). One row of the results that the page cannot read (a field of the wrong type) costs that row only: it shows as a finding on its card, "A row of this section could not be read", at the highest level its rule can give (to act on for a rule that can give one, so a page that keeps results closed keeps that card closed too), the card whose table cannot be drawn says so, and every other card and finding is on the page. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. +The header search appears only on a full report. It finds a gene or rsID across the results and opens the card that names it. A result row with a field of the wrong type produces the finding "A row of this section could not be read". It takes the highest level its rule can give. If that level is act, it keeps a child's gate shut until someone opens it. On a full report, the card says which table cannot be drawn and the other cards remain available. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. **Figure 1, the genome map.** The 24 chromosomes as rows, to scale (23 for a person with no Y), each with its bands, the centromere as a waist and the stretches few reads can map hatched (centromeres, heterochromatin, the acrocentric arms). Each finding with a place sits on its chromosome as its level's shape, with its gene beside it when the name fits; runs of homozygosity of 5 Mb or more are a line under the row, shorter ones a faint band. A finding that names a gene but no position (a medicine gene, Lp(a)) sits on that gene's span; a finding with neither stays in its section only. Click a chromosome (or, with the map focused, press `[` or `]`) and it opens in place: an overview with the part in view, a ruler in Mb, the bands with their names, a findings lane with each gene and its level's words, and a lane for the runs of homozygosity, each run with its length. Drag to move along it, scroll or press the + and − buttons beside the zoom stops to zoom in (as far as about 10 Mb across a wide screen; closer views come later), and press Esc (or click Genome) to go back. Back in the browser returns to the view before, the whole genome when the address had no place. A dashed line marks the centre, and the margin says what is under it: the chromosome, the position and the band, the findings in view and the runs. The findings count, the chromosome sentences, the runs sentence and each finding's phrase come from the map document; the page adds its own short lines: a lane or stretch with no finding, the runs in view, how to move, and why a layer is not on the map. A replay draws the runs of homozygosity, and says the sentences about them, only once their step has ended. The address follows the map: `#genome`, `#chr7`, or `#chr7:117.4-137.4M`, and a link to one of them opens that view; a place the page cannot read shows a short note and leaves the map as it was. The map comes from `/runs//map.json`, which the map plugin (`bin/owngenome_live_map.py`) builds with `og map build` from the run folder and the results the page gets (`rust/crates/og-map/README.md` describes the document). It needs og on the host (`scripts/setup.sh --og`); without it, or in a browser that cannot unpack the file (no `DecompressionStream`), the figure says so and lists the findings with a place in a table instead. A map that fails to load again after it was drawn says so the same way and draws nothing more from the map it had. A layer whose file is missing or cannot be read says so in words, never as a blank lane. **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. A child's results to act on stay closed here too, as on the child's page, and what the page shows of a child never depends on which result is closed or where it lies. While a child has any result to act on still closed, every place of the page that could carry one closes for that child in the same way, each behind one box with the same words whatever the result: the grandparents of the painted stretches (every stretch is then drawn and listed from its parent alone), the chromosome map when it comes only as an image, every de novo candidate and every compound-het pair, the child's cell in every medicines and HLA row of the side by side, and the carrier screen of a couple the child is part of (its counts, its boxes, its table, the lead's words on it and its report link). Each "Show the result" opens one result, from whichever box it is clicked; the places come back once the child's last closed result is opened, and a reload closes them again. Opening a result opens it for that child only. The page reads each child's `results.json` to know whether a result is closed; while it cannot (no run here, no answer yet, a request that failed, results the server could not build, or results with findings to act on whose `veil.spans` is missing or does not list each of them), every place of that child stays closed, and each box opens only its own place on purpose: "Show the rows" its table, "Show the cells" the side by side, "Show the grandparents" the painting, "Show the map" the image and "Show the screen" the couple screen. V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. Every row stays, with the parents' values: a child's medicines and HLA cells read "closed" while chapter IV's rule closes them (a row left out would leave a gap in a list everyone knows), and the haplogroups and blood groups, which name no gene, stay open. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). Five chapters show the pedigree, measured relationships, couple carrier screens, each child's inheritance and the members side by side. The shared data gate above applies before the family view receives child report data. Gated children have fixed opening controls and closed cell markers. Parent values and independently opened children remain available. The full family report returns for a child after that child's final act opens. -On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). On a child's page, and while the family's roles are not known, the findings to act on stay behind "Show the result" on a phone too. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. +On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). The report gate above applies at every width. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. ![The person page of DEMO-002 on a 390 px phone: the lead, the counts one line per level, the two medicine results and the nine results worth a look open, every other card one line with a plus, sections 5 to 10 each folded to one line that names what they hold, and the tab bar at the bottom](images/demo-live-phone.png){ loading=lazy width="390" } @@ -104,9 +94,7 @@ Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with it The reads cross the network to the browser that logged in: only the byte ranges igv.js asks for, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS); without it the password and the reads cross the network in clear. Opening the viewer writes one access-log row; the byte ranges igv.js reads write none. -`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title and veil tag; a finding tied to a result to act on is omitted from this legacy summary, which the closed child view does not use), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a child has any result to act on closed, the home page builds attention rows and the running tile's findings from level counts alone. Each row links to the report and uses generic text. The tile's lead is gene-free. This also hides whether a symptom ranking names a closed gene. Adults and runs with no result to act on keep their existing titles, prose and destinations. While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. - -The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (on a child's page, the places outside a result's own card that close with it, each tagged with the list of closed results it names, or `*panel*`; `spans`, the closed results with their position, one row per gene they name, from which the family page knows whether a child has a result still closed; and `panels`, the cards of a fixed panel). A structural variant's finding also lists every gene it covers in `genes`. It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. +`GET /` lists the runs. The home page reads `/runs.json`, then resolves family membership and each child's acts before projecting the run summaries. A person's loader reads `results.json` and the family routing information. Full reports contain the chapters, findings, journal, lead and source metadata built by `bin/live_results.py`. The gate passes none of that report to views while an act remains closed, apart from explicitly opened acts' own records. The family loader reads `family.json` and each child's results, and refreshes every 30 seconds. A changed act record closes again. An unchanged record keeps its opening for the current page. ## Privacy and what is stored @@ -202,4 +190,4 @@ The core files (`bin/owngenome_live.py`, `assets/live/index.html`, `home.html`, The page has a replay mode for a demo with no server: `` reads `frames.json` (`{"interval_ms": 1000, "frames": [{"t": ..., "state": }]}`) and plays it with a play button and a slider, through the same drawing code. A replay whose last frame is a finished run opens at that frame, on the finished report, until you press Play, which starts again from the first frame; with `"autoplay": true` it plays from the start at once. The bar shows each frame's moment as day, month and time in UTC ("2 Jan 10:05"), the full time on hover. -`tests/test_owngenome_live.py` checks the server, the login, the stored messages, the plugin interface and the page in a headless Chrome; `tests/test_live_results.py` checks `results.json` (every card's caveat word for word, no wording that reads as an all-clear, `results.json` behind the login); `tests/test_live_journal.py` checks the levels, the lead paragraph and the finding page's route; `tests/test_live_demo.py` builds the demo and checks every page, phone width included; the family veil browser checks run from `tests/test_owngenome_live_family.py` in a separate CI shard, with every control kept; `tests/test_live_post.sh` and `tests/fake-docker/run-all-live.sh` check what `run-all.sh` posts. +`tests/test_owngenome_live.py` checks the server, login, message storage, plugins and browser. `tests/test_live_results.py` checks card caveats, wording and login boundaries. `tests/test_live_journal.py` checks finding levels, leads and routes. `tests/test_live_demo.py` checks the demo, maps, phone layouts and report gate. `tests/test_owngenome_live_family.py` runs family checks and its browser property in a separate CI shard. `tests/test_live_post.sh` and `tests/fake-docker/run-all-live.sh` check progress messages from the launcher. diff --git a/rust/crates/og-map/README.md b/rust/crates/og-map/README.md index 5902cf646..f6aba0d9d 100644 --- a/rust/crates/og-map/README.md +++ b/rust/crates/og-map/README.md @@ -59,7 +59,7 @@ One gzip member with mtime 0 and no file name, holding JSON written as Python's "per": { "DEMO-001": { "findings": {"id": [], "level": [], "gate": [], "chrom": [], "pos": [], "s": [], "e": [], "gene": [], "title": [], - "hgvs": [], "zyg": [], "copy": [], "steps": [], "href": [], "reads": [], "phrase": [], "veil": [], "veils": []}, + "hgvs": [], "zyg": [], "copy": [], "steps": [], "href": [], "reads": [], "phrase": [], "veil": []}, "roh": {"chr1": {"ds": [], "l": [], "markers": [], "q": [], "long": [], "art": []}}, "roh_bins": {"size": 1000000, "chr1": []} } @@ -73,8 +73,9 @@ One gzip member with mtime 0 and no file name, holding JSON written as Python's - `ref.chroms`: chr1-22, X and Y. `len` is the chromosome's last band end, `cen` the span of its `acen` bands, `masked` its `acen`, `gvar` and `stalk` bands with touching neighbours merged. - `L`, a layer entry: `state` (`ok`, `missing` or `unreadable`), `step`, `step_name`, `source` (the file read or looked for, relative to GENOME_DIR), `note` (why it is missing, or `file:line: reason` when unreadable), `count` (rows on the map), `totals` (`all` rows read and `kept` rows on the map, when the file was read), `caveat`, and `empty` (the sentence for a read file with no rows). A missing input is `missing`, an input that cannot be read is `unreadable`, and a read input with no rows is `ok` with `count` 0: three answers, never a blank lane. -- `findings`: one flat column per field, rows sorted by chromosome then start. A finding's `viewer` (`chrom:pos` or `chrom:start-end`, 1-based, `chr` optional) places it: a position gives `pos` and `[pos - 1, pos)`, a span gives `[start - 1, end)` and `pos` null. Without a usable `viewer`, a finding that names a gene sits on that gene's span from the GENCODE genes file (the first in map order when a name is on several chromosomes). A finding with neither is not a row. `gate` is set for a finding the page keeps behind its own warning: the traits card holds APOE that way, so the map adds the finding `apoe` on the APOE gene, with `gate` `apoe`; the hidden text, which names the genotype, never enters the document. `zyg` (`het`, `hom`, `hemi` or `unknown`) comes from the finding's own `Genotype` evidence; no VCF is read. `hgvs` is its `Variant` evidence. `copy` is null until the family map. `href` is the finding's page relative to the person's page, `reads` the place as the reads viewer takes it (1-based), `phrase` the sentence the map shows. `veil` is the index in `dict.veil` of the first closable finding (in `dict.veil` order) whose span the row overlaps: its own for a closable finding with no earlier overlap, else -1. `veils` is the list of every such index, smallest first (`[]` for none), so its first item is `veil`. So an open finding over a closable one's span is tagged too. The layer's `count` and `totals.kept` are the rows whose `veil` is -1; `totals.all` adds the findings that are not closable and have no place, and the `note` counts those ("N findings with no place on the map stay on the page"). -- **The veil.** The document is the same for every reader. Whether a page is closed is the page's own rule; the document says only what a closed result could reveal. Closable findings are those of level `act` and every gated one (APOE closes on every page, an adult's too). Each closable finding with a place veils its span, and `dict.veil` lists their ids: gated ones first, then the act ones in the page's order. A row of a layer that can be a finding's own source (structural variants, mosaic calls and ClinVar hits, as their layers arrive), and every findings row, that overlaps a veiled span carries the index of the first such finding in `dict.veil` in a `veil` column, else -1, and the index of every such finding in a `veils` column. Gated findings come first, so a row's `veil` over both APOE and an act finding is APOE, which is closed on every page. The page keeps a row closed while any finding of its `veils` is closed: a row over two act findings never comes back with the first one opened while it still lies over the second. The page decides its own findings with its own veil, and the tag binds only the rows that are not one of the page's findings (APOE, or a row the page's results do not have yet): the page hides such a row while its result is closed. Counts, totals and density bins cover untagged rows only, so no number counts a closed row. Runs of homozygosity and bands are never a finding's source and carry no tag. +- `findings`: one flat column per field, rows sorted by chromosome then start. A finding's `viewer` (`chrom:pos` or `chrom:start-end`, 1-based, `chr` optional) places it: a position gives `pos` and `[pos - 1, pos)`, a span gives `[start - 1, end)` and `pos` null. Without a usable `viewer`, a finding that names a gene sits on that gene's span from the GENCODE genes file (the first in map order when a name is on several chromosomes). A finding with neither is not a row. `gate` is set for a finding the page keeps behind its own warning: the traits card holds APOE that way, so the map adds the finding `apoe` on the APOE gene, with `gate` `apoe`; the hidden text, which names the genotype, never enters the document. `zyg` (`het`, `hom`, `hemi` or `unknown`) comes from the finding's own `Genotype` evidence; no VCF is read. `hgvs` is its `Variant` evidence. `copy` is null until the family map. `href` is the finding's page relative to the person's page, `reads` the place as the reads viewer takes it (1-based), `phrase` the sentence the map shows. `veil` is the index in `dict.veil` of the first closable finding (in `dict.veil` order) whose span the row overlaps: its own for a closable finding with no earlier overlap, else -1. The layer's `count` and `totals.kept` are the rows whose `veil` is -1; `totals.all` adds the findings that are not closable and have no place, and the `note` counts those ("N findings with no place on the map stay on the page"). +- **The report gate.** A child's browser loader admits no map while an act remains unopened. There are no per-element tag sets. The first-tag `veil` column and `dict.veil` remain for the full report's APOE warning, opaque act controls and existing document consumers. The page decides its known findings from their own opening state. Counts, totals and density bins retain their established first-tag format. The static document remains downloadable; this is a browser boundary. + - `roh`: the `RG` rows of step 11's `bcftools roh` text (`{run}/vcf/{run}_roh.txt`, else `{run}/roh/{run}_roh.txt`). bcftools writes 1-based inclusive positions, so `s = start - 1` and `e = end`. `ST` lines are skipped; rows on chrM and other contigs count in `totals.all` only. `markers` and `q` are bcftools' columns 7 and 8. `art` indexes `dict.art`: `one X` on a male's chrX, `one Y` on a male's chrY, else `centromere` when more than half the run lies in masked spans. An unknown sex sets neither flag. `long` is 1 for a run of 5 Mb or more with no `art`. - `roh_bins`: bases covered by any run per 1 Mb bin, for each chromosome with a run. - `text`: the sentences the page shows, in pieces; `b` marks bold words. `text.findings` is the count of findings on the map in three forms (`none`, `one`, `many`): the page counts the rows it draws: each of its own findings that its veil shows at this moment, whatever the row's tag, and each other row whose tag is -1 or names a result that is open; it picks the form and puts the number in place of `{n}`. diff --git a/rust/crates/og-map/src/doc.rs b/rust/crates/og-map/src/doc.rs index be511ed1f..1df5e65de 100644 --- a/rust/crates/og-map/src/doc.rs +++ b/rust/crates/og-map/src/doc.rs @@ -188,7 +188,7 @@ fn strs<'a>(xs: impl IntoIterator) -> Value { /// The person's findings on the map: one flat column per field, rows in /// map order. `veil` is the index in `dict.veil` of the first closable -/// finding whose span the row overlaps, else -1; `veils` every such index. +/// finding whose span the row overlaps, else -1. fn findings_columns(m: &Model) -> Value { let rows = &m.findings.rows; let tags = m.findings.veil.tags(rows); @@ -255,17 +255,6 @@ fn findings_columns(m: &Model) -> Value { Value::Array(rows.iter().map(|r| Value::Str(r.phrase())).collect()), ), ("veil", ints(veil::column(&tags))), - ( - "veils", - Value::Array( - m.findings - .veil - .all_tags(rows) - .into_iter() - .map(ints) - .collect(), - ), - ), ]) } diff --git a/rust/crates/og-map/src/facts.rs b/rust/crates/og-map/src/facts.rs index 18ba81fd5..a9e10e019 100644 --- a/rust/crates/og-map/src/facts.rs +++ b/rust/crates/og-map/src/facts.rs @@ -70,17 +70,8 @@ pub struct Finding { /// The evidence row `Genotype`, when there is one. pub genotype: Option, pub steps: Vec, - /// `results.veil.findings` ties it to a closable result (a tag, the list - /// of the ids it names, holding any id but the fixed-panel one, - /// `*panel*`, which is the same whatever is closed): it closes with that - /// result, so no count of open findings takes it. - pub veiled: bool, } -/// The tag of `results.veil.findings` that names no result: a fixed-panel -/// card's own finding (bin/live_results.py `PANEL`). -pub const PANEL_TAG: &str = "*panel*"; - /// The findings part of stdin. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Findings { @@ -145,7 +136,6 @@ fn finding(v: &Value) -> Option { hgvs: evidence(f, "Variant"), genotype: evidence(f, "Genotype"), steps, - veiled: false, }) } @@ -236,19 +226,6 @@ pub fn parse(bytes: &[u8]) -> Result { None | Some((_, None)) => Findings::Missing, Some((r, Some(Value::Array(list)))) => { let mut out: Vec = list.iter().filter_map(finding).collect(); - if let Some(Value::Object(v)) = r.get("veil") - && let Some(Value::Object(tags)) = v.get("findings") - { - for f in &mut out { - f.veiled = match tags.get(&f.id) { - Some(Value::Str(t)) => t != PANEL_TAG, - Some(Value::Array(ts)) => ts - .iter() - .any(|t| matches!(t, Value::Str(t) if t != PANEL_TAG)), - _ => false, - }; - } - } let has_apoe = out .iter() .any(|f| f.id == APOE_ID || f.gate.as_deref() == Some(APOE_ID)); @@ -344,33 +321,5 @@ mod tests { panic!("not read") }; assert_eq!(f.len(), 2); - // A finding the veil ties to a closable result is veiled; the - // fixed-panel tag ties it to none. - let Findings::Read(f) = p(&body.replace( - r#""chapters""#, - r#""veil":{"findings":{"q":"sf-x","sf-x":"*panel*"}},"chapters""#, - )) else { - panic!("not read") - }; - assert_eq!( - (f[0].veiled, f[1].veiled, f[2].veiled), - (false, true, false) - ); - // A tag is the list of the ids it names: veiled while it holds any id - // but the fixed-panel one. - for (tag, want) in [ - (r#"["sf-x","sf-y"]"#, true), - (r#"["*panel*","sf-y"]"#, true), - (r#"["*panel*"]"#, false), - ("[]", false), - ] { - let Findings::Read(f) = p(&body.replace( - r#""chapters""#, - &format!(r#""veil":{{"findings":{{"q":{tag}}}}},"chapters""#), - )) else { - panic!("not read") - }; - assert_eq!(f[1].veiled, want, "{tag}"); - } } } diff --git a/rust/crates/og-map/src/findings.rs b/rust/crates/og-map/src/findings.rs index a59a50d22..d65ba737b 100644 --- a/rust/crates/og-map/src/findings.rs +++ b/rust/crates/og-map/src/findings.rs @@ -9,9 +9,7 @@ //! reads (`genes::parse_gtf`): medicine genes, HLA, LPA, a gene-level //! result. A name on several chromosomes takes the first in map order. //! - A finding with neither is not on the map. The layer note counts those -//! that no closed result hides; a closable one, and one the results' -//! veil ties to a closable result (`Finding::veiled`), is left out of -//! every count. +//! that no closed result hides; a closable one is left out of every count. //! - The layer's count and totals take the rows with no veil tag only: a //! closable finding, and an open one over a closable one's span, are in //! no number. @@ -199,7 +197,7 @@ pub fn read(findings: &facts::Findings, root: &Path, run: &str, reference: &Refe within((c, s, e, None)) }); let Some((chrom, s, e, pos)) = place else { - if !veil::closable(f) && !f.veiled { + if !veil::closable(f) { open_unplaced += 1; } continue; diff --git a/rust/crates/og-map/src/veil.rs b/rust/crates/og-map/src/veil.rs index 39202d5be..388b872d6 100644 --- a/rust/crates/og-map/src/veil.rs +++ b/rust/crates/og-map/src/veil.rs @@ -17,9 +17,7 @@ //! veiled span gets that finding's index in `dict.veil`, else -1: the //! first in `dict.veil` when several spans overlap it, where gated //! findings come first (closed on every page), then the act ones in the -//! page's order. Its `veils` holds every such index, so a page keeps the -//! row closed while any of them is (`Veil::all`): a row over two closed -//! results never comes back with the first one opened. A finding's own +//! page's order. The child report gate admits no map until every act opens. A finding's own //! row is tagged the same way, so an open finding over a closable one's //! span is tagged. Runs of homozygosity and bands are never a finding's //! source and carry no tag. @@ -96,31 +94,6 @@ impl Veil { pub fn tags(&self, rows: &[R]) -> Vec> { rows.iter().map(|r| self.tag(r.chrom(), r.span())).collect() } - - /// The index in `ids` of every closable finding whose span `row` - /// overlaps, smallest first, each once: the first of them is - /// [`Veil::tag`]. - pub fn all(&self, chrom: usize, span: (u64, u64)) -> Vec { - let (s, e) = nonempty(span); - let mut out: Vec = self - .spans - .get(chrom) - .map(|v| { - v.iter() - .filter(|&&(vs, ve, _)| s < ve && vs < e) - .map(|x| x.2) - .collect() - }) - .unwrap_or_default(); - out.sort_unstable(); - out.dedup(); - out - } - - /// [`Veil::all`] of every row, in order. - pub fn all_tags(&self, rows: &[R]) -> Vec> { - rows.iter().map(|r| self.all(r.chrom(), r.span())).collect() - } } /// A tag column as the document writes it: the index, or [`NONE`]. @@ -188,28 +161,4 @@ mod tests { assert_eq!(column(&tags), [NONE, 0, NONE]); assert_eq!(untagged(&tags), 2); } - - #[test] - fn every_id_a_row_overlaps() { - // A row over two closable findings keeps both, smallest index first, - // whatever order their spans were placed in; the first is `tag`. - let v = Veil::new([ - (2, "c", 0, (100, 101)), - (0, "a", 0, (90, 300)), - (1, "b", 0, (100, 150)), - (3, "d", 1, (100, 101)), - ]); - assert_eq!(v.all(0, (100, 101)), [0, 1, 2]); - assert_eq!(v.all(0, (120, 130)), [0, 1]); - assert_eq!(v.all(0, (200, 250)), [0]); - assert_eq!(v.all(0, (0, 10)), Vec::::new()); - assert_eq!(v.all(1, (100, 101)), [3]); - assert_eq!(v.all(5, (100, 101)), Vec::::new()); - let rows = [R(0, 100, 101), R(0, 0, 10), R(1, 100, 100)]; - let all = v.all_tags(&rows); - assert_eq!(all, [vec![0, 1, 2], vec![], vec![3]]); - // `veil` is the first of `veils`, -1 for none. - let first: Vec> = all.iter().map(|x| x.first().copied()).collect(); - assert_eq!(first, v.tags(&rows)); - } } diff --git a/rust/crates/og-map/tests/findings.rs b/rust/crates/og-map/tests/findings.rs index 5deaa5351..ad448a374 100644 --- a/rust/crates/og-map/tests/findings.rs +++ b/rust/crates/og-map/tests/findings.rs @@ -124,9 +124,6 @@ fn findings_land_on_their_place_and_the_rest_are_counted() { // APOE is gated, so it comes first in dict.veil. The open ClinVar row // at the act finding's base carries that finding's tag. assert_eq!(get(f, &["veil"]), &js("[3,1,1,2,-1,0,-1]")); - // `veils` holds every closable finding a row overlaps (one each here; - // src/veil.rs tests a row over several), its first being `veil`. - assert_eq!(get(f, &["veils"]), &js("[[3],[1],[1],[2],[],[0],[]]")); assert_eq!( get(f, &["href"]), &js( diff --git a/tests/demo/gate-property.js b/tests/demo/gate-property.js new file mode 100644 index 000000000..6ac9cffbf --- /dev/null +++ b/tests/demo/gate-property.js @@ -0,0 +1,37 @@ +// Browser-side property helper. Walk unknown fields too; callers mark only immutable controls/ownership. +(function () { + 'use strict'; + function walk(value, keep, path) { + path = path || []; + if (keep(path, value)) { return JSON.parse(JSON.stringify(value)); } + if (Array.isArray(value)) { + var items = value.map(function (v, i) { return walk(v, keep, path.concat(i)); }); + if (items.length) { items.push(walk(value[0], keep, path.concat(value.length))); } + else { items.push('future-array-value'); } + return items; + } + if (value && typeof value === 'object') { + var out = { future_payload_field: 'future-value' }; + Object.keys(value).reverse().forEach(function (k) { out[k] = walk(value[k], keep, path.concat(k)); }); + return out; + } + if (typeof value === 'string') { return value + '_CHANGED'; } + if (typeof value === 'number') { return value + 37; } + if (typeof value === 'boolean') { return !value; } + return 'changed-null'; + } + function snapshot() { + // Attribute order is part of this snapshot. No fields are removed from the gated document. + return { + dom: document.documentElement.outerHTML, + text: document.body.innerText, + title: document.title, + address: location.pathname + location.search + location.hash, + canvas: Array.from(document.querySelectorAll('canvas')).map(function (c) { return c.toDataURL(); }), + svg: Array.from(document.querySelectorAll('svg')).map(function (s) { return [s.outerHTML, s.clientWidth, s.clientHeight]; }), + storage: [Object.assign({}, localStorage), Object.assign({}, sessionStorage)], + requests: performance.getEntriesByType('resource').map(function (r) { return new URL(r.name).pathname; }).sort() + }; + } + window.GateProperty = { walk: walk, snapshot: snapshot }; +}()); diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index bc2162c3a..442d82625 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1,125 +1,9 @@ #!/usr/bin/env python3 -"""The live site's public demo (tests/demo/build_live_demo.py), built and checked. - -Checks: - 1. tests/fixtures/live/demo-001.weblog.jsonl is what tests/demo/make_demo_events.py - writes from assets/steps.tsv now; - 2. every state hook the plugins bin/owngenome_live_*.py register has a value - in the last frame of DEMO-001's page (the family page for the family - plugin's own view), none of them an error; - 3. the frames follow the run: running at the first, finished at the last with - every replayed step done; the charts and the search index appear only - once the report step has ended, the re-check only after the run; - 4. the run-health card measures the invented machine, never the machine that - builds the demo, and no frame holds a path of that machine; - 5. every link a frame gives is relative and reaches a file of the demo, and - so is every link of runs.json, results.json and family.json; - 6. every page the build writes carries the server's Content-Security-Policy - as a meta tag, and no page or script of the demo loads anything from - outside; - 7. in a headless Chrome (skipped without Chrome, except in CI), served as a - static host serves it (no policy header): the home page's people and - family, the replay to its last frame, the charts, a search, the re-check - banner, the run-health card, the family link and its pedigree, the viewer - page, and no policy violation; the person page's ten sections, its - contents, lead and ledger, #s/clinvar focusing the ClinVar card with its - caveat, chart and flag, a finding's own page with its plan and evidence; - the family page with a - clickable pedigree and no step map or report link; the widgets of the - partner's and the son's pages (the repeat gauge, the Lp(a) interval, the - CHIP VAF, the copy-number dots, the Y path) and the AnnotSV and MoChA marks - on their genome charts; the family page's relatedness grid, the son's - painted stretches (one mark per row of his segments file), his de novo - and compound-het tables and the members side by side; and at a phone's - width (390 px) no page scrolls sideways (every section open, the finding - page too) and the person page has its bottom tab bar; at 390 px the - partner's page is under 8,000 CSS px with every finding to act on, for a - prescriber, for the family or worth a look open, the rest folded, and a - link to a card two folds deep (#s/roh) or to a folded section (#s7) opens - it, while a wider page folds nothing; a section 5 card flagged with no - finding of its own stays open on a phone, the folds a reader opened (a - card's other lines too) stay open after the page turns wide and back, and - a Ctrl+click on a link to the current place opens nothing; on the son's page a - two planted CPSR results to act on name their genes and variants nowhere - (their card included) until "Show the result" is clicked, and each click - opens one result, there and on the finding's own page; they stay closed on - both pages with family.json unavailable, with a state cut short (no - plugins) and with no state at all (frames.json unavailable), and the son's - search lists no hit and suggests no term from their closed card while a - term of an open card is still found; the son's planted secondary finding - and class 5 deletion (make_demo_sample.py PLANTED) name their gene and - positions in no card, chart, chart title, link, symptom ranking or search - suggestion until each is opened, a row, mark or ranking that names both - comes back only once both are opened (the search too), and a reload - closes them again; a place that names two closed results (a lead segment, - a duplication over two genes) stays closed until both are opened, and the - page after a click is the same whichever closed result shares the opened - one's place; the home page and - every finding page of the son (an open finding's rail and related list - too) name them nowhere, and the symptom ranking's own page shows the gene - once opened on purpose; on the family page, while the son has a planted - result closed (the fixture's compound-het pair in KCNH2, a de novo - candidate and a stretch painted from a grandparent planted in family.json), - his painting's grandparents, his de novo and compound-het rows and his - medicines and HLA cells of the side by side close, one box each, and no - text or attribute names its gene, positions or that grandparent; each click - opens one result, every place comes back once both are opened, and a reload - closes them again; the page is the same, byte for byte, whichever result is - closed (KCNH2, RYR1, HLA-B, a gene alone, a position, a copy-number span, - a CHIP call) and wherever a grandparent's stretch lies, each view settled - on knowing his results; while the page cannot read his results (no - results.json, an unreadable one, one cut short, act findings without - veil.spans or with malformed ones) every place stays closed and each box - opens only its own place; with nothing of his to act on every place is - open with no box; with a second child holding the same results, opening - his leaves hers closed, and results opened close again when his run moves; - a couple screen whose partner is a child with a closed result closes whole - (counts, table, boxes, report link, the lead's words), the same page - whether that result is in a gene of the screen or not, and with the - partner's results unreadable its box and the cells' box each open only - their own place; his map as an image stays closed until each result is - opened from its box, the same page whichever result is closed, and with - his results unreadable "Show the rows" leaves it closed and "Show the map" - opens it and no table; on - his page the copy-number and CHIP cards, a fixed panel, close whole while - a result of his is closed (on a phone their line too), their box opens one - result per click and every dot, call and fact is back once all are open; - two of his pages that differ only in which result is closed (two SNV - genes, two deletions of two genes each, SMN1 against HBA1, a result with a - place on the map against one without, one ClinVar row against two, a link - to each) are the same before any click, HTML, attributes, address and map - canvas, and a closed deletion's second gene is named nowhere; DEMO-001's page - mid-replay and at the end leads with the report (a short run band above - the lead while it runs, none once finished), ends with the step timeline - folded, keeps the wire to at most eight one-sentence lines with the plain - finishes folded into one, and puts the HLA card's caveat in its margin - with its absent and undetermined lines folded. -Checks 1, 2, 3, 5, the widgets (person.js without its widget line), the family depth (family.json -with the son's stretches emptied), the phone width and the son's closed result (person.js whose cards ignore -the veil, person.js and finding.js that read a missing state as a standalone adult, and search.js -without its filter; for the planted results, results.json without its veil tags, person.js whose table -rows ignore them, person.js whose findings ignore them, charts.js that keeps every mark, search.js that ignores a hit's tag -and finding.js whose findings ignore their tags; for the family page, de novo rows, compound-het rows, a painting, -a side by side's cells, a couple screen and a map image that ignore the veil, a closed painting that keeps each -stretch's grandparent, a box that opens every result with one click, de novo rows closed only when a result lies -under one, a painting closed only when a grandparent's stretch lies over a closed result, a side by side that closes -only the closed gene's cells or stays open for a wide span, a page that never reads the son's results, one opening -shared by every box or 'Show the grandparents' opening the side by side while the results cannot be read, a failed -request read as nothing closed, malformed veil.spans read as given, results without veil.spans read as nothing -closed or as closed with nothing to act on, an opened result kept per run only or per child only, a couple screen -closed only for a result in its genes or opened by 'Show the cells', a map image closed only for a result with a -position, and de novo rows that 'Show the map' opens; for the son's page, person.js that draws a fixed-panel -card's open rows or builds its line from its facts, that puts a closed card's id back or counts the closed rows, -and a veil that tags only a deletion's first gene), the phone page (person.js that never folds, and a route that opens only the nearest fold), the -folds (a flagged section 5 card folded, a redraw that forgets the folds, a Ctrl+click that opens one) and -the person page's layout (a wire showing every line or every plain finish, a card showing -every line, a step timeline left open) also run against a planted mistake and must -fail there (2: a build with one plugin's state hook dropped): a check that cannot fail is no check. - -The family veil group described above runs from tests/test_owngenome_live_family.py in another Guard shard. -Run both files for every browser check. - -Run: python3 tests/test_live_demo.py +"""Build the public live demo and check its reports, map, phone layout and report gate. + +The gate property walks all report payloads, including unknown fields. Immutable act records, +family membership, transport routes and other people's records are its controls. It compares +whole documents, accessibility text, addresses, pixels, storage and resource requests. """ import base64 import copy @@ -373,12 +257,9 @@ def browser_checks(out): phone_checks(b, base) phone_report_checks(b, base, out) fold_checks(b, base, out) - veil_checks(b, base, out) - planted_checks(b, base, out) map_checks(b, base, out) # The family veil group runs from test_owngenome_live_family.py in a different Guard shard. - widget_veil_checks(b, base, out) - infer_checks(b, base, out) + gate_checks(b, base, out) finally: b.close() srv.shutdown() @@ -428,7 +309,20 @@ def person_checks(b, base): ("DEMO-002", "paralogs", "copies", "[data-widget=copies] li", 7), ("DEMO-003", "y-chromosome", "ypath", "[data-widget=ypath] li", 5), ("DEMO-002", "clinvar", "MoChA bar", "figure[data-chart=genome] rect.mca", 1), - ("DEMO-003", "clinvar", "AnnotSV span", "figure[data-chart=genome] rect.sv", 1)] + ("DEMO-003", "clinvar", "AnnotSV span", "figure[data-chart=genome] rect.sv", 2)] + + +def open_gate(b): + assert b.wait("document.documentElement.dataset.pgpStatus === 'ready'"), 'page did not settle: '+str(b.eval("JSON.stringify([location.pathname,document.documentElement.dataset,document.body.textContent.slice(0,200)])")) + for _ in range(10): + closed = b.eval("document.querySelector('[data-opening^=closed]') !== null") + if not closed: + return + b.eval("document.querySelector('[data-opening^=closed]').click(); true") + if not b.eval("document.querySelector('[data-opening^=closed]') !== null"): + assert b.wait("!document.documentElement.dataset.pgpGated"), 'gate did not release' + return + raise AssertionError('gate did not release in ten clicks') def widget_counts(b, base): @@ -437,12 +331,9 @@ def widget_counts(b, base): b.go("about:blank") # a new document each time, so the wait never reads the page before b.wait("location.href === 'about:blank'") b.go(f"{base}/runs/{person}/#s/{cid}") + open_gate(b) b.wait("document.documentElement.dataset.pgpResults && !!document.getElementById(%s) && 'drawn'" % json.dumps("card-" + cid)) - if kind == "ypath": - acts = json.loads(b.eval("JSON.stringify((function () { var x = new XMLHttpRequest(); x.open('GET', 'results.json', false); x.send(); return JSON.parse(x.responseText).findings.filter(function (f) { return f.level === 'act'; }).map(function (f) { return f.id; }); }()))")) - for fid in acts: - open_closed(b, fid) if kind in ("MoChA bar", "AnnotSV span"): b.wait("document.documentElement.dataset.pgpCharts") time.sleep(0.5) @@ -478,12 +369,14 @@ def family_depth(b, base): b.go("about:blank") b.wait("location.href === 'about:blank'") b.go(f"{base}/runs/{FAM}/") - b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " - "(document.documentElement.dataset.pgpKids || '').indexOf('DEMO-003:known') >= 0") - # The son's results to act on close his tables here: open each on purpose (one per click) to count them. - for _ in range(4): - b.eval("var v = document.querySelector('.veil[data-veil-child=\"DEMO-003\"] button'); v && v.click(); true") - time.sleep(0.2) + assert b.wait("document.documentElement.dataset.pgpStatus === 'ready'"), 'family never ready' + for _ in range(10): + n = b.eval("document.querySelectorAll('[data-opening^=closed]').length") + if not n: + break + b.eval("document.querySelector('[data-opening^=closed]').click(); true") + assert b.wait("document.querySelectorAll('[data-opening^=closed]').length < %d" % n), 'family opening did not settle' + assert b.wait("!document.querySelector('.report-gate')"), 'full family projection never settled' return b.wait("JSON.stringify([" "document.querySelectorAll('table.rel-grid td:not(.self):not(.none)').length, " "document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg').length, " @@ -521,2428 +414,6 @@ def family_depth_checks(b, base, out): VEIL = (("GENEQ", "c.77del"), ("GENER", "c.88dup")) -def veil_text(b, base): - """The son's pages: [the person page's text before any click, after one click on the CPSR card's 'Show the result', - the first finding's own page after its 'Show the result' click]. textContent reads closed details too.""" - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/") - b.wait("document.documentElement.dataset.pgpResults && document.documentElement.dataset.pgpStatus === 'ready' " - "&& document.querySelector('.veil') && 'drawn'") - before = b.eval("document.body.textContent") - # The card's own button: a card holding two results opens one per click, as each finding's button does. - b.eval("document.querySelector('#card-cpsr .veil button').click(); true") - time.sleep(0.5) - after = b.wait("document.body.textContent") - fid = b.eval("var a = document.querySelector('[data-finding] h3 a[href*=\"finding/cpsr-\"]'); a && a.getAttribute('href')") - page = "" - if fid: - b.go(f"{base}/runs/{OTHERS[1]}/{fid}") - b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.querySelector('.veil') && 'closed'") - b.eval("document.querySelector('.veil button').click(); true") - page = b.wait("!document.querySelector('.veil') && document.body.textContent") - return before, after, page - - -def veil_problems(before, after, page): - (g1, v1), (g2, v2) = VEIL - out = [f"{w} shown before any click" for w in (g1, v1, g2, v2) if w in (before or "")] - shown = [w for w in (g1, v1) if w in (after or "")] or [w for w in (g2, v2) if w in (after or "")] - if len(shown) != 2: - out.append(f"the first click did not open one result: {shown}") - # The other result stays closed after one click, on the person page and on the opened finding's page. - other = (g2, v2) if shown and shown[0] == g1 else (g1, v1) - out += [f"{w} opened by the click on another result" for w in other if w in (after or "")] - if not page: - out.append("no finding page of a planted result") - else: - mine = [w for pair in VEIL for w in pair if w in page] - if len(mine) != 2: - out.append(f"the finding page opened shows {mine}, want only its own gene and variant") - return out - - -def veil_checks(b, base, out): - """On a child's page each result to act on stays closed everywhere, its own section's card too, until it - is opened; opening one opens only that one, on the person page and on its own page.""" - rj = os.path.join(out, "runs", OTHERS[1], "results.json") - js = os.path.join(out, "assets", "person.js") - with open(rj) as f: - keep = f.read() - with open(js) as f: - keep_js = f.read() - made = [] - try: - with open(os.path.join(os.path.dirname(out), "root", OTHERS[1], "summary.json")) as f: - summary = json.load(f) - summary.setdefault("sections", {})["cpsr"] = {"state": "ok", "data": {"call_floor_dp": 10, "low_confidence": 0, "plp": [ - {"gene": g, "variant": v, "final": "Pathogenic", "clinvar": "Pathogenic"} for g, v in VEIL]}} - res = live_results.build(OTHERS[1], summary) - cpsr = [c for ch in res["chapters"] for c in ch["sections"] if c["id"] == "cpsr"] - check("browser: the planted CPSR card has its table, so the closed check below has rows to hide", - bool(cpsr and cpsr[0]["table"] and cpsr[0]["section"] == 1), cpsr) - with open(rj, "w") as f: - json.dump(res, f) - # Their finding pages, as build_live_demo.person_site writes one per finding: an existing page, its id swapped. - fdirs = os.path.join(out, "runs", OTHERS[1], "finding") - model = sorted(os.listdir(fdirs))[0] - with open(os.path.join(fdirs, model, "index.html")) as f: - tpl = f.read() - for fd in res["findings"]: - if fd["card"] == "cpsr": - os.makedirs(os.path.join(fdirs, fd["id"])) - made.append(os.path.join(fdirs, fd["id"])) - with open(os.path.join(fdirs, fd["id"], "index.html"), "w") as f: - f.write(tpl.replace(f'data-finding="{model}"', f'data-finding="{fd["id"]}"')) - bad = veil_problems(*veil_text(b, base)) - check("browser: on the son's page two planted results to act on name their genes and variants nowhere " - "until 'Show the result' is clicked, and each click opens one, on the person page and the finding page", - not bad, bad) - with open(js, "w") as f: - f.write(keep_js.replace("} else if (closedFs.length) {", "} else if (false) {")) - control("person.js whose cards ignore the veil", not veil_problems(*veil_text(b, base))) - with open(js, "w") as f: - f.write(keep_js) - fid = [fd["id"] for fd in res["findings"] if fd["card"] == "cpsr"][0] - unknown_checks(b, base, out, fid) - finally: - with open(rj, "w") as f: - f.write(keep) - with open(js, "w") as f: - f.write(keep_js) - for d in made: - shutil.rmtree(d, ignore_errors=True) - - -# What the son's page shows, to scan for the planted results: its text (SVG titles and closed folds included), -# every link and every title attribute. -SHOWN_JS = ("[document.body.textContent].concat(Array.prototype.map.call(document.querySelectorAll('[href],[title]'), " - "function (e) { return (e.getAttribute('href') || '') + ' ' + (e.getAttribute('title') || ''); })).join(' ')") - - -def planted_words(): - """(the gene, the hit's words, the deletion's words) of make_demo_sample.PLANTED, as the page could write them.""" - import make_demo_sample - pl = make_demo_sample.PLANTED - c, p, _, _ = pl["hit"] - sc, a, z = pl["sv"] - return pl["gene"], [f"{c}:{p}", f"{p:,}"], [f"{sc}:{a}", f"{a:,}", f"{z:,}"] - - -def son_page(b, base): - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/") - b.wait("var d = document.documentElement.dataset, i = document.querySelector('input[type=search]'); " - "d.pgpResults && d.pgpCharts && d.pgpStatus === 'ready' && document.querySelector('.veil') && i && !i.disabled " - "&& 'ready'", 15) - time.sleep(0.5) - - -def son_search(b, q): - b.eval("var i = document.querySelector('input[type=search]'); i.value = %s; i.dispatchEvent(new Event('input')); true" - % json.dumps(q)) - hits = b.wait("document.documentElement.dataset.pgpSearch", 5) - return hits, b.eval("(document.querySelector('.search-results') || {}).textContent || ''") or "" - - -def planted_closed(b, base): - """[problems] of the son's page before any click: the planted gene and positions shown anywhere, or found by - the search.""" - gene, hit, sv = planted_words() - son_page(b, base) - shown = b.eval(SHOWN_JS) or "" - out = [f"{w} shown while closed" for w in [gene] + hit + sv if w in shown] - hits, text = son_search(b, gene) - if hits != "0" or any(w in text for w in hit + sv): - out.append(f"the search for {gene} while closed: {hits} hits, {text[:200]}") - hits, text = son_search(b, gene[:3]) - if gene in text: - out.append(f"the search for {gene[:3]} suggests {gene} while closed: {text[:200]}") - return out - - -# The handle of a closed finding, as the page counts it (person.js closedOrder()): the closed findings in the order -# of the sections, then any other; a result to act on is closed until opened, a finding tagged with one while that -# result is, a finding tagged PANEL while any result to act on is, a finding tagged with no finding of the page always. -HANDLE_JS = r"""fetch('results.json', {cache: 'no-store'}).then(function (r) { return r.json(); }).then(function (r) { - var fid = %s, byId = {}, tags = (r.veil && r.veil.findings) || {}; - r.findings.forEach(function (f) { byId[f.id] = f; }); - function open(id) { return !!document.getElementById('f-' + id); } - var acts = r.findings.filter(function (f) { return f.level === 'act'; }); - var anyShut = acts.some(function (f) { return !open(f.id); }); - function shut(f) { - if (f.level === 'act') { return !open(f.id); } - var t = tags[f.id]; - if (t === undefined) { t = ['*panel*']; } - t = (Array.isArray(t) ? t : t ? [t] : []).filter(function (x) { return x !== f.id; }); - return t.some(function (x) { return x === '*panel*' ? anyShut : !byId[x] || !open(x); }); - } - var order = []; - (r.journal || []).forEach(function (s) { (s.findings || []).forEach(function (id) { - if (byId[id] && shut(byId[id]) && order.indexOf(id) < 0) { order.push(id); } }); }); - r.findings.forEach(function (f) { if (shut(f) && order.indexOf(f.id) < 0) { order.push(f.id); } }); - return order.indexOf(fid) + 1; -})""" - - -def open_closed(b, fid): - """Opens the closed result FID as a reader does who knows its place among the page's closed findings: its handle - (#f/closed-), never its id, which the page does not use to find a closed card; then its "Show the result". - False when the handle names no closed card.""" - r = b.call("Runtime.evaluate", {"expression": HANDLE_JS % json.dumps(fid), "awaitPromise": True, - "returnByValue": True}, b.session) - n = (r.get("result") or {}).get("value") or 0 - if not n: - return False - b.eval("location.hash = %s; true" % json.dumps(f"#f/closed-{n}")) - if not b.wait("location.hash === %s && document.getElementById('f-closed-%d') && location.hash" - % (json.dumps(f"#f/closed-{n}"), n), 5): - return False - b.eval("document.getElementById('f-closed-%d').querySelector('.veil button').click(); true" % n) - return True - - -def planted_problems(b, base, out_dir): - """[problems] of the planted results on the son's page: closed, then opened one at a time, then reloaded.""" - gene, hit, sv = planted_words() - out = planted_closed(b, base) - with open(os.path.join(out_dir, "runs", OTHERS[1], "results.json")) as f: - ids = " ".join(x["id"] for x in json.load(f)["findings"] if x["level"] == "act") - sf = next((i for i in ids.split() if i.startswith("sf-")), None) - import make_demo_sample - start = str(make_demo_sample.PLANTED["sv"][1]) - del_ = next((i for i in ids.split() if i.startswith("sv-") and start in i), None) - if not sf or not del_: - return out + [f"the planted results are not among the son's results to act on: {ids}"] - rows_js = ("Array.prototype.filter.call(document.querySelectorAll('#card-clinvar tbody tr'), function (r) { " - "return r.textContent.indexOf(%s) >= 0; }).length" % json.dumps(gene)) - tick_js = ("Array.prototype.filter.call(document.querySelectorAll('figure[data-chart=genome] a[data-mark^=hit-] " - "title'), function (t) { return t.textContent.indexOf(%s) >= 0; }).length" % json.dumps(gene)) - ranked_js = ("Array.prototype.filter.call(document.querySelectorAll('[data-finding^=exomiser-]'), function (e) { " - "return e.textContent.indexOf(%s) >= 0; }).length" % json.dumps(gene)) - if not open_closed(b, sf): - return out + [f"a link to {sf} lands on no closed card"] - time.sleep(0.8) - shown = b.eval(SHOWN_JS) or "" - # The planted hit lies inside the planted deletion, in its gene: its ClinVar row, its tick and the symptom - # ranking of the gene name both results, so they stay closed until the deletion is opened too. - rows, tick, ranked = b.eval(rows_js), b.eval(tick_js), b.eval(ranked_js) - if not all(w in shown for w in [gene, hit[0]]): - out.append(f"opening the secondary finding did not bring back its own card ({gene}, {hit[0]})") - if rows or tick or ranked: - out.append(f"opening the secondary finding alone brought back places that name the closed deletion too: " - f"{rows} ClinVar rows, {tick} ticks, {ranked} rankings") - out += [f"{w} shown after opening only the secondary finding" for w in sv if w in shown] - # A search hit quotes the summary's rows (prose): it stays closed while any result is, the same whichever. - hits, _ = son_search(b, gene) - if hits not in ("0", None): - out.append(f"the search for {gene} finds {hits} hits while the deletion is still closed") - if not open_closed(b, del_): - return out + [f"a link to {del_} lands on no closed card"] - time.sleep(0.8) - shown = b.eval(SHOWN_JS) or "" - spans = b.eval("document.querySelectorAll('figure[data-chart=genome] a[data-mark^=sv-]').length") - if sv[0] not in shown or spans != 2: - out.append(f"opening the deletion did not bring back its span on the genome chart: {spans} spans") - hits, _ = son_search(b, gene) - if hits in (None, "0"): - out.append(f"the search for {gene} finds nothing once every result is opened") - rows, tick, ranked = b.eval(rows_js), b.eval(tick_js), b.eval(ranked_js) - if rows != 1 or not tick or not ranked: - out.append(f"opening both results did not bring back the hit's ClinVar row, tick and ranking: {rows} rows, " - f"{tick} ticks, {ranked} rankings") - out += [f"after a reload: {x}" for x in planted_closed(b, base)] - return out - - -def planted_pages(b, base, out): - """[problems] of the son's other pages before any click: the home page and every finding page of his (an open - finding's rail and related list included) showing the planted gene or a position; then the page of the - finding that names the closed result (the symptom ranking), opened on purpose, shows the gene.""" - gene, hit, sv = planted_words() - with open(os.path.join(out, "runs", OTHERS[1], "results.json")) as f: - res = json.load(f) - ids = [f["id"] for f in res.get("findings") or []] - shown_js = "document.title + ' ' + " + SHOWN_JS - out_ = [] - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/") - b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.documentElement.dataset.pgpPeople", 15) - shown = b.eval(shown_js) or "" - out_ += [f"{w} on the home page" for w in [gene] + hit + sv if w in shown] - for fid in ids: - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/finding/{fid}/") - b.wait("document.documentElement.dataset.pgpStatus === 'ready' && 'drawn'", 10) - shown = b.eval(shown_js) or "" - out_ += [f"{w} on the finding page {fid}" for w in [gene] + hit + sv if w in shown] - named = [k for k, t in ((res.get("veil") or {}).get("findings") or {}).items() if t and k.startswith("exomiser-")] - if len(ids) < 10 or not named: - return out_ + [f"the son's findings are not the planted ones: {ids}, ranking {named}"] - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/finding/{named[0]}/") - if b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.querySelector('.veil') && 'closed'", 10): - b.eval("document.querySelector('.veil button').click(); true") - shown = b.wait("!document.querySelector('.veil') && document.body.textContent", 5) or "" - if gene not in shown: - out_.append(f"opening the page of {named[0]} did not show {gene}") - out_ += [f"{w} shown on {named[0]} after opening it" for w in sv if w in shown] - return out_ - - -def planted_checks(b, base, out): - """The son's planted secondary finding and class 5 deletion stay closed everywhere on his page: the ClinVar - card's table and viewer link, the genome chart and the review-star chart (wherever it sits), their titles, - the search; each opens on purpose with its own rows only, and a reload closes them again.""" - sd = os.path.join(out, "runs", OTHERS[1]) - rj = os.path.join(sd, "results.json") - pj, cj, sj, fj = (os.path.join(out, "assets", x) for x in ("person.js", os.path.join("plugins", "charts.js"), - os.path.join("plugins", "search.js"), "finding.js")) - keep = {} - for p in (rj, pj, cj, sj, fj): - with open(p) as f: - keep[p] = f.read() - - def write(p, text): - with open(p, "w") as f: - f.write(text) - try: - bad = planted_problems(b, base, out) - check("browser: the son's planted secondary finding and class 5 deletion name their gene and positions nowhere " - "(cards, charts, titles, links, search) until each is opened, each opens only its own rows and marks (a " - "place that names both once both are), and a reload closes them", not bad, bad) - res = json.loads(keep[rj]) - res["veil"] = {} - write(rj, json.dumps(res)) - bad = planted_closed(b, base) - check("browser: results.json without veil metadata still keeps secondary places closed", not bad, bad) - res = json.loads(keep[rj]) - def unsafe_tags(value): - if isinstance(value, dict): - return {k: unsafe_tags(v) for k, v in value.items()} - if isinstance(value, list): - return None if all(isinstance(v, str) for v in value) else [unsafe_tags(v) for v in value] - return value - for kind in ("rows", "viewer", "lines", "facts", "headline", "marks", "findings"): - if kind in res["veil"]: - res["veil"][kind] = unsafe_tags(res["veil"][kind]) - write(rj, json.dumps(res)) - control("results.json falsely exempts tagged secondary places", not planted_closed(b, base)) - write(rj, keep[rj]) - for p, a, z, desc in ( - (pj, "return !placeClosed(tags[i]);", "return true;", "person.js whose ClinVar rows ignore their tags"), - (pj, "return !!(t && tagClosed(t));", "return false;", "person.js whose findings ignore their tags"), - (cj, "return ask ? (sp.marks", "return false ? (sp.marks", "charts.js that keeps every mark"), - (sj, "it.veil || null", "null", "search.js that ignores a hit's tag")): - if a not in keep[p]: - check(f"control: {desc} (the planted line is in {os.path.basename(p)})", False, a) - continue - write(p, keep[p].replace(a, z)) - control(desc, not planted_closed(b, base)) - write(p, keep[p]) - bad = planted_pages(b, base, out) - check("browser: the home page and every finding page of the son name the planted gene and positions nowhere " - "(the rail's next link and the related list included) until opened, and the symptom ranking's page " - "opened on purpose shows it", not bad, bad) - a = "return (tagOf(x) || []).some(function (t) { return t === PANEL ? actShut() : !opened[t]; });" - if a not in keep[fj]: - check("control: finding.js whose findings ignore their tags (the planted line is in finding.js)", False, a) - else: - write(fj, keep[fj].replace(a, "return false;")) - control("finding.js whose findings ignore their tags", not planted_pages(b, base, out)) - write(fj, keep[fj]) - finally: - for p, text in keep.items(): - write(p, text) - - -# The family page: what it shows of the son while a result of his to act on is closed. -# Every text of the page and every attribute of every element: a gene, a position or a grandparent hidden in a -# title, a link, a label, a data attribute or a class is as shown as one in a cell. -FAMILY_SHOWN_JS = ("[document.body.textContent].concat(Array.prototype.map.call(document.querySelectorAll('*'), function (e) { " - "return Array.prototype.map.call(e.attributes, function (a) { return a.value; }).join(' '); })).join(' ')") -GRANDPARENT = "DEMO-GF" -PAIR_B = "150962400" # the second variant of the fixture's compound-het pair in KCNH2 -SON = OTHERS[1] -# The places that close for the son while a result of his is closed, one box each, in page order. -SON_BOXES = ["paint", "denovo", "het", "side"] - - -def no_cache(b): - """Chrome keeps a script it fetched a moment ago without asking again: the controls below swap a script, so - every request goes to the server from here on.""" - b.call("Network.enable", {}, b.session) - b.call("Network.setCacheDisabled", {"cacheDisabled": True}, b.session) - - -def family_page(b, base): - """The family page once it knows the son's results (or after 10 s, for a page that never learns them).""" - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{FAM}/") - b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " - "/DEMO-003:(known|failed)/.test(document.documentElement.dataset.pgpKids || '') && 'ready'", 10) - time.sleep(0.3) - - -def boxes_of(b, child=SON): - """The places closed for CHILD (data-veil-box of each of its closed boxes), in page order.""" - got = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('.veil[data-veil-child=\"" + child + - "\"]'), function (x) { return x.getAttribute('data-veil-box'); }))") - return json.loads(got) if got else None - - -def open_box(b, box, child=SON): - """Click the button of CHILD's closed box BOX; whether there was one.""" - ok = b.eval("var v = document.querySelector('.veil[data-veil-child=\"" + child + "\"][data-veil-box=\"" + box + "\"] button'); " - "if (v) { v.click(); } !!v") - time.sleep(0.4) - return ok - - -def son_tables(b, child=SON): - """[rows of each table] in CHILD's fold of the family page.""" - rows = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('details[data-child=\"" + child + "\"] " - "table.rows'), function (t) { return t.querySelectorAll('tbody tr').length; }))") - return json.loads(rows) if rows else None - - -def side_rows(b): - """The side by side as [[group, label, value of each member]], the group carried down.""" - got = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows tbody tr'), " - "function (r) { return Array.prototype.map.call(r.cells, function (c) { return c.textContent; }); }))") - rows, group = [], "" - for r in json.loads(got) if got else []: - group = r[0] or group - rows.append([group] + r[1:]) - return rows - - -def side_problems_of(b, closed): - """[problems] of the side by side: the son's medicines and HLA cells all "closed" when CLOSED, none otherwise; a - parent's cell never closed, the haplogroups and blood groups never closed.""" - rows = side_rows(b) - members = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows thead th'), " - "function (x) { return x.textContent; }).slice(2))") - members = json.loads(members) if members else [] - if SON not in members or not rows: - return [f"no side by side with the son: {members}"] - out = [] - for r in rows: - for m, v in zip(members, r[2:]): - want = closed and m == SON and r[0] in ("Medicines", "HLA") - if (v == "closed") != want: - out.append(f"{m}'s {r[1]} reads {v!r}") - return out[:6] - - -def family_closed(b, base): - """[problems] of the family page before any click: the son's planted gene, its positions or the grandparent of - the stretch over it named anywhere, or a place of his open: the painting's grandparents, his de novo and - compound-het rows and his medicines and HLA cells close, one box each.""" - gene, hit, sv = planted_words() - family_page(b, base) - shown = b.eval(FAMILY_SHOWN_JS) or "" - out = [f"{w} shown on the family page while closed" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] - boxes, tables = boxes_of(b), son_tables(b) - segs = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg').length") - marked = b.eval("document.querySelectorAll('[data-painting=\"DEMO-003\"] .paint-seg.closed, " - "[data-painting=\"DEMO-003\"] .paint-seg.other').length") - # Only the painting's table of stretches stays, every stretch drawn and listed from its parent, none marked. - if boxes != SON_BOXES or not tables or len(tables) != 1 or tables[0] != segs or marked: - out.append(f"closed places {boxes}, tables {tables}, {segs} stretches drawn, {marked} marked") - out += side_problems_of(b, True) - return out - - -def family_problems(b, base): - """[problems] of the family page: closed; one click on the compound-het box opens one of the son's two planted - results and leaves every place closed; a click on the de novo box opens the other and every place; a reload - closes them again.""" - gene, hit, _ = planted_words() - out = family_closed(b, base) - if out: - return out - if not open_box(b, "het"): - return ["no compound-het box"] - shown = b.eval(FAMILY_SHOWN_JS) or "" - if any(w in shown for w in [gene, GRANDPARENT, PAIR_B] + hit) or boxes_of(b) != SON_BOXES: - out.append(f"one click opened a place while the other result is still closed: {boxes_of(b)}") - open_box(b, "denovo") - shown = b.eval(FAMILY_SHOWN_JS) or "" - tables = son_tables(b) - if not all(w in shown for w in [gene, GRANDPARENT, hit[0], PAIR_B]) or not tables or tables[1:] != [3, 3] or boxes_of(b): - out.append(f"opening both results did not bring back every place: tables {tables}, closed {boxes_of(b)}") - out += side_problems_of(b, False) - out += [f"after a reload: {x}" for x in family_closed(b, base)] - return out - - -def family_view(b, base, child=SON): - """The family page's HTML before any click, or None when the page did not settle on knowing CHILD's results.""" - family_page(b, base) - kids = b.eval("document.documentElement.dataset.pgpKids || ''") or "" - if not re.search(r"\b" + re.escape(child) + r":known\b", kids): - return None - return b.eval("document.getElementById('family').innerHTML") or "" - - -# Closed results of different kinds and places, as veil.spans holds them. -CLOSED_CASES = (("RYR1, a medicines gene", [["sf-ryr1", "RYR1", "chr19", 38440000, 38440000]]), - ("HLA-B", [["sf-hlab", "HLA-B", "chr6", 31353000, 31357000]]), - ("a gene alone", [["two-hits-g0999", "G0999", None, None, None]]), - ("a position under a de novo row", [["sf-pos", None, "chr3", 48211904, 48211904]]), - ("a copy-number span in SMN1", [["sv-chr5-70925000-70953000", "SMN1", "chr5", 70925000, 70953000]]), - ("a CHIP call in TP53", [["chip-tp53", "TP53", "chr17", 7675088, 7675088]])) - - -def invariance_problems(b, base, rj, res, fj, fams, cases=None, child=SON): - """[problems]: the family page before any click differs between closed results of different genes and places - (FAMS: name -> family.json text; CASES: name -> spans set in RJ, CHILD's results, from RES, its findings to act - on renamed to match, None for RES as it is), so it would tell which is closed; or it never knew CHILD's results.""" - views = {} - for name, spans in cases or (("the planted pair in KCNH2", None),) + CLOSED_CASES: - r = copy.deepcopy(res) - if spans is not None: - act = [x for x in r.get("findings", []) if x.get("level") == "act"] - r["veil"] = dict(r.get("veil") or {}, spans=spans) - if act: - r["findings"] = [x for x in r["findings"] if x.get("level") != "act"] + [dict(act[0], id=s_[0]) for s_ in spans] - with open(rj, "w") as f: - json.dump(r, f) - for fname, text in fams.items(): - with open(fj, "w") as f: - f.write(text) - views[f"{name}, {fname}"] = family_view(b, base, child) - with open(rj, "w") as f: - json.dump(res, f) - out = [f"the page never knew {child}'s results with {k!r}" for k, v in views.items() if v is None] - first = next(iter(views)) - return out + [f"the page differs between {first!r} and {k!r}" for k, v in views.items() if v != views[first]] - - -def family_unknown(b, base, opens=True): - """[problems] of the family page while it cannot read the son's results: a place of his open or a word of his - planted results shown; then (OPENS) each box's own button, from a fresh page each, opens its own place and no - other.""" - gene, hit, sv = planted_words() - family_page(b, base) - # A page still waiting for his results keeps everything closed anyway: the check is on the state it settles in. - kids = b.eval("document.documentElement.dataset.pgpKids || ''") or "" - shown = b.eval(FAMILY_SHOWN_JS) or "" - out = [] if re.search(r"\bDEMO-003:(known|failed)\b", kids) else [f"the son's results never settled: {kids!r}"] - out += [f"{w} shown with the son's results unreadable" for w in [gene, GRANDPARENT, PAIR_B] + hit + sv[:1] if w in shown] - if boxes_of(b) != SON_BOXES or son_tables(b) is None or len(son_tables(b)) != 1: - out.append(f"closed places {boxes_of(b)}, tables {son_tables(b)}") - words = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('.veil[data-veil-child=\"DEMO-003\"] button'), " - "function (x) { return x.textContent; }))") - if json.loads(words or "[]") != ["Show the grandparents", "Show the rows", "Show the rows", "Show the cells"]: - out.append(f"the buttons read {words}") - out += side_problems_of(b, True) - if not opens or out: - return out - # What each place shows once open: the painting its grandparent, the de novo rows the planted hit (not the - # pair's second variant), the compound-het rows the pair's second variant, the side by side his cells. - for box in SON_BOXES: - family_page(b, base) - if not open_box(b, box): - out.append(f"no {box} box") - continue - shown = b.eval(FAMILY_SHOWN_JS) or "" - rest, tables = [x for x in SON_BOXES if x != box], son_tables(b) or [] - want = {"paint": GRANDPARENT in shown and PAIR_B not in shown and len(tables) == 1, - "denovo": GRANDPARENT not in shown and PAIR_B not in shown and tables[1:] == [3], - "het": GRANDPARENT not in shown and PAIR_B in shown and tables[1:] == [3], - "side": GRANDPARENT not in shown and PAIR_B not in shown and len(tables) == 1}[box] - if boxes_of(b) != rest or not want: - out.append(f"the {box} box opened another place, or not its own: closed {boxes_of(b)}, tables {tables}") - out += [f"after the {box} box: {x}" for x in side_problems_of(b, box != "side")] - return out - - -def unknown_map_problems(b, base): - """[problems] of the family page whose son has his map as an image, with his results unreadable: 'Show the - rows' of a table opening the image, 'Show the map' not opening it, or (from a fresh page) 'Show the map' opening - a table.""" - family_page(b, base) - img = "!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')" - out = [] - if not open_box(b, "denovo"): - out.append("no de novo box") - if b.eval(img): - out.append("'Show the rows' opened the map too") - if not open_box(b, "map") or not b.eval(img): - out.append("'Show the map' did not open the map") - family_page(b, base) - if not open_box(b, "map") or not b.eval(img) or boxes_of(b) != ["denovo", "het", "side"] or son_tables(b): - out.append(f"'Show the map' opened another place, or not the map: closed {boxes_of(b)}, tables {son_tables(b)}") - return out - - -def map_problems(b, base): - """[problems] of the family page whose son has his map as an image and two closed results: the image shown - before both are opened from its box (one per click), or not shown after.""" - family_page(b, base) - img = "!!document.querySelector('details[data-child=\"DEMO-003\"] img.segments')" - out = [] - if b.eval(img) or boxes_of(b) != ["map", "denovo", "het", "side"]: - out.append(f"the map shown while a result is closed, or a place open: {boxes_of(b)}") - open_box(b, "map") - if b.eval(img) or boxes_of(b) != ["map", "denovo", "het", "side"]: - out.append(f"one click opened a place while the other result is closed: {boxes_of(b)}") - open_box(b, "map") - if not b.eval(img) or boxes_of(b): - out.append(f"the map not back once both results are opened: closed {boxes_of(b)}") - return out - - -def two_children_problems(b, base): - """[problems] of a family with a second child, DEMO-004, whose run has the son's results under the same ids: - opening every result of the son leaves each of DEMO-004's places closed.""" - family_page(b, base) - out = [] - if boxes_of(b, "DEMO-004") != SON_BOXES: - out.append(f"DEMO-004's closed places before: {boxes_of(b, 'DEMO-004')}") - for _ in range(2): - open_box(b, "denovo") - if boxes_of(b): - out.append(f"the son's places not opened: {boxes_of(b)}") - other = b.eval("document.querySelector('details[data-child=\"DEMO-004\"]').textContent") or "" - if boxes_of(b, "DEMO-004") != SON_BOXES or GRANDPARENT in other or planted_words()[1][0] in other: - out.append(f"opening the son's results opened DEMO-004's: {boxes_of(b, 'DEMO-004')}") - return out - - -def couple_problems(b, base): - """[problems] of a family where DEMO-001, a partner of the couple screen, is also a child (of an unsequenced - father) with a closed result in PRF1, a gene of the screen: the screen names it (its line's counts, its - table, its boxes, its report link, the lead) before the result is opened from its box, or not after.""" - family_page(b, base) - out = [] - sect = b.eval("var s = document.getElementById('ch-couples'); s ? s.textContent + ' ' + s.innerHTML : ''") or "" - lead = b.eval("(document.getElementById('family-lead') || {}).textContent || ''") or "" - links = b.eval("Array.prototype.map.call(document.querySelectorAll('a[href]'), function (a) { return a.getAttribute('href'); }).join(' ')") or "" - bx = boxes_of(b, "DEMO-001") or [] - if ("PRF1" in sect or "70601223" in sect or "CPT2" in sect or "couple_DEMO" in links - or "carrier screen stays closed in chapter III" not in lead): - out.append("the screen of a couple whose partner has a closed result named before it is opened") - if bx != ["couple:DEMO-001 DEMO-002", "side"]: - out.append(f"DEMO-001's closed places: {bx}") - if not open_box(b, "couple:DEMO-001 DEMO-002", "DEMO-001"): - return out + ["no closed box under the screen"] - sect = b.eval("document.getElementById('ch-couples').textContent") or "" - links = b.eval("Array.prototype.map.call(document.querySelectorAll('a[href]'), function (a) { return a.getAttribute('href'); }).join(' ')") or "" - if "PRF1" not in sect or "couple_DEMO" not in links or boxes_of(b, "DEMO-001"): - out.append(f"the screen not back once the result is opened: closed {boxes_of(b, 'DEMO-001')}") - return out - - -def nothing_closed_problems(b, base): - """[problems] of the family page whose son has nothing to act on (no veil): a box of his, or a place of his - closed: the painting's grandparent, his de novo and compound-het rows, his cells.""" - family_page(b, base) - kids = b.eval("document.documentElement.dataset.pgpKids || ''") or "" - shown = b.eval(FAMILY_SHOWN_JS) or "" - tables = son_tables(b) or [] - out = [] if re.search(r"\bDEMO-003:known\b", kids) else [f"the son's results never known: {kids!r}"] - if boxes_of(b) or GRANDPARENT not in shown or len(tables) != 3 or tables[1:] != [3, 3]: - out.append(f"a place closed with nothing to act on: closed {boxes_of(b)}, tables {tables}") - return out + side_problems_of(b, False) - - -def new_run_problems(b, base, fj, before, after): - """[problems]: with family.json BEFORE, the son's results opened from the page; then family.json AFTER (his run - in another folder) is fetched by the page itself, and his places must close again. The page asks every 30 s: - here every 0.5 s.""" - b.call("Page.enable", {}, b.session) # the script below runs only with the Page domain on - fast = b.call("Page.addScriptToEvaluateOnNewDocument", {"source": "(function () { var s = window.setInterval; " - "window.setInterval = function (f, t) { return s(f, t === 30000 ? 500 : t); }; }());"}, b.session) - try: - with open(fj, "w") as f: - f.write(before) - family_page(b, base) - out = [] - for _ in range(2): - open_box(b, "denovo") - if boxes_of(b): - out.append(f"the son's places not opened: {boxes_of(b)}") - with open(fj, "w") as f: - f.write(after) - moved = b.wait("var r = Array.prototype.some.call(document.querySelectorAll('a[href]'), function (a) { " - "return a.getAttribute('href').indexOf('-again/') >= 0; }); r && " - "/DEMO-003:known/.test(document.documentElement.dataset.pgpKids || '') && 'moved'", 10) - time.sleep(1.5) - if moved != "moved": - out.append("the page never fetched the moved run") - elif boxes_of(b) != SON_BOXES: - out.append(f"opened results stayed open for the son's new run: closed {boxes_of(b)}") - return out - finally: - b.call("Page.removeScriptToEvaluateOnNewDocument", {"identifier": (fast or {}).get("identifier", "")}, b.session) - b.call("Page.disable", {}, b.session) - - -def gene_cells(b, child): - """The values of CHILD's medicines and HLA cells in the side by side.""" - rows = side_rows(b) - members = b.eval("JSON.stringify(Array.prototype.map.call(document.querySelectorAll('#ch-side-by-side table.rows thead th'), " - "function (x) { return x.textContent; }).slice(2))") - members = json.loads(members) if members else [] - if child not in members: - return [] - i = members.index(child) - return [r[2 + i] for r in rows if r[0] in ("Medicines", "HLA")] - - -def unknown_couple_problems(b, base): - """[problems] of the couple fixture while the page cannot read DEMO-001's results: from a fresh page each, 'Show - the screen' opens the screen and leaves DEMO-001's cells closed, and 'Show the cells' opens the cells and leaves - the screen closed.""" - out, box = [], "couple:DEMO-001 DEMO-002" - screen = "!!document.querySelector('#ch-couples table.rows')" - for which in (box, "side"): - family_page(b, base) - if boxes_of(b, "DEMO-001") != [box, "side"] or b.eval(screen) or set(gene_cells(b, "DEMO-001")) != {"closed"}: - out.append(f"DEMO-001's places not all closed: {boxes_of(b, 'DEMO-001')}") - continue - if not open_box(b, which, "DEMO-001"): - out.append(f"no {which} box") - continue - cells = set(gene_cells(b, "DEMO-001")) - rest = [x for x in (box, "side") if x != which] - ok = (b.eval(screen) and cells == {"closed"}) if which == box else (not b.eval(screen) and cells and "closed" not in cells) - if boxes_of(b, "DEMO-001") != rest or not ok: - out.append(f"the {which} box opened another place, or not its own: closed {boxes_of(b, 'DEMO-001')}, cells {sorted(cells)[:3]}") - return out - - -def family_veil_checks(b, base, out): - """On the family page, while a child has a result to act on still closed, or the page cannot read the child's - results, every place that could carry one closes for that child, the same whichever result it is; each known - result opens with its own click, and each place opens on its own while the results cannot be read.""" - no_cache(b) - import make_demo_sample - fj = os.path.join(out, "runs", FAM, "family.json") - rj = os.path.join(out, "runs", SON, "results.json") - rj1 = os.path.join(out, "runs", S, "results.json") - fp = os.path.join(out, "assets", "family-page.js") - keep = {} - for p in (fj, fp, rj, rj1): - with open(p) as f: - keep[p] = f.read() - - def write(p, text): - with open(p, "w") as f: - f.write(text) - - def planted_control(desc, a, z, problems): - """Run PROBLEMS against family-page.js with A replaced by Z: it must find a problem.""" - if a not in keep[fp]: - check(f"control: {desc} (the planted line is in family-page.js)", False, a) - return - write(fp, keep[fp].replace(a, z)) - try: - control(desc, not problems()) - finally: - write(fp, keep[fp]) - try: - # A de novo candidate at the planted hit, and the father's copy over it painted from a grandparent. - fam = json.loads(keep[fj]) - c, pos, ref, alt = make_demo_sample.PLANTED["hit"] - fam["de_novo"].append(dict(fam["de_novo"][0], chrom=c, pos=str(pos), ref=ref, alt=alt)) - segs = fam["segments"][SON] - over = [s_ for s_ in segs if s_["chrom"] == c and s_["side"] == "paternal" and int(s_["start"]) <= pos <= int(s_["end"])] - away = [s_ for s_ in segs if s_["chrom"] != c and s_["side"] == "paternal"] - if len(over) != 1 or not away: - check("browser: the son's stretches hold one paternal stretch over the planted hit, and others", False, over) - return - over[0]["ancestor"] = GRANDPARENT - planted = json.dumps(fam) - over[0]["ancestor"] = "." - away[0]["ancestor"] = GRANDPARENT - elsewhere = json.dumps(fam) - write(fj, planted) - bad = family_problems(b, base) - check("browser: on the family page, while the son has a result to act on closed, his painting's grandparents, " - "his de novo and compound-het rows and his medicines and HLA cells close, one box each, naming no gene, " - "position or grandparent; each click opens one result, every place comes back once both are opened, and " - "a reload closes them", not bad, bad) - for desc, a, z in ( - ("de novo rows that ignore the veil", "var dshut = shutOf(c.child, 'denovo');", "var dshut = [];"), - ("compound-het rows that ignore the veil", "var hshut = shutOf(c.child, 'het');", "var hshut = [];"), - ("a painting that ignores the veil", "var shut = anc ? shutOf(child, 'paint') : [];", "var shut = [];"), - ("a closed painting that keeps each stretch's grandparent", "ancestor: '.', basis: 'parent' };", - "ancestor: r.ancestor, basis: r.basis };"), - ("a side by side whose cells ignore the veil", "closedCells[m] = true; return 'closed';", "closedCells[m] = true; return v;")): - planted_control(desc, a, z, lambda: family_closed(b, base)) - planted_control("a box that opens every result with one click", - "function () { opened[openKey(child, shut[0])] = true; redraw(); });", - "function () { shut.forEach(function (t) { opened[openKey(child, t)] = true; }); redraw(); });", - lambda: family_problems(b, base)) - - # The same page whichever result is closed and wherever it lies. - res = json.loads(keep[rj]) - fams = {"a grandparent over the hit": planted, "a grandparent elsewhere": elsewhere} - bad = invariance_problems(b, base, rj, res, fj, fams) - check("browser: the family page is the same, byte for byte, whichever of the son's results is closed (a pair " - "in KCNH2, RYR1, HLA-B, a gene alone, a position, a copy-number span in SMN1, a CHIP call in TP53) and " - "wherever a grandparent's stretch lies, each view settled on knowing his results", not bad, bad) - for desc, a, z in ( - ("de novo rows closed only when a result lies under one", "var dshut = shutOf(c.child, 'denovo');", - "var dshut = shutOf(c.child, 'denovo').filter(function (id) { return dn.some(function (r) { return " - "kids[c.child].spans.some(function (s) { return s[0] === id && s[2] === r.chrom && +r.pos >= s[3] && " - "+r.pos <= s[4]; }); }); });"), - ("a painting closed only when a grandparent's stretch lies over a closed result", - "var anc = all.some(function (r) { return r.ancestor && r.ancestor !== '.'; });", - "var anc = all.some(function (r) { return r.ancestor && r.ancestor !== '.' && ((kids[child] || {}).spans || [])" - ".some(function (s) { return !s[2] || (s[2] === r.chrom && s[3] <= +r.end && s[4] >= +r.start); }); });"), - ("a side by side that closes only the closed gene's cells", - "if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length) {", - "if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length && (kids[m].spans || []).some(function (s) { " - "return s[1] === r.label; })) {"), - ("a side by side whose cells stay open for a span wider than 10 kb (a copy-number region)", - "if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length) {", - "if (GENE_GROUPS[r.group] && shutBy[m] && shutBy[m].length && (kids[m].spans || []).every(function (s) { " - "return !(s[4] - s[3] > 10000); })) {"), - ("a page that never reads the son's results (every view the same, all closed as unreadable)", - "U.getJSON(url + 'results.json')", "U.getJSON(url + 'missing.json')")): - planted_control(desc, a, z, lambda: invariance_problems(b, base, rj, res, fj, fams)) - write(fj, planted) - - # The page cannot read the son's results: no answer, an unreadable answer, act findings without veil.spans. - os.replace(rj, rj + ".away") - try: - bad = family_unknown(b, base) - check("browser: with the son's results.json unavailable every place of his stays closed, and each box's " - "own button (the painting's, the de novo, the compound-het, the side by side's) opens its own place " - "and no other", not bad, bad) - planted_control("one opening shared by every box while the results cannot be read", - "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", - "return opened[openKey(child, 'box:denovo')] ? [] : [UNKNOWN];", - lambda: family_unknown(b, base)) - planted_control("'Show the grandparents' that opens the side by side while the results cannot be read", - "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", - "return opened[openKey(child, box === 'side' ? 'box:paint' : 'box:' + box)] ? [] : [UNKNOWN];", - lambda: family_unknown(b, base)) - finally: - os.replace(rj + ".away", rj) - write(rj, json.dumps({"schema_version": 1, "sample": SON, "chapters": [], - "note": "The results could not be read (KeyError)."})) - bad = family_unknown(b, base, opens=False) - check("browser: with the son's results answered as unreadable (a note, no findings) every place of his stays " - "closed", not bad, bad) - write(rj, keep[rj][:5000]) - bad = family_unknown(b, base, opens=False) - check("browser: with the son's results.json cut short (a request that fails) every place of his stays closed", - not bad, bad) - planted_control("a failed request read as nothing closed", "cur.status = 'failed';", - "cur.status = 'known'; cur.spans = null; cur.acts = [];", lambda: family_unknown(b, base, opens=False)) - for desc, spans in (("[null]", [None]), ("[] while he has findings to act on", [])): - bad_spans = json.loads(keep[rj]) - bad_spans["veil"]["spans"] = spans - write(rj, json.dumps(bad_spans)) - bad = family_unknown(b, base, opens=False) - check(f"browser: with the son's veil.spans malformed ({desc}) every place of his stays closed", not bad, bad) - planted_control(f"malformed veil.spans ({desc}) read as given", "spans: spansOk ? v.spans : null", - "spans: Array.isArray(v.spans) ? v.spans : null", lambda: family_unknown(b, base, opens=False)) - clean = json.loads(keep[rj]) - clean["findings"] = [x for x in clean["findings"] if x.get("level") != "act"] - clean.pop("veil", None) - write(rj, json.dumps(clean)) - bad = nothing_closed_problems(b, base) - check("browser: with nothing of the son's to act on (no veil) every place of his is open, with no box", not bad, bad) - planted_control("results without veil.spans read as closed even with nothing to act on", - "if (!k || k.status !== 'known' || (!k.spans && k.acts.length)) {", - "if (!k || k.status !== 'known' || !k.spans) {", lambda: nothing_closed_problems(b, base)) - nospans = json.loads(keep[rj]) - nospans.pop("veil", None) - write(rj, json.dumps(nospans)) - bad = family_unknown(b, base, opens=False) - check("browser: with the son's results holding findings to act on but no veil.spans every place of his stays " - "closed", not bad, bad) - planted_control("results without veil.spans read as nothing closed", - "if (!k || k.status !== 'known' || (!k.spans && k.acts.length)) {", - "if (!k || k.status !== 'known') {", lambda: family_unknown(b, base, opens=False)) - write(rj, keep[rj]) - - # A second child whose run holds the same results under the same ids: opening is per child. - fam2 = json.loads(planted) - fam2["state"]["children"].append(dict(fam2["state"]["children"][0], child="DEMO-004")) - fam2["runs"]["DEMO-004"] = fam2["runs"][SON] - fam2["segments"]["DEMO-004"] = fam2["segments"][SON] - fam2["de_novo"] += [dict(r, child="DEMO-004") for r in fam2["de_novo"] if r["child"] == SON] - fam2["compound_het"] += [dict(r, child="DEMO-004") for r in fam2["compound_het"] if r["child"] == SON] - fam2["side_by_side"]["members"].append("DEMO-004") - for r in fam2["side_by_side"]["rows"]: - r["values"].append(r["values"][fam2["side_by_side"]["members"].index(SON)]) - if r.get("stale"): - r["stale"].append(False) - write(fj, json.dumps(fam2)) - bad = two_children_problems(b, base) - check("browser: with two children holding the same results, opening each of the son's leaves the other " - "child's places closed", not bad, bad) - planted_control("an opened result kept per run only, not per child", - "function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; }", - "function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + id; }", - lambda: two_children_problems(b, base)) - write(fj, planted) - - # The son's run moves to another folder (same results) while the page is open: what was opened closes again. - again = os.path.join(out, "runs", SON + "-again") - shutil.copytree(os.path.join(out, "runs", SON), again) - moved = json.loads(planted) - moved["runs"][SON]["url"] = moved["runs"][SON]["url"].replace(f"/{SON}/", f"/{SON}-again/") - try: - bad = new_run_problems(b, base, fj, planted, json.dumps(moved)) - check("browser: results the reader opened close again when the son's run changes while the page is open", - not bad, bad) - planted_control("an opened result kept per child only, not per run", - "function openKey(child, id) { return (kids[child] && kids[child].url) + '|' + child + '|' + id; }", - "function openKey(child, id) { return child + '|' + id; }", - lambda: new_run_problems(b, base, fj, planted, json.dumps(moved))) - finally: - shutil.rmtree(again, ignore_errors=True) - write(fj, planted) - - # A partner of the couple screen who is also a child, with a closed result in a gene of the screen. - famc = json.loads(keep[fj]) - famc["state"]["members"].insert(0, {"sample": GRANDPARENT, "sex": "male", "father": None, "mother": None, - "partner": None, "sequenced": False}) - for m in famc["state"]["members"]: - if m["sample"] == S: - m["father"] = GRANDPARENT - famc["state"]["children"].insert(0, {"child": S, "father": GRANDPARENT, "mother": None, "segments": None}) - rep = f"couple_{S}_{OTHERS[0]}.md" - famc["state"]["couples"][0]["report"] = rep - famc["state"]["reports"] = famc["state"].get("reports", []) + [{"name": rep, "url": rep}] - write(fj, json.dumps(famc)) - r1 = json.loads(keep[rj1]) - r1.setdefault("veil", {})["spans"] = [["two-hits-prf1", "PRF1", None, None, None]] - write(rj1, json.dumps(r1)) - bad = couple_problems(b, base) - check("browser: a couple screen whose partner is a child with a closed result stays closed whole (counts, " - "table, boxes, report link, the lead's words) until that result is opened from its box", not bad, bad) - planted_control("a couple screen that ignores the veil", "var cs = coupleShut(f, c);", "var cs = [];", - lambda: couple_problems(b, base)) - couple_cases = (("PRF1, a gene of the screen", [["two-hits-prf1", "PRF1", None, None, None]]), - ("RYR1, a gene the screen does not name", [["sf-ryr1", "RYR1", "chr19", 38440000, 38440000]])) - bad = invariance_problems(b, base, rj1, json.loads(keep[rj1]), fj, {"the couple": json.dumps(famc)}, - couple_cases, S) - check("browser: the family page is the same, byte for byte, whether the partner's closed result is in a gene " - "of the screen or not", not bad, bad) - planted_control("a couple screen closed only when the closed result is in one of its genes", - "var cs = coupleShut(f, c);", - "var cs = coupleShut(f, c).filter(function (x) { return (kids[x[0]].spans || []).some(function (s) { " - "return ((f.couples || {})[c.a + ' ' + c.b] || []).some(function (r) { return r.gene === s[1]; }); }); });", - lambda: invariance_problems(b, base, rj1, json.loads(keep[rj1]), fj, {"the couple": json.dumps(famc)}, - couple_cases, S)) - os.replace(rj1, rj1 + ".away") - try: - bad = unknown_couple_problems(b, base) - check("browser: with the partner's results unavailable, 'Show the screen' opens the screen and leaves the " - "partner's cells closed, and 'Show the cells' opens the cells and leaves the screen closed", not bad, bad) - planted_control("a couple screen that 'Show the cells' opens while the results cannot be read", - "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", - "return opened[openKey(child, box.indexOf('couple:') === 0 ? 'box:side' : 'box:' + box)] ? [] : [UNKNOWN];", - lambda: unknown_couple_problems(b, base)) - finally: - os.replace(rj1 + ".away", rj1) - write(rj1, keep[rj1]) - - # The drawing as an image (no stretches in family.json). - fam = json.loads(keep[fj]) - fam["segments"] = {} - fam["state"]["children"][0]["segments"] = fam["state"]["pedigree"] - write(fj, json.dumps(fam)) - bad = map_problems(b, base) - check("browser: the son's map as an image stays closed while a result is, and its box opens one result per " - "click, the map once both are opened", not bad, bad) - planted_control("a map image that ignores the veil", "var shut = shutOf(c.child, 'map');", "var shut = [];", - lambda: map_problems(b, base)) - mapfam = {"the map as an image": json.dumps(fam)} - bad = invariance_problems(b, base, rj, json.loads(keep[rj]), fj, mapfam) - check("browser: with the son's map as an image the family page is the same, byte for byte, whichever of his " - "results is closed", not bad, bad) - planted_control("a map image closed only when the closed result has a position", - "var shut = shutOf(c.child, 'map');", - "var shut = shutOf(c.child, 'map').filter(function (id) { return (kids[c.child].spans || []).some(" - "function (s) { return s[0] === id && s[2]; }); });", - lambda: invariance_problems(b, base, rj, json.loads(keep[rj]), fj, mapfam)) - write(fj, json.dumps(fam)) - os.replace(rj, rj + ".away") - try: - bad = unknown_map_problems(b, base) - check("browser: with the son's results unavailable, 'Show the rows' leaves his map as an image closed, and " - "'Show the map' opens it", not bad, bad) - planted_control("a map image that 'Show the rows' opens", - "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", - "return opened[openKey(child, 'box:denovo')] ? [] : [UNKNOWN];", - lambda: unknown_map_problems(b, base)) - planted_control("de novo rows that 'Show the map' opens", - "return opened[openKey(child, 'box:' + box)] ? [] : [UNKNOWN];", - "return opened[openKey(child, box === 'denovo' ? 'box:map' : 'box:' + box)] ? [] : [UNKNOWN];", - lambda: unknown_map_problems(b, base)) - finally: - os.replace(rj + ".away", rj) - finally: - for p, text in keep.items(): - write(p, text) - - -# What a reader of the son's page gets before any click: the whole document (every text and every attribute, the -# map's timing attributes aside), the address, where the page is scrolled to and what has the focus, and a hash of -# the map's canvas. -PAGE_JS = r"""JSON.stringify((function () { - var c = document.documentElement.cloneNode(true); - Array.prototype.forEach.call(c.querySelectorAll('[data-pgp-map-parse-ms],[data-pgp-map-paint-ms]'), function (e) { - e.removeAttribute('data-pgp-map-parse-ms'); e.removeAttribute('data-pgp-map-paint-ms'); }); - // The root's data attributes come from scripts that answer in any order: sort them by name. - var at = Array.prototype.map.call(c.attributes, function (x) { return [x.name, x.value]; }).sort(); - at.forEach(function (x) { c.removeAttribute(x[0]); }); - at.forEach(function (x) { c.setAttribute(x[0], x[1]); }); - var cv = document.querySelector('.gm-canvas'), a = document.activeElement; - return { html: c.outerHTML, href: location.href + ' scroll ' + Math.round(window.scrollY) + ' focus ' + - (a ? a.tagName + '#' + a.id : ''), canvas: cv ? cv.toDataURL() : '' }; -}()))""" - - -def page_now(b, base, hash_="", typed=False): - """The son's page before any click, once its results, its charts and its map have drawn: (lines of its HTML, - its address with its scroll position and focus, its canvas hash). TYPED: the page loads without HASH_, which is - then set as a reader types it into the address bar (a hashchange); its canvas is left out then (a hashchange - can move the map's camera).""" - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/{'' if typed else hash_}") - # The son's family answers on its own request: wait for its name in the kicker too, or a pair can differ by it. - b.wait("var d = document.documentElement.dataset; d.pgpResults && d.pgpCharts && d.pgpStatus === 'ready' && (" + MAP_READY - + ") && document.getElementById('kicker').textContent.indexOf('Family') >= 0 && 'ready'", 15) - if typed: - b.eval(f"location.hash = {json.dumps(hash_)}; true") - time.sleep(0.8) # the margin follows the camera 140 ms after it settles - got = json.loads(b.eval(PAGE_JS) or "{}") - return (got.get("html") or "").replace("><", ">\n<").split("\n"), got.get("href") or "", \ - "" if typed else hashlib.sha256((got.get("canvas") or "").encode()).hexdigest()[:12] - - -def plant_summary(out, og, change): - """The son's results.json built again by live_results from his summary.json with CHANGE(sections) applied, and - his map.json and rendered charts built again from that same summary. Returns what to write back afterwards.""" - son = OTHERS[1] - rj, mj, cj = (os.path.join(out, "runs", son, n) for n in ("results.json", "map.json", f"{son}_charts.json")) - keep = {} - for p in (rj, mj, cj): - with open(p, "rb") as fh: - keep[p] = fh.read() - root = os.path.join(os.path.dirname(out), "root") - with open(os.path.join(root, son, "summary.json")) as fh: - summary = json.load(fh) - change(summary.setdefault("sections", {})) - res = live_results.build(son, summary) - with open(rj, "w") as fh: - json.dump(res, fh) - with open(cj, "w") as fh: - json.dump(live_results.rr.charts_json(summary), fh) - doc = build_live_demo.og_map(og, ["build", "--root", root, "--run", son, "--assets", os.path.join(REPO, "assets"), - "--cache-dir", os.path.join(root, ".owngenome-live", "map")], - json.dumps({"results": res}).encode()) - with open(mj, "wb") as fh: - fh.write(doc) - return keep - - -def cpsr(gene): - """A CPSR result to act on in GENE, the same variant whatever the gene: a closed result known by its gene.""" - def change(S): - S["cpsr"] = {"state": "ok", "data": {"call_floor_dp": 10, "low_confidence": 0, "plp": [ - {"gene": gene, "variant": "c.68_69del", "final": "Pathogenic", "clinvar": "Pathogenic"}]}} - return change - - -def deep_sites(S): - """The ClinVar card's line for each known deep intronic site of assets/deep_intronic_sites.tsv, reference calls: - a fixed panel the son's demo summary leaves out.""" - with open(os.path.join(REPO, "assets", "deep_intronic_sites.tsv")) as fh: - rows = [ln.rstrip("\n").split("\t") for ln in fh if ln.strip() and not ln.startswith("#")] - S["clinvar"]["data"]["deep_intronic"] = [dict(zip(rows[0], r), call="ref", detail="GQ 50") for r in rows[1:]] - return len(rows) - 1 - - -def with_sites(gene): - """cpsr(GENE) with the known deep intronic sites' lines (deep_sites), the same on both pages.""" - def change(S): - cpsr(gene)(S) - deep_sites(S) - return change - - -def sv_over(locus, genes, carrier=True): - """A class 5 deletion over GENES, and (CARRIER) a carrier row that names only its second gene.""" - def change(S): - rows = S["annotsv"]["data"]["rows"] - S["annotsv"]["data"]["rows"] = rows + [{"sv": locus, "type": "DEL", "gt": "0/1", "length": "-100000", - "acmg_class": "5", "genes": genes, "caller": "Manta", "support": 1, - "support_callers": ["Manta"]}] - if not carrier: - return - S["carrier"]["data"]["carriers"] = [{"gene": genes[1], "variant": "c.100C>T", "genotype": "het", - "significance": "Pathogenic", "stars": 2, "inheritance": "AR"}] - return change - - -def recessive(change): - """CHANGE, plus two recessive carrier rows (CFTR and HBB, inheritance AR) on the carrier card.""" - def go(S): - change(S) - S["carrier"]["data"]["carriers"] = S["carrier"]["data"]["carriers"] + [ - {"gene": g, "variant": v, "genotype": "het", "significance": "Pathogenic", "stars": 2, "inheritance": "AR", - "condition": c} for g, v, c in (("CFTR", "c.1521_1523del", "Cystic fibrosis"), - ("HBB", "c.20A>T", "Sickle cell disease"))] - return go - - -def insertion(change): - """CHANGE, plus a class 4 insertion (type INS) over NKD2: a review finding with a place on the map.""" - def go(S): - change(S) - S["annotsv"]["data"]["rows"] = S["annotsv"]["data"]["rows"] + [ - {"sv": "chr5:1000000-1000300", "type": "INS", "gt": "0/1", "length": 300, "acmg_class": "4", - "genes": ["NKD2"], "caller": "Manta", "support": 1, "support_callers": ["Manta"]}] - return go - - -def second_hit(S): - """A second ClinVar row in the gene of the son's planted secondary finding, the card's counts kept.""" - S["clinvar"]["data"]["hits"] = S["clinvar"]["data"]["hits"] + [dict( - [h for h in S["clinvar"]["data"]["hits"] if h.get("gene") == "KCNH2"][0], pos=150970000)] - - -# Pairs of son's pages that differ only in which result is closed (or in what only a closed result names): the -# page before any click must be the same, byte for byte. The two sides of each pair differ in the thing it guards: -# a gene that a fixed-panel card's own finding names (the medicine card's CYP2D6 and NAT2) against one no card names, -# a gene of the carrier list's two-hit line (SMN1) against one it does not name, a gene of the callability card's -# list of genes not on this reference (GSTT1) or of hard genes not covered (NBPF1) against one not on it, a gene -# spelled like a word or unit the report prints (HR, MB) or like a code it prints (AR as an inheritance, INS as a -# structural-variant type, TG in a score's name) against one that is not, and one guessed address on two -# pages where it is a closed result's id on one side only, loaded with the page and typed after it. -GUESS = "#f/cpsr-brca1-c-68-69del" -INFER = (("two SNV results in different genes (BRCA1, BRCA2)", cpsr("BRCA1"), cpsr("BRCA2")), - ("a deletion over BRCA1 and NBR2 against one over CFTR and CTTNBP2", - sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"]), sv_over("chr7:117480025-117580025", ["CFTR", "CTTNBP2"])), - ("SMN1 against HBA1, the same copy numbers", cpsr("SMN1"), cpsr("HBA1")), - ("a result with a place on the map (BRCA2) against one without (MLH1)", cpsr("BRCA2"), cpsr("MLH1")), - ("a closed result with one ClinVar row against two", lambda S: None, second_hit), - ("a closed result in a gene of a known deep intronic site (ATM) against one in none (BRCA2)", with_sites("ATM"), - with_sites("BRCA2")), - ("a link to each of two closed results (#f/)", cpsr("BRCA1"), cpsr("BRCA2"), - ("#f/cpsr-brca1-c-68-69del", "#f/cpsr-brca2-c-68-69del")), - ("a gene of the medicine card's own finding (CYP2D6) against the other one (NAT2)", cpsr("CYP2D6"), cpsr("NAT2")), - ("a gene of the medicine card's own finding (CYP2D6) against one no card names (BRCA2)", cpsr("CYP2D6"), - cpsr("BRCA2")), - ("a gene the carrier list's two-hit line names (SMN1) against one it does not (BRCA2)", cpsr("SMN1"), - cpsr("BRCA2")), - ("a gene not on this reference (GSTT1) against one on it (BRCA2)", cpsr("GSTT1"), cpsr("BRCA2")), - ("a hard gene the reads did not cover (NBPF1) against one they did (BRCA2)", cpsr("NBPF1"), cpsr("BRCA2")), - ("a gene spelled like a word of the report (HR) against one that is not (BRCA2)", cpsr("HR"), - cpsr("BRCA2")), - ("a gene spelled like a unit of the report (MB) against one that is not (BRCA2)", cpsr("MB"), cpsr("BRCA2")), - ("an inheritance code (AR) against a gene (BRCA2), with recessive carrier rows", recessive(cpsr("AR")), - recessive(cpsr("BRCA2"))), - ("a deletion over AR and OPHN1 against one over BRCA1 and NBR2, with recessive carrier rows", - recessive(sv_over("chrX:67500000-67700000", ["AR", "OPHN1"], False)), - recessive(sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"], False))), - ("a deletion over AR and OPHN1 against one over BRCA1 and NBR2, each with a carrier row in its second gene " - "(OPHN1 has no place on the map, NBR2 has one)", recessive(sv_over("chrX:67500000-67700000", ["AR", "OPHN1"])), - recessive(sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"]))), - ("a structural-variant type (INS) against a gene (BRCA2), with an insertion over NKD2", insertion(cpsr("INS")), - insertion(cpsr("BRCA2"))), - ("a trait's abbreviation in a score's name (TG) against a gene (BRCA2)", cpsr("TG"), cpsr("BRCA2")), - ("one guessed address, a closed result's id on one page only (BRCA1 against BRCA2 closed)", cpsr("BRCA1"), - cpsr("BRCA2"), (GUESS, GUESS)), - ("the same guessed address typed after the page drew (BRCA1 against BRCA2 closed)", cpsr("BRCA1"), - cpsr("BRCA2"), (GUESS, GUESS), True)) - - -def infer_problems(b, base, out, og, cases=INFER): - """[problems]: for each pair of CASES, the son's page before any click differs (its HTML, an attribute, the - address or the map's canvas), with the number of differing lines and the first ones.""" - bad = [] - for what, one, two, *more in cases: - got = [] - for change, h in zip((one, two), (more or [("", "")])[0]): - keep = plant_summary(out, og, change) - try: - got.append(page_now(b, base, h, typed=more[1:] == [True])) - finally: - unplant(keep) - (h1, a1, c1), (h2, a2, c2) = got - diff = [x for x in difflib.unified_diff(h1, h2, lineterm="", n=0) if x[:1] in "+-" and x[:3] not in ("+++", "---")] - if diff or a1 != a2 or c1 != c2: - bad.append(f"{what}: {len(diff)} lines differ, address {a1 == a2}, canvas {c1 == c2}: {diff[:6]}") - return bad - - -DEEP_JS = """(function () { - var el = document.getElementById('f-closed-1'), r = el && el.getBoundingClientRect(); - var sec = el && el.closest('section.sec'), a = sec && document.querySelector('[data-toc="' + sec.id.slice(1) + '"]'); - return JSON.stringify({ found: !!el, y: Math.round(window.scrollY), inview: !!r && r.bottom > 0 && r.top < innerHeight, - current: a ? a.getAttribute('aria-current') : null }); -}())""" - - -def deep_link_problems(b, base): - """[problems]: on the son's page, #f/closed-1 and a guessed #f/ each scroll to the first closed result's card, - put it in view and mark its section in the contents.""" - bad = [] - for h in ("#f/closed-1", GUESS): - page_now(b, base, h) - got = json.loads(b.eval(DEEP_JS) or "{}") - if not (got.get("found") and got.get("y", 0) > 0 and got.get("inview") and got.get("current") == "true"): - bad.append(f"{h}: {got}") - return bad - - -def sv_gene_problems(b, base, out, og): - """[problems]: with a closed deletion over BRCA1 and NBR2, the page names NBR2 nowhere (a carrier row and its - finding name only NBR2) before any click.""" - keep = plant_summary(out, og, sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"])) - try: - html = "\n".join(page_now(b, base)[0]) - finally: - unplant(keep) - return [f"NBR2 on the page: ...{html[max(0, html.find('NBR2') - 120):html.find('NBR2') + 40]}..."] if "NBR2" in html else [] - - -def panel_words_problems(b, base, out, og): - """[problems]: with a closed result in CYP2D6, a gene the medicine card's own finding names, the medicine card shows - the fixed-panel box's words, never 'This section holds a result'.""" - keep = plant_summary(out, og, cpsr("CYP2D6")) - try: - page_now(b, base) - text = b.eval("(document.querySelector('#card-cpic .card-text') || {}).textContent || ''") - finally: - unplant(keep) - ok = "This section lists the same genes for everyone" in text and "This section holds" not in text - return [] if ok else [f"the medicine card says {text[:160]!r}"] - - -ADDRESS_JS = ("window.__seen = window.__seen || []; if (!window.__heard) { window.__heard = 1; ['hashchange', 'popstate']" - ".forEach(function (e) { window.addEventListener(e, function () { window.__seen.push(e + ' ' + location.hash); }); " - "}); } true") - - -def address_problems(b, base, out): - """[problems]: on the son's page, a result opened from its card and the reader's moves after it never put the - id of a result his page can close into the address or a history entry: #f/closed-1, Show the result, a link to - section 3, Back.""" - with open(os.path.join(out, "runs", OTHERS[1], "results.json")) as fh: - res = json.load(fh) - tags = (res.get("veil") or {}).get("findings") or {} - ids = [f["id"] for f in res["findings"] if f["level"] == "act" or f["id"] in tags] - seen = [] - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/#f/closed-1") - b.wait("document.documentElement.dataset.pgpResults && document.documentElement.dataset.pgpStatus === 'ready' && " - "document.querySelector('.finding[data-level=act] .veil button') && 'ready'", 15) - b.eval(ADDRESS_JS) - seen.append(b.eval("location.hash")) - b.eval("document.querySelector('.finding[data-level=act] .veil button').click(); true") - time.sleep(0.3) - seen.append(b.eval("location.hash")) - b.eval("location.hash = '#s3'; true") - time.sleep(0.3) - b.eval("history.back(); true") - time.sleep(0.5) - seen += [b.eval("location.hash")] + json.loads(b.eval("JSON.stringify(window.__seen)") or "[]") - bad = [h for h in seen if any(i in (h or "") for i in ids)] - if not str(seen[1]).startswith("#f/opened-"): - bad.append(f"the opened result's address: {seen[1]!r}, want #f/opened-") - return [f"addresses {seen}: {bad}"] if bad else [] - - -def home_now(b, base, out, change): - """The home page with the son's view (runs.json, live_results.overview) built from his summary with CHANGE applied: - the whole document.""" - p = os.path.join(out, "runs.json") - with open(p, "rb") as fh: - keep = fh.read() - with open(os.path.join(os.path.dirname(out), "root", OTHERS[1], "summary.json")) as fh: - summary = json.load(fh) - change(summary.setdefault("sections", {})) - idx = json.loads(keep) - idx["runs"] = [dict(r, **live_results.overview(summary)) if r["name"] == OTHERS[1] else r for r in idx["runs"]] - try: - with open(p, "w") as fh: - json.dump(idx, fh) - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/") - b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.documentElement.dataset.pgpPeople " - "&& document.querySelector('#attention li') && 'ready'", 15) - time.sleep(0.3) - return b.eval("document.documentElement.outerHTML").replace("><", ">\n<").split("\n") - finally: - with open(p, "wb") as fh: - fh.write(keep) - - -def renamed(gene): - """The son's summary with his planted gene (KCNH2) named GENE everywhere: his first result to act on, the one the - home page lists, is then in GENE.""" - def change(S): - S.update(json.loads(json.dumps(S).replace("KCNH2", gene))) - return change - - -HOME = (("his first result in BRCA1 against BRCA2", renamed("BRCA1"), renamed("BRCA2")), - ("a gene of the medicine card's own finding (CYP2D6) against BRCA2", cpsr("CYP2D6"), cpsr("BRCA2"))) - - -def home_problems(b, base, out, cases=HOME): - """[problems]: the home page is the same whichever of the son's results is closed (his tile and the list of what - needs attention: no link to a closed result's own page, whose address names its gene).""" - bad = [] - for what, one, two in cases: - h1, h2 = home_now(b, base, out, one), home_now(b, base, out, two) - links = json.loads(b.eval("JSON.stringify(Array.from(document.querySelectorAll('#attention a')).filter(function (a) { var who = a.querySelector('.who .pc'); return who && who.textContent === " + json.dumps(SON) + "; }).map(function (a) { return a.getAttribute('href'); }))") or "[]") - if any("finding/" in link for link in links): - bad.append(f"home, {what}: a closed summary links to a finding page") - diff = [x for x in difflib.unified_diff(h1, h2, lineterm="", n=0) if x[:1] in "+-" and x[:3] not in ("+++", "---")] - if diff: - bad.append(f"home, {what}: {len(diff)} lines differ: {diff[:4]}") - return bad - - -def two_hits(change): - """CHANGE, plus a two-hit gene in CFTR on the carrier list (a result to act on, two-hits-cftr), whose line prints - the inheritance code AR.""" - def go(S): - change(S) - th = S["carrier"]["data"].setdefault("two_hits", {"deletions_read": [], "deletions_not_read": []}) - th["genes"] = [{"gene": "CFTR", "verdict": "possible", "condition": "Cystic fibrosis", - "deletion": "DEL chr7:117480025-117580025", "variant": "c.1521_1523del", "genotype": "het", - "why": "a deletion and a variant on the other copy"}] - return go - - -def cpsrs(*genes): - """A closed CPSR result in each of GENES, the same variant in each.""" - def change(S): - S["cpsr"] = {"state": "ok", "data": {"call_floor_dp": 10, "low_confidence": 0, "plp": [ - {"gene": g, "variant": "c.68_69del", "final": "Pathogenic", "clinvar": "Pathogenic"} for g in genes]}} - return change - - -def dup_over(change, locus, genes): - """CHANGE, plus a class 4 duplication over GENES (a finding worth a look, with a place on the map).""" - def go(S): - change(S) - S["annotsv"]["data"]["rows"] = S["annotsv"]["data"]["rows"] + [ - {"sv": locus, "type": "DUP", "gt": "0/1", "length": 300000, "acmg_class": "4", "genes": genes, - "caller": "Manta", "support": 1, "support_callers": ["Manta"]}] - return go - - -def cid(gene): - """The id of cpsr(GENE)'s result.""" - return f"cpsr-{gene.lower()}-c-68-69del" - - -# Pairs of the son's pages that differ only in which result is closed, compared after the same click: the result -# each side opens through its handle (its place among the closed cards, as a reader who never saw its id opens it). -# A place tied to a closed result must open with the results it names and with nothing else, so the page after a -# click is again a function of the open results and the count of closed ones. The two-hit line prints the -# inheritance code AR: a closed result in AR (a CPSR result, or a deletion over AR) must not hold it closed. -CLICKS = (("a two-hit result in CFTR opened, the other closed result in AR against BRCA2", two_hits(cpsr("AR")), - two_hits(cpsr("BRCA2")), ("two-hits-cftr", "two-hits-cftr")), - ("a two-hit result in CFTR opened, a closed deletion over AR and OPHN1 against one over BRCA1 and NBR2", - two_hits(sv_over("chrX:67500000-67700000", ["AR", "OPHN1"], False)), - two_hits(sv_over("chr17:43044295-43144295", ["BRCA1", "NBR2"], False)), ("two-hits-cftr", "two-hits-cftr"))) - - -def opened_page(b, base, fid): - """The son's page after FID is opened through its handle: (lines of its HTML, its address, its canvas hash).""" - page_now(b, base) - if not open_closed(b, fid): - return None - time.sleep(0.8) - got = json.loads(b.eval(PAGE_JS) or "{}") - return (got.get("html") or "").replace("><", ">\n<").split("\n"), got.get("href") or "", \ - hashlib.sha256((got.get("canvas") or "").encode()).hexdigest()[:12] - - -def click_problems(b, base, out, og, cases=CLICKS): - """[problems]: for each pair of CASES, the son's page after each side opens its result differs (its HTML, an - attribute, the address or the map's canvas), with the number of differing lines and the first ones.""" - bad = [] - for what, one, two, fids in cases: - got = [] - for change, fid in zip((one, two), fids): - keep = plant_summary(out, og, change) - try: - got.append(opened_page(b, base, fid)) - finally: - unplant(keep) - if None in got: - bad.append(f"{what}: a side has no closed card for {fids}") - continue - (h1, a1, c1), (h2, a2, c2) = got - diff = [x for x in difflib.unified_diff(h1, h2, lineterm="", n=0) if x[:1] in "+-" and x[:3] not in ("+++", "---")] - if diff or a1 != a2 or c1 != c2: - bad.append(f"{what}: {len(diff)} lines differ, address {a1 == a2}, canvas {c1 == c2}: {diff[:6]}") - return bad - - -# Two closed results, one opened (each in turn): the other's gene must be on the page no more often than before the -# click (the search box's example names BRCA2 on every page). A place that names both (the lead's segment of the -# findings worth a look, which prints the ClinVar findings in DEMOC and DEMOA; a duplication over BRCA1 and BRCA2) -# stays closed while either is. -TWO_CLOSED = (("two CPSR results named in one segment of the lead, DEMOC opened", cpsrs("DEMOC", "DEMOA"), cid("DEMOC"), - "DEMOA"), - ("the same, DEMOA opened", cpsrs("DEMOC", "DEMOA"), cid("DEMOA"), "DEMOC"), - ("a class 4 duplication over BRCA1 and BRCA2, both closed, BRCA1 opened", - dup_over(cpsrs("BRCA1", "BRCA2"), "chr5:1000000-1300000", ["BRCA1", "BRCA2"]), cid("BRCA1"), "BRCA2"), - ("the same, BRCA2 opened", dup_over(cpsrs("BRCA1", "BRCA2"), "chr5:1000000-1300000", ["BRCA1", "BRCA2"]), - cid("BRCA2"), "BRCA1")) - - -def two_closed_problems(b, base, out, og, cases=TWO_CLOSED): - """[problems]: for each of CASES, the still closed result's gene is on the page more often after the click.""" - bad = [] - for what, change, fid, still in cases: - keep = plant_summary(out, og, change) - try: - before = "\n".join(page_now(b, base)[0]).count(still) - after = opened_page(b, base, fid) - finally: - unplant(keep) - if after is None: - bad.append(f"{what}: no closed card for {fid}") - continue - html = "\n".join(after[0]) - if html.count(still) > before: - at = html.find(still, 0) - bad.append(f"{what}: {still} {before} times before, {html.count(still)} after: " - f"...{html[max(0, at - 160):at + 40]}...") - return bad - - -def apoe_pair_problems(b, base, out, og): - """Opening the map's own gate cannot say which other result is still closed.""" - got, bad = [], [] - for gene in ("APOE", "BRCA2"): - keep = plant_summary(out, og, cpsr(gene)) - try: - stages = [page_now(b, base)] - if not b.eval("(function () { var c = document.querySelector('.gm-chip[data-closed=gate]'); " - "if (!c) { return false; } c.querySelector('button').click(); " - "document.querySelector('.gm-chip[data-closed=gate] .gm-yes').click(); return true; }())"): - return [f"{gene}: no APOE gate"] - time.sleep(0.2) - stages.append(page_snapshot(b)) - if not b.eval("!!document.querySelector('.gm-marks [data-id=apoe]')"): - bad.append(f"{gene}: opening APOE did not draw its mark") - if not b.eval("(function () { var c = Array.from(document.querySelectorAll('.gm-chip.opened')).find(" - "function (e) { return e.textContent.indexOf('Opened: APOE') === 0; }); " - "if (!c) { return false; } c.querySelector('button').click(); return true; }())"): - return bad + [f"{gene}: no close button for APOE"] - time.sleep(0.2) - stages.append(page_snapshot(b)) - if b.eval("!!document.querySelector('.gm-marks [data-id=apoe]')"): - bad.append(f"{gene}: closing APOE kept its mark") - got.append(stages) - finally: - unplant(keep) - for stage, a, z in zip(("before", "opened", "closed again"), *got): - if a != z: - bad.append(f"APOE against BRCA2 {stage}: HTML {a[0] == z[0]}, address {a[1] == z[1]}, canvas {a[2] == z[2]}") - return bad - - -def page_snapshot(b): - """The same complete document, address and canvas comparison used by page_now, after a click.""" - got = json.loads(b.eval(PAGE_JS) or "{}") - return (got.get("html") or "").replace("><", ">\n<").split("\n"), got.get("href") or "", \ - hashlib.sha256((got.get("canvas") or "").encode()).hexdigest()[:12] - - -def related_set_problems(b, base, out, og): - """On each finding's own page, opening one of two results cannot open a related finding naming both.""" - made, bad = [], [] - folder = os.path.join(out, "runs", SON, "finding") - # The demo server is static: give the planted findings the same page template as its original findings. - original = "sf-kcnh2-chr7-150960000-c-t" - with open(os.path.join(folder, original, "index.html")) as fh: - template = fh.read() - try: - for genes in (["BRCA1", "BRCA2"], ["BRCA2", "BRCA1"]): - # Related links use the finding's first gene. Reverse that as well as the CPSR order, so both - # directions can actually list the duplication and expose a first-id reader. - gene, fid = genes[0], cid(genes[0]) - keep = plant_summary(out, og, dup_over(cpsrs(*genes), "chr5:1000000-1300000", genes)) - try: - with open(os.path.join(out, "runs", SON, "results.json")) as fh: - finding = next(f for f in json.load(fh)["findings"] if f["id"] == fid) - if "sv-chr5-1000000-1300000" not in finding.get("related", []): - bad.append(f"{gene}: fixture never offers the duplication as a related result") - dest = os.path.join(folder, fid) - os.makedirs(dest) - made.append(dest) - with open(os.path.join(dest, "index.html"), "w") as fh: - fh.write(template.replace(original, fid)) - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{SON}/finding/{fid}/") - if not b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.querySelector('.veil button') " - "&& 'ready'", 10): - bad.append(f"{gene}: finding never closed") - continue - b.eval("document.querySelector('.veil button').click(); true") - if not b.wait("!document.querySelector('.veil') && document.querySelector('#related') && 'opened'", 5): - bad.append(f"{gene}: finding never opened") - related = b.eval("document.querySelector('#related').outerHTML") or "" - if "sv-chr5-1000000-1300000" in related or "Large DUP" in related: - bad.append(f"{gene} opened alone: the duplication names the other closed result in Related results") - if not b.eval("document.querySelector('#what').textContent.indexOf(" + json.dumps(gene) + ") >= 0"): - bad.append(f"{gene}: opening its finding did not show it") - finally: - unplant(keep) - finally: - for dest in made: - shutil.rmtree(dest) - return bad - - -def chart_position_problems(b, base, out, og): - """Gene-only chart tags and coordinate-only mosaic tags, including reopening with two overlapping acts.""" - def change(S): - dup_over(cpsr("KCNQ1"), "chr5:1000000-1300000", ["NKD2", "KCNQ1"])(S) - second_hit(S) # KCNH2, chr7:150970000: outside both closed results' positions, tied by gene alone. - S["mosaic"] = {"state": "ok", "data": {"status": "assessed", "floor": 0.05, "depth": 30, - "floor_read": 0.05, "floor_min": 0.05, "het_sites": 1000, "chromosomes": 22, "calls_reported": 1, - "calls": [{"chrom": "chr7", "start": 150900000, "end": 151000000, "length_mb": 0.1, - "type": "Loss", "arms": "7q", "cell_fraction": 0.2}]}} - keep = plant_summary(out, og, change) - bad, states = [], [] - try: - with open(os.path.join(out, "runs", SON, f"{SON}_charts.json")) as fh: - marks = next(sp["marks"] for sp in json.load(fh)["charts"] if sp["id"] == "genome") - hit = next(m["id"] for m in marks if m.get("kind") == "hit" and m.get("pos") == 150970000) - sv = next(m["id"] for m in marks if m.get("kind") == "sv" and m.get("pos") == 1000000) - # Inspect each surface separately. A link/title leak counts even when its visible row is absent. - expression = """JSON.stringify((function () { - var c = document.querySelector('#card-mosaic-mca'); - return [!!document.querySelector('figure[data-chart=genome] [data-mark=%s]'), - !!document.querySelector('figure[data-chart=genome] [data-mark=%s]'), - !!document.querySelector('[data-finding=mca-chr7-150900000]'), - !!(c && Array.from(c.querySelectorAll('tbody tr')).some(function (r) { return r.textContent.includes('150900000'); })), - !!document.querySelector('a[href*="150900000"]'), - !!(c && Array.from(c.querySelectorAll('li')).some(function (e) { return e.textContent.includes('150900000'); }))]; - }()))""" % (hit, sv) - page_now(b, base) - states.append(json.loads(b.eval(expression))) - for fid in ("sf-kcnh2-chr7-150960000-c-t", "sv-chr7-150955000-150965000", cid("KCNQ1")): - if not open_closed(b, fid): - return [f"no closed card for {fid}"] - states.append(json.loads(b.eval(expression))) - # Positive first: every planted mark and surface must reach the browser when opened. Otherwise a missing - # rendered charts file could make the hidden-state assertions vacuous. - if states[-1] != [True] * 6: - bad.append(f"all opened: planted hit, SV, mosaic finding, row, viewer link and line must exist: {states[-1]}") - for stage, got, want in zip(("all closed", "SNV alone opened", "SNV and deletion opened"), states, - ([False] * 6, [False] * 6, [True, False, True, True, True, True])): - if got != want: - bad.append(f"{stage}: hit, SV, mosaic finding/row/link/line {got}, want {want}") - # Reverse the overlapping results' opening order too. The SNV has no chip of its own on this demo map. - page_now(b, base) - if not open_closed(b, "sv-chr7-150955000-150965000"): - bad.append("no closed deletion after reloading") - elif json.loads(b.eval(expression)) != [False] * 6: - bad.append(f"deletion alone opened: {b.eval(expression)}") - finally: - unplant(keep) - return bad - - -def home_panel_problems(b, base, out, og): - """A panel finding's title and id stay off a closed child's home view, but remain for adults and no-act runs.""" - p = os.path.join(out, "runs.json") - with open(p, "rb") as fh: - original = fh.read() - idx = json.loads(original) - idx["runs"] = [dict(r, state="running") for r in idx["runs"] if r["name"] == SON] - bad = [] - - def change(S): - S.clear() - cpsr("TP53")(S) - S["chip"] = {"state": "ok", "data": {"calls": [{"gene": "TP53", "protein_change": "p.Demo", - "vaf": 0.1, "alt_reads": 10, "depth": 100, "status": "reported"}]}} - - try: - for case in ("child", "adult", "no act"): - view = copy.deepcopy(idx) - if case == "adult": - view["families"] = [] - with open(p, "w") as fh: - json.dump(view, fh) - - def plant(S): - change(S) - if case == "no act": - S["cpsr"]["data"]["plp"] = [] - - html = "\n".join(home_now(b, base, out, plant)) - if case == "child": - if "TP53" in html or "chip-tp53-p-demo" in html: - bad.append("closed child's home view exposes a panel finding's title or id") - elif "A blood clone in TP53" not in html or "finding/chip-tp53-p-demo/" not in html: - bad.append(f"{case}: the open panel title or finding link is missing") - finally: - with open(p, "wb") as fh: - fh.write(original) - return bad - - -def gate_regression_checks(b, base, out, og): - """Each guard has a browser control that removes exactly the reviewed rule.""" - t0 = time.time() - for desc, probe in (("APOE's gate before opening, opened and closed again gives the same HTML, address and canvas " - "with APOE or BRCA2 closed", apoe_pair_problems), - ("a finding page opened for either BRCA1 or BRCA2 keeps their duplication out of Related results", - related_set_problems), - ("a child's home view hides panel finding titles and links, while adults and no-act runs keep them", - home_panel_problems), - ("planted charts reach the browser and close by gene alone, and a gene-free mosaic finding, row, " - "viewer link and line close until both overlapping results open", chart_position_problems)): - bad = probe(b, base, out, og) - check("browser: " + desc, not bad, bad) - for file, a, z, desc, probe in ( - ("map.js", "var tag = !!r.gate ||", "var tag =", "map-owned APOE row following overlapping act tags", apoe_pair_problems), - ("finding.js", "return t.length ? t : null;", "return t.length ? t.slice(0, 1) : null;", - "finding.js tagOf keeping only the first id", related_set_problems), - ("home.js", "if (!closedSummary(r)) { return r.top || []; }", "if (true) { return r.top || []; }", - "home.js ignoring count-based child summaries", lambda *args: fail_closed_problems(*args, cases=["home"]))): - p = os.path.join(out, "assets", file) - with open(p) as fh: - source = fh.read() - try: - if source.count(a) != 1: - check("control: " + desc, False, "missing mutation anchor") - continue - with open(p, "w") as fh: - fh.write(source.replace(a, z)) - control(desc, not probe(b, base, out, og)) - finally: - with open(p, "w") as fh: - fh.write(source) - names, spans = live_results.mark_names, live_results.closable - try: - for kind, desc in (("hit", "ClinVar chart mark without its gene"), ("sv", "SV chart mark with only its first gene")): - live_results.mark_names = lambda S, sp, m: (([], "") if kind == "hit" else (names(S, sp, m)[0][:1], "")) \ - if m.get("kind") == kind else names(S, sp, m) - control(desc, not chart_position_problems(b, base, out, og)) - live_results.mark_names = names - live_results.closable = lambda found: [(fid, g, None, None, None) for fid, g, *_ in spans(found)] - control("closable without coordinates", not chart_position_problems(b, base, out, og)) - finally: - live_results.mark_names, live_results.closable = names, spans - print(f" gate regression checks: {time.time() - t0:.2f} s") - - -def fail_closed_problems(b, base, out, og, cases=None, producers=None): - """Producer plantings: home rankings, ontology facts, chart geometry and structured mitochondrial sites.""" - bad = [] - cases = cases or ("home", "hpo", "contig", "mito", "producers", "metadata") - if "home" in cases: - for n in (1, 2, 3): - def ranked(g): - def change(S): - S.clear() - cpsrs(g, *["MLH1", "MSH2"][:n - 1])(S) - S["exomiser"] = {"state": "ok", "data": {"terms": [], "genes": [ - {"gene": "BRCA1", "rank": 1, "moi": "AD", "exomiser_score": 0.9, "variants": []}]}} - return change - a, z = (home_now(b, base, out, ranked(g)) for g in ("BRCA1", "BRCA2")) - if a != z: - bad.append(f"home: {n} closed results change the whole document with the same BRCA1 ranking") - if "hpo" in cases: - import collect_summary - with tempfile.TemporaryDirectory() as td: - folder = os.path.join(td, "exomiser") - os.mkdir(folder) - with open(os.path.join(folder, f"{SON}_exomiser_top.tsv"), "w") as fh: - fh.write("rank\tgene\tmoi\texomiser_score\tphenotype_score\tvariant\tgenotype\tconsequence\thgvs\tacmg\tdisease\n" - "1\tCFTR\tAR\t0.9\t0.8\tchr7:117559590\thet\tmissense\tc.1521_1523del\tPathogenic\tDemo condition\n") - with open(os.path.join(folder, f"{SON}_exomiser_terms.tsv"), "w") as fh: - fh.write("hpo_id\tlabel\nHP:0020055\tDiminished MLH1 expression\n") - _, data = collect_summary.sec_exomiser(td, SON) - def hpo(S): - cpsrs("MLH1", "MSH2")(S) - S["exomiser"] = {"state": "ok", "data": data} - keep = plant_summary(out, og, hpo) - try: - page_now(b, base) - for stage in ("closed", "partly opened", "all opened"): - if stage == "partly opened": - assert open_closed(b, cid("MLH1")) - if stage == "all opened": - with open(os.path.join(out, "runs", SON, "results.json")) as fh: - acts = [f["id"] for f in json.load(fh)["findings"] if f["level"] == "act"] - for fid in acts: - open_closed(b, fid) - visible = b.eval("document.querySelector('#card-exomiser').textContent.includes('Diminished MLH1 expression')") - if visible != (stage == "all opened"): - bad.append(f"hpo: label visibility {visible} while {stage}") - finally: - unplant(keep) - if "contig" in cases: - views = [] - for locus in ("chrUn_KI270442v1:100-200", "chr13:32300000-32300100"): - keep = plant_summary(out, og, sv_over(locus, ["BRCA1"], False)) - try: - views.append(page_now(b, base)) - with open(os.path.join(out, "runs", SON, "results.json")) as fh: - acts = [f["id"] for f in json.load(fh)["findings"] if f["level"] == "act"] - for fid in acts: - assert open_closed(b, fid), fid - note = b.eval("document.querySelector('figure[data-chart=genome]').textContent.includes('SVs on other contigs')") - if note != locus.startswith("chrUn"): - bad.append("contig: the full chart note did not return after every act opened") - assert b.eval("(function () { var b = document.querySelector('.gm-chip.opened button'); if (!b) return false; b.click(); return true; }())") - if b.eval("document.querySelector('figure[data-chart=genome]').textContent.includes('SVs on other contigs')"): - bad.append("contig: re-closing an act leaves the chart note visible") - finally: - unplant(keep) - if views[0] != views[1]: - bad.append("contig: closed SV changes text, attributes, address, canvas or SVG geometry") - if "mito" in cases: - def mito(S): - cpsrs("MT-TL1", "MT-TL2")(S) - for row in S["cpsr"]["data"]["plp"]: - row["variant"] = "chrM:3243 A>G" - S["mito"] = {"state": "ok", "data": {"pass": 1, "heteroplasmic": 1, - "heteroplasmic_sites": [{"pos": 3243, "ref": "A", "alt": "G", "af": 0.25}]}} - keep = plant_summary(out, og, mito) - try: - page_now(b, base) - expr = "JSON.stringify([!!document.querySelector('#card-mito tbody tr'), !!document.querySelector('figure[data-chart=mito] [data-mark=m3243-G]')])" - for stage, fid in enumerate((None, "cpsr-mt-tl1-chrm-3243-a-g", "cpsr-mt-tl2-chrm-3243-a-g")): - if fid and not open_closed(b, fid): - bad.append(f"mito: no closed result {fid}") - break - got = json.loads(b.eval(expr)) - if got != [stage == 2] * 2: - bad.append(f"mito: row/mark after {stage} matching openings {got}") - finally: - unplant(keep) - if "report-metadata" in cases: - phone_at(b, 390) - try: - views = [] - for gene in ("MLH1", "BRCA2"): - build = live_results.build - def metadata(sample, summary, files=()): - token = "MLH1_METADATA_NOTE" - records = [token] * (2 if gene == "MLH1" else 1) - planted = dict(summary, not_assessed=records, not_run=records, - databases=dict(summary.get("databases") or {}, test_metadata=token)) - return build(sample, planted, files) - live_results.build = metadata - try: - keep = plant_summary(out, og, cpsr(gene)) - finally: - live_results.build = build - try: - views.append(page_now(b, base)) - if "MLH1_METADATA_NOTE" in "\n".join(views[-1][0]): - bad.append("report metadata: unclassified limitation, missing-step or version text reaches the DOM") - with open(os.path.join(out, "runs", SON, "results.json")) as fh: - acts = [f["id"] for f in json.load(fh)["findings"] if f["level"] == "act"] - for fid in acts: - assert open_closed(b, fid), fid - if "MLH1_METADATA_NOTE" not in (b.eval("document.documentElement.outerHTML") or ""): - bad.append("report metadata: text does not return after every act opens") - finally: - unplant(keep) - if views[0] != views[1]: - bad.append("report metadata: extra records change closed text, attributes, address, canvas or SVG sizes") - finally: - b.call("Emulation.clearDeviceMetricsOverride", {}, b.session) - if "trust" in cases: - def qc_note(S): - cpsr("MLH1")(S) - S["sample_qc"] = {"state": "ok", "data": {"sex_check": "not_checked", "inferred_sex": "male", - "sex_check_reason": "MLH1_TRUST_NOTE", "contamination": "not_checked"}} - keep = plant_summary(out, og, qc_note) - try: - page_now(b, base) - if "MLH1_TRUST_NOTE" in (b.eval("document.documentElement.outerHTML") or ""): - bad.append("trust: a PANEL QC fact survives in the trust-line text or title") - assert open_closed(b, cid("MLH1")) - if "MLH1_TRUST_NOTE" in (b.eval("document.documentElement.outerHTML") or ""): - bad.append("trust: opening one act exposes a QC fact while other acts remain closed") - with open(os.path.join(out, "runs", SON, "results.json")) as fh: - acts = [f["id"] for f in json.load(fh)["findings"] if f["level"] == "act"] - for fid in acts: - open_closed(b, fid) - if "MLH1_TRUST_NOTE" not in (b.eval("document.querySelector('.trustline').outerHTML") or ""): - bad.append("trust: the QC fact does not return after every act opens") - finally: - unplant(keep) - if "no-act" in cases: - def no_act(S): - S.clear() - S["exomiser"] = {"state": "ok", "data": {"terms": [], "genes": [ - {"gene": "BRCA1", "rank": 1, "moi": "AD", "exomiser_score": 0.9, "variants": []}]}} - keep = plant_summary(out, og, no_act) - try: - page_now(b, base, "#f/exomiser-brca1") - if b.eval("location.hash") != "#f/exomiser-brca1": - bad.append("no-act: missing metadata changes an open finding's address") - finally: - unplant(keep) - if "metadata" in cases: - keep = plant_summary(out, og, cpsr("MLH1")) - try: - path = os.path.join(out, "runs", SON, "results.json") - with open(path) as fh: - result = json.load(fh) - result["veil"] = {k: v for k, v in result["veil"].items() if k in ("spans", "panels")} - card = next(c for ch in result["chapters"] for c in ch["sections"] if c["id"] == "qc") - token = "MLH1_UNCLASSIFIED_NOTE" - card.update(headline=token, facts=[["Unknown", token]], lines=[token], notes=[token], - table={"columns": ["Detail"], "rows": [[token]]}) - with open(path, "w") as fh: - json.dump(result, fh) - if token in "\n".join(page_now(b, base)[0]): - bad.append("metadata: a missing tag exposes unclassified content") - for finding in result["findings"]: - if finding["level"] == "act": - assert open_closed(b, finding["id"]), finding["id"] - if token not in (b.eval("document.documentElement.outerHTML") or ""): - bad.append("metadata: content does not return after every act opens") - finally: - unplant(keep) - if "producers" in cases: - # Each adapter receives an unknown free-text field and an extra unstructured row, in turn. Neither - # the token's spelling nor its record count earns an exemption from the default PANEL rule. - for producer, adapter in list(live_results.ADAPTERS.items()): - if producers and producer not in producers: - continue - def injected(S, summary): - got = adapter(S, summary) - got.update(facts=[["Mean depth", "MLH1 sample note"]], lines=["MLH1 sample note"], - table={"columns": ["Detail"], "rows": [["MLH1"], ["extra MLH1 record"]]}) - return got - live_results.ADAPTERS[producer] = injected - try: - # Supply each adapter's real demo section. Missing sections are tested by the unit counterpart. - views = [] - for gene in ("MLH1", "BRCA2"): - keep = plant_summary(out, og, cpsr(gene)) - try: - views.append(page_now(b, base)) - if any(token in "\n".join(views[-1][0]) for token in ("MLH1 sample note", "extra MLH1 record")): - bad.append(f"producer {producer}: unclassified token reached the closed DOM") - finally: - unplant(keep) - if views[0] != views[1]: - bad.append(f"producer {producer}: unclassified planting changes the closed document") - finally: - live_results.ADAPTERS[producer] = adapter - return bad - - -def panel_finding_problems(b, base, out, og, cases=("child", "missing tags", "adult", "no acts")): - """A PANEL finding opens one act per click and reveals its content only after the last act opens.""" - folder = os.path.join(out, "runs", SON, "finding") - original, fid = "sf-kcnh2-chr7-150960000-c-t", "chip-tp53-p-demo" - dest = os.path.join(folder, fid) - with open(os.path.join(folder, original, "index.html")) as fh: - template = fh.read() - fj = os.path.join(out, "runs", FAM, "family.json") - with open(fj) as fh: - family = fh.read() - os.makedirs(dest) - bad = [] - try: - with open(os.path.join(dest, "index.html"), "w") as fh: - fh.write(template.replace(original, fid)) - for case in cases: - def change(S): - S.clear() - cpsrs("TP53", "BRCA2")(S) - if case == "no acts": - S["cpsr"]["data"]["plp"] = [] - S["chip"] = {"state": "ok", "data": {"calls": [{"gene": "TP53", "protein_change": "p.Demo", - "vaf": 0.1, "alt_reads": 10, "depth": 100, "status": "reported"}]}} - keep = plant_summary(out, og, change) - try: - rj = os.path.join(out, "runs", SON, "results.json") - with open(rj) as fh: - res = json.load(fh) - tags = (res.get("veil", {}).get("findings") or {}).get(fid) - if case != "no acts" and tags != ["*panel*"]: - bad.append(f"{case}: producer did not classify the CHIP finding as PANEL: {tags}") - if case == "missing tags": - del res["veil"]["findings"][fid] - with open(rj, "w") as fh: - json.dump(res, fh) - fam = json.loads(family) - if case == "adult": - for member in fam["state"]["members"]: - if member["sample"] == SON: - member["father"] = member["mother"] = None - with open(fj, "w") as fh: - json.dump(fam, fh) - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{SON}/finding/{fid}/") - if not b.wait("document.documentElement.dataset.pgpStatus === 'ready' && 'ready'", 10): - bad.append(f"{case}: finding never ready") - continue - for clicks in range(3 if case in ("child", "missing tags") else 1): - should_open = case in ("adult", "no acts") or clicks == 2 - visible = b.eval("document.querySelector('#what').textContent.includes('A blood clone in TP53') " - "&& document.title.includes('TP53') && !!document.querySelector('#evidence')") - shut = b.eval("!!document.querySelector('.veil button')") - if bool(visible) != should_open or bool(shut) == should_open: - bad.append(f"{case}: after {clicks} clicks PANEL finding visible={visible}, closed={shut}; " - f"expected visible={should_open}") - if not should_open: - b.eval("var button = document.querySelector('.veil button'); button && button.click(); true") - if case == "child": - b.go(f"{base}/runs/{SON}/finding/{fid}/") - if not b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.querySelector('.veil button') " - "&& !document.querySelector('#evidence') && 'closed'", 10): - bad.append("child: a reload did not close the PANEL finding again") - finally: - unplant(keep) - finally: - with open(fj, "w") as fh: - fh.write(family) - shutil.rmtree(dest) - return bad - - -def fail_closed_checks(b, base, out, og): - started = time.monotonic() - bad = panel_finding_problems(b, base, out, og) - check("browser: a PANEL finding stays closed until one click per act, with missing tags, adult and no-act controls", - not bad, bad) - finding_path = os.path.join(out, "assets", "finding.js") - with open(finding_path) as fh: - finding_reader = fh.read() - for old, new, desc in ( - ("return t === PANEL ? actShut() : !opened[t];", "return t === PANEL ? false : !opened[t];", - "a finding page that ignores PANEL"), - ("if (nextAct) { opened[nextAct.id] = true; }", - "if (nextAct) { (r.findings || []).forEach(function (x) { if (x.level === 'act') { opened[x.id] = true; } }); }", - "a PANEL finding button that opens every act in one click")): - try: - assert finding_reader.count(old) == 1 - with open(finding_path, "w") as fh: - fh.write(finding_reader.replace(old, new)) - control(desc, not panel_finding_problems(b, base, out, og, ["child"])) - finally: - with open(finding_path, "w") as fh: - fh.write(finding_reader) - for case in ("home", "hpo", "contig", "mito", "producers", "metadata", "no-act", "trust", "report-metadata"): - bad = fail_closed_problems(b, base, out, og, [case]) - check("browser: fail-closed producers " + case, not bad, bad) - prose, row, veil = live_results.prose_tag, live_results.row_tag, live_results.veil - try: - live_results.FACTS_NO_GENE["exomiser"] = "unsafe HPO labels" - control("an HPO fact declared gene-free", not fail_closed_problems(b, base, out, og, ["hpo"])) - live_results.FACTS_NO_GENE.pop("exomiser", None) - def no_mito(*args, **kwargs): - got = veil(*args, **kwargs) - got.get("rows", {}).pop("mito", None) - return got - live_results.veil = no_mito - control("mitochondrial rows without structured positions", not fail_closed_problems(b, base, out, og, ["mito"])) - live_results.veil = veil - live_results.prose_tag = lambda spans, text: None - control("unclassified producer prose left open", not fail_closed_problems(b, base, out, og, ["hpo"])) - live_results.prose_tag = prose - live_results.row_tag = lambda spans, columns, values, genes=(): \ - live_results.veil_tags(spans, live_results.gene_cell(values[0])) if columns == ["Detail"] \ - else row(spans, columns, values, genes) - control("unclassified rows read as gene cells", not fail_closed_problems(b, base, out, og, ["producers"], ["qc"])) - finally: - live_results.FACTS_NO_GENE.pop("exomiser", None) - live_results.prose_tag, live_results.row_tag, live_results.veil = prose, row, veil - path = os.path.join(out, "assets", "person.js") - with open(path) as fh: - reader = fh.read() - try: - old = "parseFromString(sp.svg_closed," - assert reader.count(old) == 1 - with open(path, "w") as fh: - fh.write(reader.replace(old, "parseFromString(sp.svg,")) - control("chart notes and geometry outside PANEL", not fail_closed_problems(b, base, out, og, ["contig"])) - finally: - with open(path, "w") as fh: - fh.write(reader) - try: - old = "tag === undefined ? tagClosed(PANEL)" - assert reader.count(old) == 1 - with open(path, "w") as fh: - fh.write(reader.replace(old, "tag === undefined ? false")) - control("missing tag metadata treated as open", not fail_closed_problems(b, base, out, og, ["metadata"])) - finally: - with open(path, "w") as fh: - fh.write(reader) - try: - old = "if (t === undefined) { t = ((results && results.findings) || []).some(function (x) { return x.level === 'act'; }) ? [PANEL] : null; }" - assert reader.count(old) == 1 - with open(path, "w") as fh: - fh.write(reader.replace(old, "if (t === undefined) { t = [PANEL]; }")) - control("a no-act page treating every unclassified finding as closable", not fail_closed_problems(b, base, out, og, ["no-act"])) - finally: - with open(path, "w") as fh: - fh.write(reader) - try: - old = "openFacts(qc)[0].filter(function (f) { return /depth|contamination|sex from/i.test(f[0]); })" - assert reader.count(old) == 1 - with open(path, "w") as fh: - fh.write(reader.replace(old, "(qc.facts || []).filter(function (f) { return /depth|contamination|sex from/i.test(f[0]); })")) - control("trust line reading QC facts outside their veil", not fail_closed_problems(b, base, out, og, ["trust"])) - finally: - with open(path, "w") as fh: - fh.write(reader) - for old, new, desc in ( - ("(sec.n === 9 || sec.n === 10) && tagClosed(PANEL)", "false", - "report metadata read outside PANEL"), - ("sec.n === 9 && tagClosed(PANEL)", "false", - "unclassified report metadata counted in the contents")): - try: - assert reader.count(old) == 1 - with open(path, "w") as fh: - fh.write(reader.replace(old, new)) - control(desc, not fail_closed_problems(b, base, out, og, ["report-metadata"])) - finally: - with open(path, "w") as fh: - fh.write(reader) - print(f" fail-closed producer checks: {time.monotonic() - started:.2f} s") - - -def infer_checks(b, base, out): - """The son's page is a function of his open results and the count of his closed ones: two pages that differ only - in which result is closed are the same before any click (INFER), and a closed deletion's second gene is named - nowhere. Controls: person.js that puts a closed card's id back, that draws a fixed-panel card's open rows, or - that counts the closed rows; a veil that tags only a deletion's first gene.""" - og = og_binary() or build_live_demo.OG - if not os.path.isfile(os.path.join(out, "runs", OTHERS[1], "map.json")): - print("[SKIP] infer checks: the demo has no map.json") - return - t0 = time.time() - gate_regression_checks(b, base, out, og) - bad = infer_problems(b, base, out, og) - check("browser: two pages of the son that differ only in which result is closed are the same before any click, " - "HTML, attributes, address and map canvas (" + "; ".join(x[0] for x in INFER) + ")", not bad, bad) - bad = click_problems(b, base, out, og) - check("browser: two pages of the son that differ only in which result is closed are the same after the same " - "result is opened through its handle (" + "; ".join(x[0] for x in CLICKS) + ")", not bad, bad) - bad = two_closed_problems(b, base, out, og) - check("browser: a place that names two closed results stays closed while either is: opening one never shows the " - "other's gene (" + "; ".join(x[0] for x in TWO_CLOSED) + ")", not bad, bad) - bad = deep_link_problems(b, base) - check("browser: #f/closed-1 and a guessed #f/ scroll to the first closed result, in view, its section marked in " - "the contents", not bad, bad) - bad = sv_gene_problems(b, base, out, og) - check("browser: a closed deletion over two genes closes the rows that name only its second gene", not bad, bad) - bad = panel_words_problems(b, base, out, og) - check("browser: a fixed-panel card whose own finding names the closed result's gene shows the panel's words", not bad, - bad) - bad = address_problems(b, base, out) - check("browser: opening a closed result and moving on never puts its id in the address or a history entry", not bad, - bad) - bad = home_problems(b, base, out) - check("browser: the home page is the same whichever of the son's results is closed (" + "; ".join(x[0] for x in HOME) - + ")", not bad, bad) - home = os.path.join(out, "assets", "home.js") - with open(home) as fh: - keep_home = fh.read() - a = "x.r.url + (homeShut(x.r, x.f) ? '' : 'finding/' + encodeURIComponent(x.f.id) + '/')" - try: - if a not in keep_home: - check("control: home.js that links a closed result's own page (the planted line is in home.js)", False, a) - else: - with open(home, "w") as fh: - fh.write(keep_home.replace(a, "x.r.url + 'finding/' + encodeURIComponent(x.f.id) + '/'")) - control("home.js that links a closed result's own page", not home_problems(b, base, out, HOME[:1])) - finally: - with open(home, "w") as fh: - fh.write(keep_home) - first = live_results.panel_finding - try: - live_results.panel_finding = lambda f: False - control("a veil that tags a fixed-panel card's own finding by the closed result's gene", - not infer_problems(b, base, out, og, [x for x in INFER if "(NAT2)" in x[0]])) - finally: - live_results.panel_finding = first - prose, two_names = live_results.prose_tag, live_results.two_hit_names - try: # the word rule back: prose tagged by its words read as gene symbols - live_results.prose_tag = lambda spans, text: live_results.veil_tags( - spans, re.findall(r"[A-Za-z0-9][A-Za-z0-9-]*", str(text).upper()), text) - control("a veil that tags the carrier list's two-hit callers line by its words", - not infer_problems(b, base, out, og, [x for x in INFER if "two-hit" in x[0]])) - live_results.two_hit_names = lambda S: {} - control("a veil that tags the two-hit gene lines by their words (AR)", not click_problems(b, base, out, og, CLICKS[:1])) - finally: - live_results.prose_tag = prose - live_results.two_hit_names = two_names - tags_, union_ = live_results.veil_tags, live_results.union - try: # one id per place, as before: the first closed result a place names - live_results.veil_tags = lambda *a, **k: (tags_(*a, **k) or [None])[:1] if tags_(*a, **k) else None - live_results.union = lambda *t: next((x[:1] for x in t if x), None) - control("a veil that keeps one id per place (the lead's segment)", not two_closed_problems(b, base, out, og, TWO_CLOSED[:1])) - control("a veil that keeps one id per place (the duplication)", not two_closed_problems(b, base, out, og, TWO_CLOSED[2:3])) - finally: - live_results.veil_tags, live_results.union = tags_, union_ - js = os.path.join(out, "assets", "person.js") - with open(js) as fh: - keep_js = fh.read() - try: - for desc, a, z, case in ( - # The guard is the pair of lines: either alone lands every guess the same way. - ("person.js that picks a closed card by a guessed id", - " if (f && !(veiled(f) && !m)) { return f; }\n return closedOrder().length ? findingById(closedOrder()[0]) : null;", - " return f || null;", [x for x in INFER if x[0].startswith("the same guessed")][0]), - ("person.js that picks a closed card by a guessed id, at load", - " if (f && !(veiled(f) && !m)) { return f; }\n return closedOrder().length ? findingById(closedOrder()[0]) : null;", - " return f || null;", [x for x in INFER if x[0].startswith("one guessed")][0]), - ("person.js that puts a closed card's id back", - "function handle(f) { return veiled(f) ? 'closed-' + (closedOrder().indexOf(f.id) + 1) : f.id; }", - "function handle(f) { return f.id; }", INFER[0]), - ("person.js that draws a fixed-panel card's open rows", "} else if (panelShut(c)) {", "} else if (false) {", - INFER[2]), - ("person.js that counts the closed rows", - "'Some of its rows or numbers stay closed with a result to act on, until that result is opened.'", - "(ot[1] + (c.lines || []).length - lines.length) + ' rows stay closed with a result to act on.'", INFER[4])): - if a not in keep_js: - check(f"control: {desc} (the planted line is in person.js)", False, a) - continue - with open(js, "w") as fh: - fh.write(keep_js.replace(a, z)) - control(desc, not infer_problems(b, base, out, og, [case])) - for desc, a, z, still in ( - ("person.js whose fixed-panel card says 'This section holds a result' first", "} else if (panelShut(c)) {", - "} else if (panelShut(c) && !closedFs.length) {", lambda: not panel_words_problems(b, base, out, og)), - ("person.js that reads only the first id of a tag", "if (Array.isArray(id)) { return id.some(tagClosed); }", - "if (Array.isArray(id)) { return tagClosed(id[0]); }", - lambda: not two_closed_problems(b, base, out, og, TWO_CLOSED[2:3])), - ("person.js that writes an opened result's id into the address", - "function addr(f) { return !closable(f) ? f.id : veiled(f) ? handle(f) : 'opened-' + (openedOrder().indexOf(f.id) + 1); }", - "function addr(f) { return veiled(f) ? handle(f) : f.id; }", lambda: not address_problems(b, base, out))): - if a not in keep_js: - check(f"control: {desc} (the planted line is in person.js)", False, a) - continue - with open(js, "w") as fh: - fh.write(keep_js.replace(a, z)) - control(desc, still()) - finally: - with open(js, "w") as fh: - fh.write(keep_js) - first = live_results.closable - try: - live_results.closable = lambda found: first([{k: v for k, v in f.items() if k != "genes"} for f in found]) - control("a veil that tags only a deletion's first gene", not sv_gene_problems(b, base, out, og)) - finally: - live_results.closable = first - cols, names, marks = live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names - try: - live_results.TABLE_COLUMNS = dict(cols, Inheritance="gene") - control("a veil that reads a table's Inheritance column as naming a gene", - not infer_problems(b, base, out, og, [x for x in INFER if x[0].startswith("an inheritance code")])) - live_results.TABLE_COLUMNS = cols - live_results.finding_names = lambda f: (re.findall(r"[A-Z0-9]+", f"{f.get('title') or ''}"), f.get("detail") or "") - control("a veil that reads a finding's title word by word", - not infer_problems(b, base, out, og, [x for x in INFER if "(INS)" in x[0]])) - live_results.finding_names = names - live_results.mark_names = lambda S, sp, m: (re.findall(r"[A-Z0-9]+", f"{m.get('label') or ''} {m.get('detail') or ''}"), "") - control("a veil that reads a chart mark's label word by word", - not infer_problems(b, base, out, og, [x for x in INFER if "(INS)" in x[0]])) - finally: - live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names = cols, names, marks - og_map = build_live_demo.og_map - - def blind(og_, args, body): - """og map without results.veil.findings: it no longer knows which open findings a closed result ties.""" - d = json.loads(body) - (d["results"].get("veil") or {}).pop("findings", None) - return og_map(og_, args, json.dumps(d).encode()) - try: - build_live_demo.og_map = blind - control("og map that counts a finding a closed result ties among the open findings with no place", - not infer_problems(b, base, out, og, [x for x in INFER if "(OPHN1 has no place" in x[0]])) - finally: - build_live_demo.og_map = og_map - with open(js) as fh: - keep_js = fh.read() - route = "{ var hf = hashFinding(); target = document.getElementById(" - try: - if route not in keep_js: - check("control: person.js whose #f/closed- lands nowhere (the planted line is in person.js)", False, route) - else: - with open(js, "w") as fh: - fh.write(keep_js.replace(route, "{ var hf = null; h = 'f/nowhere'; target = document.getElementById(")) - control("person.js whose #f/closed- lands nowhere", not deep_link_problems(b, base)) - finally: - with open(js, "w") as fh: - fh.write(keep_js) - print(f" infer checks: {time.time() - t0:.0f} s") - - -def widget_veil_checks(b, base, out): - """On the son's page the copy-number card and the CHIP card, whose rows are a fixed panel (the same genes for - everyone), close whole while any result of his to act on is closed: no dot, no call, no fact of theirs shows, - and their line on a phone is the same words whatever the result; so do the ClinVar card's lines for the known - deep intronic sites, a fixed panel inside it. Each "Show the result" of their box opens one result; once every - result is opened every dot, call, fact and deep intronic line is back.""" - no_cache(b) - rj = os.path.join(out, "runs", OTHERS[1], "results.json") - js = os.path.join(out, "assets", "person.js") - with open(rj) as f: - keep = f.read() - with open(js) as f: - keep_js = f.read() - - def drawn(): - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/") - b.wait("document.documentElement.dataset.pgpResults && document.documentElement.dataset.pgpStatus === 'ready' " - "&& document.querySelector('#card-paralogs') && document.querySelector('.veil') && 'drawn'", 15) - return shown() - - def phone_line(): - """The copy-number card's one line on a phone (its fold's summary, after the card's own title).""" - b.call("Emulation.setDeviceMetricsOverride", {"width": 390, "height": 844, "deviceScaleFactor": 2, "mobile": False}, - b.session) - try: - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/") - return b.wait("document.documentElement.dataset.pgpStatus === 'ready' && document.documentElement.dataset.pgpResults " - "&& (document.querySelector('[data-card=fold-paralogs] summary .what') || {}).textContent", 15) or "" - finally: - b.call("Emulation.setDeviceMetricsOverride", {"width": 1440, "height": 900, "deviceScaleFactor": 1, "mobile": False}, - b.session) - - def phone_problems(): - line = phone_line() - want = "The same genes for everyone, closed while a result about a child is closed." - return [] if line == want else [f"the copy-number card's line on a phone: {line!r}, want {want!r}"] - - def shown(): - return b.eval("JSON.stringify([" + SHOWN_JS + ", (document.querySelector('[data-widget=copies]') || {}).textContent || '', " - "document.querySelectorAll('.finding[data-level=act] .veil').length, " - "document.querySelectorAll('#card-paralogs tbody tr,#card-chip tbody tr').length])") - - def problems(text, opened): - page, cops, _, rows = json.loads(text) if text else ["", "", 0, 0] - out_ = [] - deep = page.count("Known deep intronic site (always reported):") - if opened and not ("SMN1" in cops and "SMN2" in cops and "p.R175H" in page and "SMN1 copies" in page - and deep == sites[0]): - out_.append(f"every result opened, yet a dot, the call, the fact or a deep intronic line ({deep} of {sites[0]}) is " - f"missing: {cops[:200]}") - if not opened and (rows or cops or "p.R175H" in page or "SMN1 copies" in page or deep): - out_.append(f"a closed result, yet the copy-number or CHIP card shows its rows or the ClinVar card " - f"{deep} deep intronic lines: {cops[:200]}") - return out_ - try: - with open(os.path.join(os.path.dirname(out), "root", OTHERS[1], "summary.json")) as f: - summary = json.load(f) - S = summary.setdefault("sections", {}) - S["cpsr"] = {"state": "ok", "data": {"call_floor_dp": 10, "low_confidence": 0, "plp": [ - {"gene": "TP53", "variant": "c.524G>A", "final": "Pathogenic", "clinvar": "Pathogenic"}, - {"gene": "SMN1", "variant": "c.1A>G", "final": "Pathogenic", "clinvar": "Pathogenic"}]}} - S["chip"] = {"state": "ok", "data": {"median_limit": 0.05, "median_depth": 72, "calls": [ - {"gene": g, "protein_change": ch, "vaf": v, "alt_reads": 6, "depth": 75, "status": "pass", "note": "a clone"} - for g, ch, v in (("DNMT3A", "p.R882H", 0.12), ("TP53", "p.R175H", 0.08))], - "genes": [{"gene": g, "detection_limit_vaf": 0.05, "mean_depth": 75, "status": "ok"} for g in ("DNMT3A", "TP53")]}} - sites = [deep_sites(S)] - S["paralogs"] = {"state": "ok", "data": {"genes": [{"gene": g, "copy_number": n, "status": "called", "detail": "."} - for g, n in (("SMN1", 2), ("SMN2", 2))]}} - res = live_results.build(OTHERS[1], summary) - with open(rj, "w") as f: - json.dump(res, f) - bad = problems(drawn(), False) + phone_problems() - if not bad: - # The copy-number card's box opens one result per click, until none is left closed. - n = json.loads(shown())[2] - for k in range(n): - b.eval("var v = document.querySelector('#card-paralogs .veil button'); v && v.click(); true") - time.sleep(0.3) - left = json.loads(shown())[2] - if left != n - k - 1: - bad.append(f"click {k + 1} on the copy-number card's box left {left} of {n} results closed") - break - bad += problems(shown(), True) - check("browser: on the son's page the copy-number and CHIP cards, a fixed panel, close whole while a result is " - "closed (their line on a phone too), and so do the ClinVar card's deep intronic lines; the box opens one " - "result per click, and every dot, call, fact and line is back once every result is opened", not bad, bad) - for desc, a, z in ( - ("person.js that draws a fixed-panel card's open rows", "} else if (panelShut(c)) {", "} else if (false) {"), - ("person.js whose card line reads a fixed-panel card's facts", ": g === 'show' && panelShut(c) ? PANEL_LINE", - ": false ? PANEL_LINE"), - ("person.js that draws a fixed panel's lines", "if (id === PANEL) { return closedPage() && closedActs().length > 0; }", - "if (id === PANEL) { return false; }")): - if a not in keep_js: - check(f"control: {desc} (the planted line is in person.js)", False, a) - continue - with open(js, "w") as f: - f.write(keep_js.replace(a, z)) - control(desc, not (problems(drawn(), False) + phone_problems())) - with open(js, "w") as f: - f.write(keep_js) - finally: - with open(rj, "w") as f: - f.write(keep) - with open(js, "w") as f: - f.write(keep_js) - - -# finding.js as it was: a missing state or one with no plugins read as a standalone adult. -OLD_FINDING = ("if (!st || !st.plugins) { return Promise.resolve(UNKNOWN); }\n var fam = st.plugins.family;", - "var fam = st && st.plugins && st.plugins.family;") - - -def closed_pages(b, base, fid): - """The son's person page and one planted finding's page, before any click: their text once each has drawn - (or after 10 s, for a page whose state never comes).""" - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/") - b.wait("document.documentElement.dataset.pgpResults && document.documentElement.dataset.pgpStatus === 'ready' " - "&& 'drawn'", 10) - time.sleep(0.5) - person = b.eval("document.body.textContent") or "" - b.go(f"{base}/runs/{OTHERS[1]}/finding/{fid}/") - b.wait("document.documentElement.dataset.pgpStatus === 'ready' && 'drawn'", 10) - page = b.eval("document.body.textContent") or "" - return person, page - - -def closed_problems(person, page): - out = [f"{w} on the person page" for pair in VEIL for w in pair if w in person] - out += [f"{w} on the finding page" for pair in VEIL for w in pair if w in page] - if "A result to act on" not in page: - out.append("the finding page does not show its closed title: " + page[:200]) - return out - - -def search_text(b, base, q): - """[hits, the search panel's text] for q typed on the son's page.""" - b.go("about:blank") - b.wait("location.href === 'about:blank'") - b.go(f"{base}/runs/{OTHERS[1]}/") - b.wait("document.documentElement.dataset.pgpResults && document.querySelector('.veil') && " - "!document.querySelector('input[type=search]').disabled && 'ready'", 15) - b.eval("var i = document.querySelector('input[type=search]'); i.value = %s; i.dispatchEvent(new Event('input')); true" - % json.dumps(q)) - hits = b.wait("document.documentElement.dataset.pgpSearch", 5) - return hits, b.eval("(document.querySelector('.search-results') || {}).textContent || ''") or "" - - -def search_problems(b, base): - out = [] - hits, text = search_text(b, base, "GENEQ") - if hits != "0" or "c.77del" in text: - out.append(f"GENEQ, a gene of a closed card: {hits} hits, {text[:200]}") - hits, text = search_text(b, base, "GENE") - if "GENEQ" in text or "GENER" in text or "GENEOPEN" not in text: - out.append(f"the suggestions for GENE: {text[:300]}") - hits, text = search_text(b, base, "GENEOPEN") - if hits != "1": - out.append(f"GENEOPEN, a gene of an open card, not found: {hits} {text[:200]}") - return out - - -def unknown_checks(b, base, out, fid): - """The son's results to act on stay closed when the page cannot learn his family role, on the person page - and on the finding's own page, and his search names nothing his closed card holds.""" - sd = os.path.join(out, "runs", OTHERS[1]) - fj = os.path.join(out, "runs", FAM, "family.json") - frames_path = os.path.join(sd, "frames.json") - idx = os.path.join(sd, f"{OTHERS[1]}_search_index.json") - pj, fjs, sj = (os.path.join(out, "assets", x) for x in ("person.js", "finding.js", os.path.join("plugins", "search.js"))) - keep = {} - for p in (frames_path, pj, fjs, sj) + ((idx,) if os.path.isfile(idx) else ()): - with open(p) as f: - keep[p] = f.read() - frames = json.loads(keep[frames_path]) - - def write(p, text): - with open(p, "w") as f: - f.write(text) - - def item(sec, text): - return {"section": sec, "anchor": sec, "title": sec, "step": "24", "state": "ok", "source": "summary.json", - "count": 1, "mentions": [{"text": text, "locus": None}]} - try: - # The search: an index naming the two planted genes in the closed CPSR card, and one gene of an open card. - write(idx, json.dumps({"schema_version": 1, "sample": OTHERS[1], "terms": { - "GENEQ": [item("cpsr", "GENEQ c.77del")], "GENER": [item("cpsr", "GENER c.88dup")], - "GENEOPEN": [item("roh", "GENEOPEN in a run of homozygosity")]}})) - with_search = copy.deepcopy(frames) - for fr in with_search["frames"]: - fr["state"].setdefault("plugins", {})["search"] = {"file": os.path.basename(idx), "mtime": 1} - write(frames_path, json.dumps(with_search)) - bad = search_problems(b, base) - check("browser: the son's search lists no hit and suggests no term of his closed card, and still finds an open one", - not bad, bad) - write(sj, keep[sj].replace("function hidden(it) { return !!(", "function hidden(it) { return false && !!(")) - control("search.js without its filter", not search_problems(b, base)) - write(sj, keep[sj]) - write(frames_path, keep[frames_path]) - - # family.json unavailable - os.replace(fj, fj + ".away") - try: - bad = closed_problems(*closed_pages(b, base, fid)) - finally: - os.replace(fj + ".away", fj) - check("browser: with family.json unavailable the son's results to act on stay closed on both pages", not bad, bad) - - # a state cut short: no plugins at all - cut = copy.deepcopy(frames) - for fr in cut["frames"]: - fr["state"].pop("plugins", None) - write(frames_path, json.dumps(cut)) - bad = closed_problems(*closed_pages(b, base, fid)) - check("browser: with a state that has no plugins the son's results to act on stay closed on both pages", not bad, bad) - write(pj, keep[pj].replace("return !state || !state.plugins || !!(fam && fam.unknown);", "return false;")) - write(fjs, keep[fjs].replace(OLD_FINDING[0], OLD_FINDING[1])) - control("person.js and finding.js that read a state with no plugins as a standalone adult", - not closed_problems(*closed_pages(b, base, fid))) - write(pj, keep[pj]) - write(fjs, keep[fjs]) - - # no state at all - os.remove(frames_path) - bad = closed_problems(*closed_pages(b, base, fid)) - check("browser: with no state at all (frames.json unavailable) the son's results to act on stay closed on both pages", - not bad, bad) - write(fjs, keep[fjs].replace(OLD_FINDING[0], OLD_FINDING[1])) - control("finding.js that reads a missing state as a standalone adult", not closed_problems(*closed_pages(b, base, fid))) - finally: - for p, text in keep.items(): - write(p, text) - if idx not in keep and os.path.isfile(idx): - os.remove(idx) - - -# The person page reads as a report: what (frame) -> the page's layout facts, read in the browser. LAYOUT_JS = """JSON.stringify((function () { var main = document.getElementById('content'), fold = document.getElementById('progress-fold'); var band = document.getElementById('runhero'), lead = document.getElementById('lead-slot'); @@ -3450,7 +921,7 @@ def chip_open(b): def parity_problems(b, base, rows): """The karyogram's marks the map does not place with the same level and chromosome within 0.01 Mb.""" bad = [] - for run in sorted({r[0] for r in rows}): + for run in sorted({r[0] for r in rows if r[0] != OTHERS[1]}): got = map_page(b, base, run) if got["state"] != "drawn": bad.append(f"{run}: map {got['state']}") @@ -3521,449 +992,6 @@ def state_problems(b, base): return bad -def map_veil_problems(b, base, out): - """On the son's closed page: his closed result to act on (the planted deletion) is in no mark, tooltip, - chip, sentence, link or address of the map until opened; the chip's Open (after its confirm) opens one - result per click in the page's order (chip_open), so his result with no place opens first, in its section; - then Open and the card's "Show the result" each bring back exactly the deletion's mark, on the map and in - the card; Close it again and a reload close it. On DEMO-001, an adult's page, APOE stays closed until - opened, from the chip or its card's fold.""" - bad = [] - son = OTHERS[1] - doc = read_map(os.path.join(out, "runs", son, "map.json")) - f = doc["per"][son]["findings"] - acts = [i for i, lv in enumerate(f["level"]) if lv == "act" and not f["gate"][i]] - if not acts: - return ["the son's map has no result to act on to close"] - i = acts[0] - fid, gene = f["id"][i], f["gene"][i] - words = [gene, f"{f['chrom'][i]}:{f['s'][i] + 1}", f"{f['s'][i] + 1:,}", fid] - x, y = genome_xy(b, doc, f["chrom"][i], (f["s"][i] + f["e"][i]) / 2) - - def leaks(c, but=""): - return [w for w in words if w not in but and (w in c["text"] or w in c["tip"] or w in c["hash"])] + \ - [m["id"] for m in c["marks"] if m["id"] == fid] - c = map_page(b, base, son) - click_canvas(b, x, y, move_only=True) - c = map_now(b) - if c["state"] != "drawn": - return [f"the son's map: {c['state']}"] - if leaks(c): - bad.append(f"closed: the map shows {leaks(c)}") - # The chip counts every result to act on the page keeps closed, with a place on the map or not. - want = b.eval("new Set(Array.prototype.filter.call(document.querySelectorAll('.finding[data-level=act]'), " - "function (a) { return a.querySelector('.veil'); }).map(function (a) { return a.dataset.finding; })).size") - label = "1 result closed" if want == 1 else f"{want} results closed" - if not want or not any(t.startswith(label + " ") for t in c["chips"]): - bad.append(f"closed: no chip counting the page's {want} closed results: {c['chips']}") - # Open opens the page's closed results one per click, in the page's order, so the son's own result with no - # place (his first closed card) opens before the deletion: the map then names it only in its section chip. - order = chips_now(b)["closed"] - if not want: - return bad + ["the son's page shows no closed result to open"] - k, at = 0, None - for k in range(want): - got, at, more = chip_open(b) - if more: - return bad + [f"Open {k + 1}: {x}" for x in more] - if at != order[k]: - return bad + [f"Open {k + 1} opened {got}, the act card at {at}, not the page's next closed one {order[k]} ({order})"] - if got == fid: - break - section = [o for o in chips_now(b)["opened"] if o["kind"] == "section"] - if [o["href"] for o in section] != [f"#f/{got}"] or "Opened in its section: " not in section[0]["text"]: - bad.append(f"Open {k + 1} opened {got}, which has no place: section chips {section}") - c = map_now(b) - if leaks(c, " ".join(o["text"] for o in section)): # the opened result names its own gene - bad.append(f"Open {k + 1} ({got}): the map shows {leaks(c)}") - c = map_now(b) - card = b.eval("!!document.querySelector('#f-%s h3') && !document.querySelector('#f-%s .veil')" % (fid, fid)) - if [m["id"] for m in c["marks"] if m["id"] == fid] != [fid] or not card: - bad.append(f"Open: mark {[m['id'] for m in c['marks']]}, card shown {card}") - if not any(t.startswith("Opened: " + f["title"][i]) for t in c["chips"]): - bad.append(f"Open: chips {c['chips']}") - if fid in c["hash"] or gene in c["hash"]: - bad.append(f"Open: the address names it: {c['hash']}") - click_canvas(b, x, y, move_only=True) - c = map_now(b) - if gene not in c["tip"]: - bad.append(f"Open: its tooltip at its place does not name it: {c['tip']!r}") - # The results opened before it in their section close again from their chips, and those chips go. - b.eval("Array.prototype.forEach.call(document.querySelectorAll('.gm-chip.opened[data-opened=section]'), " - "function () { document.querySelector('.gm-chip.opened[data-opened=section] button').click(); }); true") - if [o for o in chips_now(b)["opened"] if o["kind"] == "section"] or chips_now(b)["closed"] != order[:k]: - bad.append(f"Close it again on the section chips: chips {chips_now(b)['opened']}, closed {chips_now(b)['closed']}") - b.eval("document.querySelector('.gm-chip.opened[data-opened=\\'1\\'] button').click(); true") - c = map_now(b) - card = at in chips_now(b)["closed"] - if leaks(c) or not card: - bad.append(f"Close it again: map shows {leaks(c)}, card closed {card}") - b.eval("document.querySelectorAll('.finding[data-level=act]')[%d].querySelector('.veil button').click(); true" % at) - c = map_now(b) - if [m["id"] for m in c["marks"] if m["id"] == fid] != [fid]: - bad.append("the card's Show the result does not open it on the map") - c = map_page(b, base, son) - if leaks(c): - bad.append(f"a reload does not close it: {leaks(c)}") - # APOE on an adult's page. - c = map_page(b, base, S) - if any(m["id"] == "apoe" for m in c["marks"]) or not any(t.startswith("APOE closed") for t in c["chips"]): - bad.append(f"APOE on DEMO-001: marks {[m['id'] for m in c['marks']]}, chips {c['chips']}") - b.eval("document.querySelector('.gm-chip[data-closed=gate] button').click(); true") - time.sleep(0.3) - b.eval("document.querySelector('.gm-chip[data-closed=gate] .gm-yes').click(); true") - c = map_now(b) - fold = b.eval("(document.querySelector('details[data-gate=apoe]') || {}).open") - if not any(m["id"] == "apoe" for m in c["marks"]) or fold is not True: - bad.append(f"Open APOE: marks {[m['id'] for m in c['marks']]}, its card's fold open {fold}") - b.eval("document.querySelector('details[data-gate=apoe] summary').click(); true") - c = map_now(b) - if any(m["id"] == "apoe" for m in c["marks"]): - bad.append("closing APOE's card fold leaves it on the map") - b.eval("document.querySelector('details[data-gate=apoe] summary').click(); true") - c = map_now(b) - if not any(m["id"] == "apoe" for m in c["marks"]): - bad.append("opening APOE's card fold does not open it on the map") - c = map_page(b, base, S) - if any(m["id"] == "apoe" for m in c["marks"]): - bad.append("a reload does not close APOE") - return bad - - -def overlap_problems(b, base, out): - """On the son's closed page, a second result to act on over the first carries only the first one's tag - (rust/crates/og-map/README.md). Opening the first leaves the second closed: in no mark, sentence, link, - title or address of the map.""" - son = OTHERS[1] - path = os.path.join(out, "runs", son, "map.json") - with open(path, "rb") as fh: - keep = fh.read() - try: - doc = read_map(path) - f = doc["per"][son]["findings"] - acts = [i for i, lv in enumerate(f["level"]) if lv == "act" and not f["gate"][i]] - if not acts: - return ["the son's map has no result to act on to close"] - i, n = acts[0], len(f["id"]) - fid = f["id"][i] - for k in f: - if isinstance(f[k], list) and len(f[k]) == n: - f[k].append(f[k][i]) - plant = {"id": "planted-overlap", "gene": "PLANTEDGENE", "title": "Planted overlapping result", - "href": "finding/planted-overlap/", "phrase": "Planted overlap phrase."} - for k, v in plant.items(): - f[k][-1] = v - write_map(path, doc) - c = map_page(b, base, son, "#" + f["chrom"][i]) - if c["state"] != "drawn": - return [f"the planted map: {c['state']}"] - if not open_closed(b, fid): - return [f"a link to {fid} lands on no closed card"] - c = map_now(b) - bad = [] - if fid not in [m["id"] for m in c["marks"]]: - bad.append(f"the first result did not open: marks {[m['id'] for m in c['marks']]}") - leaked = [w for w in plant.values() if w in c["text"] or w in c["tip"] or w in c["hash"]] - leaked += [m["id"] for m in c["marks"] if m["id"] == "planted-overlap"] - if leaked: - bad.append(f"opening the first shows the second: {leaked}") - return bad - finally: - with open(path, "wb") as fh: - fh.write(keep) - - -# The figure as a reader gets it before any click: its HTML (the chips aside, which count the closed results -# on purpose, and the timing attributes), its marks, margin and layers, and a hash of the canvas's pixels. -FIGURE_JS = r"""JSON.stringify((function () { - var f = document.getElementById('genome-map'); - if (!f) { return null; } - var c = f.cloneNode(true); - Array.prototype.forEach.call(c.querySelectorAll('.gm-chips'), function (x) { x.textContent = ''; }); - c.removeAttribute('data-pgp-map-parse-ms'); - c.removeAttribute('data-pgp-map-paint-ms'); - return { html: c.outerHTML, canvas: f.querySelector('.gm-canvas').toDataURL() }; -}()))""" - - -def plant_son(out, extra, og): - """The son's results.json with the findings EXTRA added, its findings' veil tags (results.veil.findings) - computed again by live_results.veil as the server computes them, and his map.json built again from it by og - map build, as the map plugin builds it: og tags each row itself. Returns what to write back afterwards.""" - son = OTHERS[1] - rj, mj = (os.path.join(out, "runs", son, n) for n in ("results.json", "map.json")) - keep = {} - for p in (rj, mj): - with open(p, "rb") as fh: - keep[p] = fh.read() - res = json.loads(keep[rj]) - res["findings"] = res["findings"] + [dict(f) for f in extra] - sec = {"act": 1, "medicine": 2, "carrier": 3, "review": 4} - for f in extra: # each in its level's section, as live_results lists them: its card shows on the page - for j in res["journal"]: - if j["n"] == sec[f["level"]]: - j["findings"] = j["findings"] + [f["id"]] - tags = live_results.veil({}, res["findings"], []).get("findings") or {} - res["veil"] = dict(res.get("veil") or {}, findings=dict((res.get("veil") or {}).get("findings") or {}, **tags)) - with open(rj, "w") as fh: - json.dump(res, fh) - root = os.path.join(os.path.dirname(out), "root") - doc = build_live_demo.og_map(og, ["build", "--root", root, "--run", son, "--assets", os.path.join(REPO, "assets"), - "--cache-dir", os.path.join(root, ".owngenome-live", "map")], - json.dumps({"results": res}).encode()) - with open(mj, "wb") as fh: - fh.write(doc) - return keep - - -def unplant(keep): - for p, data in keep.items(): - with open(p, "wb") as fh: - fh.write(data) - - -def row_tag(out, fid): - """The id whose tag og put on FID's row of the son's map, or None.""" - son = OTHERS[1] - doc = read_map(os.path.join(out, "runs", son, "map.json")) - f = doc["per"][son]["findings"] - if fid not in f["id"]: - return None - v = f["veil"][f["id"].index(fid)] - return doc["dict"]["veil"][v] if v >= 0 else None - - -def son_figure(b, base, hash_=""): - """The son's map before any click: (the map's facts, the figure with its canvas hashed).""" - c = map_page(b, base, OTHERS[1], hash_) - fig = json.loads(b.eval(FIGURE_JS) or "null") or {} - fig["canvas"] = hashlib.sha256((fig.get("canvas") or "").encode()).hexdigest()[:12] - return c, fig - - -def act(fid, gene, viewer, title): - return {"id": fid, "level": "act", "card": "clinical-filter", "gene": gene, "viewer": viewer, "title": title, - "detail": f"{gene} {viewer}", "text": f"{title}, planted."} - - -# An open finding over a closed result whose card names another gene: og tags the open finding's row (the -# whole gene span) with the closed result, while the page shows its card. A map that hides that row only -# while the result is closed places the closed result inside that gene. Each planting: (what, the closed -# results, the open findings in both pages, the open finding to watch, chromosome). -UNDER = (("a gene-only review finding (CFTR) over a closed deletion naming another gene", - [act("pl-iota", "PLIOTA", "chr7:117400000-117700000", "Planted iota deletion")], - [{"id": "pl-kappa", "level": "review", "card": "exomiser", "gene": "CFTR", "title": "Planted kappa review", - "detail": "CFTR", "text": "A planted review finding that names CFTR only."}], "pl-kappa", "chr7"), - # The page's own CYP2D6 medicine finding closes with the medicine card while any result is closed (a fixed - # panel), so a carrier finding stands in for a gene-only finding of a card that is no panel. - ("a gene-only carrier finding (CYP2D6) over a closed variant naming another gene", - [act("pl-theta", "PLTHETA", "chr22:42125000", "Planted theta result")], - [{"id": "pl-lambda", "level": "carrier", "card": "carrier-list", "gene": "CYP2D6", - "title": "Planted lambda carrier", "detail": "CYP2D6", "text": "A planted carrier finding that names CYP2D6 only."}], - "pl-lambda", "chr22")) - - -def under_problems(b, base, out, og): - """For each planting of UNDER: the son's map before any click is the same, figure HTML, canvas, marks, - margin and layers, with the closed result and without it, at the genome level and on its chromosome; and - the open finding is drawn in both.""" - bad = [] - for what, closed, opened, watch, chrom in UNDER: - got = {} - for name, extra in (("without", opened), ("with", opened + closed)): - keep = plant_son(out, extra, og) - try: - if name == "with" and row_tag(out, watch) != closed[0]["id"]: - bad.append(f"{what}: og did not tag {watch}'s row with {closed[0]['id']} ({row_tag(out, watch)}): " - "the planting tests nothing") - got[name] = [son_figure(b, base, h) for h in ("", "#" + chrom)] - with open(os.path.join(out, "runs", OTHERS[1], "results.json")) as fh: - title = [f["title"] for f in json.load(fh)["findings"] if f["id"] == watch][0] - if not b.eval("document.body.innerText.indexOf(%s) >= 0" % json.dumps(title)): - bad.append(f"{what}: the page does not show {watch}'s card ({title}): the planting tests nothing") - finally: - unplant(keep) - for (c0, f0), (c1, f1), lv in zip(got["without"], got["with"], ("genome", chrom)): - if watch not in [m["id"] for m in c0["marks"]]: - bad.append(f"{what}, {lv}: {watch} is not on the map even with no closed result") - if f0 != f1: - diff = [k for k in ("html", "canvas") if f0.get(k) != f1.get(k)] - bad.append(f"{what}, {lv}: the figure differs with the closed result ({diff}): marks " - f"{[m['id'] for m in c0['marks']]} vs {[m['id'] for m in c1['marks']]}") - return bad - - -def second_open_problems(b, base, out, og): - """Two overlapping closed results on the son's page (og tags the second with the first): opening the - second by its own card draws exactly it on the map, never the first.""" - two = [act("pl-alpha", "PLALPHA", "chr17:43050000-43060000", "Planted alpha result"), - act("pl-beta", "PLBETA", "chr17:43055000", "Planted beta result")] - keep = plant_son(out, two, og) - try: - if row_tag(out, "pl-beta") != "pl-alpha": - return [f"og did not tag pl-beta's row with pl-alpha ({row_tag(out, 'pl-beta')}): the planting tests nothing"] - c = map_page(b, base, OTHERS[1], "#chr17") - ids = [m["id"] for m in c["marks"]] - if "pl-alpha" in ids or "pl-beta" in ids: - return [f"closed: marks {ids}"] - if not open_closed(b, "pl-beta"): - return ["a link to pl-beta lands on no closed card"] - c = map_now(b) - ids = [m["id"] for m in c["marks"]] - bad = [] - if "pl-beta" not in ids: - bad.append(f"opened by its card, pl-beta is not on the map: marks {ids}, chips {c['chips']}") - if "pl-alpha" in ids or "PLALPHA" in c["text"]: - bad.append(f"opening pl-beta shows pl-alpha: marks {ids}") - return bad - finally: - unplant(keep) - - -# A finding the server tags with a closed result because it names its gene in its genes (results.veil.findings: -# an open structural variant over two genes), once with og's tag on its row too (CFTR, inside the deletion's span) -# and once with none (BRCA1). -NAMED = (act("pl-x", "PLXGENE", "chr7:117400000-117700000", "Planted x deletion"), - {"id": "pl-q", "level": "review", "card": "exomiser", "gene": "CFTR", "genes": ["CFTR", "PLXGENE"], - "title": "Symptom ranking: PLXGENE ranks 1", "detail": "CFTR", - "text": "A planted ranking that names the deletion's gene."}, - {"id": "pl-r", "level": "review", "card": "exomiser", "gene": "BRCA1", "genes": ["BRCA1", "PLXGENE"], - "title": "Symptom ranking: PLXGENE ranks 2", "detail": "BRCA1", - "text": "A planted ranking that names the deletion's gene."}) -# A row of the map with no finding in results.json (a newer finding the map holds before the page has it), -# on the closed deletion's place and with its tag. -GHOST = {"id": "pl-ghost", "level": "review", "gene": "GHOSTGENE", "title": "Ghost review", "href": "finding/pl-ghost/", - "phrase": "Ghost review phrase."} - - -def named_problems(b, base, out, og): - """On the son's closed page, with the planted deletion pl-x closed: the findings the server closes with it - (pl-q, pl-r) and a tagged map row the page has no finding for (pl-ghost) are in no mark, sentence, link, - title or address, and the figure is the one of the page without them; opening pl-x by its card draws pl-x, - pl-q and pl-r.""" - son = OTHERS[1] - _, before = son_figure(b, base) - keep = plant_son(out, NAMED, og) - try: - with open(os.path.join(out, "runs", son, "results.json")) as fh: - tags = (json.load(fh).get("veil") or {}).get("findings") or {} - if tags.get("pl-q") != ["pl-x"] or tags.get("pl-r") != ["pl-x"] or row_tag(out, "pl-r"): - return [f"the server's tags {tags}, og's tag on pl-r {row_tag(out, 'pl-r')}: the planting tests nothing"] - path = os.path.join(out, "runs", son, "map.json") - doc = read_map(path) - f = doc["per"][son]["findings"] - if "pl-x" not in f["id"] or "pl-x" not in doc["dict"]["veil"]: - return ["og did not place or tag pl-x: the planting tests nothing"] - i, n = f["id"].index("pl-x"), len(f["id"]) - for k in f: - if isinstance(f[k], list) and len(f[k]) == n: - f[k].append(f[k][i]) - for k, v in GHOST.items(): - f[k][-1] = v - f["veil"][-1] = doc["dict"]["veil"].index("pl-x") - f["veils"][-1] = [f["veil"][-1]] - write_map(path, doc) - c, fig = son_figure(b, base) - bad = [] - ids = [m["id"] for m in c["marks"]] - shown = [x for x in ("pl-x", "pl-q", "pl-r", "pl-ghost") if x in ids] - shown += [w for w in ("PLXGENE", "Ghost", "GHOSTGENE") if w in c["text"] or w in c["tip"] or w in c["hash"]] - if shown: - bad.append(f"closed: the map shows {shown}") - if fig != before: - bad.append(f"closed: the figure differs from the page without them " - f"({[k for k in ('html', 'canvas') if fig.get(k) != before.get(k)]})") - if not open_closed(b, "pl-x"): - return bad + ["a link to pl-x lands on no closed card"] - c = map_now(b) - ids = [m["id"] for m in c["marks"]] - if not {"pl-x", "pl-q", "pl-r"} <= set(ids): - bad.append(f"opened by its card: marks {ids}") - return bad - finally: - unplant(keep) - - -def chip_problems(b, base, out, og): - """The son's map before any click is the same, its chips included, whether his planted closed result has a - place on the map (chr13) or none (no viewer, a gene not in GENCODE): the chip counts the results the page - keeps closed, never only the placed ones.""" - got, bad = {}, [] - for name, fid, viewer in (("no place", "pl-nowhere", ""), ("a place", "pl-somewhere", "chr13:32340000")): - keep = plant_son(out, [act(fid, "PL" + fid[3:].upper(), viewer, "Planted closed result")], og) - try: - placed = fid in read_map(os.path.join(out, "runs", OTHERS[1], "map.json"))["per"][OTHERS[1]]["findings"]["id"] - if placed != bool(viewer): - bad.append(f"{name}: og placed {fid} {placed}: the planting tests nothing") - c, fig = son_figure(b, base) - got[name] = (c["chips"], fig) - finally: - unplant(keep) - if got["no place"] != got["a place"]: - bad.append(f"the chips {got['no place'][0]} with no place, {got['a place'][0]} with one" - + ("" if got["no place"][1] == got["a place"][1] else ", and the figure differs")) - return bad - - -# Two planted closed results, one with a place on the map and one without, in both orders of the page. -OPEN_ORDER = (("unplaced pl-a, placed pl-b", [act("pl-a", "PLAGENE", "", "Planted a result"), - act("pl-b", "PLBGENE", "chr13:32340000", "Planted b result")]), - ("placed pl-a, unplaced pl-b", [act("pl-a", "PLAGENE", "chr13:32340000", "Planted a result"), - act("pl-b", "PLBGENE", "", "Planted b result")])) - - -def open_order_problems(b, base, out, og): - """For each planting of OPEN_ORDER, the chip's Open clicked until nothing is closed (chip_open checks each - click): the ids it opens, click by click, are the same in both, so the order never tells which closed result - has a place. A result with no place opens with an "Opened in its section" chip that links its card; its - Close it again closes it on the page and takes the chip away. Before that, Open then Keep it closed opens - nothing and gives the keyboard back to the chips.""" - seqs, bad = {}, [] - for k, (what, extra) in enumerate(OPEN_ORDER): - keep = plant_son(out, extra, og) - try: - placed = read_map(os.path.join(out, "runs", OTHERS[1], "map.json"))["per"][OTHERS[1]]["findings"]["id"] - if [x["id"] in placed for x in extra] != [bool(x["viewer"]) for x in extra]: - return [f"{what}: og placed {[x['id'] for x in extra if x['id'] in placed]}: the planting tests nothing"] - map_page(b, base, OTHERS[1]) - first = chips_now(b) - if k == 0: - b.eval("var o = document.querySelector('.gm-chip[data-closed=act] button'); o.focus(); o.click(); true") - b.eval("document.querySelector('.gm-chip[data-closed=act] .gm-no').click(); true") - kept = chips_now(b) - if kept["closed"] != first["closed"] or kept["confirm"] or kept["focus"] != "first": - return [f"{what}: Keep it closed: closed {kept['closed']}, confirm {kept['confirm']}, " - f"keyboard on {kept['focus']!r}"] - seq = [] - for _ in first["closed"]: - got, _, more = chip_open(b) - if more: - return bad + [f"{what}, Open {len(seq) + 1}: {x}" for x in more] - seq.append(got) - opened = chips_now(b)["opened"] - section = [o["href"] for o in opened if o["kind"] == "section"] - if (got in placed) == (f"#f/{got}" in section): - return [f"{what}: {got} (placed {got in placed}) opened with section chips {section}"] - seqs[what] = seq - if k == 0: - gone = [x["id"] for x in extra if not x["viewer"]][0] - at = chips_now(b)["ids"].index(gone) if gone in chips_now(b)["ids"] else -1 - was = [o for o in chips_now(b)["opened"] if o["kind"] == "section"] - raw = [o["raw"] for o in was if o["href"] == f"#f/{gone}"] - b.eval("var o = document.querySelector('.gm-chip.opened[data-opened=section] a[href=\"%s\"]'); " - "o.parentNode.parentNode.querySelector('button').click(); true" % (raw[0] if raw else "")) - after = chips_now(b) - left = [o for o in after["opened"] if o["kind"] == "section"] - if after["closed"] != [at] or len(left) != len(was) - 1: - bad.append(f"{what}: Close it again on {gone}'s section chip: closed {after['closed']}, " - f"opened chips {after['opened']}") - finally: - unplant(keep) - if len(set(map(tuple, seqs.values()))) != 1: - bad.append(f"the order Open opens them tells which has a place: {seqs}") - return bad - - def runs_sentence(b, base): """Whether DEMO-001's planted runs sentence on chr1 is in the margin at the first frame of the replay and at the last: (first, last, the level and chromosome at the first frame).""" @@ -4072,13 +1100,10 @@ def missing_layer_problems(b, base, out): def fallback_problems(b, base, out): """A browser with no DecompressionStream: the sentence and the findings with a place, each through the veil.""" son = OTHERS[1] - doc = read_map(os.path.join(out, "runs", son, "map.json")) - f = doc["per"][son]["findings"] - gene = [f["gene"][i] for i, lv in enumerate(f["level"]) if lv == "act"][0] b.call("Page.enable", session=b.session) # a script for each new document needs the Page domain on sid = b.call("Page.addScriptToEvaluateOnNewDocument", {"source": "delete window.DecompressionStream;"}, b.session)["identifier"] try: - c = map_page(b, base, son) + c = map_page(b, base, S) rows = b.eval("document.querySelectorAll('.gm-fallback table tbody tr').length") finally: b.call("Page.removeScriptToEvaluateOnNewDocument", {"identifier": sid}, b.session) @@ -4087,8 +1112,6 @@ def fallback_problems(b, base, out): bad.append(f"no DecompressionStream: {c['state']}") if not rows: bad.append("no DecompressionStream: no findings table") - if gene in c["text"]: - bad.append(f"no DecompressionStream: the table names the closed {gene}") return bad @@ -4101,13 +1124,22 @@ def map_screens(b, base, dest): b.call("Emulation.setDeviceMetricsOverride", {"width": w, "height": h, "deviceScaleFactor": 1, "mobile": False}, b.session) for run, link, name in ((S, "", "genome"), (S, "#chr16", "chromosome"), (OTHERS[1], "", "closed")): - got = map_page(b, base, run, link) - want = "chromosome" if link else "genome" - if got["state"] != "drawn" or got["level"] != want: - print(f"ERROR: {run}{link}: the map is {got['state']} at {got['level'] or 'no'} level, " - f"want drawn at {want}: no screenshot") - continue - b.eval("document.getElementById('genome-map').scrollIntoView({block: 'start'}); true") + if name == 'closed': + b.go('about:blank'); b.wait("location.href === 'about:blank'") + b.go(base+'/runs/'+run+'/') + if not b.wait("document.documentElement.dataset.pgpGated === 'true'"): + print(f"ERROR: {run}: no closed report gate: no screenshot") + continue + target = 'report-gate' + else: + got = map_page(b, base, run, link) + want = "chromosome" if link else "genome" + if got["state"] != "drawn" or got["level"] != want: + print(f"ERROR: {run}{link}: the map is {got['state']} at {got['level'] or 'no'} level, " + f"want drawn at {want}: no screenshot") + continue + target = 'genome-map' + b.eval("document.getElementById(%s).scrollIntoView({block: 'start'}); true" % json.dumps(target)) time.sleep(0.5) png = b.call("Page.captureScreenshot", {"format": "png"}, b.session)["data"] p = os.path.join(dest, f"map-{name}-{w}x{h}.png") @@ -4161,114 +1193,9 @@ def map_checks(b, base, out): js = os.path.join(out, "assets", "person.js") with open(js) as fh: keep_js = fh.read() - bad = map_veil_problems(b, base, out) - check("map: the son's closed result is in no mark, tooltip, chip, sentence, link or address until opened; Open " - "(after its confirm) opens one result per click, in the page's order, and the count falls by one; Open and " - "the card's Show the result each bring back exactly its mark there and in the card; " - "Close it again and a reload close it; APOE stays closed on an adult's page until opened, from the chip " - "or its card's fold", not bad, bad) - mjs = os.path.join(out, "assets", "map.js") + mjs = os.path.join(out, 'assets', 'map.js') with open(mjs) as fh: keep_mjs = fh.read() - one = "opts.open(id); // " - every = "f.level === 'act' && veiled(f); })" - try: - with open(js, "w") as fh: - fh.write(keep_js.replace("function mapVeil(id, gated) {", "function mapVeil(id, gated) { return 'open';")) - control("person.js whose map veil keeps nothing closed", not map_veil_problems(b, base, out)) - with open(js, "w") as fh: - fh.write(keep_js.replace(every, "f.level === 'act'; })")) - control("person.js whose map chip counts every result to act on, closed or open", keep_js.count(every) != 1 or - not map_veil_problems(b, base, out)) - with open(js, "w") as fh: - fh.write(keep_js) - with open(mjs, "w") as fh: - fh.write(keep_mjs.replace(one, "(gated ? [id] : closedActs()).forEach(function (x) { opts.open(x); }); // ")) - control("map.js whose Open opens every closed result at once", keep_mjs.count(one) != 1 or - not map_veil_problems(b, base, out)) - finally: - with open(js, "w") as fh: - fh.write(keep_js) - with open(mjs, "w") as fh: - fh.write(keep_mjs) - bad = overlap_problems(b, base, out) - check("map: a second closed result to act on over the first stays closed when the first is opened", not bad, bad) - own = "var own = M.closable.indexOf(r.id) < 0 || stateOf(r.id) !== 'closed';" - try: - with open(mjs, "w") as fh: - fh.write(keep_mjs.replace(own, "var own = true;")) - control("map.js whose rows ask only their tag, never their own result", keep_mjs.count(own) != 1 or - not overlap_problems(b, base, out)) - finally: - with open(mjs, "w") as fh: - fh.write(keep_mjs) - og = og_binary() or build_live_demo.OG - bad = under_problems(b, base, out, og) - check("map: an open finding over a closed result naming another gene stays on the map; before any click the " - "figure is the same with the closed result and without it", not bad, bad) - bad = second_open_problems(b, base, out, og) - check("map: the second of two overlapping closed results, opened by its own card, is drawn, and the first " - "is not", not bad, bad) - known = "(opts.known && opts.known(r.id)) || " - try: - with open(mjs, "w") as fh: - fh.write(keep_mjs.replace(known, "")) - control("map.js whose page findings follow another result's tag", keep_mjs.count(known) != 1 or - not under_problems(b, base, out, og)) - control("map.js whose page findings follow another result's tag (the second result opened)", - keep_mjs.count(known) != 1 or not second_open_problems(b, base, out, og)) - finally: - with open(mjs, "w") as fh: - fh.write(keep_mjs) - bad = named_problems(b, base, out, og) - check("map: a finding the server closes with a closed result, and a tagged map row the page has no finding " - "for, stay off the map until the result is opened", not bad, bad) - now = "&& (!opts.now || opts.now(r.id));" - try: - with open(mjs, "w") as fh: - fh.write(keep_mjs.replace(now, "&& true;")) - control("map.js whose rows ignore what the page shows now", keep_mjs.count(now) != 1 or - not named_problems(b, base, out, og)) - with open(mjs, "w") as fh: - fh.write(keep_mjs.replace(known, "true || ")) - control("map.js that takes every row for a finding of the page", keep_mjs.count(known) != 1 or - not named_problems(b, base, out, og)) - finally: - with open(mjs, "w") as fh: - fh.write(keep_mjs) - bad = chip_problems(b, base, out, og) - check("map: the closed-results chip counts the results the page keeps closed, with a place on the map or " - "without", not bad, bad) - try: - with open(js, "w") as fh: - fh.write(keep_js.replace("closed: mapClosed, ", "")) - control("person.js that does not tell the map which results it keeps closed", - keep_js.count("closed: mapClosed, ") != 1 or not chip_problems(b, base, out, og)) - finally: - with open(js, "w") as fh: - fh.write(keep_js) - bad = open_order_problems(b, base, out, og) - check("map: the chip's Open opens one closed result per click in the page's order, whether it has a place or " - "not; one with no place gets an Opened in its section chip until closed again; the keyboard follows", - not bad, bad) - concat = "if (page) { return page.filter(function (id) { return !M.gated[id]; }); }" - placed_first = ("if (page) { var placed = M.closable.filter(function (id) { return !M.gated[id] && stateOf(id) === " - "'closed' && page.indexOf(id) >= 0; }); return placed.concat(page.filter(function (id) { return " - "placed.indexOf(id) < 0; })); }") - alive = "stateOf(id) !== 'closed' && opts.title && opts.title(id);" - pushed = "inSection.push(id);" - focus = "(b || canvas).focus({ preventScroll: true });" - try: - for desc, a, z in (("map.js whose Open opens the closed results with a place first", concat, placed_first), - ("map.js whose section chip stays for a closed result", alive, "true;"), - ("map.js with no section chip for a result with no place", pushed, ""), - ("map.js that leaves the keyboard where the chips were rebuilt", focus, "void b;")): - with open(mjs, "w") as fh: - fh.write(keep_mjs.replace(a, z)) - control(desc, keep_mjs.count(a) != 1 or not open_order_problems(b, base, out, og)) - finally: - with open(mjs, "w") as fh: - fh.write(keep_mjs) bad = replay_apoe_problems(b, base) check("map: before the traits step a replay has no APOE chip, switch or mark", not bad, bad) gated = "if (M.gated[id] && opts.now && !opts.now(id)) { return; }" @@ -4327,6 +1254,431 @@ def map_checks(b, base, out): print("map screenshots: " + ", ".join(map_screens(b, base, os.environ["MAP_SCREENS"]))) +def gate_checks(b, base, out, family_only=False): + """A single ownership property over every payload, including fields added after this test.""" + import itertools + start = time.monotonic() + son = OTHERS[1] + helper = open(os.path.join(REPO, 'tests', 'demo', 'gate-property.js')).read() + paths = [os.path.join(out, 'runs', son, n) for n in os.listdir(os.path.join(out, 'runs', son)) + if n.endswith('.json')] + paths += [os.path.join(out, 'runs.json'), os.path.join(out, 'runs', FAM, 'family.json')] + saved = {p: open(p, 'rb').read() for p in paths} + result_path = os.path.join(out, 'runs', son, 'results.json') + original = json.loads(saved[result_path]) + acts = [f for f in original['findings'] if f['level'] == 'act'] + # Include placed, unplaced and multi-gene acts with stable identities. Other records are not controls. + acts = (acts + [dict(acts[0], id='gate-third', gene='HBA1', genes=['HBA1', 'HBA2'], viewer=None)])[:3] + + def write(p, d): + if p.endswith('/map.json'): + write_map(p, d) + else: + with open(p, 'w') as f: + json.dump(d, f) + + def walk(d): + b.eval(helper) + return b.eval('GateProperty.walk(' + json.dumps(d) + ', function () { return false; })') + + payload_cache = {} + + def payloads(count, changed=False, keep_first=False): + cache_key = (count, changed, keep_first if changed else False) + if cache_key in payload_cache: + for p, raw in payload_cache[cache_key].items(): + with open(p, 'wb') as f: + f.write(raw) + return + for p, raw in saved.items(): + d = json.loads(gzip.decompress(raw)) if p.endswith('/map.json') else json.loads(raw) + if p == result_path: + d['findings'] = [f for f in d['findings'] if f['level'] != 'act'] + acts[:count] + d.setdefault('veil', {})['acts'] = [f['id'] for f in acts[:count]] + if changed: + nonacts = [f for f in original['findings'] if f['level'] != 'act'] + d = walk(d) + changed_acts = d['findings'][len(nonacts):len(nonacts)+count] + d['findings'] = d['findings'][:len(nonacts)] + [d['findings'][-1]] + [ + f if keep_first and i == 0 else dict(changed_acts[i], id=f['id'], level='act') + for i, f in enumerate(acts[:count])] + [None] + d['veil']['acts'] = [f['id'] for f in acts[:count]] + elif changed and '/'+son+'/' in p: + if p.endswith('/frames.json'): + # Family membership and URLs are routing controls, not report content. + family_plugin = json.loads(raw)['frames'][-1]['state']['plugins']['family'] + d = walk(d) + for fr in d['frames']: + fr['state']['plugins']['family'] = family_plugin + else: + d = walk(d) + elif changed and p.endswith('/runs.json'): + for i, run in enumerate(d['runs']): + if run['name'] == son: + d['runs'][i] = dict(walk(run), name=son, url=run['url']) + elif changed and p.endswith('/family.json'): + # One generic walk, then restore the other open owners and immutable routing controls. + changed_family = walk(d) + for j, child in enumerate(d['state']['children']): + if child['child'] == son: + d['state']['children'][j] = dict(changed_family['state']['children'][j], child=son) + for key in ('relationships', 'ibd_pairs'): + for j, row in enumerate(d.get(key, [])): + if son in (row.get('sample_a'), row.get('sample_b')): + d[key][j] = dict(changed_family[key][j], sample_a=row['sample_a'], sample_b=row['sample_b']) + for key in ('trio_files', 'trio_file_state'): + if key in d: + d[key] = changed_family[key] + d['state']['plan'] = changed_family['state']['plan'] + d['state']['reports'] = changed_family['state']['reports'] + for key in ('segments', 'trio_status'): + if son in d.get(key, {}): + d[key][son] = changed_family[key][son] + for key in ('de_novo', 'compound_het'): + d[key] = [dict(changed_family[key][i], child=son) if r.get('child') == son else r + for i, r in enumerate(d.get(key, []))] + d[key] += [dict(child=son, future_child_field='new child record')] + sb = d['side_by_side']; i = sb['members'].index(son) + for j, row in enumerate(sb['rows']): + changed_row = changed_family['side_by_side']['rows'][j] + row['values'][i] = changed_row['values'][i] + row['stale'][i] = changed_row['stale'][i] + row['future_child_note'] = changed_row['future_payload_field'] + sb['rows'].append(dict(group='Future child group', label='Future child field', + values=['no summary yet' if m != son else 'new child value' for m in sb['members']], + stale=[False]*len(sb['members']))) + sb['rows'].append(copy.deepcopy(sb['rows'][0])) + sb['rows'].reverse() + if son in d.get('runs', {}): + d['runs'][son] = dict(changed_family['runs'][son], url=d['runs'][son]['url']) + d['future_child_payload'] = changed_family['future_payload_field'] + d = {k: d[k] for k in reversed(d)} + write(p, d) + payload_cache[cache_key] = {p: open(p, 'rb').read() for p in saved} + + def visit(route): + b.go('about:blank') + assert b.wait("location.href === 'about:blank'"), 'navigation did not start' + b.go(base + route) + assert b.wait("document.readyState === 'complete' && document.documentElement.dataset.pgpStatus === 'ready'"), route + ' never ready' + # The gate is synchronous once its inputs settle; no arbitrary multi-second settling sleep. + if route.startswith('/runs/'+son+'/'): + assert b.wait("document.documentElement.dataset.pgpGated === 'true'"), route + ' not gated' + b.eval(helper) + + def snap(): + b.eval(helper) + s = b.eval('GateProperty.snapshot()') + ax = b.call('Accessibility.getFullAXTree', {}, b.session) + s['accessibility'] = [(n.get('role', {}).get('value'), n.get('name', {}).get('value')) + for n in ax.get('nodes', []) if not n.get('ignored')] + return s + + routes = ['/runs/'+son+'/', '/', '/runs/'+FAM+'/', + '/runs/'+son+'/finding/'+acts[0]['id']+'/', '/runs/'+son+'/#chr7'] + if family_only: + routes = ['/runs/'+FAM+'/'] + b.call('Network.enable', {}, b.session) + b.call('Network.setCacheDisabled', {'cacheDisabled': True}, b.session) + bad = [] + try: + b.eval(helper) + future = walk({'future_field': {'s': 'new', 'n': 2, 'b': True, 'a': []}}) + check('gate walker changes a new field, primitives, array length and key order', + future != {'future_field': {'s': 'new', 'n': 2, 'b': True, 'a': []}} + and future['future_field']['n'] != 2 and len(future['future_field']['a']) == 1, future) + for width, media in ((1440, 'screen'), (390, 'screen'), (1440, 'print')): + b.call('Emulation.setDeviceMetricsOverride', dict(width=width, height=900, deviceScaleFactor=1, mobile=False), b.session) + b.call('Emulation.setEmulatedMedia', {'media': media, 'features': [{'name':'prefers-color-scheme', 'value':'dark'}]}, b.session) + for route in routes: + snapshots, partials = [], [] + for changed in (False, True): + payloads(3, changed); visit(route); snapshots.append(snap()) + if route != '/': + payloads(3, changed, keep_first=True); visit(route) + completed = b.eval("performance.getEntriesByType('resource').filter(function(r){return r.name.endsWith('/"+son+"/results.json');}).length") + b.eval("document.querySelector('[data-opening^=closed]').click(); true") + assert b.wait("!!document.querySelector('[data-opening=opened-1]')"), 'partial opening never settled' + if route == '/runs/'+FAM+'/': + assert b.wait("performance.getEntriesByType('resource').filter(function(r){return r.name.endsWith('/"+son+"/results.json');}).length > "+str(completed)), 'family projection refresh never settled' + partials.append(snap()) + if snapshots[0] != snapshots[1]: + bad.append((width, media, route, [k for k in snapshots[0] if snapshots[0][k] != snapshots[1][k]])) + if partials and partials[0] != partials[1]: + bad.append((width, media, route, 'partially opened', [k for k in partials[0] if partials[0][k] != partials[1][k]])) + print(' gate matrix time: %.2f s' % (time.monotonic()-start)) + check('gate property: every mutable payload produces identical DOM, AX text, address, pixels, SVG, storage and requests', not bad, bad) + if not family_only: + b.call('Emulation.setEmulatedMedia', {'media': 'screen'}, b.session) + b.call('Emulation.setDeviceMetricsOverride', dict(width=1440, height=900, deviceScaleFactor=1, mobile=False), b.session) + transitions = [] + for count in (1, 2, 3): + payloads(count) + for order in itertools.permutations(range(count)): + visit('/runs/'+son+'/') + for stage, i in enumerate(order): + before = snap() + b.eval("document.querySelectorAll('[data-opening]')[%d].click()" % i) + if stage < count - 1: + assert b.wait("document.documentElement.dataset.pgpGated === 'true'"), 'released early' + b.eval("document.querySelectorAll('[data-opening]')[%d].click()" % i) + if snap() != before: + transitions.append((count, order, stage, 'close-again differs')) + b.eval("document.querySelectorAll('[data-opening]')[%d].click()" % i) + else: + assert b.wait("!document.documentElement.dataset.pgpGated && !!document.querySelector('#results .finding')"), 'last opening never released' + # Search something while open; closing must replace this whole document and its caches. + b.wait("document.querySelector('input[type=search]') && !document.querySelector('input[type=search]').disabled") + b.eval("var s=document.querySelector('input[type=search]'); s.value='KCNH2'; s.dispatchEvent(new Event('input')); true") + b.eval("PGPUI.closeAct(%s); true" % json.dumps(acts[i]['id'])) + assert b.wait("document.documentElement.dataset.pgpGated === 'true'"), 'close did not clean up' + if b.eval("!!document.querySelector('input[type=search],.search-results,.search-status,canvas,svg,img')"): + transitions.append((count, order, stage, 'old view survives')) + b.call('Page.reload', {}, b.session) + assert b.wait("document.querySelectorAll('[data-opening^=closed]').length === %d" % count), 'reload kept opening' + print(' gate matrix and transitions time: %.2f s' % (time.monotonic()-start)) + check('gate transitions: 1/2/3 acts, every order, close-again at every stage, search then close and reload', not transitions, transitions) + payloads(3) + guesses = [] + for h in ('#f/'+acts[0]['id'], '#f/closed-99', '#f/opened-1', '#f/closed-1', '#s/exomiser', '?chr=17&start=43050000#f/closed-99'): + visit('/runs/'+son+'/'+h) + guesses.append(b.eval('location.hash')) + assert b.eval('location.search') == '', 'a gated address kept a data query' + check('gate handles: typed and guessed ids share the first opaque closed handle', len(set(guesses)) == 1 and guesses[0] == '#f/closed-1', guesses) + visit('/runs/'+son+'/') + b.eval("location.hash='f/closed-2'; true") + assert b.wait("location.hash === '#f/closed-2'") + b.eval("location.hash='f/closed-3'; true") + assert b.wait("location.hash === '#f/closed-3'") + b.eval("history.back(); true") + assert b.wait("location.hash === '#f/closed-2'") + b.eval("history.forward(); true") + assert b.wait("location.hash === '#f/closed-3'") + b.eval("document.querySelector('[data-opening=closed-1]').click(); true") + check('gate handles follow their act after another act opens', b.eval("location.hash") == '#f/closed-2') + + finally: + for p, raw in saved.items(): + with open(p, 'wb') as f: + f.write(raw) + b.call('Emulation.clearDeviceMetricsOverride', {}, b.session) + b.call('Emulation.setEmulatedMedia', {'media': ''}, b.session) + if family_only: + gate_family_independence(b, base, out) + if not family_only: + gate_controls(b, base, out) + gate_transparency(b, base, out) + print('gate browser time: %.2f s' % (time.monotonic()-start)) + + +def gate_controls(b, base, out): + """Each gate guard must fail when its production implementation is deliberately broken.""" + son = OTHERS[1] + js = os.path.join(out, 'assets', 'core.js') + rj = os.path.join(out, 'runs', son, 'results.json') + source = open(js).read(); raw = open(rj).read() + data = json.loads(raw); data['future_payload_field'] = 'FUTURE_GATE_SENTINEL' + with open(rj, 'w') as f: + json.dump(data, f) + + def visit(): + b.go('about:blank'); assert b.wait("location.href === 'about:blank'") + b.go(base+'/runs/'+son+'/') + assert b.wait("document.documentElement.dataset.pgpGated === 'true'"), 'fixture never gated' + + def partial(): + visit() + b.eval("document.querySelector('[data-opening^=closed]').click(); true") + return bool(b.wait("document.documentElement.dataset.pgpGated === 'true' && document.querySelectorAll('[data-opening^=closed]').length === 1", timeout=2)) + + def raw_view(): + visit() + # Both requests have settled when the optional deliberately broken view signals it is ready. + b.wait("document.readyState === 'complete'", timeout=2) + if 'data-gate-raw-read' in open(js).read(): + assert b.wait("document.documentElement.dataset.gateRawRead === 'done'", timeout=2), 'raw probe did not settle' + return not b.eval("document.documentElement.outerHTML.includes('FUTURE_GATE_SENTINEL')") + + def cleanup(): + visit(); open_gate(b) + assert b.wait("!!document.querySelector('#results .finding')"), 'full report never loaded' + assert b.wait("document.querySelector('input[type=search]') && !document.querySelector('input[type=search]').disabled"), 'search never loaded' + b.eval("var s=document.querySelector('input[type=search]'); s.value='KCNH2'; s.dispatchEvent(new Event('input')); true") + fid = next(f['id'] for f in data['findings'] if f['level']=='act') + return b.eval('PGPUI.closeAct('+json.dumps(fid)+"); PGPUI.gateOwnsPage() && !document.querySelector('input[type=search],.search-results,.search-status,canvas,svg,img')") + + def address(): + b.go('about:blank'); assert b.wait("location.href === 'about:blank'") + b.go(base+'/runs/'+son+'/?chr=17&start=43050000#f/closed-99') + assert b.wait("document.documentElement.dataset.pgpGated === 'true'") + return b.eval("location.search === '' && location.hash === '#f/closed-1'") + + def stale_control(): + b.go('about:blank'); assert b.wait("location.href === 'about:blank'") + b.go(base+'/runs/'+FAM+'/') + assert b.wait("!!document.querySelector('[data-gate-child=DEMO-003] [data-opening^=closed]')") + b.eval("window.gateOldControl=document.querySelector('[data-gate-child=DEMO-003] [data-opening^=closed]'); true") + changed = copy.deepcopy(data) + next(f for f in changed['findings'] if f['level']=='act')['text'] += ' REPLACEMENT_ACT_RECORD' + try: + with open(rj, 'w') as f: + json.dump(changed, f) + b.eval("window.gateRefreshDone=false; PGPUI.getJSON('family.json').then(function(){window.gateRefreshDone=true;}); true") + assert b.wait('window.gateRefreshDone'), 'family refresh never settled' + return b.eval("gateOldControl.click(); !document.querySelector('[data-gate-child=DEMO-003] .opened-act')") + finally: + with open(rj, 'w') as f: + json.dump(data, f) + + def refresh_cleanup(): + visit(); open_gate(b) + assert b.wait("!!document.querySelector('#results .finding')") + # Hold document replacement so the guard must remove the old report synchronously. + b.eval("window.gateTransferStarted=false; crypto.subtle.digest=function(){window.gateTransferStarted=true; return new Promise(function(){});}; true") + changed=copy.deepcopy(data) + next(f for f in changed['findings'] if f['level']=='act')['text'] += ' REFRESHED_ACT_RECORD' + try: + with open(rj, 'w') as f: + json.dump(changed, f) + b.eval("PGPUI.getJSON('results.json'); true") + assert b.wait('window.gateTransferStarted'), 'refresh never reached document replacement' + return b.eval("PGPUI.gateOwnsPage() && !document.querySelector('#results,canvas')") + finally: + with open(rj, 'w') as f: + json.dump(data, f) + + mutations = [ + ('stale control opens a replacement act', 'if (!f || JSON.stringify(f) !== JSON.stringify(rec.acts[i]))', 'if (!f)', stale_control), + ('refresh leaves the old report visible', 'if (shut(current)) { ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer();', 'if (shut(current)) { transfer();', refresh_cleanup), + ('keep a map query in the closed address', 'location.pathname + to', 'location.pathname + location.search + to', address), + ('release one act early', "rec.acts.some(function (f) { return !rec.opened[f.id]; })", "rec.acts.filter(function (f) { return !rec.opened[f.id]; }).length > 1", partial), + ('one view reads the raw payload', 'add(document.body, wrap);', "add(document.body, wrap); jsonAt(current.url + 'results.json').then(function (r) { add(wrap, el('p', {}, JSON.stringify(r))); document.documentElement.setAttribute('data-gate-raw-read', 'done'); });", raw_view), + ('skip cleanup after closing', 'delete current.opened[id]; ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return true;', 'delete current.opened[id]; return true;', cleanup)] + try: + for desc, before, after, probe in mutations: + assert source.count(before)==1, 'mutation anchor moved: '+desc + check('gate guard: '+desc, probe()) + with open(js, 'w') as f: + f.write(source.replace(before, after)) + still_passes = probe() + control('gate: '+desc, still_passes) + if desc == 'stale control opens a replacement act': + assert b.eval("document.querySelector('[data-gate-child=DEMO-003] .opened-act').textContent.includes('REPLACEMENT_ACT_RECORD')"), 'control failed without opening the replacement' + elif desc == 'refresh leaves the old report visible': + assert b.eval("!!document.querySelector('#results .finding')"), 'control failed without retaining the report' + elif desc == 'release one act early': + assert b.eval("!!document.documentElement.dataset.pgpResults && !document.documentElement.dataset.pgpGated"), 'control failed without releasing the report' + elif desc == 'skip cleanup after closing': + assert b.eval("document.querySelector('input[type=search]').value === 'KCNH2'"), 'control failed without retaining the query' + print(' observed guard failure: '+desc) + + with open(js, 'w') as f: + f.write(source) + finally: + with open(js, 'w') as f: + f.write(source) + with open(rj, 'w') as f: + f.write(raw) + + +def gate_transparency(b, base, out): + """The gate must leave adult and no-act documents byte-equal to the unmediated full renderer.""" + js = os.path.join(out, 'assets', 'core.js') + rj = os.path.join(out, 'runs', OTHERS[1], 'results.json') + source = open(js).read(); original = open(rj).read() + # The same current renderer with the loader absent is the transparent reference. + bare = source[:source.index(' // The report gate owns raw payloads.')] + '''\n PGPUI.ready = Promise.resolve(); + PGPUI.gateOwnsPage = function () { return false; }; + PGPUI.openedActs = function () { return {}; }; + PGPUI.closeAct = function () { return false; }; +}()); +''' + helper = open(os.path.join(REPO, 'tests', 'demo', 'gate-property.js')).read() + + def capture(run, timeout=20): + b.go('about:blank'); assert b.wait("location.href === 'about:blank'") + b.go(base+'/runs/'+run+'/') + assert b.wait("document.documentElement.dataset.pgpResults && !!document.querySelector('[data-pgp-map=drawn]')", timeout=timeout), 'open report not drawn' + assert b.wait("!!document.querySelector('#rail a[href*=DEMO-FAMILY]') && !!document.documentElement.dataset.pgpCharts"), 'family or charts never settled' + time.sleep(0.2) + b.eval(helper) + return b.eval("""(function () { + var clone=document.documentElement.cloneNode(true); + clone.querySelectorAll('[data-pgp-map-paint-ms],[data-pgp-map-parse-ms]').forEach(function (n) { + n.removeAttribute('data-pgp-map-paint-ms'); n.removeAttribute('data-pgp-map-parse-ms'); + }); + return [clone.outerHTML, location.hash, Array.from(document.querySelectorAll('canvas')).map(function(c){return c.toDataURL();})]; + }())""") + try: + for label, run in (('adult', S), ('child with no act', OTHERS[1])): + if run == OTHERS[1]: + data=json.loads(original); data['findings']=[f for f in data['findings'] if f['level'] != 'act']; data['veil']={'acts':[]} + with open(rj, 'w') as f: + json.dump(data, f) + pair=[] + for text in (bare, source): + with open(js, 'w') as f: + f.write(text) + pair.append(capture(run)) + detail = [i for i in range(3) if pair[0][i] != pair[1][i]] + if 0 in detail: + at = next((i for i,(a,z) in enumerate(zip(pair[0][0],pair[1][0])) if a!=z),0) + detail += [pair[0][0][max(0,at-80):at+120], pair[1][0][max(0,at-80):at+120]] + check('gate transparency: '+label+' has identical DOM, attributes, address and canvas', pair[0] == pair[1], detail) + assert source.count('ownsPage = shut(current);') == 1 + with open(js, 'w') as f: + f.write(source.replace('ownsPage = shut(current);', 'ownsPage = true;')) + try: + still_equal = capture(OTHERS[1], timeout=2) == pair[1] + except AssertionError: + still_equal = False + control('gate closing an adult or no-act report', still_equal) + + finally: + with open(js, 'w') as f: + f.write(source) + with open(rj, 'w') as f: + f.write(original) + + +def gate_family_independence(b, base, out): + """Two children may have the same act IDs; opening one must never open the other.""" + son, other = OTHERS[1], 'DEMO-004' + fj = os.path.join(out, 'runs', FAM, 'family.json') + raw = open(fj).read(); family=json.loads(raw) + member = next(m for m in family['state']['members'] if m['sample']==son) + family['state']['members'].append(dict(member, sample=other)) + child = next(c for c in family['state']['children'] if c['child']==son) + family['state']['children'].append(dict(child, child=other)) + family['runs'][other]=dict(family['runs'][son], url='../../runs/'+other+'/') + sb=family['side_by_side']; i=sb['members'].index(son); sb['members'].append(other) + for row in sb['rows']: + row['values'].append(row['values'][i]); row['stale'].append(row['stale'][i]) + target=os.path.join(out,'runs',other) + assert not os.path.exists(target), 'independent-child fixture already exists' + os.makedirs(target) + results=json.load(open(os.path.join(out,'runs',son,'results.json'))) + results['sample']=other; results['person']['sample']=other + with open(os.path.join(target,'results.json'),'w') as f: + json.dump(results,f) + try: + with open(fj,'w') as f: + json.dump(family,f) + b.go('about:blank'); assert b.wait("location.href === 'about:blank'") + b.go(base+'/runs/'+FAM+'/') + assert b.wait("!!document.querySelector('[data-gate-child=DEMO-004]')"), 'second child never gated' + before=b.eval("document.querySelector('[data-gate-child=DEMO-004]').outerHTML") + for remaining in range(sum(f['level']=='act' for f in results['findings']),0,-1): + b.eval("document.querySelector('[data-gate-child=DEMO-003] [data-opening^=closed]').click(); true") + assert b.wait("document.querySelectorAll('[data-gate-child=DEMO-003] [data-opening^=closed]').length === %d" % (remaining-1)), 'opening did not settle' + after=b.eval("document.querySelector('[data-gate-child=DEMO-004]').outerHTML") + check('family gate openings are independent for children sharing act IDs', before==after) + finally: + with open(fj,'w') as f: + f.write(raw) + shutil.rmtree(target) + + def screens_main(dest, site): """--map-screens DEST SITE: the map's screenshots of an already built demo (the docs workflow's artifact).""" chrome = find_chrome() diff --git a/tests/test_live_results.py b/tests/test_live_results.py index da0d3f94e..2b1481d66 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -31,43 +31,7 @@ secondary-findings list, a carrier list not made and an unparsed PharmCAT report as sentences (never zero counts), an unsized "normal" locus as not sized, AnnotSV's exon-edge check, a genotype rule and a deep intronic call; - 8. the veil's tags on DEMO-003 (make_demo_sample.py --sample DEMO-003, with its - planted secondary finding and class 5 deletion): a tag is the list of every - closed result a place names; the ClinVar row and viewer link of the planted - hit, its genome-chart tick and its review-star bar carry the secondary - finding's id, the deletion's span its own, every search item the fixed-panel - tag (a hit quotes the summary's rows, prose), and nothing else is tagged; - DEMO-001, with no result to act on, has no tag; every known deep intronic - site's line carries the fixed-panel tag, never the id of a closed result in - its gene (the card prints each site whether hit or not, so one missing line - would name the gene); prose (a line, a fact, a headline) is tagged by the - fixed-panel tag whatever it names, unless its card is declared as printing - no gene there (LINES_NO_GENE, FACTS_NO_GENE, HEADLINES_NO_GENE), so no word - is ever read as a gene; a gene cell is read as its list of symbols; a - ClinVar row closed by its gene alone closes its "See the reads at" link with - it; every other card's row that names the gene (the son's symptom ranking - ranks it) carries one of the ids and no other row does, the ranking's - finding carries the secondary finding's id, and so does a segment of the - lead that prints the ranking, and a segment that prints two closed results' - findings carries both ids; the home page's view of the run (overview) lists - no such finding among its first ones and its closed lead names none; a CHIP - call, a copy-number dot and the carrier list's two-hit fact and line carry - the id of the closed result in their gene (the CHIP chart's panel of genes - never does), and results.json carries the closed results as veil.spans (the - family page closes a child's places while one is closed); a closed - structural variant over two genes closes the rows and the finding that name - only its second gene, a place over two closed results carries both ids, and - veil.panels lists the cards of a fixed panel (live_results.FIXED_PANELS). -Each check runs again on a planted mistake and must fail there: 1 with VNTYPER's -entry dropped, 2 with one card's caveat blanked, 3 with "ok" planted in a card -and "All clear" in a script, 4 with the hits emptied, 5 with the results route -registered without the login, 6 with a row printed as its Python dict, 7 with the Slivar adapter returning nothing, -8 with a veil rule that tags nothing (rows, lines, facts and drawings), one that tags every row, one that tags -only a structural variant's first gene, one that keeps only the first id of a place, one that reads prose word by -word, and each entry of the declared lists dropped. A check -that cannot fail is no check. - -Run: python3 tests/test_live_results.py + 8. results.veil carries only the act identities needed by the shared browser data gate. """ import contextlib import base64 @@ -360,513 +324,6 @@ def server_problems(port, state): return out -def veil_problems(res, idx): - """What is wrong with the veil's tags of DEMO-003's results.json and search index.""" - import make_demo_sample - gene = make_demo_sample.PLANTED["gene"] - c, p, _, _ = make_demo_sample.PLANTED["hit"] - sv_c, a, b = make_demo_sample.PLANTED["sv"] - act = [f["id"] for f in res["findings"] if f["level"] == "act"] - sf = next((i for i in act if i.startswith("sf-")), None) - sv = next((i for i in act if i.startswith("sv-")), None) - if not sf or not sv: - return [f"no planted secondary finding and deletion among the results to act on: {act}"] - v = res.get("veil") or {} - out = [] - card = next(x for ch in res["chapters"] for x in ch["sections"] if x["id"] == "clinvar") - # The planted hit sits inside the planted deletion and names its gene: its places name both closed results. - both = sorted([sf, sv]) - rows = (v.get("rows") or {}).get("clinvar") or [] - want = [both if r[0] == gene and r[1].startswith(f"{c}:{p} ") else None for r in card["table"]["rows"]] - if [sorted(t) if t else None for t in rows] != want: - out.append(f"ClinVar rows tagged {rows}, want {want}") - view = (v.get("viewer") or {}).get("clinvar") or [] - if [sorted(t) if t else None for t in view] != [both if x == f"{c}:{p}" else None for x in card.get("viewer") or []]: - out.append(f"ClinVar viewer links tagged {view}") - marks = {k: {m: sorted(t) for m, t in x.items() if t and t != [live_results.PANEL]} for k, x in (v.get("marks") or {}).items()} - marks = {k: x for k, x in marks.items() if x} - specs = {sp["id"]: sp for sp in rr.chart_specs(res["_summary"])} - hit = [m["id"] for m in specs["genome"]["marks"] if m.get("kind") == "hit" and m["pos"] == p and m["chrom"] == c] - span = [m["id"] for m in specs["genome"]["marks"] if m.get("kind") == "sv" and (m["chrom"], m["pos"], m["end"]) == (sv_c, a, b)] - bar = [m["id"] for m in specs["clinvar-stars"]["marks"] if gene in (m.get("detail") or "")] - want = {"genome": {**{h: both for h in hit}, **{s_: both for s_ in span}}, "clinvar-stars": {x: both for x in bar}} - if not hit or not span or not bar or marks != want: - out.append(f"chart marks tagged {marks}, want {want}") - # Every other card's rows: a row that names the gene carries one of the two ids, a row that does not carries - # none; the symptom ranking's row of the gene carries the secondary finding's, and so does its finding. - for cid, rows_ in ((x["id"], (x.get("table") or {}).get("rows") or []) for ch in res["chapters"] for x in ch["sections"]): - if cid == "clinvar" or not rows_: - continue - got = (v.get("rows") or {}).get(cid) or [None] * len(rows_) - bad = [(r, t) for r, t in zip(rows_, got) if bool(set(t or ()) & {sf, sv}) != (gene in " ".join(map(str, r)))] - if bad or len(got) != len(rows_): - out.append(f"{cid} rows tagged wrong: {bad or got}") - exo = (v.get("rows") or {}).get("exomiser") or [] - ranked = [f["id"] for f in res["findings"] if f["level"] != "act" and gene in f["title"]] - # A fixed-panel card's own findings (a medicine gene) carry the panel tag, whatever the closed result. - named = {k: t for k, t in (v.get("findings") or {}).items() if t and t != [live_results.PANEL]} - if not any(sf in (t or ()) for t in exo) or not ranked or set(named) != set(ranked) \ - or any(sf not in named[i] for i in ranked): - out.append(f"the symptom ranking of {gene}: rows tagged {exo}, findings tagged {v.get('findings')}, want {ranked} -> {sf}") - # A search item quotes the summary's rows: prose, so every one carries the fixed-panel tag. - items = [it for items in idx["terms"].values() for it in items] - if not items or any(it.get("veil") != [live_results.PANEL] for it in items): - out.append(f"search items tagged {sorted({str(it.get('veil')) for it in items})}, want every one {[live_results.PANEL]}") - return out - - -def veil_lines_problems(): - """What is wrong with the veil's tags of the ClinVar card's lines: a known deep intronic site is printed - whether hit or not, a fixed panel, so every such line carries the panel tag (live_results.PANEL), the one in - a closed secondary finding's gene and the others alike: a line closed alone would name the gene.""" - S = {"clinvar": {"state": "ok", "data": {"hits": [], "deep_intronic": [ - {"gene": "GLA", "hgvs": "NM_000169.3:c.640-801G>A", "chrom": "chrX", "pos": 101399747, "ref": "C", - "alt": "T", "call": "hom", "detail": "GQ 40"}, - {"gene": "GENE5", "hgvs": "c.100+500A>G", "chrom": "chr1", "pos": 1, "ref": "A", "alt": "G", - "call": "het", "detail": "GQ 40"}]}}} - lines = live_results.a_clinvar(S, {"sections": S}).get("lines") or [] - found = [{"id": "sf-gla-test", "level": "act", "gene": "GLA", "detail": "GLA"}] - v = live_results.veil({"sections": S}, found, [{"sections": [{"id": "clinvar", "lines": lines}]}]) - panel = [live_results.PANEL] - want = [panel for ln in lines] - got = (v.get("lines") or {}).get("clinvar") - out = [] if want.count(panel) == 2 and got == want else [f"lines {lines} tagged {got}, want {want}"] - # A segment of the lead is tagged by the findings it prints (lead()'s names), beside the act names the page - # closes itself; a segment printing two closed results' findings carries both ids. - rank = {"id": "exomiser-gla", "level": "review", "card": "exomiser", "gene": "GLA", "title": "Symptom ranking: GLA ranks 1"} - two = found + [{"id": "sf-tp53-test", "level": "act", "gene": "TP53", "detail": "TP53"}, - {"id": "exomiser-tp53", "level": "review", "card": "exomiser", "gene": "TP53", - "title": "Symptom ranking: TP53 ranks 2"}] - lead = [[": Symptom ranking: GLA ranks 1 and Symptom ranking: TP53 ranks 2. ", False], - [": Secondary finding in GLA. ", False, "act"], ["Mean depth", False]] - got = live_results.veil({"sections": S}, two + [rank], [], lead, [[rank, two[2]], [found[0]], []]).get("lead") - if got != [["sf-gla-test", "sf-tp53-test"], None, None]: - out.append(f"the lead's segments tagged {got}, want both ids on the segment that prints both rankings") - # A finding known by its gene alone (no position): its ClinVar row is closed by the gene, and the row's - # "See the reads at" link, which shows only the position, with it. - S["clinvar"]["data"]["hits"] = [{"gene": "GLA", "chrom": "chrX", "pos": 101398000, "ref": "G", "alt": "A"}, - {"gene": "GENE5", "chrom": "chr1", "pos": 5, "ref": "A", "alt": "G"}] - card = dict(live_results.a_clinvar(S, {"sections": S}), id="clinvar") - gene_only = [{"id": "carrier-gla-test", "level": "act", "gene": "GLA"}] - v = live_results.veil({"sections": S}, gene_only, [{"sections": [card]}]) - rows, view = (v.get("rows") or {}).get("clinvar"), (v.get("viewer") or {}).get("clinvar") - if rows != [["carrier-gla-test"], None] or view != rows: - out.append(f"a gene-only finding: ClinVar rows tagged {rows}, viewer links {view}, want both " - "['carrier-gla-test', None]") - return out - - -def widget_problems(): - """What is wrong with the veil's tags of a card's facts and drawing: a CHIP call and a copy-number dot, and a - fact, that name a closed result's gene carry its id; the CHIP chart's list of genes, the same panel for - everyone, is never tagged; nothing else is.""" - found = [{"id": "cpsr-tp53-test", "level": "act", "gene": "TP53", "detail": "TP53 c.524G>A"}, - {"id": "two-hits-smn1-test", "level": "act", "gene": "SMN1"}] - S = {"chip": {"state": "ok", "data": {"median_limit": 0.05, "median_depth": 72, "calls": [ - {"gene": "DNMT3A", "protein_change": "p.R882H", "vaf": 0.12, "alt_reads": 9, "depth": 75, "status": "pass", "note": "a clone"}, - {"gene": "TP53", "protein_change": "p.R175H", "vaf": 0.08, "alt_reads": 6, "depth": 75, "status": "pass", "note": "a clone"}], - "genes": [{"gene": "DNMT3A", "detection_limit_vaf": 0.05, "mean_depth": 75, "status": "ok"}, - {"gene": "TP53", "detection_limit_vaf": 0.06, "mean_depth": 70, "status": "ok"}]}}, - "paralogs": {"state": "ok", "data": {"genes": [ - {"gene": "SMN1", "copy_number": 1, "status": "called", "detail": "."}, - {"gene": "SMN2", "copy_number": 2, "status": "called", "detail": "."}]}}} - summ = {"sections": S} - chip = dict(live_results.a_chip(S, summ), id="chip") - par = dict(live_results.a_paralogs(S, summ), id="paralogs") - par["facts"] = [["SMN1 copies", "1"], ["Genes read", "2"]] - v = live_results.veil(summ, found, [{"sections": [chip, par]}]) - out = [] - want = {"chip": [None, ["cpsr-tp53-test"]], "paralogs": [["two-hits-smn1-test"], None]} - if v.get("widget") != want: - out.append(f"drawings tagged {v.get('widget')}, want {want}") - # A fixed-panel card's facts are prose no list declares: the panel tag, whatever they name. - if (v.get("facts") or {}) != {c: [[live_results.PANEL], [live_results.PANEL]] for c in ("chip", "paralogs")}: - out.append(f"facts tagged {v.get('facts')}") - if v.get("spans") != [list(x) for x in live_results.closable(found)]: - out.append(f"spans {v.get('spans')}") - # The carrier list's two-hit fact and line are tagged by the two-hit row's gene field, never by their words: the - # line prints the inheritance code AR, and a closed result in AR leaves both open. - th = {"genes": [{"gene": "CFTR", "verdict": "possible", "condition": "Cystic fibrosis", "deletion": "DEL chr7:117480025-117580025", - "variant": "c.1521_1523del", "genotype": "het", "why": "a deletion and a variant"}], - "deletions_read": ["AnnotSV"], "deletions_not_read": []} - S2 = {"carrier": {"state": "ok", "data": {"genes_on_list": 2, "carriers": [], "two_hits": th}}} - car = dict(live_results.a_carrier(S2, {"sections": S2}), id="carrier-list") - for g, want in (("CFTR", ["two-hits-cftr"]), ("AR", None)): - found2 = [{"id": "two-hits-cftr", "level": "act", "gene": "CFTR"}] if g == "CFTR" else \ - [{"id": "cpsr-ar", "level": "act", "gene": "AR", "detail": "AR c.1A>G"}] - v2 = live_results.veil({"sections": S2}, found2, [{"sections": [car]}]) - facts, lines = (v2.get("facts") or {}).get("carrier-list") or [], (v2.get("lines") or {}).get("carrier-list") or [] - if (facts[2:3] or [None])[0] != want or (lines[:1] or [None])[0] != want or (lines[1:2] or [None])[0] != [live_results.PANEL]: - out.append(f"a closed result in {g}: the two-hit fact tagged {facts}, lines {lines}, want {want} and the panel tag") - return out - - -def sv_genes_problems(): - """What is wrong with the veil's tags of a closed structural variant over two genes: a ClinVar row and a carrier - row that name only its second gene (the hit sits elsewhere, so only the gene can tie it) carry its id, and so - does the carrier finding; veil.spans lists it once per gene; veil.panels lists the cards of a fixed panel.""" - S = {"annotsv": {"state": "ok", "data": {"rows": [ - {"sv": "chr17:43044295-43170245", "type": "deletion", "length": -125950, "acmg_class": "5", - "genes": ["BRCA1", "NBR2"], "gt": "0/1", "support": 2, "support_callers": ["Manta", "Delly"]}]}}, - "clinvar": {"state": "ok", "data": {"hits": [ - {"gene": "NBR2", "chrom": "chr1", "pos": 5, "ref": "A", "alt": "G", "significance": "Pathogenic"}, - {"gene": "GENE5", "chrom": "chr1", "pos": 9, "ref": "A", "alt": "G", "significance": "Pathogenic"}]}}, - "carrier": {"state": "ok", "data": {"genes_on_list": 2, "carriers": [ - {"gene": "NBR2", "variant": "c.1A>G", "genotype": "het", "significance": "Pathogenic"}]}}, - "paralogs": {"state": "ok", "data": {"genes": [{"gene": "SMN1", "copy_number": 2, "status": "called"}]}}} - res = live_results.build("T", {"sections": S}) - v = res.get("veil") or {} - sv = [f["id"] for f in res["findings"] if f["card"] == "structural-variants"] - car = [f["id"] for f in res["findings"] if f["card"] == "carrier-list"] - out = [] - if len(sv) != 1 or len(car) != 1: - return [f"the planted findings: {[f['id'] for f in res['findings']]}"] - if (v.get("rows") or {}).get("clinvar") != [[sv[0]], None]: - out.append(f"ClinVar rows tagged {(v.get('rows') or {}).get('clinvar')}, want the deletion's id on the NBR2 row") - if (v.get("rows") or {}).get("carrier-list") != [[sv[0]]]: - out.append(f"carrier rows tagged {(v.get('rows') or {}).get('carrier-list')}") - if (v.get("findings") or {}).get(car[0]) != [sv[0]]: - out.append(f"the NBR2 carrier finding tagged {(v.get('findings') or {}).get(car[0])}") - if sorted(s[1] for s in v.get("spans") or [] if s[0] == sv[0]) != ["BRCA1", "NBR2"]: - out.append(f"spans {v.get('spans')}") - if v.get("panels") != ["paralogs"]: - out.append(f"panels {v.get('panels')}, want the copy-number card, the only fixed panel here") - # A place that names two closed results keeps both ids: a class 4 duplication over BRCA1 and BRCA2 with a closed - # CPSR result in each, so opening one never brings back a place that still names the other. - S["annotsv"]["data"]["rows"] = [{"sv": "chr5:1000000-1300000", "type": "DUP", "length": 300000, "acmg_class": "4", - "genes": ["BRCA1", "BRCA2"], "gt": "0/1", "support": 1, "caller": "Manta", - "support_callers": ["Manta"]}] - S["cpsr"] = {"state": "ok", "data": {"plp": [{"gene": g, "variant": "c.68_69del", "final": "Pathogenic"} - for g in ("BRCA1", "BRCA2")]}} - res = live_results.build("T", {"sections": S}) - v = res.get("veil") or {} - acts = [f["id"] for f in res["findings"] if f["card"] == "cpsr"] - dup = [f["id"] for f in res["findings"] if f["card"] == "structural-variants"] - want = sorted(acts) - row = ((v.get("rows") or {}).get("structural-variants") or [None])[0] - if len(acts) != 2 or len(dup) != 1 or sorted((v.get("findings") or {}).get(dup[0]) or []) != want \ - or sorted(row or []) != want: - out.append(f"a duplication over two closed results: finding {(v.get('findings') or {}).get(dup[0] if dup else '')}, " - f"row {row}, want {want}") - return out - - -def gene_and_position_problems(): - """Chart genes without position overlap, and a gene-free call with position overlap, need independent tags.""" - summary = {"sections": { - "roh": {"state": "ok", "data": {}}, - "annotsv": {"state": "ok", "data": {"rows": [ - {"sv": "chr17:43044295-43170245", "type": "DEL", "length": -125950, "acmg_class": "5", - "genes": ["BRCA1", "NBR2"], "gt": "0/1", "support": 1, "caller": "Manta"}, - {"sv": "chr5:1000000-1300000", "type": "DUP", "length": 300000, "acmg_class": "4", - "genes": ["NKD2", "NBR2"], "gt": "0/1", "support": 1, "caller": "Manta"}]}}, - "clinvar": {"state": "ok", "data": {"hits": [ - {"gene": "NBR2", "chrom": "chr1", "pos": 5, "ref": "A", "alt": "G", "significance": "Pathogenic"}]}}, - "mosaic": {"state": "ok", "data": {"status": "assessed", "calls": [ - {"chrom": "chr17", "start": 43050000, "end": 43100000, "length_mb": 0.05, - "type": "Loss", "arms": "17q", "cell_fraction": 0.2}]}}}} - res = live_results.build("T", summary) - v, out = res["veil"], [] - closed = "sv-chr17-43044295-43170245" - for mark in ("hit-0", "sv-1"): - got = v.get("marks", {}).get("genome", {}).get(mark) - if got != [closed]: - out.append(f"gene-only chart {mark}: {got}, want {[closed]}") - if v.get("findings", {}).get("mca-chr17-43050000") != [closed]: - out.append("coordinate-only mosaic finding not tagged with the deletion") - if v.get("rows", {}).get("mosaic-mca") != [[closed]]: - out.append("coordinate-only mosaic table row not tagged with the deletion") - card = next(c for ch in res["chapters"] for c in ch["sections"] if c["id"] == "mosaic-mca") - lines = [(ln, tag) for ln, tag in zip(card["lines"], v.get("lines", {}).get("mosaic-mca", [])) - if "chr17:43050000-43100000" in ln] - if not lines or any(tag != [closed] for _, tag in lines): - out.append(f"coordinate-only mosaic lines: {lines}") - return out - - -# The veil's tables, pinned here so that an entry dropped from one turns this file red (each is checked by what -# veil() does with it, not by reading the table back): the fixed-panel cards, the fixed lines of a card (a sample -# line or row naming a gene, its card, and whether it is a row), the cards that name no gene, the cards whose lines, -# facts or headline name no gene, and prose that names a gene or a word spelled like one. -WANT_PANELS = ("mei", "expansions", "vntyper", "lpa", "chip", "paralogs", "cpic", "cyp2d6", "hla", "locityper", - "traits-panel", "bloodtype") -WANT_LINES = (("clinvar", "Known deep intronic site (always reported): GLA c.640-801G>A chrX:1 C>T: hom", False), - ("callability", ["GSTT1", "not on this reference", ".", "ALT-only"], True), - ("callability", ["NBPF1", "not read", "69%", "segdup, collapsed"], True), - ("roh", "Imprinted 6q24 (PLAGL1, HYMAI): from the mother: -; from the father: Transient neonatal diabetes " - "mellitus (6q24-related) (PMID 20301706)", False)) -WANT_NO_GENE = () -WANT_LINES_NO_GENE = () -WANT_FACTS_NO_GENE = () -WANT_FACT_PROSE = (("clinical-filter", "High-impact rare variants in its genes"),) -WANT_HEADLINES_NO_GENE = () -# Prose, each with the gene it names or the gene its word is spelled like: the tag must be the panel tag either way. -# The carrier list's summary of the two-hit callers (it names SMN1, HBA1 and HBA2 as the callers it read) is prose of -# a card with no gene-free lines: the panel tag, as every other line of that card but a two-hit gene's own. -WANT_PROSE_LINES = (("carrier-list", "Two hits across callers (a deletion plus a variant on the other copy): none found; " - "deletions read from Parascopy's SMN1 count (step 35)"),) -WANT_PROSE = (("KCNH2", "A change in KCNH2."), ("C9ORF72", "C9orf72 repeat"), ("HLA-A", "HLA-A*02:01"), - ("SET", "the same set of genes"), ("MB", "0.0 MB in runs of 5 MB or more"), ("AR", "CFTR (Cystic fibrosis; AR)"), - ("HP", "Seizure (HP:0001250)"), ("GC", "reads per million with 48-52% GC")) -WANT_GENE_CELLS = (("NKD2, SLC12A7 and 2 more", ["NKD2", "SLC12A7"]), ("SMN1 (Parascopy)", ["SMN1"]), - ("HLA-A", ["HLA-A"]), ("BRCA1, BRCA2", ["BRCA1", "BRCA2"]), (".", [])) - - -def gene_of(text): - """The first upper-case word of TEXT (a list is a table row) that looks like a gene symbol.""" - t = " ".join(map(str, text)) if isinstance(text, list) else text - return next(w for w in re.findall(r"[A-Z][A-Z0-9]{2,}", t.split(":", 1)[-1] if not isinstance(text, list) else t) - if w not in ("ALT", "SMN")) - - -def table_problems(): - """What is wrong with veil()'s tables (WANT_*): a fixed-panel card missing from veil.panels, a fixed line or row - tagged by the gene it names instead of the panel tag, a card that names no gene tagged, a line, fact or headline - of a card declared gene-free there tagged by the panel tag (or one of a card not declared not), prose read word by - word, or a gene cell not read as its list of symbols.""" - out = [] - for name in ("NAMES_NO_GENE", "LINES_NO_GENE", "FACTS_NO_GENE", "HEADLINES_NO_GENE"): - if getattr(live_results, name): - out.append(f"{name}: no card-wide free-text exemption passed the producer audit") - panel = [live_results.PANEL] - found = lambda g: [{"id": "sf-" + g.lower(), "level": "act", "gene": g, "detail": g}] - chapters = [{"sections": [{"id": c, "state": "ok"} for c in WANT_PANELS]}] - got = live_results.veil({"sections": {}}, found("BRCA2"), chapters).get("panels") or [] - out += [f"card {c} is not in veil.panels" for c in WANT_PANELS if c not in got] - for card, item, is_row in WANT_LINES: - g = gene_of(item) - sec = {"id": card, "table": {"columns": ["Gene", "Status", "Bases read", "Why"], "rows": [item]}} if is_row else {"id": card, "lines": [item]} - v = live_results.veil({"sections": {}}, found(g), [{"sections": [sec]}]) - tag = ((v.get("rows" if is_row else "lines") or {}).get(card) or [None])[0] - if tag != panel: - out.append(f"{card}: {str(item)[:60]} tagged {tag} with a closed result in {g}, want the panel tag") - for card, line in WANT_PROSE_LINES: - v = live_results.veil({"sections": {}}, found("SMN1"), [{"sections": [{"id": card, "lines": [line]}]}]) - if ((v.get("lines") or {}).get(card) or [None])[0] != panel: - out.append(f"{card}: {line[:60]} tagged {(v.get('lines') or {}).get(card)}, want the panel tag") - for card, row, g in WANT_NO_GENE: - sec = {"id": card, "table": {"head": ["a"] * len(row), "rows": [row]}, "lines": [" ".join(row)], "facts": [row[:2]]} - v = live_results.veil({"sections": {}}, found(g), [{"sections": [sec]}]) - if any((v.get(k) or {}).get(card) for k in ("rows", "lines", "facts")): - out.append(f"{card} names no gene, yet a closed result in {g} tags it: {v}") - # A line or fact of a card declared gene-free is untagged even when it spells the closed gene; one of a card not - # declared carries the panel tag whatever it says. A headline likewise, by the summary key it comes from. - for card in sorted(set(WANT_LINES_NO_GENE) | {"cpsr", "slivar", "archaic"}): - v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [{"id": card, "lines": ["a line, BRCA2"]}]}]) - tag = ((v.get("lines") or {}).get(card) or [None])[0] - if tag != (None if card in WANT_LINES_NO_GENE else panel): - out.append(f"the line of {card} tagged {tag}") - for card in sorted(set(WANT_FACTS_NO_GENE) | set(WANT_PANELS)): - v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [{"id": card, "facts": [["A count", "BRCA2"]]}]}]) - tag = ((v.get("facts") or {}).get(card) or [None])[0] - if tag != (None if card in WANT_FACTS_NO_GENE else panel): - out.append(f"the fact of {card} tagged {tag}") - for card, label in WANT_FACT_PROSE: - v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [{"id": card, "facts": [[label, "x"]]}]}]) - if ((v.get("facts") or {}).get(card) or [None])[0] != panel: - out.append(f"the fact {label!r} of {card} is prose, yet not tagged by the panel tag") - for key in sorted(set(WANT_HEADLINES_NO_GENE) | {"couple"}): - sec = {"id": "c", "keys": [key], "headline": "BRCA2 and more"} - v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [sec]}]) - if ((v.get("headline") or {}).get("c")) != (None if key in WANT_HEADLINES_NO_GENE else panel): - out.append(f"the headline from {key} tagged {(v.get('headline') or {}).get('c')}") - for card in live_results.ADAPTERS: - sec = {"id": card, "keys": ["unknown-producer"], "headline": "MLH1 sample note", - "facts": [["Unclassified", "MLH1"]], "lines": ["MLH1 sample note"], - "table": {"columns": ["Future field"], "rows": [["MLH1"], ["extra MLH1 record"]]}} - v = live_results.veil({"sections": {}}, found("BRCA2"), [{"sections": [sec]}]) - for kind in ("facts", "lines", "rows"): - if not (v.get(kind) or {}).get(card) or any(t != panel for t in v[kind][card]): - out.append(f"{card}: unclassified {kind} did not default to PANEL") - if (v.get("headline") or {}).get(card) != panel: - out.append(f"{card}: unknown headline did not default to PANEL") - # Prose read with no gene field: the panel tag whatever it names, so the tag never tells the words apart. - for g, text in WANT_PROSE: - for spans in (live_results.closable(found(g)), live_results.closable(found("BRCA2"))): - if live_results.veil_id(spans, text) != panel: - out.append(f"{text!r} tagged {live_results.veil_id(spans, text)}, want the panel tag") - for cell, want in WANT_GENE_CELLS: - if live_results.gene_cell(cell) != want: - out.append(f"gene cell {cell!r} read as {live_results.gene_cell(cell)}, want {want}") - return out - - -def table_controls(): - """Each entry of the veil's tables dropped in turn, and the word rule back: table_problems() must fail every time.""" - names = ("FIXED_PANELS", "FIXED_LINES", "NAMES_NO_GENE", "LINES_NO_GENE", "FACTS_NO_GENE", "FACT_NAMES", - "HEADLINES_NO_GENE") - keep = {n: getattr(live_results, n) for n in names} - first_prose = live_results.prose_tag - try: - for c in WANT_PANELS: - live_results.FIXED_PANELS = {k: v for k, v in keep["FIXED_PANELS"].items() if k != c} - control(f"FIXED_PANELS without {c}", not table_problems()) - live_results.FIXED_PANELS = keep["FIXED_PANELS"] - for card, words in keep["FIXED_LINES"].items(): - for w in words: - live_results.FIXED_LINES = dict(keep["FIXED_LINES"], **{card: tuple(x for x in words if x != w)}) - control(f"FIXED_LINES[{card}] without {w!r}", not table_problems()) - live_results.FIXED_LINES = keep["FIXED_LINES"] - live_results.NAMES_NO_GENE = {"prs": "an unsafe blanket exemption"} - control("NAMES_NO_GENE with an unsafe exemption", not table_problems()) - live_results.NAMES_NO_GENE = keep["NAMES_NO_GENE"] - for n in ("LINES_NO_GENE", "FACTS_NO_GENE", "FACT_NAMES", "HEADLINES_NO_GENE"): - for k in keep[n]: - setattr(live_results, n, {x: y for x, y in keep[n].items() if x != k}) - control(f"{n} without {k}", not (table_problems() or widget_problems())) - setattr(live_results, n, keep[n]) - live_results.LINES_NO_GENE = dict(keep["LINES_NO_GENE"], cpsr="its lines") - control("LINES_NO_GENE with the CPSR card, whose lines name genes", not table_problems()) - live_results.LINES_NO_GENE = keep["LINES_NO_GENE"] - live_results.prose_tag = lambda spans, text: live_results.veil_tags( - spans, re.findall(r"[A-Za-z0-9][A-Za-z0-9-]*", str(text).upper()), text) - control("a veil that reads prose word by word, the word rule", not table_problems()) - control("a veil that reads prose word by word, the two-hit line's AR", not widget_problems()) - finally: - for n in names: - setattr(live_results, n, keep[n]) - live_results.prose_tag = first_prose - - -# Every adapter is a fixed panel (live_results.FIXED_PANELS) or listed here as one whose rows say something about this -# genome alone: a new or renamed adapter turns this file red until someone decides which it is. -WANT_NOT_PANELS = ("qc", "callability", "variants", "clinvar", "clinical-filter", "cpsr", "slivar", "exomiser", - "structural-variants", "mito", "mosaic", "mosaic-mca", "carrier-list", "couple", "pharmcat", - "ancestry-fine", "mt-haplogroup", "y-chromosome", "archaic", "ancient", "roh", "prs", "telomere", - "viral", "consumer-export") - - -def table_columns(): - """Every column of every table() call in bin/live_results.py, read from its source.""" - import ast - with open(os.path.join(REPO, "bin", "live_results.py")) as fh: - tree = ast.parse(fh.read()) - return {e.value for n in ast.walk(tree) if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "table" - and n.args and isinstance(n.args[0], ast.List) for e in n.args[0].elts if isinstance(e, ast.Constant)} - - -def declared_problems(adapters=None): - """What the veil's tables leave undeclared: a table column not in TABLE_COLUMNS, a fixed-panel card that is no - adapter, an adapter in neither FIXED_PANELS nor WANT_NOT_PANELS.""" - adapters = live_results.ADAPTERS if adapters is None else adapters - out = [f"column {c!r} is not in TABLE_COLUMNS" for c in sorted(table_columns() - set(live_results.TABLE_COLUMNS))] - out += [f"FIXED_PANELS card {c} is no adapter" for c in live_results.FIXED_PANELS if c not in adapters] - out += [f"adapter {c} is neither a fixed panel nor in WANT_NOT_PANELS" for c in adapters - if c not in live_results.FIXED_PANELS and c not in WANT_NOT_PANELS] - return out - - -def vocabulary_summary(gene): - """A summary with two recessive carrier rows (inheritance AR), an insertion over NKD2 (type INS) and the score of - 'Triglycerides (TG)', plus a closed CPSR result in GENE.""" - car = [{"gene": g, "variant": v, "genotype": "het", "significance": "Pathogenic", "stars": 2, "inheritance": "AR", - "condition": c} for g, v, c in (("CFTR", "c.1521_1523del", "Cystic fibrosis"), - ("HBB", "c.20A>T", "Sickle cell disease"))] - return {"sections": { - "carrier": {"state": "ok", "data": {"genes_on_list": 2, "carriers": car}}, - "annotsv": {"state": "ok", "data": {"count": 1, "rows": [ - {"sv": "chr5:1000000-1000300", "type": "INS", "gt": "0/1", "length": 300, "acmg_class": "4", "caller": "Manta", - "genes": ["NKD2"], "support": 1, "support_callers": ["Manta"]}]}}, - "exomiser": {"state": "ok", "data": {"terms": [], "genes": [ - {"rank": 1, "gene": "GJB2", "moi": "AR", "exomiser_score": 0.9, "variants": [{"hgvs": "c.35del"}]}]}}, - "roh": {"state": "ok", "data": {"autosomal_over_5mb": []}}, - "prs": {"state": "ok", "data": {"adjusted": True, "scores": [ - {"pgs_id": "PGS000066", "condition": "Triglycerides (TG)", "percentile": "63.3"}]}}, - "cpsr": {"state": "ok", "data": {"plp": [{"gene": gene, "variant": "c.68_69del", "final": "Pathogenic"}]}}}} - - -def named_tags(v): - """The veil's places (rows, lines, facts, marks, ...) whose tag names a result, never the panel tag alone: the - same nesting, emptied of the rest.""" - if v is None or (isinstance(v, list) and all(isinstance(x, str) for x in v)): # a tag - return v if v and v != [live_results.PANEL] else None - if isinstance(v, dict): - got = {k: named_tags(x) for k, x in v.items()} - return {k: x for k, x in got.items() if x} or None - got = [named_tags(x) for x in v] - return got if any(got) else None - - -def vocabulary_problems(): - """What is wrong with the veil's tags of the report's own vocabulary: a closed result in AR (an inheritance code), - INS (a structural-variant type) or TG (a trait's abbreviation in a score's name) tags nothing, while one in CFTR - tags the CFTR carrier row, one in GJB2 the symptom ranking's row and one in NKD2 the insertion's row, finding and - chart mark.""" - marks = {(sp["id"], m["id"]) for sp in rr.chart_specs(vocabulary_summary("AR")) for m in sp["marks"]} - out = [f"chart mark {m} not drawn" for m in (("genome", "sv-0"), ("prs", "pgs-PGS000066")) if m not in marks] - for g in ("AR", "INS", "TG"): - res = live_results.build("T", vocabulary_summary(g)) - v = {k: x for k, x in (res.get("veil") or {}).items() if k not in ("spans", "panels")} - v = named_tags(v) or {} - if any(v.values()): - out.append(f"a closed result in {g} tags {v}") - for g, where in (("CFTR", ("rows", "carrier-list")), ("GJB2", ("rows", "exomiser")), - ("NKD2", ("rows", "structural-variants")), ("NKD2", ("marks", "genome"))): - v = live_results.build("T", vocabulary_summary(g)).get("veil") or {} - if not any((v.get(where[0]) or {}).get(where[1]) or []): - out.append(f"a closed result in {g} leaves {where} untagged: {v.get(where[0])}") - v = live_results.build("T", vocabulary_summary("NKD2")).get("veil") or {} - sv = [k for k in (v.get("findings") or {}) if k.startswith("sv-")] - if not sv: - out.append(f"a closed result in NKD2 leaves the insertion's finding untagged: {v.get('findings')}") - return out - - -def vocabulary_controls(): - """Each guard of vocabulary_problems() dropped in turn: the check must fail every time.""" - cols, names, marks = live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names - try: - live_results.TABLE_COLUMNS = dict(cols, Inheritance="gene") - control("TABLE_COLUMNS that reads the Inheritance column as naming a gene", not vocabulary_problems()) - live_results.TABLE_COLUMNS = dict(cols, Type="gene") - control("TABLE_COLUMNS that reads the SV Type column as naming a gene", not vocabulary_problems()) - live_results.TABLE_COLUMNS = cols - live_results.finding_names = lambda f: (re.findall(r"[A-Z0-9]+", f"{f.get('title') or ''}"), f.get("detail") or "") - control("a veil that reads a finding's title word by word", not vocabulary_problems()) - live_results.finding_names = names - live_results.mark_names = lambda S, sp, m: (re.findall(r"[A-Z0-9]+", f"{m.get('label') or ''} {m.get('detail') or ''}"), "") - control("a veil that reads a chart mark's label word by word", not vocabulary_problems()) - finally: - live_results.TABLE_COLUMNS, live_results.finding_names, live_results.mark_names = cols, names, marks - try: - live_results.TABLE_COLUMNS = {k: x for k, x in cols.items() if k != "Inheritance"} - control("TABLE_COLUMNS without the Inheritance column", not declared_problems()) - finally: - live_results.TABLE_COLUMNS = cols - renamed = {("traits" if k == "traits-panel" else k): f for k, f in live_results.ADAPTERS.items()} - control("the traits-panel adapter renamed traits", not declared_problems(renamed)) - control("a new adapter nobody classified", not declared_problems(dict(live_results.ADAPTERS, newcard=None))) - - -def overview_problems(): - """What is wrong with the home page's view of a child's run (overview): a finding that names a closed result - (a symptom ranking of its gene) is listed among the first findings, or the closed lead still names it.""" - r = {"attention": 0, "levels": {}, "chapters": [], - "findings": [{"id": "sf-gla-test", "level": "act", "title": "Secondary finding in GLA"}, - {"id": "exomiser-gla", "level": "review", "title": "Symptom ranking: GLA ranks 1"}, - {"id": "pgx-demo", "level": "medicine", "title": "Medicine panel finding"}, - {"id": "chip-tp53", "level": "review", "title": "A blood clone in TP53"}], - "lead": [["Of everything, ", False], [": Secondary finding in GLA. ", False, "act"], - ["One result is worth a look", True], [": Symptom ranking: GLA ranks 1.", False]], - "veil": {"findings": {"exomiser-gla": ["sf-gla-test"], "pgx-demo": [live_results.PANEL], - "chip-tp53": [live_results.PANEL]}, "lead": [None, None, None, ["sf-gla-test"]]}} - keep = live_results.build - try: - live_results.build = lambda *a, **k: r - o = live_results.overview({}) - finally: - live_results.build = keep - out = [] - if [f["id"] for f in o["top"]] != ["sf-gla-test", "pgx-demo", "chip-tp53"]: - out.append(f"the home page's first findings: {o['top']}") - for f in o["top"]: - if f["id"] in ("pgx-demo", "chip-tp53") and f.get("veil") != [live_results.PANEL]: - out.append(f"home panel finding has no panel tag: {f}") - if "GLA" in o["lead_closed"] or "worth a look" not in o["lead_closed"]: - out.append(f"the closed lead: {o['lead_closed']}") - return out - - def main(): work = tempfile.mkdtemp(prefix="test_live_results.") srvs = [] @@ -943,7 +400,7 @@ def main(): # 8. import results_index sys.path.insert(0, os.path.join(REPO, "tests", "demo")) - check("8. DEMO-001, with no result to act on, has no veil tag", res.get("veil") == {}, res.get("veil")) + check("8. DEMO-001, with no result to act on, has no veil tag", res.get("veil") == {"acts": []}, res.get("veil")) subprocess.run([sys.executable, os.path.join(REPO, "tests", "demo", "make_demo_sample.py"), "--genome-dir", root, "--records", "2000", "--sample", "DEMO-003", "--sex", "male", "--seed", "3"], check=True, stdout=subprocess.DEVNULL) @@ -954,97 +411,9 @@ def tagged(sample_dir): r3 = live_results.build("DEMO-003", s3) r3["_summary"] = s3 return r3, results_index.build(s3, sample_dir, rr.SECTION_OF) - bad = veil_lines_problems() - check("8. every known deep intronic site's line carries the fixed-panel tag, not a closed result's id, a ClinVar " - "row closed by its gene closes its reads link, and a lead segment carries the ids of every closed result " - "its findings name", not bad, bad) - fixed = live_results.FIXED_LINES - prose = live_results.prose_tag - try: # the card still prints its lines; veil() no longer knows they are a panel - live_results.FIXED_LINES = type("Unknown", (dict,), {"get": lambda self, k, d=None: None})(fixed) - live_results.prose_tag = lambda spans, text: live_results.veil_tags(spans, re.findall(r"[A-Z][A-Z0-9]+", text), text) - control("a veil that tags a known deep intronic site's line by its gene", not veil_lines_problems()) - finally: - live_results.FIXED_LINES = fixed - live_results.prose_tag = prose - bad = table_problems() - check("8. every fixed-panel card, fixed line and gene-free card of the veil's tables closes as a panel or never, " - "prose is the panel tag whatever it names unless declared gene-free, and a gene cell is its symbols", not bad, bad) - table_controls() - bad = declared_problems() - check("8. every table column is declared as naming a gene or not (TABLE_COLUMNS), and every adapter as a fixed " - "panel or not", not bad, bad) - bad = vocabulary_problems() - check("8. the report's own codes (AR as an inheritance, INS as an SV type, TG in a score's name) tag nothing; a " - "row, finding or mark is tagged by its gene and place only", not bad, bad) - vocabulary_controls() - bad = overview_problems() - check("8. a child's home tile lists no finding that names a closed result, and its closed lead names none", - not bad, bad) - overview = live_results.overview - try: - def no_panel(summary): - r = overview(summary) - for f in r["top"]: - f.pop("veil", None) - return r - live_results.overview = no_panel - control("overview drops panel tags from medicine and review findings", not overview_problems()) - finally: - live_results.overview = overview - bad = widget_problems() - check("8. a CHIP call, a copy-number dot and the two-hit fact and line carry the id of a closed result in their " - "gene field, never by their words (AR), the CHIP chart's gene panel none, a fixed-panel card's facts the " - "panel tag, and results.json carries the closed results as veil.spans", not bad, bad) - bad = sv_genes_problems() - check("8. a closed structural variant over two genes closes the rows and the finding that name only its second " - "gene, veil.spans lists it per gene, veil.panels lists the fixed-panel cards, and a duplication over two " - "closed results carries both ids", not bad, bad) - bad = gene_and_position_problems() - check("8. gene-only ClinVar and second-gene SV chart marks, and coordinate-only mosaic findings, rows and lines " - "carry the closed result's tag", not bad, bad) - names, spans = live_results.mark_names, live_results.closable - try: - for kind in ("hit", "sv"): - live_results.mark_names = lambda S, sp, m: (([], "") if kind == "hit" else (names(S, sp, m)[0][:1], "")) \ - if m.get("kind") == kind else names(S, sp, m) - control(f"{kind} chart mark without every gene", not gene_and_position_problems()) - live_results.mark_names = names - live_results.closable = lambda found: [(fid, g, None, None, None) for fid, g, *_ in spans(found)] - control("closable without positions", not gene_and_position_problems()) - finally: - live_results.mark_names, live_results.closable = names, spans - first = live_results.closable - try: - live_results.closable = lambda found: first([{k: v for k, v in f.items() if k != "genes"} for f in found]) - control("a veil that tags only a structural variant's first gene", not sv_genes_problems()) - finally: - live_results.closable = first - bad = veil_problems(*tagged(d3)) - check("8. the planted ClinVar hit's row, viewer link, tick and star bar carry its secondary finding's id, the " - "deletion's span its own, every other card's row of the gene and the symptom ranking's finding one of them, " - "and every search item of the gene one of them", not bad, bad) - # The planted mistakes: a rule that tags nothing, and one that tags every row with the first result. - keep = live_results.veil_tags - try: - live_results.veil_tags = lambda spans, *a, **k: None - control("a veil rule that tags nothing", not veil_problems(*tagged(d3))) - control("a veil rule that tags no line", not veil_lines_problems()) - control("a veil rule that tags no fact or drawing", not widget_problems()) - live_results.veil_tags = lambda spans, *a, **k: [spans[0][0]] if spans else None - control("a veil rule that tags every row", not veil_problems(*tagged(d3))) - live_results.veil_tags = lambda spans, *a, **k: (keep(spans, *a, **k) or [None])[:1] if keep(spans, *a, **k) else None - control("a veil rule that keeps only the first id of a place", not veil_problems(*tagged(d3))) - control("a veil rule that keeps only the first id of a place, two closed results", not sv_genes_problems()) - finally: - live_results.veil_tags = keep - first_union = live_results.union - try: - live_results.union = lambda *t: next((x[:1] for x in t if x), None) - control("a veil rule that keeps only the first id of a lead segment", not veil_lines_problems()) - finally: - live_results.union = first_union - + acts = live_results.finding_rows(collect_summary.collect("DEMO-003", d3)) + gate = tagged(d3)[0]["veil"] + check("8. the gate exports only act identities", gate == {"acts": [f["id"] for f in acts if f["level"] == "act"]}, gate) # 3, the map documents: og map build of DEMO-001 and DEMO-003, with the results each page gets. map_docs = map_documents(root, [(SAMPLE, res), ("DEMO-003", tagged(d3)[0])]) if map_docs is None: diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index 3fbf4477f..e013613f5 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -183,8 +183,7 @@ def family_browser_checks(work): b = demo.Chrome(chrome, os.path.join(work, "family-browser", "chrome")) before, t0 = len(demo.FAILS), demo.time.time() try: - demo.family_veil_checks(b, f"http://127.0.0.1:{srv.server_address[1]}", out) - demo.fail_closed_checks(b, f"http://127.0.0.1:{srv.server_address[1]}", out, og) + demo.gate_checks(b, f"http://127.0.0.1:{srv.server_address[1]}", out, family_only=True) FAILS.extend(demo.FAILS[before:]) finally: b.close() @@ -377,49 +376,22 @@ def browser_checks(port, work): loaded = b.wait("var i = document.querySelector('img.pedigree'); i && i.complete && i.naturalWidth > 0 && " "i.naturalWidth", 10) check("browser: the pedigree image loads", loaded, loaded) - # father_a and mother_a are also children (of gf_a and gm_a) whose results this server cannot read, so the - # couple screen stays closed whole until the button of each opens it. - shut = open_screen(b) - check("browser: the couple screen of a partner who is a child stays closed until 'Show the screen'", shut, shut) - couple = b.eval("var p = document.querySelector('p[data-couple=\\'father_a mother_a\\']'); p && p.textContent") - check("browser: the couple line counts 1 shared and 1 X-linked and links the report", - couple and "1 gene(s) where both carry a finding, 1 X-linked" in couple - and b.eval("!!document.querySelector('p[data-couple=\\'father_a mother_a\\'] a')"), couple) - b.eval("document.querySelector('details[data-child=child_a]').open = true; true") - # child_a's results cannot be built here (its summary is cut short), so the page cannot tell whether a - # result to act on is closed: the map stays closed until "Show the map" is clicked. - # Only once the page has given up on child_a's results: a page still waiting keeps the map closed anyway. - shut = b.wait("/\\bchild_a:failed\\b/.test(document.documentElement.dataset.pgpKids || '') && " - "(function () { var v = Array.prototype.filter.call(document.querySelectorAll('details[data-child=child_a] .veil'), " - "function (x) { return x.textContent.indexOf('Show the map') >= 0; })[0]; " - "return v && !document.querySelector('details[data-child=child_a] img') && (v.querySelector('button').click(), true); }())", 10) - check("browser: child_a's chromosome map stays closed while its results cannot be read", shut, shut) - seg = b.wait("var i = document.querySelector('details[data-child=child_a] img'); " - "i && i.complete && i.naturalWidth > 0 && i.getAttribute('src')", 10) - check("browser: child_a's chromosome map loads", seg == "/family/family_a/child_a_segments.svg", seg) + # Unreadable child reports cannot be opened a place at a time under the data gate. + check("browser: unreadable child reports keep the couple screen and chromosome image closed", + b.eval("!!document.querySelector('.report-gate') && !document.querySelector('#ch-couples table') && !document.querySelector('img.segments')")) blocked = b.eval("document.documentElement.dataset.pgpBlocked || ''") check("browser: the policy blocked nothing on the family page", not blocked, blocked) depth_checks(b, base, os.path.join(work, "genome")) b.go(base + "/runs/child_a/") - href = b.wait("var a = document.querySelector('section[data-plugin=family] a'); a && a.getAttribute('href')") - check("browser: child_a's page links to its family", href == "/runs/family_a/", href) + closed = b.wait("!!document.querySelector('#report-gate')") + check("browser: an unreadable child's page shows the fixed gate", bool(closed), closed) + finally: b.close() -def open_screen(b): - """Click the closed boxes of father_a's and mother_a's couple screen, one per partner, once they show with no - table or link: whether they did and the screen then opened.""" - shut = b.wait("!/:pending\\b/.test(document.documentElement.dataset.pgpKids || ':pending') && " - "document.querySelectorAll('#ch-couples .veil').length === 2 && !document.querySelector('#ch-couples table') " - "&& !document.querySelector('#ch-couples p[data-couple] a') && 'shut'", 10) - for who in ("father_a", "mother_a"): - b.eval("var v = document.querySelector('#ch-couples .veil[data-veil-child=%s] button'); v && v.click(); true" % who) - return bool(shut) and bool(b.wait("!!document.querySelector('#ch-couples table') && !document.querySelector('#ch-couples .veil')", 5)) - - # A stale blood-group section with two systems past ABO and RhD, for child_a's summary.json. -STALE_SUMMARY = {"sections": {"bloodtype": {"state": "stale", "data": { +STALE_SUMMARY = {"sections": {"bloodtype": {"state": "stale", "note": "older than its input", "data": { "abo": "A1", "rhd": "D+", "systems": [{"system": "ABO", "phenotype": "A1"}, {"system": "RhD", "phenotype": "D+"}, {"system": "Kell", "phenotype": "K-k+"}, {"system": "Duffy", "phenotype": "Fy(a+b-)"}]}}, "haplogroup": {"state": "ok", "data": {"haplogroup": "H1"}}}} @@ -443,10 +415,22 @@ def depth_checks(b, base, root): os.symlink(secret, os.path.join(fam_dir, "compound_het.tsv")) with open(os.path.join(root, "child_a", "summary.json"), "w") as f: json.dump(STALE_SUMMARY, f) + # Give the formatting fixtures readable no-act results. Unknown results deliberately cannot open. + log = os.path.join(root, "child_a", "logs", "run_status.tsv") + with open(log) as f: + old_log = f.read() + with open(log, "a") as f: + f.write("step\t06\tok\n") + parent_dirs = [] + for parent in ("father_a", "mother_a"): + d = os.path.join(root, parent) + os.makedirs(os.path.join(d, "logs"), exist_ok=True) + with open(os.path.join(d, "logs", "run_status.tsv"), "w") as f: + f.write("step\t06\tok\n") + parent_dirs.append(d) clin = os.path.join(root, "child_a", "clinical") try: b.go(base + "/runs/family_a/") - open_screen(b) ready = b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " "!!document.querySelector('div[data-shared=GENE4]')") four = b.eval("var k = document.querySelector('div[data-shared=GENE1] .kids4 .grid4'); k ? k.children.length : -1") @@ -497,6 +481,10 @@ def comp_line(): os.remove(os.path.join(fam_dir, "compound_het.tsv")) os.remove(os.path.join(root, "child_a", "summary.json")) shutil.rmtree(clin, ignore_errors=True) + with open(log, "w") as f: + f.write(old_log) + for d in parent_dirs: + shutil.rmtree(d) if __name__ == "__main__": From 9be17ad6a6db13cf86092ae848703beedb57fb37 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:13:15 +0100 Subject: [PATCH 054/255] test(live): fix fallback favicon request races --- tests/test_live_demo.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 442d82625..5795a2f86 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1378,6 +1378,14 @@ def snap(): '/runs/'+son+'/finding/'+acts[0]['id']+'/', '/runs/'+son+'/#chr7'] if family_only: routes = ['/runs/'+FAM+'/'] + # Full Chrome asks for a fallback favicon after load, outside page readiness. Give the static + # fixture a fixed inline icon. Every request the page makes still stays in the snapshot. + html_paths = {os.path.join(out, route.split('#')[0].lstrip('/'), 'index.html') for route in routes} + html_saved = {p: open(p, 'rb').read() for p in html_paths} + for p, raw in html_saved.items(): + assert b'' in raw, 'fixture has no head: '+p + with open(p, 'wb') as f: + f.write(raw.replace(b'', b'', 1)) b.call('Network.enable', {}, b.session) b.call('Network.setCacheDisabled', {'cacheDisabled': True}, b.session) bad = [] @@ -1403,7 +1411,8 @@ def snap(): assert b.wait("performance.getEntriesByType('resource').filter(function(r){return r.name.endsWith('/"+son+"/results.json');}).length > "+str(completed)), 'family projection refresh never settled' partials.append(snap()) if snapshots[0] != snapshots[1]: - bad.append((width, media, route, [k for k in snapshots[0] if snapshots[0][k] != snapshots[1][k]])) + keys = [k for k in snapshots[0] if snapshots[0][k] != snapshots[1][k]] + bad.append((width, media, route, keys, [v['requests'] for v in snapshots] if 'requests' in keys else [])) if partials and partials[0] != partials[1]: bad.append((width, media, route, 'partially opened', [k for k in partials[0] if partials[0][k] != partials[1][k]])) print(' gate matrix time: %.2f s' % (time.monotonic()-start)) @@ -1458,7 +1467,7 @@ def snap(): check('gate handles follow their act after another act opens', b.eval("location.hash") == '#f/closed-2') finally: - for p, raw in saved.items(): + for p, raw in dict(saved, **html_saved).items(): with open(p, 'wb') as f: f.write(raw) b.call('Emulation.clearDeviceMetricsOverride', {}, b.session) From c47cd892115bdc1c0c55897d72bce3845b86004d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:17:11 +0100 Subject: [PATCH 055/255] feat(summary): whole summary.json and its post pass in og summary collect Without --only, og summary collect reads all three groups over one walk of the folder and runs the post pass: the readings that join sections, the cyp2d6 block, not_run, not_assessed, the database re-check and new_since. A part of the post pass that fails makes only its own section unreadable; the summary is still written. Bead: pgp-bs9u.44 --- rust/Cargo.lock | 1 + rust/crates/og-summary/Cargo.toml | 1 + rust/crates/og-summary/src/collect.rs | 257 ++++++++----- rust/crates/og-summary/src/collect/group_b.rs | 108 +++--- rust/crates/og-summary/src/collect/group_c.rs | 46 ++- .../og-summary/src/collect/group_c/repeats.rs | 15 +- rust/crates/og-summary/src/collect/model.rs | 14 + rust/crates/og-summary/src/collect/post.rs | 339 +++++++++++++++++ .../og-summary/src/collect/post/tests.rs | 355 ++++++++++++++++++ rust/crates/og-summary/src/collect/tests.rs | 12 +- 10 files changed, 974 insertions(+), 174 deletions(-) create mode 100644 rust/crates/og-summary/src/collect/post.rs create mode 100644 rust/crates/og-summary/src/collect/post/tests.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c5d9ae54a..a99006d13 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -846,6 +846,7 @@ dependencies = [ "og-clinvar", "og-cn", "og-io", + "og-runlog", "og-steps", "pycompat", ] diff --git a/rust/crates/og-summary/Cargo.toml b/rust/crates/og-summary/Cargo.toml index 325b1f475..dedbfb695 100644 --- a/rust/crates/og-summary/Cargo.toml +++ b/rust/crates/og-summary/Cargo.toml @@ -15,6 +15,7 @@ og-steps.workspace = true og-ancestry.workspace = true og-chip.workspace = true og-cn.workspace = true +og-runlog.workspace = true indexmap.workspace = true memchr.workspace = true diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index e351fd308..13b22eb76 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -14,6 +14,10 @@ //! ancestry, structural variants, repeats, telomeres, runs of //! homozygosity, the haplogroups, the opt-in step tables, chrM, traits with //! the APOL1 G2 deletion, and the couple screen ([`group_c`]). +//! Without `--only` it reads all three groups and runs the post pass +//! ([`post`]): the readings that join sections, the steps not run, what no +//! step assessed, the last database re-check and what changed since the +//! previous run. That is the whole summary.json. //! //! How it reads a folder: one walk of the folder ([`scan::FolderIndex`]) //! instead of a `glob` per candidate path; each input read once (the @@ -32,6 +36,7 @@ pub mod group_b; pub mod group_c; pub mod model; pub mod pgx; +pub mod post; pub mod scan; pub mod sections; pub mod text; @@ -47,17 +52,17 @@ use std::io::Write; use std::path::Path; const USAGE: &str = - "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR --only {a,b,c} - --out OUT [--declared-sex DECLARED_SEX] [--assets ASSETS] + "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR [--out OUT] + [--declared-sex DECLARED_SEX] [--assets ASSETS] [--only {a,b,c}] "; const OPTS: &[Opt] = &[ Opt("--sample", Kind::Value, true), Opt("--sample-dir", Kind::Value, true), Opt("--declared-sex", Kind::Value, false), - Opt("--out", Kind::Value, true), + Opt("--out", Kind::Value, false), Opt("--assets", Kind::Value, false), - Opt("--only", Kind::Choice(&["a", "b", "c"]), true), + Opt("--only", Kind::Choice(&["a", "b", "c"]), false), ]; /// What one collect reads. @@ -70,7 +75,7 @@ pub struct Input { pub declared_sex: Option, /// The repository's `assets/` folder. pub assets: String, - /// The group of sections to read: `a`, `b` or `c`. + /// The group of sections to read: `a`, `b`, `c`, or all of them. pub group: Group, /// The folder of the two tables group c reads from `bin/`. pub tables: String, @@ -82,6 +87,8 @@ pub enum Group { A, B, C, + /// Every section and the post pass: the whole summary.json. + All, } /// `og summary collect`: exit 0 when summary.json is written (whatever @@ -102,28 +109,39 @@ pub fn cli(argv: &[String], out: &mut impl Write, err: &mut impl Write) -> u8 { declared_sex: a.get("--declared-sex").map(str::to_owned), assets: a.get("--assets").map_or_else(default_assets, str::to_owned), tables: group_c::util::default_tables(), - group: match a.text("--only") { - "b" => Group::B, - "c" => Group::C, - _ => Group::A, + group: match a.get("--only") { + Some("a") => Group::A, + Some("b") => Group::B, + Some("c") => Group::C, + _ => Group::All, }, }; // A part of the sections must never replace the sample's complete - // summary.json, so there is no default path while `--only` exists. - let target = a.text("--out").to_owned(); - let read = match a.text("--only") { - "b" => group_b::collect, - _ => collect, - }; - let summary = match read(&input) { - Ok(s) => s, - Err(e) => return crate::finish(Err(e), err), + // summary.json, so a part is written only where `--out` says. + let target = match a.get("--out") { + Some(t) => t.to_owned(), + None if input.group == Group::All => crate::join(&input.sample_dir, "summary.json"), + None => { + let _ = write!(err, "{USAGE}"); + let _ = writeln!( + err, + "og summary collect: error: --only needs --out: a part of the sections never replaces summary.json" + ); + return 2; + } }; // The sections' own data is freed before the text is made, and the // tree before the file is written: a large ClinVar list is in memory in // one form at a time. - let value = summary.to_json(); - drop(summary); + let read = match input.group { + Group::All => post::collect(&input), + Group::B => group_b::collect(&input).map(|s| s.to_json()), + Group::A | Group::C => collect(&input).map(|s| s.to_json()), + }; + let value = match read { + Ok(v) => v, + Err(e) => return crate::finish(Err(e), err), + }; let body = match dumps( &value, &Dump { @@ -161,47 +179,83 @@ fn default_assets() -> String { .to_string() } -/// Reads the folder into the summary of group a. An error is a run status -/// or a manifest that is there and cannot be read: without it a result of -/// an earlier run would be called current, so no summary is written. +/// What every group reads first: one walk of the folder, the run status +/// and the manifest. +pub struct Folder { + pub index: FolderIndex, + pub run: Option, + pub manifest: Option, +} + +impl Folder { + /// Walks the folder and reads its run status and manifest. An error is + /// a run status or a manifest that is there and cannot be read: without + /// it a result of an earlier run would be called current, so no summary + /// is written. + pub fn open(input: &Input) -> Result { + let dir = Path::new(&input.sample_dir); + let index = FolderIndex::walk(dir); + let run = + read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); + let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); + Ok(Folder { + index, + run, + manifest, + }) + } +} + +/// Reads the folder into the summary of group a or group c. pub fn collect(input: &Input) -> Result { - let dir = Path::new(&input.sample_dir); - let index = FolderIndex::walk(dir); - let run = - read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); - let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); - let s = pycompat::mtime::escape(&input.sample); - let find = |pats: &[String]| index.first(pats).map(str::to_owned); - let file = |rel: String| -> (String, String) { (crate::join(&input.sample_dir, &rel), rel) }; - if input.group == Group::C { + let folder = Folder::open(input)?; + let (sections, declared_sex) = if input.group == Group::C { // The declared sex the chrY filter of CNVpytor reads comes from // step 33's table when the run status and manifest have none. - let qc = find(&[format!("qc/{s}_sample_qc.tsv")]).map(file); - let qc_declared = qc.and_then(|(p, _)| { + let s = pycompat::mtime::escape(&input.sample); + let qc = folder + .index + .first(&[format!("qc/{s}_sample_qc.tsv")]) + .map(|rel| crate::join(&input.sample_dir, rel)); + let qc_declared = qc.and_then(|p| { let rows = sections::read_tsv(&p).ok()?; SampleQc::read(&p, &rows) .ok()? .declared_sex() .map(str::to_owned) }); - let declared_sex = declared( + let declared_sex = declared(input, &folder, qc_declared.as_deref()); + let mut sections = read_c(input, &folder); + post_c( input, - run.as_ref(), - manifest.as_ref(), - qc_declared.as_deref(), + &folder.index, + &mut sections, + &group_c::Others::default(), + declared_sex.as_deref(), ); - let sections = read_group_c(input, &index, run.as_ref(), declared_sex.as_deref()); - return Ok(Summary { - sample: input.sample.clone(), - generated_utc: time::utc_minute(now()).unwrap_or_default(), - run, - manifest, - declared_sex, - sections, - cyp2d6: None, - }); - } + (sections, declared_sex) + } else { + let sections = read_a(input, &folder); + let declared_sex = declared(input, &folder, qc_declared(§ions)); + (sections, declared_sex) + }; + Ok(Summary { + sample: input.sample.clone(), + generated_utc: time::utc_minute(now()).unwrap_or_default(), + run: folder.run, + manifest: folder.manifest, + declared_sex, + sections, + cyp2d6: None, + }) +} +/// Reads the sections of group a, in [`GROUP_A`] order: the sample VCF on +/// its own thread while the other six are read. +pub fn read_a(input: &Input, folder: &Folder) -> Vec
{ + let s = pycompat::mtime::escape(&input.sample); + let find = |pats: &[String]| folder.index.first(pats).map(str::to_owned); + let file = |rel: String| -> (String, String) { (crate::join(&input.sample_dir, &rel), rel) }; let variants_at = find(&[format!("vcf/{s}.vcf.gz")]); let (variants, rest) = std::thread::scope(|scope| { let v = scope.spawn(|| { @@ -233,29 +287,23 @@ pub fn collect(input: &Input) -> Result { read_count, variants, ]; - let sections: Vec
= GROUP_A + GROUP_A .iter() .zip(reads) - .map(|(spec, r): (&Spec, _)| Section::new(*spec, r, run.as_ref())) - .collect(); + .map(|(spec, r): (&Spec, _)| Section::new(*spec, r, folder.run.as_ref())) + .collect() +} - let qc_declared = sections +/// The declared sex step 33's table records, from its section (read when +/// the section is ok or stale). +pub fn qc_declared(sections: &[Section]) -> Option<&str> { + sections .iter() .find(|s| s.spec.key == "sample_qc") .and_then(|s| match &s.data { Some(Data::SampleQc(q)) => q.declared_sex(), _ => None, - }); - let declared_sex = declared(input, run.as_ref(), manifest.as_ref(), qc_declared); - Ok(Summary { - sample: input.sample.clone(), - generated_utc: time::utc_minute(now()).unwrap_or_default(), - run, - manifest, - declared_sex, - sections, - cyp2d6: None, - }) + }) } fn now() -> f64 { @@ -266,16 +314,14 @@ fn now() -> f64 { /// The declared sex: the command line's, the run status's, the /// manifest's, then step 33's; the first that is not empty. -fn declared( - input: &Input, - run: Option<&RunStatus>, - manifest: Option<&Manifest>, - qc: Option<&str>, -) -> Option { +pub fn declared(input: &Input, folder: &Folder, qc: Option<&str>) -> Option { [ input.declared_sex.as_deref(), - run.and_then(|r| r.declared_sex.as_deref()), - manifest.and_then(|m| m.value("run", "declared_sex")), + folder.run.as_ref().and_then(|r| r.declared_sex.as_deref()), + folder + .manifest + .as_ref() + .and_then(|m| m.value("run", "declared_sex")), qc, ] .into_iter() @@ -284,39 +330,56 @@ fn declared( .map(str::to_owned) } -/// Reads the sections of group c, then runs the parts of the post pass -/// that change them. -fn read_group_c( +/// Reads the sections of group c, in [`group_c::GROUP_C`] order. +pub fn read_c(input: &Input, folder: &Folder) -> Vec
{ + let c = ctx_c(input, &folder.index); + group_c::read_all(&c) + .into_iter() + .zip(group_c::GROUP_C.iter()) + .map(|(r, spec)| { + let r = r.and_then(|found| match found { + None => Ok(None), + Some((f, v)) => with_mtime(&f.path, f.rel, Data::Json(v)), + }); + Section::new(*spec, r, folder.run.as_ref()) + }) + .collect() +} + +/// The parts of the post pass that change group c's sections: the repeat +/// readings, Locityper's primary calls and CNVpytor's chrY filter. A part +/// that fails makes its own section unreadable, with a note that names the +/// file and the reading; the other sections and the summary stay. +pub fn post_c( input: &Input, index: &FolderIndex, - run: Option<&RunStatus>, + sections: &mut [Section], + others: &group_c::Others, declared_sex: Option<&str>, -) -> Vec
{ - let c = group_c::Ctx { +) { + let c = ctx_c(input, index); + let failed = { + let mut view: Vec<(Spec, model::State, Option<&mut Data>)> = sections + .iter_mut() + .map(|s| (s.spec, s.state, s.data.as_mut())) + .collect(); + group_c::post(&c, &mut view, others, declared_sex) + }; + for (key, why) in failed { + if let Some(s) = sections.iter_mut().find(|s| s.spec.key == key) { + s.make_unreadable(&why); + } + } +} + +fn ctx_c<'a>(input: &'a Input, index: &'a FolderIndex) -> group_c::Ctx<'a> { + group_c::Ctx { sample: &input.sample, dir: &input.sample_dir, assets: &input.assets, tables: &input.tables, index, - }; - let reads = group_c::read_all(&c); - let mut sections: Vec
= group_c::GROUP_C - .iter() - .zip(reads) - .map(|(spec, r)| { - let r = r.and_then(|found| match found { - None => Ok(None), - Some((f, v)) => with_mtime(&f.path, f.rel, Data::Json(v)), - }); - Section::new(*spec, r, run) - }) - .collect(); - let mut view: Vec<(Spec, model::State, Option<&mut Data>)> = sections - .iter_mut() - .map(|s| (s.spec, s.state, s.data.as_mut())) - .collect(); - group_c::post(&c, &mut view, &group_c::Others::default(), declared_sex); - sections + } } /// What one reader gave: the file (relative), its mtime and its data. diff --git a/rust/crates/og-summary/src/collect/group_b.rs b/rust/crates/og-summary/src/collect/group_b.rs index cb0ba8a1e..394178cf5 100644 --- a/rust/crates/og-summary/src/collect/group_b.rs +++ b/rust/crates/og-summary/src/collect/group_b.rs @@ -14,13 +14,12 @@ use super::carrier::Carrier; use super::clinical::{Clinical, Slivar}; use super::clinvar::{ClinVar, Floor, HitsVcf}; use super::cpsr::{self, Cpsr}; -use super::model::{Manifest, RunStatus, Section, Spec, Summary}; +use super::model::{Section, Spec, Summary}; use super::pgx::{Cpic, Cyp2d6, Cyrius, PharmCat, Pypgx}; use super::scan::FolderIndex; use super::sections::{Data, SampleQc}; use super::two_hits::{self, DeletionInputs}; -use super::{Input, Read, read_at, read_optional, with_mtime}; -use std::path::Path; +use super::{Folder, Input, Read, read_at, with_mtime}; /// The sections `og summary collect --only b` reads, in summary.json's order. pub const GROUP_B: [Spec; 9] = [ @@ -74,12 +73,42 @@ pub const GROUP_B: [Spec; 9] = [ /// Reads the folder into the summary of group b. An error is a run status /// or a manifest that is there and cannot be read. pub fn collect(input: &Input) -> Result { - let dir = Path::new(&input.sample_dir); + let folder = Folder::open(input)?; + let sections = read(input, &folder); + let cyp2d6 = cyp2d6(input, &folder.index, §ions); + let s = pycompat::mtime::escape(&input.sample); + // The run block is group a's: the declared sex from the command line, + // the run status, the manifest or step 33's table, the first that has one. + let qc_declared = folder + .index + .first(&[format!("qc/{s}_sample_qc.tsv")]) + .and_then(|rel| { + let p = crate::join(&input.sample_dir, rel); + let rows = super::sections::read_tsv(&p).ok()?; + SampleQc::read(&p, &rows) + .ok()? + .declared_sex() + .map(str::to_owned) + }); + let declared_sex = super::declared(input, &folder, qc_declared.as_deref()); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()); + Ok(Summary { + sample: input.sample.clone(), + generated_utc: super::time::utc_minute(now).unwrap_or_default(), + run: folder.run, + manifest: folder.manifest, + declared_sex, + sections, + cyp2d6, + }) +} + +/// Reads the sections of group b, in [`GROUP_B`] order. +pub fn read(input: &Input, folder: &Folder) -> Vec
{ let d = input.sample_dir.as_str(); - let index = FolderIndex::walk(dir); - let run = - read_optional(&dir.join("logs").join("run_status.tsv"))?.map(|t| RunStatus::parse(&t)); - let manifest = read_optional(&dir.join("run_manifest.tsv"))?.map(|t| Manifest::parse(&t)); + let index = &folder.index; let s = pycompat::mtime::escape(&input.sample); let raw = input.sample.as_str(); let file = |rel: &str| -> (String, String) { (crate::join(d, rel), rel.to_owned()) }; @@ -122,7 +151,7 @@ pub fn collect(input: &Input) -> Result { )?; with_mtime(&p, rel, Data::Clinical(data)) }); - let rest = read_rest(input, &s, &index, &find, &hits, &deletions); + let rest = read_rest(input, &s, index, &find, &hits, &deletions); let join = |h: std::thread::ScopedJoinHandle<'_, Read>| { h.join() .unwrap_or_else(|_| Err("internal error: the section reader stopped".into())) @@ -133,55 +162,26 @@ pub fn collect(input: &Input) -> Result { let reads = [ clinvar, carrier, pharmcat, cpic, pypgx, cyrius, cpsr, clinical, slivar, ]; - let sections: Vec
= GROUP_B + GROUP_B .iter() .zip(reads) - .map(|(spec, r)| Section::new(*spec, r, run.as_ref())) - .collect(); + .map(|(spec, r)| Section::new(*spec, r, folder.run.as_ref())) + .collect() +} + +/// The `cyp2d6` block: CPIC, pypgx and Cyrius side by side, with step 36's +/// row. `None` when `sections` lack one of the three. +pub fn cyp2d6(input: &Input, index: &FolderIndex, sections: &[Section]) -> Option { + let s = pycompat::mtime::escape(&input.sample); + let d = input.sample_dir.as_str(); let sec = |k: &str| sections.iter().find(|x| x.spec.key == k); - let cyp2d6 = match (sec("cpic"), sec("pypgx"), sec("cyrius")) { - (Some(c), Some(p), Some(y)) => Some(Cyp2d6::new( - c, - p, - y, - one(format!("pgx_consensus/{s}_pgx_consensus.tsv")), - d, - )), + let consensus = index + .first(&[format!("pgx_consensus/{s}_pgx_consensus.tsv")]) + .map(|rel| (crate::join(d, rel), rel.to_owned())); + match (sec("cpic"), sec("pypgx"), sec("cyrius")) { + (Some(c), Some(p), Some(y)) => Some(Cyp2d6::new(c, p, y, consensus, d)), _ => None, - }; - - // The run block is group a's: the declared sex from the command line, - // the run status, the manifest or step 33's table, the first that has one. - let qc_declared = one(format!("qc/{s}_sample_qc.tsv")).and_then(|(p, _)| { - let rows = super::sections::read_tsv(&p).ok()?; - SampleQc::read(&p, &rows) - .ok()? - .declared_sex() - .map(str::to_owned) - }); - let declared_sex = [ - input.declared_sex.clone(), - run.as_ref().and_then(|r| r.declared_sex.clone()), - manifest - .as_ref() - .and_then(|m| m.value("run", "declared_sex").map(str::to_owned)), - qc_declared, - ] - .into_iter() - .flatten() - .find(|v| !v.is_empty()); - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0.0, |d| d.as_secs_f64()); - Ok(Summary { - sample: input.sample.clone(), - generated_utc: super::time::utc_minute(now).unwrap_or_default(), - run, - manifest, - declared_sex, - sections, - cyp2d6, - }) + } } /// Finds a section's file: the full path and the path in the folder. diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 15ab27782..3450f6b5d 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -9,7 +9,8 @@ //! readings with the traits and CPIC states, Locityper's primary calls, and //! the chrY filter of CNVpytor's count for a declared female. What they //! need from group b (the CPIC state, pypgx and Cyrius) comes in -//! [`Others`]; `--only c` reads none of it, so it is empty there. +//! [`Others`]; `--only c` reads none of it, so it is empty there. A part +//! that fails makes only its own section unreadable. mod chrm; mod cn; @@ -236,13 +237,15 @@ pub fn read_all(c: &Ctx<'_>) -> Vec { /// The post pass over the read sections (`sections` in [`GROUP_C`] /// order, each with its state and data): the repeat readings with the /// traits and CPIC states, each Locityper locus's primary call, and the -/// chrY filter of CNVpytor's count. +/// chrY filter of CNVpytor's count. Each part runs on its own: one that +/// fails (an error or a panic) is returned as its section's key and why, +/// and the others still run. pub fn post( c: &Ctx<'_>, sections: &mut [(Spec, State, Option<&mut Data>)], others: &Others, declared_sex: Option<&str>, -) { +) -> Vec<(&'static str, String)> { let state = |key: &str, secs: &[(Spec, State, Option<&mut Data>)]| { secs.iter() .find(|(s, ..)| s.key == key) @@ -257,19 +260,42 @@ pub fn post( .and_then(|x| x.2.as_deref()) .and_then(json) .cloned(); + let mut failed = Vec::new(); for (spec, st, data) in sections.iter_mut() { let live = matches!(st, State::Ok | State::Stale); let Some(Data::Json(v)) = data.as_deref_mut() else { continue; }; - match spec.key { - "expansions" if live => repeats::post(c, v, traits_state, cpic_state), - "locityper" if live => { - tables::locityper_post(v, hla_state, hla_data.as_ref(), others); + let part = || -> Result<(), String> { + match spec.key { + "expansions" if live => repeats::post(c, v, traits_state, cpic_state) + .map_err(|e| format!("the repeat readings could not be made: {e}")), + "locityper" if live => { + tables::locityper_post(v, hla_state, hla_data.as_ref(), others); + Ok(()) + } + "cnvpytor" => { + sv::cnvpytor_sex_filter(v, declared_sex); + Ok(()) + } + _ => Ok(()), } - "cnvpytor" => sv::cnvpytor_sex_filter(v, declared_sex), - _ => {} - } + }; + failed.extend(contain(spec.key, part)); + } + failed +} + +/// Runs one part of the post pass: `None` when it went through, else its +/// section's key and why it failed, for an error and for a panic alike. +pub fn contain( + key: &'static str, + part: impl FnOnce() -> Result<(), String>, +) -> Option<(&'static str, String)> { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { + Ok(Ok(())) => None, + Ok(Err(e)) => Some((key, e)), + Err(_) => Some((key, format!("internal error: the post pass of {key} stopped"))), } } diff --git a/rust/crates/og-summary/src/collect/group_c/repeats.rs b/rust/crates/og-summary/src/collect/group_c/repeats.rs index 9d9610fd0..3081543d3 100644 --- a/rust/crates/og-summary/src/collect/group_c/repeats.rs +++ b/rust/crates/og-summary/src/collect/group_c/repeats.rs @@ -910,9 +910,10 @@ fn readings( } /// The post pass: the readings again, with the traits and CPIC states, over -/// the loci as the section lists them. -pub fn post(c: &Ctx<'_>, v: &mut Value, traits: State, cpic: State) { - let Value::Object(m) = v else { return }; +/// the loci as the section lists them. An error is a reading that could +/// not be made; the section's data is then left as it was. +pub fn post(c: &Ctx<'_>, v: &mut Value, traits: State, cpic: State) -> Result<(), String> { + let Value::Object(m) = v else { return Ok(()) }; let loci: Vec<(String, Obj)> = match m.get("loci") { Some(Value::Array(xs)) => xs .iter() @@ -924,10 +925,10 @@ pub fn post(c: &Ctx<'_>, v: &mut Value, traits: State, cpic: State) { _ => None, }) .collect(), - _ => return, + _ => return Ok(()), }; let by = by_locus(loci.iter().map(|(k, o)| (k, o))); - if let Ok(r) = readings(c, &by, traits, cpic) { - m.insert("readings".into(), r); - } + let r = readings(c, &by, traits, cpic)?; + m.insert("readings".into(), r); + Ok(()) } diff --git a/rust/crates/og-summary/src/collect/model.rs b/rust/crates/og-summary/src/collect/model.rs index c6d6e22a3..2d6e7c994 100644 --- a/rust/crates/og-summary/src/collect/model.rs +++ b/rust/crates/og-summary/src/collect/model.rs @@ -178,6 +178,20 @@ impl Section { s } + /// Marks a section whose reading failed after its file was read (a + /// part of the post pass): `unreadable`, as a reader's error leaves it, + /// with a note that names the file the section came from. + pub fn make_unreadable(&mut self, why: &str) { + self.note = Some(match self.source.as_deref() { + Some(src) => format!("{}: {why}", crate::basename(src)), + None => why.to_owned(), + }); + self.state = State::Unreadable; + self.source = None; + self.file_date = None; + self.data = None; + } + pub fn to_json(&self) -> Value { let mut m = IndexMap::new(); m.insert("title".into(), Value::Str(self.spec.title.into())); diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs new file mode 100644 index 000000000..42b9fe1ad --- /dev/null +++ b/rust/crates/og-summary/src/collect/post.rs @@ -0,0 +1,339 @@ +//! The whole summary.json: every section of groups a, b and c, read over +//! one walk of the folder, then the post pass. +//! +//! The post pass is what joins sections after each is read: +//! +//! - the repeat readings with the traits and CPIC states, Locityper's +//! primary calls with HLA, pypgx and Cyrius, and CNVpytor's chrY filter +//! with the declared sex ([`super::post_c`]); +//! - the `cyp2d6` block over CPIC, pypgx and Cyrius; +//! - `not_run`: the steps whose section has no file; +//! - `not_assessed`: what no step looked at, with the lines that depend on +//! the polygenic scores, HBA1/HBA2, mobile elements, VNtyper and MoChA; +//! - `recheck`: the last database re-check of the sample; +//! - `new_since`: what changed since the previous run kept in `history/`. +//! +//! Each part that changes a section runs on its own: when it fails, only +//! its section becomes `unreadable`, with a note naming the file, and the +//! summary is still written. The Python collector ran this pass outside +//! its per-section error handling, so one failure there lost summary.json +//! and both reports. A previous run that cannot be read leaves `new_since` +//! out, as before; the comparison itself has no error to hide. + +use super::group_c::Others; +use super::model::{Section, State, Summary}; +use super::sections::Data; +use super::{Folder, Input, declared, group_b, post_c, qc_declared, read_a, read_c, time}; +use og_runlog::{delta, files}; +use pycompat::json::Value; +use pycompat::value::truthy; +use std::path::Path; + +/// summary.json's sections, in its order. +pub const ORDER: [&str; 48] = [ + "coverage", + "callability", + "sex_check", + "sample_qc", + "mosaic_sex_loss", + "read_count", + "variants", + "clinvar", + "couple", + "carrier", + "pharmcat", + "cpic", + "pypgx", + "cyrius", + "parascopy", + "paralogs", + "hla", + "prs", + "ancestry_fine", + "manta", + "delly", + "cnvpytor", + "sv_consensus", + "annotsv", + "expansions", + "telomere", + "roh", + "haplogroup", + "y_haplogroup", + "y_tree", + "y_str", + "consumer_export", + "bloodtype", + "lpa", + "vntyper", + "locityper", + "mito", + "traits", + "archaic", + "ancient", + "viral", + "exomiser", + "chip", + "mei", + "mosaic", + "cpsr", + "clinical", + "slivar", +]; + +/// Sections that are no step of their own, so never in `not_run`: the read +/// count is written by alignment, which a run from a BAM or a VCF never +/// does; the couple screen and the carrier list come with other steps. +const NOT_A_STEP: [&str; 3] = ["read_count", "couple", "carrier"]; + +/// What no step assesses. +const NOT_ASSESSED: [&str; 5] = [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", +]; + +/// Added unless step 25 ran with an ancestry reference panel. +const PRS_NOT_ADJUSTED: &str = + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums"; + +/// Added unless step 35's depth count (short reads) or step 35b +/// (Paraphase, PacBio HiFi, both genes) made a call. +const HBA_NOT_ASSESSED: &str = "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count \ +(short reads) nor step 35b (Paraphase, PacBio HiFi) made a call"; + +/// Added when step 50 (opt-in, MEGAnE) did not run. A failed step 50 is +/// listed under not run, and an unreadable table under its section. +const MEI_NOT_ASSESSED: &str = "Mobile element insertions (Alu, L1 and SVA, the FKTN, TAF1 and BBS1 SVA insertions among them): \ +not looked for (step 50, MEGAnE, is opt-in)"; + +/// Added unless step 52 (VNtyper) gave a positive, possible or negative +/// reading: no other step sees a frameshift inside the MUC1 VNTR. +const VNTYPER_NOT_ASSESSED: &str = "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; \ +no standard caller sees them"; + +/// Added unless step 51 (MoChA) assessed the sample. +const MCA_NOT_ASSESSED: &str = "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share \ +of the blood cells): step 51 (MoChA) did not assess this sample"; + +/// Reads the whole summary of a sample folder. The three groups are read +/// at once over one walk of the folder. An error is a run status or a +/// manifest that is there and cannot be read. +pub fn collect(input: &Input) -> Result { + let folder = Folder::open(input)?; + let (a, b, mut c) = std::thread::scope(|scope| { + let a = scope.spawn(|| read_a(input, &folder)); + let b = scope.spawn(|| group_b::read(input, &folder)); + let c = read_c(input, &folder); + let stopped = |specs: &[super::model::Spec]| -> Vec
{ + specs + .iter() + .map(|s| { + Section::new( + *s, + Err("internal error: the section reader stopped".into()), + folder.run.as_ref(), + ) + }) + .collect() + }; + let a = a + .join() + .unwrap_or_else(|_| stopped(&super::model::GROUP_A)); + let b = b.join().unwrap_or_else(|_| stopped(&group_b::GROUP_B)); + (a, b, c) + }); + let declared_sex = declared(input, &folder, qc_declared(&a)); + post_c( + input, + &folder.index, + &mut c, + &others(&b), + declared_sex.as_deref(), + ); + let cyp2d6 = group_b::cyp2d6(input, &folder.index, &b); + let sections = in_order(a.into_iter().chain(b).chain(c).collect()); + let not_run = not_run(§ions); + let not_assessed = not_assessed(§ions); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()); + let summary = Summary { + sample: input.sample.clone(), + generated_utc: time::utc_minute(now).unwrap_or_default(), + run: folder.run, + manifest: folder.manifest, + declared_sex, + sections, + cyp2d6, + }; + let mut v = summary.to_json(); + drop(summary); + let Value::Object(m) = &mut v else { + return Err("internal error: the summary is not an object".into()); + }; + m.insert( + "not_run".into(), + Value::Array(not_run.into_iter().map(Value::Str).collect()), + ); + m.insert( + "not_assessed".into(), + Value::Array( + not_assessed + .into_iter() + .map(|x| Value::Str(x.to_owned())) + .collect(), + ), + ); + let dir = Path::new(&input.sample_dir); + if let Some(r) = delta::read_recheck(&dir.join("recheck").join("state.json")) { + m.insert("recheck".into(), Value::Object(r)); + } + if let Some(n) = new_since(dir, &v) { + if let Value::Object(m) = &mut v { + m.insert("new_since".into(), n); + } + } + Ok(v) +} + +/// What the post pass of group c needs from group b: the CPIC state, +/// pypgx's CYP2D6 call and Cyrius's genotype and Filter. +fn others(b: &[Section]) -> Others { + let sec = |k: &str| b.iter().find(|s| s.spec.key == k); + let live = |s: &Section| matches!(s.state, State::Ok | State::Stale); + Others { + cpic: sec("cpic").map(|s| s.state), + traits: None, + pypgx: sec("pypgx") + .filter(|s| live(s)) + .and_then(|s| match &s.data { + Some(Data::Pypgx(p)) => Some((s.state, p.cyp2d6.clone())), + _ => None, + }), + cyrius: sec("cyrius") + .filter(|s| live(s)) + .and_then(|s| match &s.data { + Some(Data::Cyrius(c)) => Some((s.state, Some(c.genotype.clone()), c.filter.clone())), + _ => None, + }), + } +} + +/// The sections in summary.json's order ([`ORDER`]). +fn in_order(mut sections: Vec
) -> Vec
{ + sections.sort_by_key(|s| ORDER.iter().position(|k| *k == s.spec.key)); + sections +} + +/// The steps whose section has no file, with " (step N failed)" for a step +/// that failed in the last run. +fn not_run(sections: &[Section]) -> Vec { + sections + .iter() + .filter(|s| matches!(s.state, State::Missing | State::Failed)) + .filter(|s| !NOT_A_STEP.contains(&s.spec.key)) + .map(|s| { + if s.state == State::Failed { + format!("{} (step {} failed)", s.spec.title, s.spec.step) + } else { + s.spec.title.to_owned() + } + }) + .collect() +} + +/// What no step assessed for this sample. +fn not_assessed(sections: &[Section]) -> Vec<&'static str> { + let sec = |k: &str| sections.iter().find(|s| s.spec.key == k); + let state = |k: &str| sec(k).map_or(State::Missing, |s| s.state); + // A section's data is there when it is ok or stale; else it reads {}. + let field = |k: &str, key: &str| -> Option<&Value> { + match sec(k).and_then(|s| s.data.as_ref()) { + Some(Data::Json(Value::Object(m))) => m.get(key), + _ => None, + } + }; + let mut out = NOT_ASSESSED.to_vec(); + if !field("prs", "adjusted").is_some_and(truthy) { + out.push(PRS_NOT_ADJUSTED); + } + // Step 35b on a region subset (--paraphase_regions) may leave hba out. + let hba_genes: Vec<&str> = match field("paralogs", "genes") { + Some(Value::Array(genes)) => genes + .iter() + .filter(|g| is(at(Some(g), "region"), "hba") && is(at(Some(g), "status"), "called")) + .filter_map(|g| match at(Some(g), "gene") { + Some(Value::Str(x)) => Some(x.as_str()), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + let paraphase_hba = ["HBA1", "HBA2"].iter().all(|g| hba_genes.contains(g)); + if !(paraphase_hba || is(at(field("parascopy", "hba"), "status"), "called")) { + out.push(HBA_NOT_ASSESSED); + } + if state("mei") == State::Missing { + out.push(MEI_NOT_ASSESSED); + } + let vntyper_read = state("vntyper") == State::Ok + && ["positive", "possible", "negative"] + .iter() + .any(|w| is(field("vntyper", "status"), w)); + if !vntyper_read { + out.push(VNTYPER_NOT_ASSESSED); + } + if !is(field("mosaic", "status"), "assessed") { + out.push(MCA_NOT_ASSESSED); + } + out +} + +/// True when `v` is the string `word`. +fn is(v: Option<&Value>, word: &str) -> bool { + matches!(v, Some(Value::Str(x)) if x == word) +} + +/// `v[key]` when `v` is an object that has it. +fn at<'a>(v: Option<&'a Value>, key: &str) -> Option<&'a Value> { + match v { + Some(Value::Object(m)) => m.get(key), + _ => None, + } +} + +/// `{previous_utc, sections}`: what changed since the newest run kept in +/// `history/` that holds a summary.json (generate-report.sh keeps one per +/// run), or `None` without one. A previous run whose summary or manifest +/// cannot be read, or whose summary is not shaped as a summary, counts as +/// none. +fn new_since(dir: &Path, cur: &Value) -> Option { + let prev_dir = delta::newest_history(dir)?; + let bytes = std::fs::read(prev_dir.join("summary.json")).ok()?; + let prev = pycompat::json::loads(std::str::from_utf8(&bytes).ok()?).ok()?; + let prev_manifest = files::read_manifest(Some(&prev_dir.join("run_manifest.tsv"))).ok()?; + let manifest = files::read_manifest(Some(&dir.join("run_manifest.tsv"))).ok()?; + let changes = delta::compare(&prev, &prev_manifest, cur, &manifest).ok()?; + let previous = match &prev { + Value::Object(m) => m.get("generated_utc").cloned().unwrap_or(Value::Null), + _ => Value::Null, + }; + let mut out = indexmap::IndexMap::new(); + out.insert("previous_utc".to_owned(), previous); + out.insert( + "sections".to_owned(), + Value::Array( + delta::changed_sections(&changes) + .into_iter() + .map(Value::Str) + .collect(), + ), + ); + Some(Value::Object(out)) +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/og-summary/src/collect/post/tests.rs b/rust/crates/og-summary/src/collect/post/tests.rs new file mode 100644 index 000000000..e33709b92 --- /dev/null +++ b/rust/crates/og-summary/src/collect/post/tests.rs @@ -0,0 +1,355 @@ +use super::super::group_c::{self, GROUP_C}; +use super::super::model::{GROUP_A, Spec}; +use super::super::{Group, Input, group_b}; +use super::*; +use crate::testutil::{Dir, run}; +use pycompat::json::{Value, loads}; + +fn input(d: &Dir) -> Input { + Input { + sample: "S".into(), + sample_dir: d.path("S"), + declared_sex: None, + assets: d.path("assets"), + group: Group::All, + tables: d.path("tables"), + } +} + +fn summary(d: &Dir) -> Value { + std::fs::create_dir_all(d.path("S")).unwrap(); + collect(&input(d)).unwrap() +} + +fn at<'a>(v: &'a Value, path: &[&str]) -> &'a Value { + path.iter().fold(v, |v, k| match v { + Value::Object(m) => m.get(*k).unwrap_or_else(|| panic!("no {k} in {path:?}")), + Value::Array(a) => &a[k.parse::().unwrap()], + _ => panic!("{path:?}"), + }) +} + +fn has(v: &Value, key: &str) -> bool { + matches!(v, Value::Object(m) if m.contains_key(key)) +} + +fn texts(v: &Value) -> Vec { + match v { + Value::Array(a) => a + .iter() + .map(|x| match x { + Value::Str(s) => s.clone(), + _ => panic!("{x:?}"), + }) + .collect(), + _ => panic!("{v:?}"), + } +} + +fn word(v: &Value) -> &str { + match v { + Value::Str(s) => s, + _ => panic!("{v:?}"), + } +} + +#[test] +fn the_order_holds_every_section_once() { + let mut keys: Vec<&str> = GROUP_A + .iter() + .chain(group_b::GROUP_B.iter()) + .chain(GROUP_C.iter()) + .map(|s: &Spec| s.key) + .collect(); + assert_eq!(keys.len(), ORDER.len()); + keys.sort_unstable(); + let mut order = ORDER.to_vec(); + order.sort_unstable(); + assert_eq!(keys, order); +} + +#[test] +fn an_empty_folder_lists_every_step_and_every_line() { + let d = Dir::new("post-empty"); + let v = summary(&d); + let Value::Object(m) = &v else { panic!() }; + let keys: Vec<&str> = m.keys().map(String::as_str).collect(); + assert_eq!( + keys, + [ + "schema_version", + "sample", + "generated_utc", + "run", + "databases", + "manifest", + "sections", + "cyp2d6", + "not_run", + "not_assessed" + ] + ); + let Value::Object(sections) = at(&v, &["sections"]) else { + panic!() + }; + assert_eq!(sections.keys().map(String::as_str).collect::>(), ORDER); + let not_run = texts(at(&v, &["not_run"])); + assert_eq!(not_run.len(), 45); + assert_eq!(not_run[0], "Coverage (mosdepth)"); + assert!(!not_run.contains(&"Reads kept by alignment".to_owned())); + assert!(!not_run.contains(&"Carrier list".to_owned())); + assert!(!not_run.contains(&"Couple carrier screen".to_owned())); + let lines = texts(at(&v, &["not_assessed"])); + let mut expected: Vec<&str> = NOT_ASSESSED.to_vec(); + expected.extend([ + PRS_NOT_ADJUSTED, + HBA_NOT_ASSESSED, + MEI_NOT_ASSESSED, + VNTYPER_NOT_ASSESSED, + MCA_NOT_ASSESSED, + ]); + assert_eq!(lines, expected); +} + +#[test] +fn a_failed_step_says_so_under_not_run() { + let d = Dir::new("post-failed"); + d.put( + "S/logs/run_status.tsv", + "meta\tstarted_epoch\t1577836800\nstep\t06\tfailed (exit 1)\nstep\t02\tfailed (exit 1)\n", + ); + let v = summary(&d); + let not_run = texts(at(&v, &["not_run"])); + assert!(not_run.contains(&"ClinVar screen (step 06 failed)".to_owned())); + // The read count is no step of its own, failed or not; the carrier + // list shares step 06 with ClinVar. + assert!(!not_run.iter().any(|x| x.starts_with("Reads kept"))); + assert!(!not_run.iter().any(|x| x.starts_with("Carrier list"))); + assert_eq!(word(at(&v, &["sections", "carrier", "state"])), "failed"); +} + +#[test] +fn the_lines_go_when_their_step_assessed_the_sample() { + let d = Dir::new("post-assessed"); + d.put( + "S/vntyper/S_vntyper.tsv", + "key\tvalue\nstatus\tnegative\n", + ); + d.put( + "S/paralogs/S_hba_copy_number.tsv", + "metric\tvalue\nstatus\tcalled\ncopies\t4\nreading\tfour\n", + ); + let v = summary(&d); + let lines = texts(at(&v, &["not_assessed"])); + let vntyper = at(&v, &["sections", "vntyper"]); + if word(at(vntyper, &["state"])) == "ok" && word(at(vntyper, &["data", "status"])) == "negative" { + assert!(!lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); + } else { + assert!(lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); + } + let parascopy = at(&v, &["sections", "parascopy"]); + if word(at(parascopy, &["state"])) == "ok" + && word(at(parascopy, &["data", "hba", "status"])) == "called" + { + assert!(!lines.contains(&HBA_NOT_ASSESSED.to_owned())); + } +} + +#[test] +fn a_stale_vntyper_reading_is_not_assessed() { + // Only an ok section counts for VNtyper; any other state leaves the line. + let mk = |state: State, status: &str| { + let mut s = Section::new( + GROUP_C.iter().copied().find(|s| s.key == "vntyper").unwrap(), + Ok(None), + None, + ); + s.state = state; + s.data = Some(Data::Json( + loads(&format!(r#"{{"status": "{status}"}}"#)).unwrap(), + )); + s + }; + let lines = not_assessed(&[mk(State::Ok, "positive")]); + assert!(!lines.contains(&VNTYPER_NOT_ASSESSED)); + let lines = not_assessed(&[mk(State::Stale, "positive")]); + assert!(lines.contains(&VNTYPER_NOT_ASSESSED)); + let lines = not_assessed(&[mk(State::Ok, "failed")]); + assert!(lines.contains(&VNTYPER_NOT_ASSESSED)); +} + +#[test] +fn paraphase_needs_both_hba_genes_called() { + let paralogs = |genes: &str| { + let mut s = Section::new( + GROUP_C.iter().copied().find(|s| s.key == "paralogs").unwrap(), + Ok(None), + None, + ); + s.state = State::Ok; + s.data = Some(Data::Json( + loads(&format!(r#"{{"genes": {genes}}}"#)).unwrap(), + )); + s + }; + let both = r#"[{"region": "hba", "gene": "HBA1", "status": "called"}, + {"region": "hba", "gene": "HBA2", "status": "called"}]"#; + let one = r#"[{"region": "hba", "gene": "HBA1", "status": "called"}, + {"region": "hba", "gene": "HBA2", "status": "no call"}]"#; + // A row without a region, the shape that stopped the Python collector. + let odd = r#"[{"gene": "HBA1", "status": "called"}, 7]"#; + assert!(!not_assessed(&[paralogs(both)]).contains(&HBA_NOT_ASSESSED)); + assert!(not_assessed(&[paralogs(one)]).contains(&HBA_NOT_ASSESSED)); + assert!(not_assessed(&[paralogs(odd)]).contains(&HBA_NOT_ASSESSED)); +} + +#[test] +fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { + // An error and a panic alike come back as the section's key and why. + assert_eq!(group_c::contain("cnvpytor", || Ok(())), None); + assert_eq!( + group_c::contain("expansions", || Err("no table".into())), + Some(("expansions", "no table".to_owned())) + ); + let panicked = group_c::contain("locityper", || panic!("planted")); + assert_eq!( + panicked, + Some(( + "locityper", + "internal error: the post pass of locityper stopped".to_owned() + )) + ); + // The section is then unreadable, with no data, and names its file. + let mut s = Section::new( + GROUP_C.iter().copied().find(|s| s.key == "locityper").unwrap(), + Ok(Some(( + "locityper/S_locityper.tsv".into(), + 1_600_000_000.0, + Data::Json(Value::Object(indexmap::IndexMap::new())), + ))), + None, + ); + s.make_unreadable("internal error: the post pass of locityper stopped"); + let j = s.to_json(); + assert_eq!(word(at(&j, &["state"])), "unreadable"); + assert_eq!( + word(at(&j, &["note"])), + "S_locityper.tsv: internal error: the post pass of locityper stopped" + ); + assert_eq!(at(&j, &["source"]), &Value::Null); + assert_eq!(at(&j, &["data"]), &Value::Object(indexmap::IndexMap::new())); +} + +#[test] +fn an_infinite_telomere_read_length_keeps_the_summary() { + // The Python collector stopped on this table with an OverflowError and + // wrote no summary.json; here only the telomere section is unreadable. + let d = Dir::new("post-inf"); + d.put( + "S/telomere/S/S/S_summary.tsv", + "PID\tsample\ttotal_reads\tread_lengths\tintratel_reads\ttel_content\nS\ttumor\t1000\tinf\t40\t1.5\n", + ); + d.put( + "S/mosdepth/S.mosdepth.summary.txt", + "chrom\tlength\tbases\tmean\tmin\tmax\ntotal\t100\t3000\t30\t0\t60\n", + ); + let v = summary(&d); + assert_eq!(word(at(&v, &["sections", "telomere", "state"])), "unreadable"); + assert_eq!(word(at(&v, &["sections", "coverage", "state"])), "ok"); +} + +#[test] +fn a_declared_female_drops_cnvpytor_chry_calls_across_groups() { + // The declared sex comes from step 33's table (group a) and changes + // CNVpytor's count (group c). + let d = Dir::new("post-sex"); + d.put( + "S/qc/S_sample_qc.tsv", + "key\tvalue\nsample\tS\ndeclared_sex\tfemale\ninferred_sex\tfemale\nsex_check\tok\n", + ); + d.put( + "S/cnvpytor/S_cnvs.txt", + "deletion\tchrY:1-200000\t200000\t0.4\t0.00001\t1\t1\t1\t0.00001\t0.00001\t0.1\n\ +deletion\tchr2:1-200000\t200000\t0.4\t0.00001\t1\t1\t1\t0.00001\t0.00001\t0.1\n", + ); + let v = summary(&d); + assert_eq!(word(at(&v, &["run", "declared_sex"])), "female"); + let cnv = at(&v, &["sections", "cnvpytor"]); + if word(at(cnv, &["state"])) == "ok" { + assert_eq!(word(at(cnv, &["data", "declared_sex"])), "female"); + assert!(has(at(cnv, &["data", "dropped"]), "chry_female")); + } +} + +const PREV: &str = r#"{"schema_version": 2, "generated_utc": "2020-01-01 00:00 UTC", + "sections": {"coverage": {"state": "ok", "data": {}}, "clinvar": {"state": "missing", "data": {}}}}"#; + +#[test] +fn new_since_compares_with_the_newest_kept_run() { + let d = Dir::new("post-history"); + d.put("S/history/20200101T000000/summary.json", PREV); + // A newer folder without a summary.json is a half-copied one: passed over. + d.put("S/history/20990101T000000/run_manifest.tsv", "run\tx\ty\n"); + let v = summary(&d); + assert_eq!( + at(&v, &["new_since", "previous_utc"]), + &Value::Str("2020-01-01 00:00 UTC".into()) + ); + assert!(texts(at(&v, &["new_since", "sections"])).contains(&"coverage".to_owned())); +} + +#[test] +fn a_previous_run_that_cannot_be_read_leaves_new_since_out() { + for body in ["{not json", "[]", r#"{"sections": 7}"#] { + let d = Dir::new("post-history-bad"); + d.put("S/history/20200101T000000/summary.json", body); + let v = summary(&d); + assert!(!has(&v, "new_since"), "{body}"); + } +} + +#[test] +fn the_recheck_block_comes_from_its_state_file() { + let d = Dir::new("post-recheck"); + d.put( + "S/recheck/state.json", + r#"{"checked_utc": "2026-01-02 03:04:05 UTC", "items": [{"item": "clinvar", "class": "newer", "x": 1}, 5], + "affected_steps": ["06"]}"#, + ); + let v = summary(&d); + assert_eq!( + word(at(&v, &["recheck", "checked_utc"])), + "2026-01-02 03:04:05 UTC" + ); + assert_eq!(word(at(&v, &["recheck", "items", "0", "item"])), "clinvar"); + // A check older than the run's manifest is of an earlier run. + d.put( + "S/run_manifest.tsv", + "run\twritten_utc\t2027-01-01 00:00:00 UTC\n", + ); + assert!(!has(&summary(&d), "recheck")); +} + +#[test] +fn without_only_the_command_writes_summary_json_in_the_folder() { + let d = Dir::new("post-cli"); + std::fs::create_dir_all(d.path("S")).unwrap(); + let dir = d.path("S"); + let assets = d.path("assets"); + let (code, out, err) = run( + crate::collect::cli, + &["--sample", "S", "--sample-dir", &dir, "--assets", &assets], + ); + assert_eq!((code, err.as_str()), (0, "")); + let target = format!("{dir}/summary.json"); + assert_eq!(out, format!("Summary written: {target}\n")); + let v = loads(&d.read("S/summary.json")).unwrap(); + assert!(has(&v, "not_assessed")); + // A part of the sections never takes summary.json's place. + let (code, _, err) = run( + crate::collect::cli, + &["--sample", "S", "--sample-dir", &dir, "--only", "a"], + ); + assert_eq!(code, 2); + assert!(err.contains("--only needs --out"), "{err}"); +} diff --git a/rust/crates/og-summary/src/collect/tests.rs b/rust/crates/og-summary/src/collect/tests.rs index 75031f745..4f622c77f 100644 --- a/rust/crates/og-summary/src/collect/tests.rs +++ b/rust/crates/og-summary/src/collect/tests.rs @@ -448,22 +448,22 @@ fn the_command_line() { let text = d.read("out.json"); assert!(text.starts_with("{\n \"schema_version\": 2,\n \"sample\": \"S\",\n")); assert!(text.ends_with("}\n")); - // --only is required until every section is ported. + // Without --only it writes every section and the post pass. let (code, _, err) = run( cli, &["--sample", "S", "--sample-dir", &d.path("S"), "--out", &out], ); - assert_eq!(code, 2); - assert!(err.contains("required: --only"), "{err}"); - // So is --out: a part of the sections must not land on the sample's - // complete summary.json. + assert_eq!((code, err.as_str()), (0, "")); + assert!(d.read("out.json").contains("\n \"not_assessed\": [\n")); + // --only needs --out: a part of the sections must not land on the + // sample's complete summary.json. std::fs::write(d.path("S/summary.json"), "complete\n").unwrap(); let (code, _, err) = run( cli, &["--sample", "S", "--sample-dir", &d.path("S"), "--only", "a"], ); assert_eq!(code, 2); - assert!(err.contains("required: --out"), "{err}"); + assert!(err.contains("--only needs --out"), "{err}"); assert_eq!(d.read("S/summary.json"), "complete\n"); let (code, _, err) = run( cli, From 5e22114a3ad3a25009b7e76bf3211631d567d375 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:34:15 +0100 Subject: [PATCH 056/255] test(parity): compare the whole summary.json with the Python collector Nine cases run og summary collect without --only against bin/collect_summary.py on folders that join the three groups' fixtures, with failed steps, an empty folder, kept runs in history/ and a database re-check. A telomere table with an infinite read length stopped the Python with an uncaught OverflowError; og keeps the summary with that section unreadable (the all-inf golden). Bead: pgp-bs9u.44 --- rust/crates/og-summary/src/collect.rs | 3 +- rust/crates/og-summary/src/collect/group_c.rs | 5 +- rust/crates/og-summary/src/collect/post.rs | 16 +- .../og-summary/src/collect/post/tests.rs | 56 +- tests/parity/cases.tsv | 9 + tests/parity/gen/summary-collect-full.sh | 107 +++ .../summary-collect/all-inf/all-inf.json | 820 ++++++++++++++++++ .../golden/summary-collect/all-inf/exit | 1 + .../golden/summary-collect/all-inf/stderr | 0 .../golden/summary-collect/all-inf/stdout | 1 + 10 files changed, 981 insertions(+), 37 deletions(-) create mode 100644 tests/parity/gen/summary-collect-full.sh create mode 100644 tests/parity/golden/summary-collect/all-inf/all-inf.json create mode 100644 tests/parity/golden/summary-collect/all-inf/exit create mode 100644 tests/parity/golden/summary-collect/all-inf/stderr create mode 100644 tests/parity/golden/summary-collect/all-inf/stdout diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index 13b22eb76..4bbc9529f 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -51,8 +51,7 @@ use sections::{Callability, Coverage, Data, Mosaic, ReadCount, SampleQc, SexChec use std::io::Write; use std::path::Path; -const USAGE: &str = - "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR [--out OUT] +const USAGE: &str = "usage: og summary collect --sample SAMPLE --sample-dir SAMPLE_DIR [--out OUT] [--declared-sex DECLARED_SEX] [--assets ASSETS] [--only {a,b,c}] "; diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 3450f6b5d..29f66b870 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -295,7 +295,10 @@ pub fn contain( match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { Ok(Ok(())) => None, Ok(Err(e)) => Some((key, e)), - Err(_) => Some((key, format!("internal error: the post pass of {key} stopped"))), + Err(_) => Some(( + key, + format!("internal error: the post pass of {key} stopped"), + )), } } diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 42b9fe1ad..54c5c36da 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -139,9 +139,7 @@ pub fn collect(input: &Input) -> Result { }) .collect() }; - let a = a - .join() - .unwrap_or_else(|_| stopped(&super::model::GROUP_A)); + let a = a.join().unwrap_or_else(|_| stopped(&super::model::GROUP_A)); let b = b.join().unwrap_or_else(|_| stopped(&group_b::GROUP_B)); (a, b, c) }); @@ -191,10 +189,10 @@ pub fn collect(input: &Input) -> Result { if let Some(r) = delta::read_recheck(&dir.join("recheck").join("state.json")) { m.insert("recheck".into(), Value::Object(r)); } - if let Some(n) = new_since(dir, &v) { - if let Value::Object(m) = &mut v { - m.insert("new_since".into(), n); - } + if let Some(n) = new_since(dir, &v) + && let Value::Object(m) = &mut v + { + m.insert("new_since".into(), n); } Ok(v) } @@ -216,7 +214,9 @@ fn others(b: &[Section]) -> Others { cyrius: sec("cyrius") .filter(|s| live(s)) .and_then(|s| match &s.data { - Some(Data::Cyrius(c)) => Some((s.state, Some(c.genotype.clone()), c.filter.clone())), + Some(Data::Cyrius(c)) => { + Some((s.state, Some(c.genotype.clone()), c.filter.clone())) + } _ => None, }), } diff --git a/rust/crates/og-summary/src/collect/post/tests.rs b/rust/crates/og-summary/src/collect/post/tests.rs index e33709b92..bcf38afd5 100644 --- a/rust/crates/og-summary/src/collect/post/tests.rs +++ b/rust/crates/og-summary/src/collect/post/tests.rs @@ -92,7 +92,10 @@ fn an_empty_folder_lists_every_step_and_every_line() { let Value::Object(sections) = at(&v, &["sections"]) else { panic!() }; - assert_eq!(sections.keys().map(String::as_str).collect::>(), ORDER); + assert_eq!( + sections.keys().map(String::as_str).collect::>(), + ORDER + ); let not_run = texts(at(&v, &["not_run"])); assert_eq!(not_run.len(), 45); assert_eq!(not_run[0], "Coverage (mosdepth)"); @@ -129,30 +132,17 @@ fn a_failed_step_says_so_under_not_run() { } #[test] -fn the_lines_go_when_their_step_assessed_the_sample() { +fn step_35s_hba_count_assesses_hba() { let d = Dir::new("post-assessed"); - d.put( - "S/vntyper/S_vntyper.tsv", - "key\tvalue\nstatus\tnegative\n", - ); d.put( "S/paralogs/S_hba_copy_number.tsv", "metric\tvalue\nstatus\tcalled\ncopies\t4\nreading\tfour\n", ); let v = summary(&d); + assert_eq!(word(at(&v, &["sections", "parascopy", "state"])), "ok"); let lines = texts(at(&v, &["not_assessed"])); - let vntyper = at(&v, &["sections", "vntyper"]); - if word(at(vntyper, &["state"])) == "ok" && word(at(vntyper, &["data", "status"])) == "negative" { - assert!(!lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); - } else { - assert!(lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); - } - let parascopy = at(&v, &["sections", "parascopy"]); - if word(at(parascopy, &["state"])) == "ok" - && word(at(parascopy, &["data", "hba", "status"])) == "called" - { - assert!(!lines.contains(&HBA_NOT_ASSESSED.to_owned())); - } + assert!(!lines.contains(&HBA_NOT_ASSESSED.to_owned())); + assert!(lines.contains(&VNTYPER_NOT_ASSESSED.to_owned())); } #[test] @@ -160,7 +150,11 @@ fn a_stale_vntyper_reading_is_not_assessed() { // Only an ok section counts for VNtyper; any other state leaves the line. let mk = |state: State, status: &str| { let mut s = Section::new( - GROUP_C.iter().copied().find(|s| s.key == "vntyper").unwrap(), + GROUP_C + .iter() + .copied() + .find(|s| s.key == "vntyper") + .unwrap(), Ok(None), None, ); @@ -182,7 +176,11 @@ fn a_stale_vntyper_reading_is_not_assessed() { fn paraphase_needs_both_hba_genes_called() { let paralogs = |genes: &str| { let mut s = Section::new( - GROUP_C.iter().copied().find(|s| s.key == "paralogs").unwrap(), + GROUP_C + .iter() + .copied() + .find(|s| s.key == "paralogs") + .unwrap(), Ok(None), None, ); @@ -221,7 +219,11 @@ fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { ); // The section is then unreadable, with no data, and names its file. let mut s = Section::new( - GROUP_C.iter().copied().find(|s| s.key == "locityper").unwrap(), + GROUP_C + .iter() + .copied() + .find(|s| s.key == "locityper") + .unwrap(), Ok(Some(( "locityper/S_locityper.tsv".into(), 1_600_000_000.0, @@ -254,7 +256,10 @@ fn an_infinite_telomere_read_length_keeps_the_summary() { "chrom\tlength\tbases\tmean\tmin\tmax\ntotal\t100\t3000\t30\t0\t60\n", ); let v = summary(&d); - assert_eq!(word(at(&v, &["sections", "telomere", "state"])), "unreadable"); + assert_eq!( + word(at(&v, &["sections", "telomere", "state"])), + "unreadable" + ); assert_eq!(word(at(&v, &["sections", "coverage", "state"])), "ok"); } @@ -275,10 +280,9 @@ deletion\tchr2:1-200000\t200000\t0.4\t0.00001\t1\t1\t1\t0.00001\t0.00001\t0.1\n" let v = summary(&d); assert_eq!(word(at(&v, &["run", "declared_sex"])), "female"); let cnv = at(&v, &["sections", "cnvpytor"]); - if word(at(cnv, &["state"])) == "ok" { - assert_eq!(word(at(cnv, &["data", "declared_sex"])), "female"); - assert!(has(at(cnv, &["data", "dropped"]), "chry_female")); - } + assert_eq!(word(at(cnv, &["state"])), "ok"); + assert_eq!(word(at(cnv, &["data", "declared_sex"])), "female"); + assert!(has(at(cnv, &["data", "dropped"]), "chry_female")); } const PREV: &str = r#"{"schema_version": 2, "generated_utc": "2020-01-01 00:00 UTC", diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 7c21916ad..14edaa05e 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -870,6 +870,15 @@ summary-collect cb-notbi gen:summary-clinical.sh,summary-collect/oracle.sh,summa summary-collect cb-notgz gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-notgz --assets assets --out cb-notgz.json --only b cb-notgz.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-empty gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-empty --assets assets --out cb-empty.json --only b cb-empty.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE summary-collect cb-no-assets gen:summary-clinical.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir cb-full --assets nowhere --out cb-no-assets.json --only b cb-no-assets.json json:paths=schema_version,sample,run,databases,manifest,sections.clinvar,sections.carrier,sections.pharmcat,sections.cpic,sections.pypgx,sections.cyrius,sections.cpsr,sections.clinical,sections.slivar,cyp2d6 PYTHON_IMAGE +summary-collect all gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir all --assets assets --out all.json all.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-nf gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir nf --assets assets --out all-nf.json all-nf.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-bad gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir bad --assets assets --out all-bad.json all-bad.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-failed gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir failed --assets assets --out all-failed.json all-failed.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-empty gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir empty --assets assets --out all-empty.json all-empty.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-history gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir history --assets assets --out all-history.json all-history.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-history-bad gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir history-bad --assets assets --out all-history-bad.json all-history-bad.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-inf gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir inf --assets assets --out all-inf.json all-inf.json json:mask=generated_utc PYTHON_IMAGE +summary-collect all-default-out gen:summary-collect-full.sh,summary-collect/oracle.sh,summary-collect/cb_vep.vcf.gz,summary-collect/cb_vep.vcf.gz.tbi --sample S --sample-dir history-bad --assets assets history-bad/summary.json json:mask=generated_utc PYTHON_IMAGE y-tree-positions small gen:y-inputs.sh --tree-dir t pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions big gen:y-inputs.sh --tree-dir ./big/ pos.txt pos.txt bytes PYTHON_IMAGE y-tree-positions odd gen:y-inputs.sh --tree-dir odd pos.txt pos.txt bytes PYTHON_IMAGE diff --git a/tests/parity/gen/summary-collect-full.sh b/tests/parity/gen/summary-collect-full.sh new file mode 100644 index 000000000..94d4db167 --- /dev/null +++ b/tests/parity/gen/summary-collect-full.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# summary-collect-full.sh: writes the sample folders of the whole-file +# summary-collect parity cases (og summary collect without --only) in the +# current directory, the same every time. It runs the generators of the +# three groups (summary-collect.sh, summary-clinical.sh and +# summary-collect-c.sh) and joins their folders, so one folder holds every +# section and the post pass reads across the groups: the repeat readings +# with the CPIC state, Locityper's primary calls with pypgx and Cyrius, +# CNVpytor's chrY filter with step 33's declared sex, the cyp2d6 block, +# not_run and not_assessed. On top: a run with failed steps, an empty +# folder, previous runs kept in history/ (one half-copied, one whose +# summary is no JSON) with a database re-check, and a telomere table whose +# infinite read length stopped the Python collector (a golden: og keeps +# the summary and marks that one section unreadable). +# py/ holds the modules the oracle imports at the oracle's commit, with the +# real run_delta.py for recheck and new_since. The case stages +# cb_vep.vcf.gz and its index beside the folders. +# scripts/ci/oracle-diff.sh runs it. +set -euo pipefail +export LC_ALL=C TZ=UTC +here=$(cd "$(dirname "$0")" && pwd) +root=$(cd "${here}/../../.." && pwd) +ORACLE_SHA=98aa9252ff77825a8a797d388b449931fa9a234e + +put() { mkdir -p "$(dirname "$1")"; printf '%b' "$2" > "$1"; at "$NEW" "$1"; } +at() { touch -t "$1" "${@:2}"; } # [[CC]YY]MMDDhhmm[.SS], in UTC +NEW=202103040506.07 # after the run of the three groups' folders + +for g in a b c; do mkdir -p "parts/$g"; done +(cd parts/a && bash "${here}/summary-collect.sh") +(cd parts/b && bash "${here}/summary-clinical.sh") +(cd parts/c && bash "${here}/summary-collect-c.sh") + +# merge DEST SRC...: the folders' files, the later folder's file winning; +# times and symlinks kept. +merge() { + local dest=$1 src + shift + mkdir -p "$dest" + for src in "$@"; do cp -pPR "parts/${src}/." "$dest"/; done +} + +mkdir -p assets py pymods +for g in a b c; do cp -pR "parts/${g}/assets/." assets/; done +cp -pR parts/b/pymods/. pymods/ +cp -pR parts/c/py/. py/ +git -C "$root" cat-file -e "${ORACLE_SHA}^{commit}" 2> /dev/null \ + || git -C "$root" fetch -q --depth=1 origin "$ORACLE_SHA" +git -C "$root" show "${ORACLE_SHA}:bin/run_delta.py" > py/run_delta.py + +# all: every section of the three groups, the bash names. +merge all a/full b/cb-full c/full +# nf: the Nextflow names, steps failed and from an earlier run. +merge nf a/nf b/cb-nf c/nf +# bad: an unreadable file in sections of each group. +merge bad a/bad-a b/cb-bad c/bad1 +# failed: steps that failed and wrote nothing, under not_run. +merge failed a/failed b/cb-failed +mkdir -p empty + +# history: all, with three earlier runs kept. The newest has no summary +# (a half-copied folder) and is passed over; the next is compared: its +# ClinVar, CPIC, pypgx and PRS results and its section states differ, and +# so do the inputs of its manifest. A database re-check newer than the +# run's manifest. +merge history a/full b/cb-full c/full +put history/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.10.0\ndata\tclinvar_file_date\t2021-03-01\n" +put history/history/20200101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2019-12-01 00:00 UTC", "sections": {}}' +put history/history/20210101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2021-01-01 00:00 UTC", + "sections": { + "coverage": {"title": "Coverage (mosdepth)", "state": "missing", "data": {}}, + "clinvar": {"title": "ClinVar screen", "state": "ok", "data": {"hits": [ + {"chrom": "chr1", "pos": "45331556", "ref": "C", "alt": "T", "gene": "MUTYH", "significance": "Likely_pathogenic", "stars": 1, "review_status": "x", "genotype": "1/1"}, + {"chrom": "chr9", "pos": "1", "ref": "A", "alt": "G", "gene": "X", "significance": "Pathogenic", "stars": 2}]}}, + "cpic": {"title": "CPIC", "state": "ok", "data": {"genes": [{"gene": "CYP2D6", "diplotype": "*1/*1", "phenotype": "Normal"}]}}, + "pypgx": {"title": "pypgx", "state": "ok", "data": {"cyp2d6": "*1/*1"}}, + "prs": {"title": "PRS", "state": "ok", "data": {"scores": [{"pgs_id": "PGS000001", "condition": "x", "percentile": 10}]}}, + "mei": {"title": "Mobile element insertions (MEGAnE)", "state": "ok", "data": {}}}, + "cyp2d6": {"consensus": {"result": "*1/*1"}}}\n' +put history/history/20210101T000000/run_manifest.tsv "run\twritten_utc\t2021-01-01 00:00:00 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.9.0\ndata\tclinvar_file_date\t2020-12-01\n" +mkdir -p history/history/20220101T000000 +put history/history/20220101T000000/run_manifest.tsv "run\twritten_utc\t2022-01-01 00:00:00 UTC\n" +put history/recheck/state.json '{"checked_utc": "2021-03-05 00:00:00 UTC", "items": [ + {"item": "clinvar", "label": "ClinVar", "class": "newer-release", "ran_with": "2021-03-01", "upstream": "2021-03-04", "action": "rerun step 06", "extra": 1}, + {"item": "vep", "label": "VEP", "class": "no-newer-release"}, 7], + "affected_steps": ["06"], "rerun": {"command": "run-all.sh"}}\n' + +# history-bad: the only kept run has a summary that is no JSON, so nothing +# is compared; a re-check older than the run's manifest is of an earlier +# run and left out. +mkdir -p history-bad +put history-bad/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\n" +put history-bad/history/20210101T000000/summary.json '{"schema_version": 2,' +put history-bad/recheck/state.json '{"checked_utc": "2021-03-01 00:00:00 UTC", "items": []}\n' + +# inf: a telomere table whose read length is infinite, beside the +# coverage, step 33 and repeat sections. The Python collector stopped with +# an uncaught OverflowError and wrote no summary; og writes it with the +# telomere section unreadable (a golden). +mkdir -p inf +for f in mosdepth/S.mosdepth.summary.txt qc/S_sample_qc.tsv logs/run_status.tsv; do + mkdir -p "inf/$(dirname "$f")" && cp -p "parts/a/full/$f" "inf/$f" +done +cp -pR parts/c/full/expansion_hunter inf/ +put inf/telomere/S/S/S_summary.tsv "PID\tsample\ttotal_reads\tread_lengths\tintratel_reads\ttel_content\nS\ttumor\t1000\tinf\t40\t1.5\n" + +rm -rf parts diff --git a/tests/parity/golden/summary-collect/all-inf/all-inf.json b/tests/parity/golden/summary-collect/all-inf/all-inf.json new file mode 100644 index 000000000..99b10ea8d --- /dev/null +++ b/tests/parity/golden/summary-collect/all-inf/all-inf.json @@ -0,0 +1,820 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-13 23:22 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "male" + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": 31.25 + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "declared_sex": "male", + "mosaic_call": "not_detected", + "mosaic_depth_ratio": "0.98", + "mosaic_depth_ratio_expected": "1", + "mosaic_loss_depth": "0.02", + "mosaic_loss_depth_low": "-0.01", + "mosaic_loss_depth_high": "0.05", + "mosaic_loss_af": "NA", + "mosaic_floor": "0.1", + "mosaic_x_het_sites": "0", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "call": "not_detected", + "depth_ratio": 0.98, + "depth_ratio_expected": 1.0, + "loss_depth": 0.02, + "loss_depth_low": -0.01, + "loss_depth_high": 0.05, + "loss_af": null, + "floor": 0.1, + "x_het_sites": 0, + "loss_af_low": null, + "loss_af_high": null + } + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "not_read", + "r117h": ".", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. R117H was not read (no step 42 traits row for rs78655421 was called), so whether it sits on the 5T copy is not known." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": ".", + "agreement": "not_compared", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). Not compared with PharmCAT (no unambiguous UGT1A1 call from step 27)." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "OverflowError: cannot round", + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Gene callability (mosdepth)", + "Sex check (indexcov)", + "Variant calling", + "ClinVar screen", + "PharmCAT", + "CPIC drug-gene recommendations", + "pypgx", + "Cyrius (CYP2D6)", + "SMN1/SMN2 copy number (Parascopy, short reads)", + "Paralog genes (Paraphase, PacBio HiFi)", + "HLA typing (T1K)", + "Polygenic risk scores", + "Fine-grained ancestry (ADMIXTURE, RFMix)", + "Structural variants (Manta)", + "Structural variants (Delly)", + "Copy number (CNVpytor)", + "SV consensus merge", + "Structural variants, class 4 and 5 (AnnotSV)", + "Runs of homozygosity", + "Mitochondrial haplogroup", + "Y-chromosome haplogroup (Yleaf)", + "Y placement on the current public tree", + "Y-STR markers (HipSTR)", + "Raw-data export (23andMe v5 layout)", + "Blood groups (predicted)", + "Lp(a): LPA KIV-2 repeat count", + "MUC1 VNTR frameshifts (VNtyper)", + "Locityper second opinion at hard loci", + "Mitochondrial variants", + "Traits panel", + "Archaic ancestry (IBDmix)", + "Ancient populations (AADR PCA)", + "Viral DNA in the unmapped reads", + "Symptom-driven ranking (Exomiser)", + "Clonal haematopoiesis (CHIP)", + "Mobile element insertions (MEGAnE)", + "Mosaic chromosomal alterations (MoChA)", + "Cancer predisposition (CPSR)", + "Clinical variant filter", + "Variant prioritization (slivar)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "Mobile element insertions (Alu, L1 and SVA, the FKTN, TAF1 and BBS1 SVA insertions among them): not looked for (step 50, MEGAnE, is opt-in)", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/parity/golden/summary-collect/all-inf/exit b/tests/parity/golden/summary-collect/all-inf/exit new file mode 100644 index 000000000..573541ac9 --- /dev/null +++ b/tests/parity/golden/summary-collect/all-inf/exit @@ -0,0 +1 @@ +0 diff --git a/tests/parity/golden/summary-collect/all-inf/stderr b/tests/parity/golden/summary-collect/all-inf/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/summary-collect/all-inf/stdout b/tests/parity/golden/summary-collect/all-inf/stdout new file mode 100644 index 000000000..437515d43 --- /dev/null +++ b/tests/parity/golden/summary-collect/all-inf/stdout @@ -0,0 +1 @@ +Summary written: all-inf.json From 88dd793198d73798bb50013332594a1bc61a1031 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:04:00 +0100 Subject: [PATCH 057/255] docs(summary): og summary collect writes the whole summary.json Bead: pgp-bs9u.44 --- CHANGELOG.md | 3 +++ docs/rust.md | 2 +- rust/README.md | 2 +- rust/og/src/main.rs | 2 +- tests/parity/inputs/summary-collect/oracle.sh | 4 ++-- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7489faaa..0a14652ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The Rust `og` binary now writes the whole summary.json: `og summary collect` without `--only` reads all 48 sections over one walk of the sample folder, then runs the post pass. The post pass joins sections: the repeat readings with the traits and CPIC states, Locityper's primary calls with HLA, pypgx and Cyrius, and CNVpytor's chrY filter with the declared sex. It also builds the CYP2D6 block and writes `not_run`, `not_assessed`, `recheck` and `new_since`. Without `--out` it writes `summary.json` in the sample folder, as the Python did; `--only` still needs `--out`. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The whole file is the same as the Python's on 8 of 9 new parity cases, compared as JSON with key order and only `generated_utc` masked; the ninth holds the deliberate fix below. On a synthetic folder with every section it takes 0.01 to 0.03 s and 15 MB, where the Python took 0.15 to 0.21 s and 42 to 44 MB. With a 1,000,000-record VCF it takes 0.44 to 0.62 s and 17 to 20 MB, where the Python took 1.5 s and 42 to 44 MB. What differs from the Python: + - A part of the post pass that fails makes only its own section unreadable, with a note that names the file. The Python ran the post pass outside its per-section error handling, so one failure there lost summary.json and both reports. + - A section reader that stops on an error the Python did not catch keeps the summary too. A telomere table with an infinite read length stopped the Python with an OverflowError and no summary.json; og marks only the telomere section unreadable. - Step 42 reads traits and genotype rules with `og traits table`, exports WhatsHap sites with `og traits phase-sites`, and checks the catalogs with `og traits check`. The replaced Python evaluator is removed. Report columns and readings stay the same. Genotypes outside the catalog are streamed past, and write errors now report a controlled error with the path instead of a traceback. Both tables are staged before either is published. A staging failure or a directory destination leaves both previous tables intact. Each file is then replaced by a separate atomic rename. The pair is not a transaction: a failure of the second rename can leave the first table updated. Catalogs and output remain plain TSV; quotes never hide a tab or bypass validation. - The unused Python copy-number engine is removed. Its historical catalog tests materialise a pinned test oracle and keep testing the Rust commands. diff --git a/docs/rust.md b/docs/rust.md index 57471737d..528d0a63e 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -21,7 +21,7 @@ rust/ crates/og-qc/ og qc sex-loss: mosaic loss of Y or X for step 33 crates/og-setup/ what setup.sh runs once: the STRchive catalog, zips, the genetic map crates/og-steps/ the step helpers: constraint-join, prs format, step 40's consumer export; og family (step 38); og pgx outside-calls|hla-concordance|pypgx-summary (steps 08, 32, 36); og paraphase (step 35b) - crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect --only a|b|c, QC, clinical and the remaining summary sections (no caller yet); og roh summary (step 11) + crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect, the whole summary.json with its post pass, or one group with --only a|b|c (no caller yet); og roh summary (step 11) crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read diff --git a/rust/README.md b/rust/README.md index 449ce5867..14724865c 100644 --- a/rust/README.md +++ b/rust/README.md @@ -10,7 +10,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically, and the tools that read BAMs: the reads at a variant (`reads`: one CIGAR walk per read, SA pieces, filters per tool), `og cdna check`, `og cdna scan`, `og comphet phase`, `og bam contigs`, the CYP2D6 depth check and the RFC1 motif count | | `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-qc` | `og qc sex-loss`: step 33's mosaic loss of Y or X, with CPython's medians, `fmean` and exact `variance` | -| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), beside the Python collector, which still writes summary.json; `og roh summary` (step 11) | +| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), and without `--only` the whole summary.json with its post pass (the readings that join sections, the CYP2D6 block, `not_run`, `not_assessed`, `recheck`, `new_since`), beside the Python collector, which still writes summary.json; `og roh summary` (step 11) | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | | `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-bloodtype` | step 41, blood groups: `og bloodtype report`, `og bloodtype check-markers`, `og bloodtype systems` | diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 6b4833c7b..82e29d218 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -50,7 +50,7 @@ const USAGE: &str = "usage: og [args] og prs format [args] og prs table [args] og mito annotate|deletions|copies [args] - og summary collect --only {a,b,c} [args] + og summary collect [--only {a,b,c}] [args] og roh summary --sample SAMPLE --roh ROH [--out OUT] og chip prepare|report|check-table [args] og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] diff --git a/tests/parity/inputs/summary-collect/oracle.sh b/tests/parity/inputs/summary-collect/oracle.sh index 3e33b6992..7add40e40 100644 --- a/tests/parity/inputs/summary-collect/oracle.sh +++ b/tests/parity/inputs/summary-collect/oracle.sh @@ -6,8 +6,8 @@ # --only is og's alone (the Python writes every section, and the case # compares the group's subtrees). The modules collect_summary imports are # stubbed, except those the group b and c generators stage in pymods/ -# and py/. No other module is reached for these sample folders, except -# run_delta's two readers of history/ and recheck/, which find nothing here. +# and py/. run_delta is stubbed to find no history/ and no recheck/, unless +# py/ brings the real one (the whole-file cases, summary-collect-full.sh). prog=$1 shift d=$(mktemp -d) From 888cd36fcdb34eeba513bb77fbec943e763997e6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:05:22 +0100 Subject: [PATCH 058/255] test(summary): plant four post-pass faults (red control, reverted next) not_run keeps the sections that are no step, a stale VNtyper reading counts as assessed, new_since takes the current run's date, and a panic in a post-pass part is no longer contained. The next commit removes them. Bead: pgp-bs9u.44 --- rust/crates/og-summary/src/collect/group_c.rs | 3 ++- rust/crates/og-summary/src/collect/post.rs | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 29f66b870..dcc7380fd 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -292,7 +292,8 @@ pub fn contain( key: &'static str, part: impl FnOnce() -> Result<(), String>, ) -> Option<(&'static str, String)> { - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { + // Planted fault (red control): the panic is no longer caught. + match Ok::<_, ()>(part()) { Ok(Ok(())) => None, Ok(Err(e)) => Some((key, e)), Err(_) => Some(( diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 54c5c36da..faec5a064 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -234,7 +234,7 @@ fn not_run(sections: &[Section]) -> Vec { sections .iter() .filter(|s| matches!(s.state, State::Missing | State::Failed)) - .filter(|s| !NOT_A_STEP.contains(&s.spec.key)) + .filter(|s| !NOT_A_STEP[..0].contains(&s.spec.key)) // planted fault (red control) .map(|s| { if s.state == State::Failed { format!("{} (step {} failed)", s.spec.title, s.spec.step) @@ -279,7 +279,7 @@ fn not_assessed(sections: &[Section]) -> Vec<&'static str> { if state("mei") == State::Missing { out.push(MEI_NOT_ASSESSED); } - let vntyper_read = state("vntyper") == State::Ok + let vntyper_read = matches!(state("vntyper"), State::Ok | State::Stale) && ["positive", "possible", "negative"] .iter() .any(|w| is(field("vntyper", "status"), w)); @@ -317,7 +317,7 @@ fn new_since(dir: &Path, cur: &Value) -> Option { let prev_manifest = files::read_manifest(Some(&prev_dir.join("run_manifest.tsv"))).ok()?; let manifest = files::read_manifest(Some(&dir.join("run_manifest.tsv"))).ok()?; let changes = delta::compare(&prev, &prev_manifest, cur, &manifest).ok()?; - let previous = match &prev { + let previous = match cur { Value::Object(m) => m.get("generated_utc").cloned().unwrap_or(Value::Null), _ => Value::Null, }; From af005e271db960a44135b5208b8a587b5496e515 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:05:55 +0100 Subject: [PATCH 059/255] test(summary): remove the planted post-pass faults Reverts 888cd36f, the red control of this PR. Bead: pgp-bs9u.44 --- rust/crates/og-summary/src/collect/group_c.rs | 3 +-- rust/crates/og-summary/src/collect/post.rs | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index dcc7380fd..29f66b870 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -292,8 +292,7 @@ pub fn contain( key: &'static str, part: impl FnOnce() -> Result<(), String>, ) -> Option<(&'static str, String)> { - // Planted fault (red control): the panic is no longer caught. - match Ok::<_, ()>(part()) { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { Ok(Ok(())) => None, Ok(Err(e)) => Some((key, e)), Err(_) => Some(( diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index faec5a064..54c5c36da 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -234,7 +234,7 @@ fn not_run(sections: &[Section]) -> Vec { sections .iter() .filter(|s| matches!(s.state, State::Missing | State::Failed)) - .filter(|s| !NOT_A_STEP[..0].contains(&s.spec.key)) // planted fault (red control) + .filter(|s| !NOT_A_STEP.contains(&s.spec.key)) .map(|s| { if s.state == State::Failed { format!("{} (step {} failed)", s.spec.title, s.spec.step) @@ -279,7 +279,7 @@ fn not_assessed(sections: &[Section]) -> Vec<&'static str> { if state("mei") == State::Missing { out.push(MEI_NOT_ASSESSED); } - let vntyper_read = matches!(state("vntyper"), State::Ok | State::Stale) + let vntyper_read = state("vntyper") == State::Ok && ["positive", "possible", "negative"] .iter() .any(|w| is(field("vntyper", "status"), w)); @@ -317,7 +317,7 @@ fn new_since(dir: &Path, cur: &Value) -> Option { let prev_manifest = files::read_manifest(Some(&prev_dir.join("run_manifest.tsv"))).ok()?; let manifest = files::read_manifest(Some(&dir.join("run_manifest.tsv"))).ok()?; let changes = delta::compare(&prev, &prev_manifest, cur, &manifest).ok()?; - let previous = match cur { + let previous = match &prev { Value::Object(m) => m.get("generated_utc").cloned().unwrap_or(Value::Null), _ => Value::Null, }; From 7b0a8e735331d3e7e8411ec670bcb9918b38c079 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:23:59 +0100 Subject: [PATCH 060/255] test(parity): name no image in the whole-summary fixture manifests The images check reads any image name in a tracked file as one that bypasses versions.env; the fixture's DEEPVARIANT_IMAGE rows now hold plain version text. Bead: pgp-bs9u.44 --- tests/parity/gen/summary-collect-full.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/parity/gen/summary-collect-full.sh b/tests/parity/gen/summary-collect-full.sh index 94d4db167..accdcdf73 100644 --- a/tests/parity/gen/summary-collect-full.sh +++ b/tests/parity/gen/summary-collect-full.sh @@ -64,7 +64,7 @@ mkdir -p empty # so do the inputs of its manifest. A database re-check newer than the # run's manifest. merge history a/full b/cb-full c/full -put history/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.10.0\ndata\tclinvar_file_date\t2021-03-01\n" +put history/run_manifest.tsv "run\twritten_utc\t2021-03-04 05:06:07 UTC\nimage\tDEEPVARIANT_IMAGE\tv1.10.0\ndata\tclinvar_file_date\t2021-03-01\n" put history/history/20200101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2019-12-01 00:00 UTC", "sections": {}}' put history/history/20210101T000000/summary.json '{"schema_version": 2, "sample": "S", "generated_utc": "2021-01-01 00:00 UTC", "sections": { @@ -77,7 +77,7 @@ put history/history/20210101T000000/summary.json '{"schema_version": 2, "sample" "prs": {"title": "PRS", "state": "ok", "data": {"scores": [{"pgs_id": "PGS000001", "condition": "x", "percentile": 10}]}}, "mei": {"title": "Mobile element insertions (MEGAnE)", "state": "ok", "data": {}}}, "cyp2d6": {"consensus": {"result": "*1/*1"}}}\n' -put history/history/20210101T000000/run_manifest.tsv "run\twritten_utc\t2021-01-01 00:00:00 UTC\nimage\tDEEPVARIANT_IMAGE\tgoogle/deepvariant:1.9.0\ndata\tclinvar_file_date\t2020-12-01\n" +put history/history/20210101T000000/run_manifest.tsv "run\twritten_utc\t2021-01-01 00:00:00 UTC\nimage\tDEEPVARIANT_IMAGE\tv1.9.0\ndata\tclinvar_file_date\t2020-12-01\n" mkdir -p history/history/20220101T000000 put history/history/20220101T000000/run_manifest.tsv "run\twritten_utc\t2022-01-01 00:00:00 UTC\n" put history/recheck/state.json '{"checked_utc": "2021-03-05 00:00:00 UTC", "items": [ From 8dd88d6bcc29808a3454e7743d514a8c4965e530 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:46:19 +0100 Subject: [PATCH 061/255] docs(ancestry): give the archaic table's measured cost on the real table Chromosomes 1 to 6 and 21 of the installed table, rebuilt by og from the same public inputs, equal the Python-built table after decompression. The synthetic table figure understated mask memory, so the changelog now gives the measured chromosome 1 cost. --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d74fe2a71..30fe87bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On synthetic inputs of real size: the archaic table of one chromosome (3 million sites) takes 11 s and 4 MB where the Python took 25 s and 21 MB; step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: - - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes, so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. +- Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On the real archaic table, `og archaic table` merges chromosome 1 (4.1 million sites) in 18 s and 190 MB where the Python took 76 s and 1.25 GB; bcftools and Picard still take most of each chromosome's setup. On a synthetic input of real size, step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: + - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes (chromosomes 1 to 6 and 21 of the real table, rebuilt from the same public files, are the same after decompression), so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. From a3f08e08baa9a849eac0643faa8ea92f9e1e91f0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:36:49 +0100 Subject: [PATCH 062/255] fix(summary): claim only error containment in the post pass The release profile aborts on a panic, so catch_unwind in contain() caught nothing in the shipped og. contain() now maps the error a part returns, and the docs say the parts read with defaults instead of promising to catch a panic. --- rust/crates/og-summary/src/collect/group_c.rs | 17 ++++++----------- rust/crates/og-summary/src/collect/post.rs | 8 +++++--- .../crates/og-summary/src/collect/post/tests.rs | 17 +++-------------- rust/crates/og-summary/src/lib.rs | 4 ++-- 4 files changed, 16 insertions(+), 30 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index 29f66b870..d9c8c9506 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -238,8 +238,10 @@ pub fn read_all(c: &Ctx<'_>) -> Vec { /// order, each with its state and data): the repeat readings with the /// traits and CPIC states, each Locityper locus's primary call, and the /// chrY filter of CNVpytor's count. Each part runs on its own: one that -/// fails (an error or a panic) is returned as its section's key and why, -/// and the others still run. +/// returns an error is returned as its section's key and why, and the +/// others still run. The parts read with defaults, so a row of an +/// unexpected shape is no error. A panic is not caught: the release binary +/// aborts on one (`panic = "abort"`). pub fn post( c: &Ctx<'_>, sections: &mut [(Spec, State, Option<&mut Data>)], @@ -287,19 +289,12 @@ pub fn post( } /// Runs one part of the post pass: `None` when it went through, else its -/// section's key and why it failed, for an error and for a panic alike. +/// section's key and the error it returned. pub fn contain( key: &'static str, part: impl FnOnce() -> Result<(), String>, ) -> Option<(&'static str, String)> { - match std::panic::catch_unwind(std::panic::AssertUnwindSafe(part)) { - Ok(Ok(())) => None, - Ok(Err(e)) => Some((key, e)), - Err(_) => Some(( - key, - format!("internal error: the post pass of {key} stopped"), - )), - } + part().err().map(|e| (key, e)) } fn json(d: &Data) -> Option<&Value> { diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 54c5c36da..6065d5402 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -13,9 +13,11 @@ //! - `recheck`: the last database re-check of the sample; //! - `new_since`: what changed since the previous run kept in `history/`. //! -//! Each part that changes a section runs on its own: when it fails, only -//! its section becomes `unreadable`, with a note naming the file, and the -//! summary is still written. The Python collector ran this pass outside +//! Each part that changes a section runs on its own: when it returns an +//! error, only its section becomes `unreadable`, with a note naming the +//! file, and the summary is still written. The parts read with defaults, so +//! a row of an unexpected shape is no error; a panic is not caught, as the +//! release binary aborts on one. The Python collector ran this pass outside //! its per-section error handling, so one failure there lost summary.json //! and both reports. A previous run that cannot be read leaves `new_since` //! out, as before; the comparison itself has no error to hide. diff --git a/rust/crates/og-summary/src/collect/post/tests.rs b/rust/crates/og-summary/src/collect/post/tests.rs index bcf38afd5..34d18cc61 100644 --- a/rust/crates/og-summary/src/collect/post/tests.rs +++ b/rust/crates/og-summary/src/collect/post/tests.rs @@ -203,20 +203,12 @@ fn paraphase_needs_both_hba_genes_called() { #[test] fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { - // An error and a panic alike come back as the section's key and why. + // An error comes back as the section's key and why. assert_eq!(group_c::contain("cnvpytor", || Ok(())), None); assert_eq!( group_c::contain("expansions", || Err("no table".into())), Some(("expansions", "no table".to_owned())) ); - let panicked = group_c::contain("locityper", || panic!("planted")); - assert_eq!( - panicked, - Some(( - "locityper", - "internal error: the post pass of locityper stopped".to_owned() - )) - ); // The section is then unreadable, with no data, and names its file. let mut s = Section::new( GROUP_C @@ -231,13 +223,10 @@ fn a_part_of_the_post_pass_that_fails_stays_in_its_section() { ))), None, ); - s.make_unreadable("internal error: the post pass of locityper stopped"); + s.make_unreadable("no table"); let j = s.to_json(); assert_eq!(word(at(&j, &["state"])), "unreadable"); - assert_eq!( - word(at(&j, &["note"])), - "S_locityper.tsv: internal error: the post pass of locityper stopped" - ); + assert_eq!(word(at(&j, &["note"])), "S_locityper.tsv: no table"); assert_eq!(at(&j, &["source"]), &Value::Null); assert_eq!(at(&j, &["data"]), &Value::Object(indexmap::IndexMap::new())); } diff --git a/rust/crates/og-summary/src/lib.rs b/rust/crates/og-summary/src/lib.rs index ac4040411..4aa53eeca 100644 --- a/rust/crates/og-summary/src/lib.rs +++ b/rust/crates/og-summary/src/lib.rs @@ -7,8 +7,8 @@ //! table from pgsc_calc's output (`prs/_prs_summary.tsv`, //! `ancestry/_ancestry.tsv`). //! - [`mito`]: `og mito annotate|deletions|copies`, step 20's chrM tables. -//! - [`collect`]: `og summary collect`, summary.json, ported one group of -//! sections at a time beside the Python collector (`--only a`). +//! - [`collect`]: `og summary collect`, the whole summary.json, or one +//! group with `--only a|b|c`, beside the Python collector (no caller yet). //! //! Each tool exits 0 when it wrote its table, 1 with `ERROR: ` //! when an input cannot be read or the table cannot be written, and 2 with From 66668d14cb837b156cfddb3ee0ec4a833aa6eb58 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:12:55 +0100 Subject: [PATCH 063/255] fix(live): keep every producer output as it is on main The report gate reads only which findings are results to act on, so the producers need none of the round's changes. bin/live_results.py, bin/report_charts.py, og-map's comments and tests/test_live_results.py are main's again: adults' home top entries, the search index tags, the charts and the SV findings are byte for byte main's, and overview_problems guards the top entries again. The untrue facts.rs sentence goes with it. --- bin/live_results.py | 221 ++++++++++++++++++++++++++------ bin/report_charts.py | 13 +- rust/crates/og-map/README.md | 5 +- rust/crates/og-map/src/facts.rs | 4 +- rust/crates/og-map/src/veil.rs | 14 +- tests/test_live_results.py | 196 +++++++++++++++++++++++++++- 6 files changed, 383 insertions(+), 70 deletions(-) diff --git a/bin/live_results.py b/bin/live_results.py index 6d0911341..3f0a3a603 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -20,8 +20,9 @@ or a couple's shared or X-linked gene. Nothing says a section is fine: a card without attention is "for information". -The veil metadata lists act identities. The shared browser loader withholds a child's entire -report from views while any act remains unopened. No individual field is classified here. +Beside the cards, "veil" tags the rows that name a result to act on outside its own card (every card's +table rows and lines, the viewer links, the charts' marks, the other findings and the lead's names) with +that result's id, so a child's page closes them with it (veil() below). Nothing here reads a file or computes anything about genetics; the adapters pick fields the summary already holds. A section id with no adapter falls back to its @@ -792,8 +793,8 @@ class _Row: cannot read (a field of the wrong type, a stars value that is not a number) costs that row only: it becomes a finding on its card that says the row could not be read, and every other row and card still renders. Its words name no gene or variant: the row is unreadable. Its level is the highest the rule - can give (level "act" for a rule that can give one). Such an act keeps a child's report gate shut - until someone opens it. The full report may still draw the row the rule could not read.""" + can give (level "act" for a rule that can give one): a child's page veils a card that holds a finding + to act on, and the card's table may still draw the row the rule could not read.""" def __init__(self, out, card_id, level="review"): self.out, self.card_id, self.level = out, card_id, level @@ -887,9 +888,6 @@ def gene(x): gene=(x.get("genes") or [None])[0], detail=x.get("sv"), evidence=[["Class", x.get("acmg_class")], ["Callers", rr.annotsv_support_text(x)], ["Genotype", x.get("gt")]], viewer=str(x.get("sv") or "").replace(" ", "") or None)) - # Every gene covered by this structural variant belongs on its own finding record. - gl = x.get("genes") - out[-1]["genes"] = [str(g) for g in gl if g] if isinstance(gl, list) else [] # 6. Repeat expansions outside Stranger's "normal", or in an intermediate zone. for l in _items(_data(S, "expansions").get("loci")): with _Row(out, "expansions", "act"): @@ -1100,63 +1098,181 @@ def screened(summary): def lead(found, levels, summary): """The report's lead paragraph: [[text, bold]] segments (a third item "act" marks the names of the - findings to act on), plain sentences that name the findings by level. With no finding of a level, - it says what was assessed. The legacy act marker remains part of the document format.""" + findings to act on), plain sentences that name the findings by level. It never says a section is fine: with no finding of a level it says what was assessed.""" S = summary.get("sections") or {} by = {lv: [f for f in found if f["level"] == lv] for lv in LEVELS} out = [] - - def seg(text, bold, act=False): - out.append([text, bold] + (["act"] if act else [])) if by["trust"]: - seg("Read the data-quality section first: ", True) - seg(_names(by["trust"]) + ". ", False) + out += [["Read the data-quality section first: ", True], [_names(by["trust"]) + ". ", False]] if by["act"]: n = len(by["act"]) - seg("Of everything this genome was screened for, ", False) - seg(_count(n, "finding should", "findings should") + " go to a doctor", True) - seg(": " + _names(by["act"]) + ". ", False, act=True) # "act": the names a child's page keeps closed + out += [["Of everything this genome was screened for, ", False], + [_count(n, "finding should", "findings should") + " go to a doctor", True], + [": " + _names(by["act"]) + ". ", False, "act"]] # "act": the names a child's page keeps closed elif screened(summary): - seg("No finding in what these steps assessed needs a doctor; ", True) - seg("sections 8 and 9 say what they do not see. ", False) + out += [["No finding in what these steps assessed needs a doctor; ", True], + ["sections 8 and 9 say what they do not see. ", False]] else: # no clinical screen has a result: saying nothing needs a doctor would read as an all-clear - seg("The clinical screens have no result yet; ", True) - seg("section 9 lists the steps that did not run. ", False) + out += [["The clinical screens have no result yet; ", True], + ["section 9 lists the steps that did not run. ", False]] if by["medicine"]: genes = sorted({f["gene"] for f in by["medicine"] if f["gene"]}) - seg(_count(len(by["medicine"]), "medicine result", "medicine results").capitalize(), True) - seg(f" bear{'s' if len(by['medicine']) == 1 else ''} on how drugs work ({', '.join(genes)}): tell whoever prescribes. ", - False) + out += [[_count(len(by["medicine"]), "medicine result", "medicine results").capitalize(), True], + [f" bear{'s' if len(by['medicine']) == 1 else ''} on how drugs work ({', '.join(genes)}): tell whoever prescribes. ", False]] if by["carrier"]: - seg(f"A carrier of {_names(by['carrier'], 2)}", True) - seg(": carriers are not affected; it matters when a partner carries the same gene. ", False) + out += [[f"A carrier of {_names(by['carrier'], 2)}", True], + [": carriers are not affected; it matters when a partner carries the same gene. ", False]] if by["review"]: - seg(_count(len(by["review"]), "result is", "results are").capitalize() + " worth a look", True) - seg(": " + _names(by["review"]) + ". ", False) + out += [[_count(len(by["review"]), "result is", "results are").capitalize() + " worth a look", True], + [": " + _names(by["review"]) + ". ", False]] cov = _data(S, "coverage").get("mean_depth") q = _data(S, "sample_qc") checked = q.get("contamination") == "ok" and q.get("sex_check") == "ok" # both ran and raised nothing if not by["trust"] and isinstance(cov, (int, float)): - seg(f"Mean depth {cov:.1f}×" + ("; the identity and contamination checks raised no flag." if checked - else "; the identity and contamination checks have no result here: section 5 says why."), False) + out.append([f"Mean depth {cov:.1f}×" + ("; the identity and contamination checks raised no flag." if checked + else "; the identity and contamination checks have no result here: section 5 says why."), False]) if out: out[-1][0] = out[-1][0].rstrip() return out -# The gate needs only the act list. It never classifies individual fields or map spans. -def veil(found): - return {"acts": [f["id"] for f in found if f.get("level") == "act"]} +# --- the veil's tags ----------------------------------------------------------------------------- +# A child's page keeps each result to act on closed (assets/live/person.js veiled()). The page closes a +# finding's own card with it, but other places can name the same gene or variant: any card's table and +# lines, the charts' marks and their titles, another finding (a symptom ranking), the lead, the search index. Each such row carries the id of the +# result it belongs to, so the page closes it with that result and opens it with it. The rule: a row +# belongs to a closed result when it names the result's gene, or its position falls inside the result's +# position or span; a row that is the result's own position or span belongs to that result first, else +# the first such result in finding order. The veil lives in the browser: the files still hold every row. +LOCUS_RE = re.compile(r"(?= a for lc, ls, le in locs): + return fid + return None + + +def veil(summary, found, chapters, lead_parts=()): + """The rows a child's page closes with a result to act on, by the result's id: each card's table rows + (rows: card -> one id or None per row; the ClinVar card's by each hit's own position), each card's viewer + links (viewer: card -> one per link), each card's lines (lines: card -> one per line; a known deep + intronic site is printed whether hit or not), the marks of the report's charts (marks: chart -> {mark id: + id}), every other finding that names a closed result (findings: finding id -> id; a symptom ranking can + rank the gene) and the lead's segments that name one (lead: one per segment of LEAD_PARTS, the "act" + names left out: the page closes those itself), each card's facts (facts: card -> one per fact) and the items + of its drawing (widget: card -> one per CHIP call or copy-number dot). spans is closable() itself: the family + page (assets/live/family-page.js) reads which results are closed from it, and closes every place of the child + that could carry one while one is. Empty when nothing can be closed.""" + spans = closable(found) + if not spans: + return {} + out = {"rows": {}, "viewer": {}, "lines": {}, "facts": {}, "widget": {}, "marks": {}, "findings": {}, "lead": [], + "spans": [list(s) for s in spans]} + S = summary.get("sections") or {} + hits = [h for h in _items(_data(S, "clinvar").get("hits"))[:50]] + tags = [veil_id(spans, str(h.get("gene") or ""), h.get("chrom"), h.get("pos")) if isinstance(h, dict) else None + for h in hits] + if any(tags): + out["rows"]["clinvar"] = tags + for ch in chapters: + for c in ch["sections"]: + rows = (c.get("table") or {}).get("rows") or [] + if c["id"] != "clinvar" and rows: + tags = [veil_id(spans, " ".join(str(x) for x in r) if isinstance(r, (list, tuple)) else str(r)) + for r in rows] + if any(tags): + out["rows"][c["id"]] = tags + tags = [veil_id(spans, v) for v in c.get("viewer") or []] + if c["id"] == "clinvar": # one link per row: a row closed by its gene alone closes its link too + rt = out["rows"].get("clinvar") or [] + tags = [(rt[i] if i < len(rt) else None) or t for i, t in enumerate(tags)] + if any(tags): + out["viewer"][c["id"]] = tags + tags = [veil_id(spans, ln) for ln in c.get("lines") or []] + if any(tags): + out["lines"][c["id"]] = tags + tags = [veil_id(spans, " ".join(str(x) for x in f)) for f in c.get("facts") or []] + if any(tags): + out["facts"][c["id"]] = tags + # A drawing's items: the CHIP calls (gene and change) and the copy-number dots (gene). The CHIP + # chart's list of genes with their detection limits is the same panel for everyone: never tagged. + w = c.get("widget") or {} + items = w.get("calls") if w.get("kind") == "vaf" else w.get("genes") if w.get("kind") == "copies" else None + tags = [veil_id(spans, f"{x.get('gene') or ''} {x.get('change') or ''}") for x in items or []] + if any(tags): + out["widget"][c["id"]] = tags + try: + specs = rr.chart_specs(summary) + except (KeyError, TypeError, ValueError, AttributeError, IndexError, ZeroDivisionError): + specs = [] + for sp in specs: + for m in sp.get("marks") or []: + if m.get("kind") == "roh": # a run of homozygosity is no result's source + continue + fid = veil_id(spans, f"{m.get('label') or ''} {m.get('detail') or ''}", m.get("chrom"), m.get("pos"), + m.get("end")) + if fid: + out["marks"].setdefault(sp["id"], {})[m["id"]] = fid + for f in found: + if f.get("level") != "act": + fid = veil_id(spans, f"{f.get('gene') or ''} {f.get('detail') or ''} {f.get('title') or ''} " + f"{f.get('viewer') or ''}") + if fid: + out["findings"][f["id"]] = fid + tags = [None if len(x) > 2 else veil_id(spans, x[0]) for x in lead_parts or ()] + out["lead"] = tags if any(tags) else [] + return {k: v for k, v in out.items() if v} def build(sample, summary, files=()): @@ -1183,7 +1299,7 @@ def build(sample, summary, files=()): "not_assessed": [plain(x) for x in summary.get("not_assessed") or []], "not_run": [plain(x) for x in summary.get("not_run") or []], "new_since": summary.get("new_since") or None, - "charts": CHART_SECTION, "veil": veil(found)} + "charts": CHART_SECTION, "veil": veil(summary, found, chapters, lead_parts)} def attention_count(summary): @@ -1197,10 +1313,33 @@ def overview(summary): r = build("", summary) order = {lv: i for i, lv in enumerate(LEVELS)} order["trust"] = -1 # a problem with the data comes before any finding read from it - top = sorted((f for f in r["findings"] if f["level"] != "info"), key=lambda f: order[f["level"]])[:4] + # A finding that names a result to act on (veil: a symptom ranking of its gene) is not listed: a child's tile + # closes only the results to act on themselves, and the result it names already comes first. + named = (r.get("veil") or {}).get("findings") or {} + top = sorted((f for f in r["findings"] if f["level"] != "info" and f["id"] not in named), + key=lambda f: order[f["level"]])[:4] + tags = (r.get("veil") or {}).get("lead") or [] return {"attention": r["attention"], "levels": r["levels"], "screened": screened(summary), "top": [{"id": f["id"], "level": f["level"], "title": f["title"]} for f in top], - "lead": "".join(x[0] for x in r["lead"]), "lead_closed": "Results closed until opened on purpose."} + "lead": "".join(x[0] for x in r["lead"]), + # The same lead with the names of the findings to act on left out, and every other segment that names + # one of them: a child's tile shows this one. + "lead_closed": "".join(": closed until opened on purpose. " if x[2:] == ["act"] + else closed_lead(x[0]) if i < len(tags) and tags[i] else x[0] + for i, x in enumerate(r["lead"]))} + + +def closed_lead(t): + """A segment of the lead that names a result to act on, without the names (person.js closedLead()): the + list after a colon, the genes in brackets, or what follows "of".""" + w = "closed until opened on purpose" + if t.lstrip().startswith(":"): + return f": {w}. " + if re.search(r"\(.*\)", t): + return re.sub(r"\(.*\)", f"({w})", t) + if " of " in t: + return re.sub(r" of .*$", f" result, {w}", t) + return f"{w}. " def strings(obj): diff --git a/bin/report_charts.py b/bin/report_charts.py index 7ed86620b..bd463e730 100755 --- a/bin/report_charts.py +++ b/bin/report_charts.py @@ -324,13 +324,6 @@ def build(summary, notes, helpers): for x in out: if x is not None: x["svg"] = svg(x) - # The live child's drawing keeps structured marks, but no data-derived note, empty-state - # sentence, legend choice or variable dimension. Render its geometry from a stable spec. - # The complete SVG above remains the report's and an opened/adult page's drawing. - if x["kind"] in ("ideogram", "scatter"): - closed = dict(x, not_drawn=[], axis=dict(x["axis"], sites_missing=0), veil_geometry=True) - closed["marks"] = list(x["marks"]) - x["svg_closed"] = svg(closed) return [x for x in out if x is not None] @@ -508,7 +501,7 @@ def inside(a, w): + _text(x0 + 258, y + 8, "ClinVar hit", 11) + f'' + _text(x0 + 348, y + 8, "low-confidence ClinVar call", 11)) - kinds = {"sv", "mca"} if sp.get("veil_geometry") else {m.get("kind") for m in sp["marks"]} + kinds = {m.get("kind") for m in sp["marks"]} if kinds & {"sv", "mca"}: # A second legend row, drawn only when the chart has these marks. y += 18 @@ -538,9 +531,7 @@ def svg_scatter(sp): for lo, hi in ((303, 315), (16184, 16193)): body.append(f'') - if sp.get("veil_geometry"): - pass # empty-state prose is PANEL too - elif sp["axis"].get("sites_missing"): + if sp["axis"].get("sites_missing"): body.append(_text((x0 + x1) / 2, (y0 + y1) / 2, "The positions of the heteroplasmic sites are not in this " "summary", 12, "middle")) elif not sp["marks"]: diff --git a/rust/crates/og-map/README.md b/rust/crates/og-map/README.md index f6aba0d9d..b2906b391 100644 --- a/rust/crates/og-map/README.md +++ b/rust/crates/og-map/README.md @@ -73,9 +73,8 @@ One gzip member with mtime 0 and no file name, holding JSON written as Python's - `ref.chroms`: chr1-22, X and Y. `len` is the chromosome's last band end, `cen` the span of its `acen` bands, `masked` its `acen`, `gvar` and `stalk` bands with touching neighbours merged. - `L`, a layer entry: `state` (`ok`, `missing` or `unreadable`), `step`, `step_name`, `source` (the file read or looked for, relative to GENOME_DIR), `note` (why it is missing, or `file:line: reason` when unreadable), `count` (rows on the map), `totals` (`all` rows read and `kept` rows on the map, when the file was read), `caveat`, and `empty` (the sentence for a read file with no rows). A missing input is `missing`, an input that cannot be read is `unreadable`, and a read input with no rows is `ok` with `count` 0: three answers, never a blank lane. -- `findings`: one flat column per field, rows sorted by chromosome then start. A finding's `viewer` (`chrom:pos` or `chrom:start-end`, 1-based, `chr` optional) places it: a position gives `pos` and `[pos - 1, pos)`, a span gives `[start - 1, end)` and `pos` null. Without a usable `viewer`, a finding that names a gene sits on that gene's span from the GENCODE genes file (the first in map order when a name is on several chromosomes). A finding with neither is not a row. `gate` is set for a finding the page keeps behind its own warning: the traits card holds APOE that way, so the map adds the finding `apoe` on the APOE gene, with `gate` `apoe`; the hidden text, which names the genotype, never enters the document. `zyg` (`het`, `hom`, `hemi` or `unknown`) comes from the finding's own `Genotype` evidence; no VCF is read. `hgvs` is its `Variant` evidence. `copy` is null until the family map. `href` is the finding's page relative to the person's page, `reads` the place as the reads viewer takes it (1-based), `phrase` the sentence the map shows. `veil` is the index in `dict.veil` of the first closable finding (in `dict.veil` order) whose span the row overlaps: its own for a closable finding with no earlier overlap, else -1. The layer's `count` and `totals.kept` are the rows whose `veil` is -1; `totals.all` adds the findings that are not closable and have no place, and the `note` counts those ("N findings with no place on the map stay on the page"). -- **The report gate.** A child's browser loader admits no map while an act remains unopened. There are no per-element tag sets. The first-tag `veil` column and `dict.veil` remain for the full report's APOE warning, opaque act controls and existing document consumers. The page decides its known findings from their own opening state. Counts, totals and density bins retain their established first-tag format. The static document remains downloadable; this is a browser boundary. - +- `findings`: one flat column per field, rows sorted by chromosome then start. A finding's `viewer` (`chrom:pos` or `chrom:start-end`, 1-based, `chr` optional) places it: a position gives `pos` and `[pos - 1, pos)`, a span gives `[start - 1, end)` and `pos` null. Without a usable `viewer`, a finding that names a gene sits on that gene's span from the GENCODE genes file (the first in map order when a name is on several chromosomes). A finding with neither is not a row. `gate` is set for a finding the page keeps behind its own warning: the traits card holds APOE that way, so the map adds the finding `apoe` on the APOE gene, with `gate` `apoe`; the hidden text, which names the genotype, never enters the document. `zyg` (`het`, `hom`, `hemi` or `unknown`) comes from the finding's own `Genotype` evidence; no VCF is read. `hgvs` is its `Variant` evidence. `copy` is null until the family map. `href` is the finding's page relative to the person's page, `reads` the place as the reads viewer takes it (1-based), `phrase` the sentence the map shows. `veil` is the index in `dict.veil` of the first closable finding (in `dict.veil` order) whose span the row overlaps: its own for a closable finding with no earlier overlap, else -1. So an open finding over a closable one's span is tagged too. The layer's `count` and `totals.kept` are the rows whose `veil` is -1; `totals.all` adds the findings that are not closable and have no place, and the `note` counts those ("N findings with no place on the map stay on the page"). +- **The veil.** The document is the same for every reader. Whether a page is closed is the page's own rule; the document says only what a closed result could reveal. Closable findings are those of level `act` and every gated one (APOE closes on every page, an adult's too). Each closable finding with a place veils its span, and `dict.veil` lists their ids: gated ones first, then the act ones in the page's order. A row of a layer that can be a finding's own source (structural variants, mosaic calls and ClinVar hits, as their layers arrive), and every findings row, that overlaps a veiled span carries the index of the first such finding in `dict.veil` in a `veil` column, else -1. A row holds one tag only. Gated findings come first, so a row over both APOE and an act finding carries APOE, which is closed on every page; a row over two act findings carries the first, and both close under the same rule. The page decides its own findings with its own veil, and the tag binds only the rows that are not one of the page's findings (APOE, or a row the page's results do not have yet): the page hides such a row while its result is closed. Counts, totals and density bins cover untagged rows only, so no number counts a closed row. Runs of homozygosity and bands are never a finding's source and carry no tag. - `roh`: the `RG` rows of step 11's `bcftools roh` text (`{run}/vcf/{run}_roh.txt`, else `{run}/roh/{run}_roh.txt`). bcftools writes 1-based inclusive positions, so `s = start - 1` and `e = end`. `ST` lines are skipped; rows on chrM and other contigs count in `totals.all` only. `markers` and `q` are bcftools' columns 7 and 8. `art` indexes `dict.art`: `one X` on a male's chrX, `one Y` on a male's chrY, else `centromere` when more than half the run lies in masked spans. An unknown sex sets neither flag. `long` is 1 for a run of 5 Mb or more with no `art`. - `roh_bins`: bases covered by any run per 1 Mb bin, for each chromosome with a run. - `text`: the sentences the page shows, in pieces; `b` marks bold words. `text.findings` is the count of findings on the map in three forms (`none`, `one`, `many`): the page counts the rows it draws: each of its own findings that its veil shows at this moment, whatever the row's tag, and each other row whose tag is -1 or names a result that is open; it picks the form and puts the number in place of `{n}`. diff --git a/rust/crates/og-map/src/facts.rs b/rust/crates/og-map/src/facts.rs index a9e10e019..c96539ce1 100644 --- a/rust/crates/og-map/src/facts.rs +++ b/rust/crates/og-map/src/facts.rs @@ -5,9 +5,7 @@ //! when it is male or female; else the results body's `person.inferred_sex`, //! then `person.declared_sex`; else unknown. //! - Findings: `results.findings`, one object per finding, in the page's -//! order. Each keeps the fields the map places and labels it with, and -//! whether `results.veil.findings` ties it to a result the page can close -//! (a carrier row in a closed deletion's second gene). +//! order. Each keeps the fields the map places and labels it with. //! - The gated APOE result: the traits card keeps APOE behind its own //! warning (`hidden` on the card `traits-panel`). It is not a row of //! `results.findings`, so the map adds it as one finding, `apoe`, with diff --git a/rust/crates/og-map/src/veil.rs b/rust/crates/og-map/src/veil.rs index 388b872d6..70187d37a 100644 --- a/rust/crates/og-map/src/veil.rs +++ b/rust/crates/og-map/src/veil.rs @@ -14,13 +14,13 @@ //! its span, or its gene's span when it has no position. //! 3. Row tags: a row of a layer that can be a finding's own source (a //! structural variant, a mosaic call, a ClinVar hit) that overlaps a -//! veiled span gets that finding's index in `dict.veil`, else -1: the -//! first in `dict.veil` when several spans overlap it, where gated -//! findings come first (closed on every page), then the act ones in the -//! page's order. The child report gate admits no map until every act opens. A finding's own -//! row is tagged the same way, so an open finding over a closable one's -//! span is tagged. Runs of homozygosity and bands are never a finding's -//! source and carry no tag. +//! veiled span gets that finding's index in `dict.veil`, else -1. A row +//! holds one tag: when several spans overlap it, the first in +//! `dict.veil`, where gated findings come first (closed on every page), +//! then the act ones in the page's order. A finding's own row is tagged +//! the same way, so an open finding over a closable one's span is +//! tagged. Runs of homozygosity and bands are never a finding's source +//! and carry no tag. //! 4. Counts, totals and density bins cover untagged rows only, so no number on //! the map counts a closed row. diff --git a/tests/test_live_results.py b/tests/test_live_results.py index 2b1481d66..f5bec4523 100644 --- a/tests/test_live_results.py +++ b/tests/test_live_results.py @@ -31,7 +31,32 @@ secondary-findings list, a carrier list not made and an unparsed PharmCAT report as sentences (never zero counts), an unsized "normal" locus as not sized, AnnotSV's exon-edge check, a genotype rule and a deep intronic call; - 8. results.veil carries only the act identities needed by the shared browser data gate. + 8. the veil's tags on DEMO-003 (make_demo_sample.py --sample DEMO-003, with its + planted secondary finding and class 5 deletion): the ClinVar row and viewer + link of the planted hit, its genome-chart tick and its review-star bar carry + the secondary finding's id, the deletion's span carries its own, the search + index's items that name the gene carry one of them, and nothing else is + tagged; DEMO-001, with no result to act on, has no tag; and a known deep + intronic site's line in the gene of a closed secondary finding carries + that finding's id (the card prints each site whether hit or not), a gene + named before a full stop is that gene, and a ClinVar row closed by its gene + alone closes its "See the reads at" link with it; every other card's row + that names the gene (the son's symptom ranking ranks it) carries one of + the ids and no other row does, the ranking's finding carries the secondary + finding's id, and so does a segment of the lead that names the gene; the + home page's view of the run (overview) lists no such finding among its + first ones and its closed lead names none; a CHIP call, a copy-number dot + and a fact that name a closed result's gene carry its id (the CHIP chart's + panel of genes never does), and results.json carries the closed results as + veil.spans (the family page closes a child's places while one is closed). +Each check runs again on a planted mistake and must fail there: 1 with VNTYPER's +entry dropped, 2 with one card's caveat blanked, 3 with "ok" planted in a card +and "All clear" in a script, 4 with the hits emptied, 5 with the results route +registered without the login, 6 with a row printed as its Python dict, 7 with the Slivar adapter returning nothing, +8 with a veil rule that tags nothing (rows, lines, facts and drawings) and one that tags every row. A check +that cannot fail is no check. + +Run: python3 tests/test_live_results.py """ import contextlib import base64 @@ -324,6 +349,144 @@ def server_problems(port, state): return out +def veil_problems(res, idx): + """What is wrong with the veil's tags of DEMO-003's results.json and search index.""" + import make_demo_sample + gene = make_demo_sample.PLANTED["gene"] + c, p, _, _ = make_demo_sample.PLANTED["hit"] + sv_c, a, b = make_demo_sample.PLANTED["sv"] + act = [f["id"] for f in res["findings"] if f["level"] == "act"] + sf = next((i for i in act if i.startswith("sf-")), None) + sv = next((i for i in act if i.startswith("sv-")), None) + if not sf or not sv: + return [f"no planted secondary finding and deletion among the results to act on: {act}"] + v = res.get("veil") or {} + out = [] + card = next(x for ch in res["chapters"] for x in ch["sections"] if x["id"] == "clinvar") + rows = (v.get("rows") or {}).get("clinvar") or [] + want = [sf if r[0] == gene and r[1].startswith(f"{c}:{p} ") else None for r in card["table"]["rows"]] + if rows != want: + out.append(f"ClinVar rows tagged {rows}, want {want}") + view = (v.get("viewer") or {}).get("clinvar") or [] + if view != [sf if x == f"{c}:{p}" else None for x in card.get("viewer") or []]: + out.append(f"ClinVar viewer links tagged {view}") + marks = v.get("marks") or {} + specs = {sp["id"]: sp for sp in rr.chart_specs(res["_summary"])} + hit = [m["id"] for m in specs["genome"]["marks"] if m.get("kind") == "hit" and m["pos"] == p and m["chrom"] == c] + span = [m["id"] for m in specs["genome"]["marks"] if m.get("kind") == "sv" and (m["chrom"], m["pos"], m["end"]) == (sv_c, a, b)] + bar = [m["id"] for m in specs["clinvar-stars"]["marks"] if gene in (m.get("detail") or "")] + want = {"genome": {**{h: sf for h in hit}, **{s_: sv for s_ in span}}, "clinvar-stars": {x: sf for x in bar}} + if not hit or not span or not bar or marks != want: + out.append(f"chart marks tagged {marks}, want {want}") + # Every other card's rows: a row that names the gene carries one of the two ids, a row that does not carries + # none; the symptom ranking's row of the gene carries the secondary finding's, and so does its finding. + for cid, rows_ in ((x["id"], (x.get("table") or {}).get("rows") or []) for ch in res["chapters"] for x in ch["sections"]): + if cid == "clinvar" or not rows_: + continue + got = (v.get("rows") or {}).get(cid) or [None] * len(rows_) + bad = [(r, t) for r, t in zip(rows_, got) if (t in (sf, sv)) != (gene in " ".join(map(str, r)))] + if bad or len(got) != len(rows_): + out.append(f"{cid} rows tagged wrong: {bad or got}") + exo = (v.get("rows") or {}).get("exomiser") or [] + ranked = [f["id"] for f in res["findings"] if f["level"] != "act" and gene in f["title"]] + if sf not in exo or not ranked or v.get("findings") != {i: sf for i in ranked}: + out.append(f"the symptom ranking of {gene}: rows tagged {exo}, findings tagged {v.get('findings')}, want {ranked} -> {sf}") + tagged = {(t, it["section"]): it.get("veil") for t, items in idx["terms"].items() for it in items if it.get("veil")} + named = {(gene, it["section"]) for it in idx["terms"].get(gene) or []} + if set(tagged) != named or not named or any(x not in (sf, sv) for x in tagged.values()): + out.append(f"search items tagged {tagged}, want every item of {gene}: {sorted(named)}") + return out + + +def veil_lines_problems(): + """What is wrong with the veil's tags of the ClinVar card's lines: a known deep intronic site is printed + whether hit or not, so its line names the gene of a closed secondary finding in that gene.""" + S = {"clinvar": {"state": "ok", "data": {"hits": [], "deep_intronic": [ + {"gene": "GLA", "hgvs": "NM_000169.3:c.640-801G>A", "chrom": "chrX", "pos": 101399747, "ref": "C", + "alt": "T", "call": "hom", "detail": "GQ 40"}, + {"gene": "GENE5", "hgvs": "c.100+500A>G", "chrom": "chr1", "pos": 1, "ref": "A", "alt": "G", + "call": "het", "detail": "GQ 40"}]}}} + lines = live_results.a_clinvar(S, {"sections": S}).get("lines") or [] + found = [{"id": "sf-gla-test", "level": "act", "gene": "GLA", "detail": "GLA"}] + v = live_results.veil({"sections": S}, found, [{"sections": [{"id": "clinvar", "lines": lines}]}]) + want = ["sf-gla-test" if "GLA " in ln else None for ln in lines] + got = (v.get("lines") or {}).get("clinvar") + out = [] if "sf-gla-test" in want and None in want and got == want else [f"lines {lines} tagged {got}, want {want}"] + # A gene named at the end of a sentence ("in GLA.") is that gene. + if live_results.veil_id(live_results.closable(found), "A change in GLA.") != "sf-gla-test": + out.append("'A change in GLA.' carries no tag") + # A segment of the lead that names the gene, beside the act names the page closes itself. + lead = [[": Symptom ranking: GLA ranks 1. ", False], [": Secondary finding in GLA. ", False, "act"], ["Mean depth", False]] + got = live_results.veil({"sections": S}, found, [], lead).get("lead") + if got != ["sf-gla-test", None, None]: + out.append(f"the lead's segments tagged {got}, want the names that are not the act names") + # A finding known by its gene alone (no position): its ClinVar row is closed by the gene, and the row's + # "See the reads at" link, which shows only the position, with it. + S["clinvar"]["data"]["hits"] = [{"gene": "GLA", "chrom": "chrX", "pos": 101398000, "ref": "G", "alt": "A"}, + {"gene": "GENE5", "chrom": "chr1", "pos": 5, "ref": "A", "alt": "G"}] + card = dict(live_results.a_clinvar(S, {"sections": S}), id="clinvar") + gene_only = [{"id": "carrier-gla-test", "level": "act", "gene": "GLA"}] + v = live_results.veil({"sections": S}, gene_only, [{"sections": [card]}]) + rows, view = (v.get("rows") or {}).get("clinvar"), (v.get("viewer") or {}).get("clinvar") + if rows != ["carrier-gla-test", None] or view != rows: + out.append(f"a gene-only finding: ClinVar rows tagged {rows}, viewer links {view}, want both " + "['carrier-gla-test', None]") + return out + + +def widget_problems(): + """What is wrong with the veil's tags of a card's facts and drawing: a CHIP call and a copy-number dot, and a + fact, that name a closed result's gene carry its id; the CHIP chart's list of genes, the same panel for + everyone, is never tagged; nothing else is.""" + found = [{"id": "cpsr-tp53-test", "level": "act", "gene": "TP53", "detail": "TP53 c.524G>A"}, + {"id": "two-hits-smn1-test", "level": "act", "gene": "SMN1"}] + S = {"chip": {"state": "ok", "data": {"median_limit": 0.05, "median_depth": 72, "calls": [ + {"gene": "DNMT3A", "protein_change": "p.R882H", "vaf": 0.12, "alt_reads": 9, "depth": 75, "status": "pass", "note": "a clone"}, + {"gene": "TP53", "protein_change": "p.R175H", "vaf": 0.08, "alt_reads": 6, "depth": 75, "status": "pass", "note": "a clone"}], + "genes": [{"gene": "DNMT3A", "detection_limit_vaf": 0.05, "mean_depth": 75, "status": "ok"}, + {"gene": "TP53", "detection_limit_vaf": 0.06, "mean_depth": 70, "status": "ok"}]}}, + "paralogs": {"state": "ok", "data": {"genes": [ + {"gene": "SMN1", "copy_number": 1, "status": "called", "detail": "."}, + {"gene": "SMN2", "copy_number": 2, "status": "called", "detail": "."}]}}} + summ = {"sections": S} + chip = dict(live_results.a_chip(S, summ), id="chip") + par = dict(live_results.a_paralogs(S, summ), id="paralogs") + par["facts"] = [["SMN1 copies", "1"], ["Genes read", "2"]] + v = live_results.veil(summ, found, [{"sections": [chip, par]}]) + out = [] + want = {"chip": [None, "cpsr-tp53-test"], "paralogs": ["two-hits-smn1-test", None]} + if v.get("widget") != want: + out.append(f"drawings tagged {v.get('widget')}, want {want}") + if (v.get("facts") or {}) != {"paralogs": ["two-hits-smn1-test", None]}: + out.append(f"facts tagged {v.get('facts')}") + if v.get("spans") != [list(x) for x in live_results.closable(found)]: + out.append(f"spans {v.get('spans')}") + return out + + +def overview_problems(): + """What is wrong with the home page's view of a child's run (overview): a finding that names a closed result + (a symptom ranking of its gene) is listed among the first findings, or the closed lead still names it.""" + r = {"attention": 0, "levels": {}, "chapters": [], + "findings": [{"id": "sf-gla-test", "level": "act", "title": "Secondary finding in GLA"}, + {"id": "exomiser-gla", "level": "review", "title": "Symptom ranking: GLA ranks 1"}], + "lead": [["Of everything, ", False], [": Secondary finding in GLA. ", False, "act"], + ["One result is worth a look", True], [": Symptom ranking: GLA ranks 1.", False]], + "veil": {"findings": {"exomiser-gla": "sf-gla-test"}, "lead": [None, None, None, "sf-gla-test"]}} + keep = live_results.build + try: + live_results.build = lambda *a, **k: r + o = live_results.overview({}) + finally: + live_results.build = keep + out = [] + if [f["id"] for f in o["top"]] != ["sf-gla-test"]: + out.append(f"the home page's first findings: {o['top']}") + if "GLA" in o["lead_closed"] or "worth a look" not in o["lead_closed"]: + out.append(f"the closed lead: {o['lead_closed']}") + return out + + def main(): work = tempfile.mkdtemp(prefix="test_live_results.") srvs = [] @@ -400,7 +563,7 @@ def main(): # 8. import results_index sys.path.insert(0, os.path.join(REPO, "tests", "demo")) - check("8. DEMO-001, with no result to act on, has no veil tag", res.get("veil") == {"acts": []}, res.get("veil")) + check("8. DEMO-001, with no result to act on, has no veil tag", res.get("veil") == {}, res.get("veil")) subprocess.run([sys.executable, os.path.join(REPO, "tests", "demo", "make_demo_sample.py"), "--genome-dir", root, "--records", "2000", "--sample", "DEMO-003", "--sex", "male", "--seed", "3"], check=True, stdout=subprocess.DEVNULL) @@ -411,9 +574,32 @@ def tagged(sample_dir): r3 = live_results.build("DEMO-003", s3) r3["_summary"] = s3 return r3, results_index.build(s3, sample_dir, rr.SECTION_OF) - acts = live_results.finding_rows(collect_summary.collect("DEMO-003", d3)) - gate = tagged(d3)[0]["veil"] - check("8. the gate exports only act identities", gate == {"acts": [f["id"] for f in acts if f["level"] == "act"]}, gate) + bad = veil_lines_problems() + check("8. a known deep intronic site's line in a closed secondary finding's gene carries its id, a gene " + "before a full stop is that gene, a ClinVar row closed by its gene closes its reads link, and a lead " + "segment naming the gene carries its id", not bad, bad) + bad = overview_problems() + check("8. a child's home tile lists no finding that names a closed result, and its closed lead names none", + not bad, bad) + bad = widget_problems() + check("8. a CHIP call, a copy-number dot and a fact that name a closed result's gene carry its id, the CHIP " + "chart's gene panel none, and results.json carries the closed results as veil.spans", not bad, bad) + bad = veil_problems(*tagged(d3)) + check("8. the planted ClinVar hit's row, viewer link, tick and star bar carry its secondary finding's id, the " + "deletion's span its own, every other card's row of the gene and the symptom ranking's finding one of them, " + "and every search item of the gene one of them", not bad, bad) + # The planted mistakes: a rule that tags nothing, and one that tags every row with the first result. + keep = live_results.veil_id + try: + live_results.veil_id = lambda spans, *a, **k: None + control("a veil rule that tags nothing", not veil_problems(*tagged(d3))) + control("a veil rule that tags no line", not veil_lines_problems()) + control("a veil rule that tags no fact or drawing", not widget_problems()) + live_results.veil_id = lambda spans, *a, **k: spans[0][0] if spans else None + control("a veil rule that tags every row", not veil_problems(*tagged(d3))) + finally: + live_results.veil_id = keep + # 3, the map documents: og map build of DEMO-001 and DEMO-003, with the results each page gets. map_docs = map_documents(root, [(SAMPLE, res), ("DEMO-003", tagged(d3)[0])]) if map_docs is None: From 9a3b917e218271a60c59f174afa97c28bb35c682 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:13:10 +0100 Subject: [PATCH 064/255] fix(live): make the gated page depend only on the opened results and the count - Closed buttons sit in one block; which result each opens is drawn at random on every load. Opened records follow, in the order opened. - The opening ticket is read and removed as core.js starts and is bound to the next load by a one-use value in window.name, with no time limit, so a slow server no longer loses the last opening. - The server answers every finding id of a child with a result to act on with the same page; the page shows the run's address. After release the report keeps its own #f/ addresses. - Adult and no-act pages make main's requests: the gate reads results.json only for a child and hands each view its first read from its copy, and projects family.json only on the family page and runs.json only at home. - Home decides from runs.json alone and keeps the run's progress; the family page keeps the parents' plan rows and couple reports, matches members exactly, and gates only children with a run here. - Removes the half-dead per-element veil code (drawLead's closed, the map's closed-act chip) and writes "1 result" / "N results". --- assets/live/core.js | 450 ++++++++++++++++++++-------------- assets/live/family-page.js | 24 +- assets/live/finding.js | 14 +- assets/live/home.js | 34 +-- assets/live/map.js | 53 +--- assets/live/person.js | 110 ++------- bin/owngenome_live_results.py | 27 +- 7 files changed, 357 insertions(+), 355 deletions(-) diff --git a/assets/live/core.js b/assets/live/core.js index 7546874dd..6bdae7386 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -150,59 +150,100 @@ facts: facts, table: table, tablist: tablist, firstSentence: firstSentence, level: level, findingLevel: findingLevel, LEVEL_WORDS: LEVEL_WORDS, LEVEL_SHORT: LEVEL_SHORT, LEVEL_ORDER: LEVEL_ORDER, hm: hm, dm: dm, hours: hours }; - // The report gate owns raw payloads. Renderers only see its projection until the final act opens. + // --- the report gate ---------------------------------------------------------------------------------- + // A child's report stays closed while any of the child's results to act on (findings of level "act") is + // unopened. The page is then a function of the opened results alone and of how many are closed: no view (person.js, + // finding.js, map.js, family-page.js, home.js, live.js, the plugins) receives the child's report, because every + // view reads through fetch and EventSource and this gate holds both. Adults and children with no result to act on + // pass through untouched: each file is fetched as often, and in the same order, as without the gate. var originalFetch = window.fetch.bind(window); - var records = {}, current = null, ownsPage = false, eventSources = []; + var records = {}, current = null, ownsPage = false, eventSources = [], shared = {}, cleaned = false; var CLOSED_TEXT = "This child's report stays closed until every result to act on in adult life is opened on purpose."; - var TRANSFER = 'og-report-opening'; + var TRANSFER = 'og-report-opening', NONCE = 'og-report-opening:'; + var findingPath = /\/finding\//.test(location.pathname) ? location.pathname : null; + // The opening ticket of the navigation that replaced the last document (transfer below): read and removed at + // once, before any network load, and valid only for this navigation (its one-use nonce in window.name). + var ticket = null; + try { + var text = sessionStorage.getItem(TRANSFER), t = text ? JSON.parse(text) : null; + sessionStorage.removeItem(TRANSFER); + if (window.name.indexOf(NONCE) === 0) { + if (t && typeof t.nonce === 'string' && window.name === NONCE + t.nonce && t.path === location.pathname) { ticket = t; } + window.name = t && typeof t.name === 'string' ? t.name : ''; + } + } catch (e) { ticket = null; } + function absolute(url, base) { return new URL(url, base || location.href).href; } + function copy(x) { return x === undefined ? null : JSON.parse(JSON.stringify(x)); } + function read(r) { + if (r.status === 401) { window.location.href = '/login?next=' + encodeURIComponent(location.pathname); } + return r.ok ? r.json() : null; + } function jsonAt(url) { + return originalFetch(url, { credentials: 'same-origin', cache: 'no-store' }).then(read).catch(function () { return null; }); + } + // A file the gate reads before the views: kept once, so the view's own first read of it costs no request. + function sharedJSON(url) { + url = absolute(url); return originalFetch(url, { credentials: 'same-origin', cache: 'no-store' }).then(function (r) { - if (r.status === 401) { window.location.href = '/login?next=' + encodeURIComponent(location.pathname); } - return r.ok ? r.json() : null; + return r.text().then(function (body) { + if (r.ok) { shared[url] = { body: body, status: r.status, type: r.headers.get('Content-Type') || 'application/json' }; } + return read({ status: r.status, ok: r.ok, json: function () { return Promise.resolve(JSON.parse(body)); } }); + }); }).catch(function () { return null; }); } - function copy(x) { return x === undefined ? null : JSON.parse(JSON.stringify(x)); } - function absolute(url, base) { return new URL(url, base || location.href).href; } + function random() { var b = new Uint32Array(1); crypto.getRandomValues(b); return b[0] / 4294967296; } + // The results to act on, one record each: the findings of level "act" (bin/live_results.py). null when unreadable. function actsOf(r) { if (!r || !Array.isArray(r.findings)) { return null; } - // The producer's act manifest is independent of every other record. Ignore unrelated malformed rows. - // Level reconciliation supports older documents and refuses to lose an act omitted from a manifest. - var ids = r.veil ? r.veil.acts : undefined; - if (ids !== undefined && (!Array.isArray(ids) || ids.some(function (id) { return typeof id !== 'string'; }))) { return null; } - ids = (ids || []).slice(); - r.findings.forEach(function (f) { if (f && f.level === 'act' && typeof f.id === 'string' && ids.indexOf(f.id) < 0) { ids.push(f.id); } }); - var acts = ids.map(function (id) { return r.findings.filter(function (f) { return f && f.id === id; })[0]; }); - return acts.every(Boolean) ? acts : null; + var acts = [], seen = {}; + r.findings.forEach(function (f) { + if (f && f.level === 'act' && typeof f.id === 'string' && !seen[f.id]) { seen[f.id] = true; acts.push(f); } + }); + return acts; } - function shut(rec) { return rec.child && (!rec.acts || rec.acts.some(function (f) { return !rec.opened[f.id]; })); } - function record(url, child) { + function byId(rec, id) { return (rec.acts || []).filter(function (f) { return f.id === id; })[0]; } + function same(a, b) { return !!a && !!b && JSON.stringify(a) === JSON.stringify(b); } + // perm: the acts in a random order drawn for this page load. A closed control is named by its place in it + // (closed-1, closed-2), so no control's place says anything about the act behind it. order: the opened acts in + // the order they were opened (opened-1, opened-2). + function shuffled(ids) { + var out = ids.slice(); + for (var i = out.length - 1; i > 0; i--) { var j = Math.floor(random() * (i + 1)), x = out[i]; out[i] = out[j]; out[j] = x; } + return out; + } + function makeRecord(url, child, r, old) { + var acts = actsOf(r), rec = { url: url, child: child, acts: acts, opened: {}, order: [], perm: [] }; + if (!acts) { return rec; } + // An act whose record changed is closed again; an unchanged one keeps its opening for this page. + rec.order = old ? old.order.filter(function (id) { return same(byId(old, id), byId(rec, id)); }) : []; + rec.order.forEach(function (id) { rec.opened[id] = true; }); + var ids = acts.map(function (f) { return f.id; }); + rec.perm = old && same(ids, (old.acts || []).map(function (f) { return f.id; })) ? old.perm : shuffled(ids); + return rec; + } + function record(url, child, cached) { url = absolute(url); - var old = records[url]; - return jsonAt(url + 'results.json').then(function (r) { - var nextActs = actsOf(r); - var opened = {}; - (nextActs || []).forEach(function (f) { - if (old && old.opened[f.id] && (old.acts || []).some(function (prev) { return JSON.stringify(prev) === JSON.stringify(f); })) { opened[f.id] = true; } - }); - var rec = { url: url, child: child !== undefined ? child : old ? old.child : true, acts: nextActs, opened: opened }; + return (cached ? sharedJSON : jsonAt)(url + 'results.json').then(function (r) { + var old = records[url]; + var rec = makeRecord(url, child !== undefined ? child : old ? old.child : true, r, old); records[url] = rec; return rec; }); } + function shut(rec) { return rec.child && (!rec.acts || rec.acts.some(function (f) { return !rec.opened[f.id]; })); } function projection(rec) { - // No data-derived default: even missing/unreadable results have only the fixed explanation. - return { gated: true, acts: (rec.acts || []).map(function (f) { - var closed = rec.acts.filter(function (a) { return !rec.opened[a.id]; }); - var opened = rec.acts.filter(function (a) { return rec.opened[a.id]; }); - return rec.opened[f.id] ? { handle: 'opened-' + (opened.indexOf(f) + 1), open: true, finding: copy(f) } - : { handle: 'closed-' + (closed.indexOf(f) + 1), open: false }; - }) }; - } - function normalizeHash(p) { + var closed = rec.perm.filter(function (id) { return !rec.opened[id]; }); + return { closed: closed.map(function (id, k) { return { handle: 'closed-' + (k + 1), id: id }; }), + opened: rec.order.map(function (id, k) { return { handle: 'opened-' + (k + 1), id: id, finding: copy(byId(rec, id)) }; }) }; + } + function handles(rec) { var p = projection(rec); return p.closed.concat(p.opened); } + // The address of a closed page names a handle or nothing: any other #f/ (an id, a guess) and any query + // land on the first closed handle, so a typed address tells nothing about which result is closed. + function normalizeHash(rec) { if (!location.hash && !location.search) { return; } - var h = location.hash.slice(3); - var found = /^#f\/(closed|opened)-[1-9][0-9]*$/.test(location.hash) && p.acts.some(function (a) { return a.handle === h; }); - var to = found ? location.hash : p.acts.length ? '#f/' + (p.acts.filter(function (a) { return !a.open; })[0] || p.acts[0]).handle : ''; + var all = handles(rec), h = location.hash.slice(3); + var found = /^#f\/(closed|opened)-[1-9][0-9]*$/.test(location.hash) && all.some(function (a) { return a.handle === h; }); + var to = found ? location.hash : all.length ? '#f/' + all[0].handle : ''; if (location.hash !== to || location.search) { history.replaceState(null, '', location.pathname + to); } } function actRecord(f) { @@ -213,48 +254,53 @@ add(box, plan, facts(f.evidence), caveat(f.caveat)); return box; } - function controls(rec, changed, only) { - var p = projection(rec), box = el('section', { 'class': 'report-gate' }); + // The controls of a closed report: the fixed text, the count, one block of identical closed controls, then each + // opened act's own record, in the order it was opened. PREFIX keeps element ids apart on a page with two children. + function controls(rec, changed, prefix) { + var p = projection(rec), box = el('section', { 'class': 'report-gate' }), n = (rec.acts || []).length; add(box, el('p', { 'class': 'lede' }, CLOSED_TEXT)); - add(box, el('p', {}, rec.acts ? p.acts.length + ' results to act on' : 'The results cannot be read yet. Reload to try again.')); - p.acts.forEach(function (a, i) { - var row = el('article', { id: 'f-' + a.handle, 'data-finding': a.handle, 'class': 'finding' }); - if (a.open && (!only || only === a.finding.id)) { add(row, actRecord(a.finding)); } - var b = el('button', { type: 'button', 'class': 'btn', 'data-opening': a.handle }, a.open ? 'Close it again' : 'Show the result'); - b.addEventListener('click', function () { - var active = records[rec.url] || rec, id = rec.acts[i].id; - var f = (active.acts || []).filter(function (item) { return item.id === id; })[0]; - if (!f || JSON.stringify(f) !== JSON.stringify(rec.acts[i])) { changed(); return; } // stale controls cannot open replacement records - var at = p.acts.map(function (item) { return '#f/' + item.handle; }).indexOf(location.hash); - var addressed = at >= 0 ? rec.acts[at].id : null; - if (a.open) { delete active.opened[f.id]; } else { active.opened[f.id] = true; } - if (addressed) { - var next = (active.acts || []).map(function (item) { return item.id; }).indexOf(addressed); - history.replaceState(null, '', next >= 0 ? '#f/' + projection(active).acts[next].handle : location.pathname); - } - changed(); - }); - add(row, b); add(box, row); - }); + add(box, el('p', {}, rec.acts ? n + (n === 1 ? ' result' : ' results') + ' to act on' : 'The results cannot be read yet. Reload to try again.')); + function row(a, open) { + var art = el('article', { id: prefix + 'f-' + a.handle, 'data-finding': a.handle, 'class': 'finding' }); + if (open) { add(art, actRecord(a.finding)); } + var b = el('button', { type: 'button', 'class': 'btn', 'data-opening': a.handle }, open ? 'Close it again' : 'Show the result'); + b.addEventListener('click', function () { toggle(rec, a.id, open, changed, !prefix); }); + return add(art, b); + } + if (p.closed.length) { add(box, add.apply(null, [el('div', { 'class': 'closed-acts' })].concat(p.closed.map(function (a) { return row(a, false); })))); } + p.opened.forEach(function (a) { add(box, row(a, true)); }); return box; } + function toggle(rec, id, open, changed, addressed) { + var active = records[rec.url] || rec; + // A control drawn for an act record that has changed since opens nothing: the page draws again instead. + if (!same(byId(active, id), byId(rec, id))) { changed(); return; } + var at = addressed ? handles(active).filter(function (a) { return '#f/' + a.handle === location.hash; })[0] : null; + if (open) { delete active.opened[id]; active.order = active.order.filter(function (x) { return x !== id; }); } + else { active.opened[id] = true; active.order.push(id); } + if (at) { + // The address follows the act it named, under its new handle. + var now = handles(active).filter(function (a) { return a.id === at.id; })[0]; + history.replaceState(null, '', location.pathname + (now ? '#f/' + now.handle : '')); + } + changed(); + } function cleanPage() { - // A gated page has no renderer, search widget, replay, canvas, progress view or external report link. + // A closed page has no renderer, search widget, replay, canvas, progress view or external report link. document.body.textContent = ''; Array.prototype.slice.call(root.attributes).forEach(function (a) { if (a.name.indexOf('data-pgp-') === 0 || a.name === 'data-finding') { root.removeAttribute(a.name); } }); document.title = 'Results closed: OwnGenome'; - root.removeAttribute('data-pgp-search'); } function renderGate() { cleanPage(); - var p = projection(current); - normalizeHash(p); + // A finding's address names its finding: a closed page shows the run's address instead, the same for any id. + if (findingPath) { history.replaceState(null, '', new URL('../../', absolute(findingPath)).pathname + location.hash); } + normalizeHash(current); var wrap = el('main', { 'class': 'wrap', id: 'report-gate' }); add(wrap, el('h1', {}, 'Results about a child')); - var requested = /\/finding\//.test(location.pathname) ? decodeURIComponent(location.pathname.split('/finding/')[1].split('/')[0]) : null; - add(wrap, controls(current, function () { if (shut(current)) { renderGate(); } else { transfer(); } }, requested)); + add(wrap, controls(current, function () { if (shut(current)) { renderGate(); } else { transfer(); } }, '')); add(document.body, wrap); root.setAttribute('data-pgp-status', 'ready'); root.setAttribute('data-pgp-gated', 'true'); @@ -263,169 +309,196 @@ var bytes = new TextEncoder().encode(JSON.stringify(rec.acts)); return crypto.subtle.digest('SHA-256', bytes).then(function (h) { return Array.from(new Uint8Array(h)).map(function (v) { return v.toString(16).padStart(2, '0'); }).join(''); }); } + // Replacing the document discards every view closure, image, chart, search cache and in-flight callback. Only + // which acts are open crosses it, with a digest of their records and a nonce for the next navigation only. function transfer() { - // Only opaque booleans and a digest cross the document replacement. The receiver consumes them once. - // Replacing the document discards every view closure, image, chart, search cache and in-flight callback. proof(current).then(function (stamp) { - sessionStorage.setItem(TRANSFER, JSON.stringify({ path: location.pathname, stamp: stamp, time: Date.now(), - opened: current.acts.map(function (f) { return !!current.opened[f.id]; }) })); - location.replace(location.pathname); // requested map/section returns to the default view + var ids = current.acts.map(function (f) { return f.id; }), b = new Uint8Array(16); + crypto.getRandomValues(b); + var nonce = Array.from(b).map(function (v) { return v.toString(16).padStart(2, '0'); }).join(''); + var target = findingPath || location.pathname; + sessionStorage.setItem(TRANSFER, JSON.stringify({ path: target, stamp: stamp, nonce: nonce, + name: window.name.indexOf(NONCE) === 0 ? '' : window.name, order: current.order.map(function (id) { return ids.indexOf(id); }) })); + window.name = NONCE + nonce; + location.replace(target); // a map or section address returns to the default view }); } function consume(rec) { - var text = sessionStorage.getItem(TRANSFER); sessionStorage.removeItem(TRANSFER); - if (!text) { return Promise.resolve(); } + var t = ticket; + ticket = null; + if (!t || !rec.acts || !Array.isArray(t.order)) { return Promise.resolve(); } return proof(rec).then(function (stamp) { - try { - var t = JSON.parse(text); - if (t.path === location.pathname && t.stamp === stamp && Date.now() - t.time < 5000 && t.opened.length === rec.acts.length) { - rec.acts.forEach(function (f, i) { if (t.opened[i] === true) { rec.opened[f.id] = true; } }); - } - } catch (e) { /* An invalid or stale ticket opens nothing. */ } + if (t.stamp !== stamp) { return; } + t.order.forEach(function (i) { + var f = typeof i === 'number' ? rec.acts[i] : null; + if (f && !rec.opened[f.id]) { rec.opened[f.id] = true; rec.order.push(f.id); } + }); }); } + // The person and finding pages: the run's state, its family's roles and, for a child, its results, read in the + // order the views read them (the finding page asks for its results at once, the person page once the state has + // answered). An adult's results are not read here: the views read them as they would without the gate. function preparePerson() { - var url = absolute(/\/finding\//.test(location.pathname) ? '../../' : './'); + var url = absolute(findingPath ? '../../' : './'); var replaying = root.getAttribute('data-mode') === 'replay'; - return Promise.all([record(url), jsonAt(url + (replaying ? 'frames.json' : 'state.json'))]).then(function (x) { - current = x[0]; - var state = replaying ? x[1] && x[1].frames && x[1].frames.slice(-1)[0] : null; - state = replaying ? state && state.state : x[1]; + var stateRead = sharedJSON(url + (replaying ? 'frames.json' : 'state.json')); + var results = findingPath ? record(url, undefined, true) : null; + return stateRead.then(function (raw) { + var state = replaying ? raw && raw.frames && raw.frames.slice(-1)[0] : raw; + state = replaying ? state && state.state : state; var fam = state && state.plugins && state.plugins.family; var entry = fam && fam.kind === 'member' && (fam.families || [])[0]; - if (!entry) { current.child = !state || !state.plugins || !!(fam && fam.unknown); return; } - return jsonAt(absolute(entry.url, url) + 'family.json').then(function (f) { + return entry ? sharedJSON(absolute(entry.url, url) + 'family.json').then(function (f) { var member = f && f.state && (f.state.members || []).filter(function (m) { return m.sample === root.getAttribute('data-run'); })[0]; - current.child = !member || !!(member.father || member.mother); - }); - }).then(function () { return consume(current); }).then(function () { + // A family that cannot be read, or does not list this person, cannot say they are not a child. + return !member || !!(member.father || member.mother); + }) : !state || !state.plugins || !!(fam && fam.unknown); + }).then(function (child) { + if (!child && !results) { return { url: url, child: false, acts: [], opened: {}, order: [], perm: [] }; } + return (results || record(url, child, true)).then(function (rec) { rec.child = child; return rec; }); + }).then(function (rec) { + current = records[url] = rec; + return consume(current); + }).then(function () { ownsPage = shut(current); if (ownsPage) { renderGate(); } }).catch(function () { - current = { child: true, acts: null, opened: {} }; ownsPage = true; renderGate(); + current = { child: true, acts: null, opened: {}, order: [], perm: [] }; ownsPage = true; renderGate(); }); } + // A family's children with a run here; a member who was never sequenced, or has no run, has no report to close. function familyRecords(f, base) { - return Promise.all(((f.state && f.state.members) || []).filter(function (m) { return m.father || m.mother; }).map(function (m) { - var run = f.runs && f.runs[m.sample]; - return run && run.url ? record(absolute(run.url, base), true).then(function (r) { return [m.sample, r]; }) : Promise.resolve([m.sample, { child: true, acts: null, opened: {} }]); + return Promise.all(((f.state && f.state.members) || []).filter(function (m) { + return (m.father || m.mother) && f.runs && f.runs[m.sample] && f.runs[m.sample].url; + }).map(function (m) { + return record(absolute(f.runs[m.sample].url, base), true).then(function (r) { return [m.sample, r]; }); })).then(function (entries) { var out = {}; entries.forEach(function (e) { out[e[0]] = e[1]; }); return out; }); } + // A run's progress on the home and family pages: how far the run is, never what it found. + var PROGRESS = ['name', 'kind', 'url', 'state', 'started_utc', 'last_heard_utc', 'finished_utc', 'planned', 'done', 'running', 'failed']; + function progressOf(r) { var out = {}; PROGRESS.forEach(function (k) { if (r[k] !== undefined) { out[k] = r[k]; } }); return out; } function projectFamily(raw, recs) { var closed = Object.keys(recs).filter(function (k) { return shut(recs[k]); }); if (!closed.length) { return raw; } - // An ownership allowlist, not a classifier of genes, prose or individual report fields. - // Anything a new producer adds stays out until its owner is explicit here. + function gated(s) { return closed.indexOf(s) >= 0; } + // An ownership allowlist, not a classifier of genes, prose or individual report fields: a field a producer adds + // tomorrow stays out until it is listed here. var f = { state: { kind: raw.state.kind, name: raw.state.name, pedigree: raw.state.pedigree, - members: (raw.state.members || []).map(function (m) { return { sample: m.sample, sex: m.sex, father: m.father, mother: m.mother, partner: m.partner, sequenced: m.sequenced }; }), children: [], couples: copy(raw.state.couples || []), plan: [], reports: [] }, + members: (raw.state.members || []).map(function (m) { return { sample: m.sample, sex: m.sex, father: m.father, mother: m.mother, partner: m.partner, sequenced: m.sequenced }; }), + children: [], couples: [], plan: [], reports: [] }, layout: raw.layout ? { width: raw.layout.width, height: raw.layout.height, size: raw.layout.size, lines: copy(raw.layout.lines), - people: (raw.layout.people || []).map(function (m) { return { sample: m.sample, sex: m.sex, sequenced: m.sequenced, x: m.x, y: m.y }; }) } : null, relationships: copy(raw.relationships || []), ibd_pairs: copy(raw.ibd_pairs || []), - couples: copy(raw.couples || {}), couple_note: raw.couple_note, - segments: copy(raw.segments || {}), de_novo: copy(raw.de_novo || []), compound_het: copy(raw.compound_het || []), - trio_status: copy(raw.trio_status || {}), side_by_side: copy(raw.side_by_side), chrom_lengths: copy(raw.chrom_lengths), - runs: copy(raw.runs || {}) }; - // Ownership cut: discard child report objects wholesale. Unknown child fields are discarded too. - f.gated = closed; - f.openings = {}; closed.forEach(function (k) { f.openings[k] = projection(recs[k]); }); + people: (raw.layout.people || []).map(function (m) { return { sample: m.sample, sex: m.sex, sequenced: m.sequenced, x: m.x, y: m.y }; }) } : null, + couple_note: raw.couple_note, chrom_lengths: copy(raw.chrom_lengths), gated: closed, runs: {} }; f.state.children = (raw.state.members || []).filter(function (m) { return m.father || m.mother; }).map(function (m) { - return closed.indexOf(m.sample) >= 0 ? { child: m.sample, gated: true } : (raw.state.children || []).filter(function (c) { return c.child === m.sample; })[0]; + return gated(m.sample) ? { child: m.sample, gated: true } : copy((raw.state.children || []).filter(function (c) { return c.child === m.sample; })[0]); }).filter(Boolean); - ['segments', 'trio_status'].forEach(function (key) { closed.forEach(function (k) { if (f[key]) { delete f[key][k]; } }); }); - ['de_novo', 'compound_het'].forEach(function (key) { f[key] = (f[key] || []).filter(function (r) { return closed.indexOf(r.child) < 0; }); }); - f.relationships = (f.relationships || []).filter(function (r) { return !closed.some(function (c) { return JSON.stringify(r).indexOf(c) >= 0; }); }); - f.state.plan = []; f.state.reports = []; - closed.forEach(function (k) { if (f.runs && f.runs[k]) { f.runs[k] = { url: f.runs[k].url, gated: true }; } }); - f.state.couples = (f.state.couples || []).filter(function (c) { return closed.indexOf(c.a) < 0 && closed.indexOf(c.b) < 0; }); - Object.keys(f.couples || {}).forEach(function (key) { if (key.split(' ').some(function (k) { return closed.indexOf(k) >= 0; })) { delete f.couples[key]; } }); - f.ibd_pairs = (f.ibd_pairs || []).filter(function (r) { return closed.indexOf(r.sample_a) < 0 && closed.indexOf(r.sample_b) < 0; }); - if (f.side_by_side) { - f.side_by_side = { members: f.side_by_side.members, rows: f.side_by_side.rows }; - var sb = f.side_by_side, seenRows = Object.create(null); - // Presence comes only from actual open-member values, never an API absence marker. - sb.rows = sb.rows.filter(function (row) { return row.values.some(function (v, i) { return closed.indexOf(sb.members[i]) < 0 && v !== null && v !== undefined && typeof v !== 'object' && ['', '.', '—', 'no summary yet', 'summary not readable'].indexOf(v) < 0; }); }).map(function (row) { - row.values = row.values.map(function (v, i) { return closed.indexOf(sb.members[i]) >= 0 ? 'closed' : v; }); - row.stale = (row.stale || []).map(function (v, i) { return closed.indexOf(sb.members[i]) >= 0 ? false : v; }); - return { group: row.group, label: row.label, values: row.values, stale: row.stale }; - }).filter(function (row) { - var key = JSON.stringify(row); - if (seenRows[key]) { return false; } seenRows[key] = true; return true; - }).sort(function (a, b) { - var x = JSON.stringify([a.group, a.label]), y = JSON.stringify([b.group, b.label]); - return x < y ? -1 : x > y ? 1 : 0; + // A row belongs to a gated child when it names that child as one of its members, by exact name. + function names(v) { return String(v || '').split(/[\s,;]+/); } + f.state.couples = copy((raw.state.couples || []).filter(function (c) { return !gated(c.a) && !gated(c.b); })); + // The parents' own items stay: the plan rows that name no gated child, and the couple reports of couples with + // no gated partner. Any other report of the family folder may hold a child's data and stays out. + f.state.plan = copy((raw.state.plan || []).filter(function (r) { return !names(r.members).some(gated); })); + var reports = f.state.couples.map(function (c) { return c.report; }).filter(Boolean); + f.state.reports = copy((raw.state.reports || []).filter(function (r) { return reports.indexOf(r.url) >= 0; })); + function free(r) { return !gated(r.sample_a) && !gated(r.sample_b) && !gated(r.a) && !gated(r.b); } + f.relationships = copy((raw.relationships || []).filter(free)); + f.ibd_pairs = copy((raw.ibd_pairs || []).filter(free)); + f.couples = {}; + Object.keys(raw.couples || {}).forEach(function (key) { if (!key.split(' ').some(gated)) { f.couples[key] = copy(raw.couples[key]); } }); + ['segments', 'trio_status', 'trio_files', 'trio_file_state'].forEach(function (key) { + if (!raw[key] || typeof raw[key] !== 'object') { return; } + f[key] = {}; + Object.keys(raw[key]).forEach(function (k) { if (!gated(k)) { f[key][k] = copy(raw[key][k]); } }); + }); + ['de_novo', 'compound_het'].forEach(function (key) { f[key] = copy((raw[key] || []).filter(function (r) { return !gated(r.child); })); }); + Object.keys(raw.runs || {}).forEach(function (k) { + f.runs[k] = gated(k) ? Object.assign(progressOf(raw.runs[k]), { url: raw.runs[k].url, gated: true }) : copy(raw.runs[k]); + }); + f.openings = {}; closed.forEach(function (k) { f.openings[k] = (recs[k].acts || []).length; }); + if (raw.side_by_side) { + var sb = { members: copy(raw.side_by_side.members), rows: [] }, seenRows = Object.create(null); + // A row is there when an open member has a value in it, never because a gated child has one; the gated + // child's cell reads "closed". Rows come in a fixed order, so neither a row nor its place depends on the child. + (raw.side_by_side.rows || []).forEach(function (row) { + if (!row.values.some(function (v, i) { return !gated(sb.members[i]) && v !== null && v !== undefined && typeof v !== 'object' && ['', '.', '—', 'no summary yet', 'summary not readable'].indexOf(v) < 0; })) { return; } + var out = { group: row.group, label: row.label, + values: row.values.map(function (v, i) { return gated(sb.members[i]) ? 'closed' : v; }), + stale: (row.stale || []).map(function (v, i) { return gated(sb.members[i]) ? false : v; }) }; + var key = JSON.stringify(out); + if (!seenRows[key]) { seenRows[key] = true; sb.rows.push(out); } }); + sb.rows.sort(function (a, b) { var x = JSON.stringify([a.group, a.label]), y = JSON.stringify([b.group, b.label]); return x < y ? -1 : x > y ? 1 : 0; }); + f.side_by_side = sb; } return f; } - var familyGates = {}; + // The home page decides from runs.json alone, as it always has: a run is a child's when a readable family lists + // one of its parents (or, while a family's members cannot be read, when no readable family lists it), and its + // summary is closed when runs.json counts or names a result to act on. + function projectRuns(data) { + var idx = copy(data), fams = idx.families || [], child = {}, listed = {}; + fams.forEach(function (f) { + (f.members || []).forEach(function (m) { + listed[m.sample] = true; + if ((f.members || []).some(function (x) { return x.sample === m.father || x.sample === m.mother; })) { child[m.sample] = true; } + }); + }); + var unknown = fams.some(function (f) { return !(f.members || []).length; }); + idx.runs = (idx.runs || []).map(function (r) { + var n = (r.levels && r.levels.act) || (r.top || []).filter(function (x) { return x && x.level === 'act'; }).length; + if (!(child[r.name] || (unknown && !listed[r.name])) || !n) { return r; } + return Object.assign(progressOf(r), { gated: true, results: true, screened: true, attention: null, levels: { act: n }, + top: [{ id: '', level: 'act', title: CLOSED_TEXT }], lead: CLOSED_TEXT, lead_closed: CLOSED_TEXT }); + }); + return idx; + } + var familyGates = {}, familyKids = []; + // The two files the gate projects before a view reads them: family.json on the family page and runs.json on the + // home page. Every other response reaches its view as it came. + var familyPage = document.body.classList.contains('page-family'); + function projected(path) { return familyPage ? path.endsWith('/family.json') : path.endsWith('/runs.json'); } function loadedJSON(url, data) { var u = new URL(url, location.href); - if (u.pathname.endsWith('/family.json') && document.body.classList.contains('page-family')) { + if (familyPage) { + familyKids = ((data.state && data.state.children) || []).map(function (c) { return c.child; }); return familyRecords(data, u.href).then(function (recs) { familyGates = recs; return projectFamily(data, recs); }); } - if (u.pathname.endsWith('/runs.json')) { - var idx = copy(data); - var families = idx.families || [], listed = {}; - families.forEach(function (f) { (f.members || []).forEach(function (m) { listed[m.sample] = true; }); }); - var unknown = families.some(function (f) { return !(f.members || []).length; }); - var unknownRuns = unknown ? (idx.runs || []).filter(function (r) { return !listed[r.name]; }).map(function (r) { - return record(absolute(r.url), true).then(function (rec) { var out = {}; out[r.name] = rec; return out; }); - }) : []; - return Promise.all(unknownRuns.concat(families.map(function (entry) { - return jsonAt(absolute(entry.url) + 'family.json').then(function (f) { - if (!f || !f.state) { - var runs = {}; - (idx.runs || []).forEach(function (r) { runs[r.name] = { url: absolute(r.url) }; }); - f = { state: { members: entry.members || [] }, runs: runs }; - } - return familyRecords(f, absolute(entry.url)); - }); - }))).then(function (all) { - all.forEach(function (recs) { - (idx.runs || []).forEach(function (r, i) { - if (recs[r.name] && shut(recs[r.name])) { - idx.runs[i] = { name: r.name, url: r.url, gated: true, state: 'closed', levels: { act: recs[r.name].acts ? recs[r.name].acts.length : 0 }, - top: [{ level: 'act', title: CLOSED_TEXT }], lead: CLOSED_TEXT, lead_closed: CLOSED_TEXT }; - } - }); - }); - return idx; - }); - } - return Promise.resolve(data); + return Promise.resolve(projectRuns(data)); } var ready = Promise.resolve(); if (document.body.classList.contains('page-person') || document.body.classList.contains('page-finding')) { ready = preparePerson(); } // Permission is checked after parsing too: a response started while open may finish after closing. function viewResponse(response) { - var read = response.json.bind(response); - response.json = function () { return read().then(function (data) { return ownsPage ? new Promise(function () {}) : data; }); }; + var parse = response.json.bind(response); + response.json = function () { return parse().then(function (data) { return ownsPage ? new Promise(function () {}) : data; }); }; return response; } - // All fetch-based readers (including plugins) share this boundary. An unrouted view receives nothing. + function closedRun(url) { return Object.keys(records).some(function (key) { return url.indexOf(key) === 0 && shut(records[key]); }); } + // Every view reads through here (and through EventSource below); one that reads a child's closed run gets nothing. window.fetch = function (url, options) { return ready.then(function () { var requestURL = absolute(url && url.url || url); - var closedRun = Object.keys(records).some(function (key) { return requestURL.indexOf(key) === 0 && shut(records[key]); }); - if (ownsPage || closedRun) { return new Promise(function () {}); } - return originalFetch(url, options).then(function (response) { + if (ownsPage || closedRun(requestURL)) { return new Promise(function () {}); } + var kept = shared[requestURL]; + delete shared[requestURL]; + var got = kept ? Promise.resolve(new Response(kept.body, { status: kept.status, headers: { 'Content-Type': kept.type } })) + : originalFetch(url, options); + return got.then(function (response) { if (ownsPage) { return new Promise(function () {}); } - var path = new URL(url && url.url || url, location.href).pathname; + var path = new URL(requestURL).pathname; if (response.ok && current && current.child && path === new URL(current.url + 'results.json').pathname) { return response.clone().json().then(function (r) { - var nextActs = actsOf(r); - if (JSON.stringify(nextActs) !== JSON.stringify(current.acts)) { - var opened = {}; - (nextActs || []).forEach(function (f) { - if (current.opened[f.id] && (current.acts || []).some(function (p) { return JSON.stringify(p) === JSON.stringify(f); })) { opened[f.id] = true; } - }); - current.acts = nextActs; current.opened = opened; + var next = makeRecord(current.url, true, r, current); + if (!same(next.acts, current.acts)) { + current = records[current.url] = next; + // A result to act on that appeared or changed while the report was open closes it at once. if (shut(current)) { ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return new Promise(function () {}); } } return viewResponse(response); }); } - if (response.ok && (path.endsWith('/family.json') || path.endsWith('/runs.json'))) { - return response.json().then(function (data) { return loadedJSON(url, data); }).then(function (data) { + if (response.ok && projected(path)) { + return response.json().then(function (data) { return loadedJSON(requestURL, data); }).then(function (data) { return viewResponse(new Response(JSON.stringify(data), { status: response.status, headers: response.headers })); }); } @@ -434,17 +507,19 @@ }); }; window.addEventListener('hashchange', function () { if (ownsPage) { renderGate(); } }); - window.addEventListener('pagehide', function () { if (current && current.child && current.acts && current.acts.length) { ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); } }); - window.addEventListener('pageshow', function (e) { if (e.persisted) { location.reload(); } }); - // live.js also has an event reader. Delay it at the same gate; it never sees child events while closed. + // A child's open report never stays in the back-forward cache: it is cleared on the way out, and a page shown + // again from that cache loads afresh (closed). Other pages come back from the cache as they were. + window.addEventListener('pagehide', function () { + if (current && current.child && current.acts && current.acts.length) { ownsPage = true; cleaned = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); } + }); + window.addEventListener('pageshow', function (e) { if (e.persisted && cleaned) { location.reload(); } }); var NativeEvents = window.EventSource; window.EventSource = function (url, options) { var proxy = this, source = null; proxy.readyState = 0; proxy.close = function () { if (source) { source.close(); } proxy.readyState = 2; }; ready.then(function () { - var requestURL = absolute(url); - if (ownsPage || Object.keys(records).some(function (key) { return requestURL.indexOf(key) === 0 && shut(records[key]); }) || proxy.readyState === 2) { return; } + if (ownsPage || closedRun(absolute(url)) || proxy.readyState === 2) { return; } source = new NativeEvents(url, options); eventSources.push(source); source.onmessage = function (e) { if (!ownsPage && proxy.onmessage) { proxy.onmessage(e); } }; source.onerror = function (e) { proxy.readyState = source.readyState; if (proxy.onerror) { proxy.onerror(e); } }; @@ -455,18 +530,25 @@ window.PGPUI.ready = ready; window.PGPUI.gateOwnsPage = function () { return ownsPage; }; window.PGPUI.closedText = CLOSED_TEXT; + // The acts the gate released, for person.js's map switches: every act of a child's open report, else none. window.PGPUI.openedActs = function () { return current && !shut(current) ? copy(current.opened) : {}; }; + // Closing one act of a child's open report hands the page back to the gate, which closes it whole. window.PGPUI.closeAct = function (id) { - if (!current || !current.child || !current.acts || !current.acts.some(function (f) { return f.id === id; })) { return false; } - delete current.opened[id]; ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return true; + if (!current || !current.child || !byId(current, id)) { return false; } + delete current.opened[id]; + current.order = current.order.filter(function (x) { return x !== id; }); + ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return true; + }; + // What the family page knows of each child's results (for the tests): "DEMO-003:known", "DEMO-005:failed" for a + // child with no run here. + window.PGPUI.familyStatus = function () { + return familyKids.slice().sort().map(function (k) { return k + ':' + (familyGates[k] && familyGates[k].acts ? 'known' : 'failed'); }).join(' '); }; - window.PGPUI.familyStatus = function () { return Object.keys(familyGates).sort().map(function (k) { return k + ':' + (familyGates[k].acts ? 'known' : 'failed'); }).join(' '); }; window.PGPUI.familyControls = function (child, redraw) { function build() { - var box = controls(familyGates[child], function () { box.replaceWith(build()); redraw(); }); + var box = controls(familyGates[child], function () { box.replaceWith(build()); redraw(); }, child + '-'); box.setAttribute('data-gate-child', child); return box; } return build(); }; - }()); diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 1d10edfe4..8c93c9557 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -6,12 +6,10 @@ // phone), each child's stretches painted by parent, the de novo and compound-het candidates, and the // members side by side. It draws what the steps wrote and computes nothing about genetics. Text through textContent, // the drawing through createElementNS; images from this server only. -// A child's results to act on stay closed here as on the child's own page (person.js): each child's results.json -// is read by the shared gate, which passes no report data of a child with an unopened act to this view. +// A child's report stays closed here as on the child's own page: the shared gate (core.js) reads each child's +// results.json and gives this view the family without that child's report data while a result to act on is unopened. (function () { 'use strict'; - window.PGPUI.ready.then(function () { - if (window.PGPUI.gateOwnsPage()) { return; } var U = window.PGPUI; if (!U) { return; } var el = U.el, add = U.add, root = document.documentElement; @@ -137,7 +135,7 @@ if (m.father || m.mother) { rel.push('child of ' + [m.father, m.mother].filter(Boolean).join(' and ')); } if (m.partner) { rel.push('partner of ' + m.partner); } if (rel.length) { add(txt, el('span', { 'class': 'person-place' }, rel.join('; '))); } - var state = r && r.gated ? U.closedText : !m.sequenced ? 'Not sequenced' : !r ? 'No run on this server' + var state = !m.sequenced ? 'Not sequenced' : !r ? 'No run on this server' : r.state === 'finished' ? 'Finished' : (r.state === 'stopped' ? 'Stopped, ' : '') + r.done + ' of ' + r.planned + ' steps'; add(txt, el('span', { 'class': 'person-state' }, state)); if (r && r.results) { @@ -370,8 +368,6 @@ if (rows && rows.length) { add(d, painting(f, c.child, rows)); } else if (c.segments) { - // The drawing as an image is coloured by grandparent too, and cannot close one stretch: it stays closed - // whole, as the painting does. add(d, el('img', { src: c.segments, alt: 'Chromosome map of ' + c.child, 'class': 'segments' })); } else { add(d, el('p', { 'class': 'dim' }, 'No chromosome map yet: step 38 draws one for a child with a sequenced parent.')); @@ -381,10 +377,9 @@ if (dn.length) { add(d, el('p', { 'class': 'dim' }, 'In the child and in neither parent. Most candidates in a genome are genotype errors: ' + 'check the reads before reading anything into one.')); - // Every row closes while a result is: a row closed on its own would say where the result lies. - add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], - rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, - r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); + add(d, U.table({ columns: ['Variant', 'Child', 'Father', 'Mother', 'Child DP / GQ', 'Allele balance', 'Quality'], + rows: dn.map(function (r) { return [r.chrom + ':' + r.pos + ' ' + r.ref + '>' + r.alt, r.child_gt, r.father_gt, + r.mother_gt, r.child_dp + ' / ' + r.child_gq, r.child_ab, r.quality]; }) }, 8)); } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'de_novo' }, empty(c.child, 'de_novo', 'de_novo.tsv'))); } @@ -392,8 +387,8 @@ add(d, el('h3', {}, 'Compound-het pairs')); if (ch.length) { add(d, el('p', { 'class': 'dim' }, 'Two candidate variants in one gene, in trans first: one from each parent.')); - add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], - rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); + add(d, U.table({ columns: ['Gene', 'Phase', 'Variant A', 'From', 'Variant B', 'From'], + rows: ch.map(function (r) { return [r.gene, r.phase, r.variant_a, r.a_from, r.variant_b, r.b_from]; }) }, 8)); } else { add(d, el('p', { 'class': 'dim', 'data-trio': 'compound_het' }, empty(c.child, 'compound_het', 'compound_het.tsv'))); } @@ -444,7 +439,6 @@ } function draw(f) { - var main = document.getElementById('family'); // A redraw (a result opened, a child's results answered) keeps each child's fold as the reader left it. var folds = {}; @@ -490,7 +484,6 @@ root.setAttribute('data-pgp-status', 'ready'); } - // The page asks again every 30 s and redraws only when family.json changed (a member's run moved on, // step 39 finished). A failed first request says so, with a retry; a later one keeps what is shown. var drawnKey = null; @@ -516,5 +509,4 @@ window.setInterval(function () { if (!document.hidden) { fetchFamily(); } }, 30000); } if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', start); } else { start(); } - }); }()); diff --git a/assets/live/finding.js b/assets/live/finding.js index 9b0041875..f7a3c23f0 100644 --- a/assets/live/finding.js +++ b/assets/live/finding.js @@ -3,7 +3,7 @@ // the explanation, the plan under it, then the evidence (the numbers, the frequency on a log strip, the // reads with a link to the genome viewer), the caveat, the related findings and the family. Everything // comes from the run's results.json (bin/live_results.py findings); nothing is computed here about -// genetics. A result to act on about a child of the family stays closed until it is opened on purpose. +// genetics. A child's finding page draws only once the report gate (core.js) has released the child's report. (function () { 'use strict'; window.PGPUI.ready.then(function () { @@ -74,14 +74,10 @@ root.setAttribute('data-pgp-status', 'ready'); return; } - // known, not family: the click that opens this finding draws again with family as it came, UNKNOWN kept. var unknown = family === UNKNOWN; var known = unknown ? null : family; - var me = known && (known.state.members || []).filter(function (m) { return m.sample === r.sample; })[0]; - // A family that does not list this person cannot say they are not a child: closed as well. - var title = f.title; - document.title = title + ': ' + r.sample + ': OwnGenome'; - document.getElementById('finding-crumb').textContent = title; + document.title = f.title + ': ' + r.sample + ': OwnGenome'; + document.getElementById('finding-crumb').textContent = f.title; var sec = (r.journal || []).filter(function (s) { return s.n === f.section; })[0]; var same = (r.findings || []).filter(function (x) { return x.level === f.level; }); var order = (r.findings || []).filter(function (x) { return x.level !== 'info'; }); @@ -114,7 +110,7 @@ add(lv, el('span', { 'class': 'dim' }, same.length === 1 ? 'The only finding at this level in ' + r.sample + '’s genome' : same.length + ' findings at this level in ' + r.sample + '’s genome')); add(cover, lv); - add(cover, el('h1', { 'class': 'h-display' }, title)); + add(cover, el('h1', { 'class': 'h-display' }, f.title)); var chips = el('div', { 'class': 'chips' }); var cs = []; if (f.gene) { cs.push(f.gene); } @@ -223,7 +219,7 @@ var f = fam && fam.kind === 'member' && (fam.families || [])[0]; if (!f) { return Promise.resolve(null); } familyUrl = f.url; - // A family that does not answer: roles unknown, so a result to act on stays closed (draw()). + // A family that does not answer: roles unknown (the family part of the page is left out). return U.getJSON(abs(f.url) + 'family.json').then(function (d) { return d && d.state ? d : UNKNOWN; }).catch(function () { return UNKNOWN; }); } function failed() { diff --git a/assets/live/home.js b/assets/live/home.js index 803fc0522..fed41b05c 100644 --- a/assets/live/home.js +++ b/assets/live/home.js @@ -6,8 +6,6 @@ // member of a family keeps one colour on this page. The plain list of runs is what shows without scripts. (function () { 'use strict'; - window.PGPUI.ready.then(function () { - if (window.PGPUI.gateOwnsPage()) { return; } var U = window.PGPUI; if (!U) { return; } var el = U.el, add = U.add; @@ -45,8 +43,9 @@ if (!row.childNodes.length) { add(row, el('span', { 'class': 'dim' }, 'No finding in what was assessed; read the caveats')); } return row; } - // The loader has already replaced each gated child's summary. - function homeTop(r) { return r.top || []; } + // The runs whose results to act on stay closed on this page: a child in any family, and, while a family's + // members cannot be read, every run that no readable family lists (it may be that family's child). + var closedRuns = {}; function tile(m, fam, r, colour) { var div = el('div', { 'class': 'ptile person-card' + (r ? ' has-run' : ' norun') + (r && r.state === 'running' ? ' running' : ''), 'data-sample': m.sample }); @@ -55,12 +54,11 @@ var h = el('h3', {}); add(h, r ? el('a', { href: r.url, 'class': 'person-link' }, m.sample) : el('span', {}, m.sample)); add(div, h); - if (r && r.gated) { add(div, el('p', { 'class': 'lead' }, U.closedText)); return div; } var meta = el('div', { 'class': 'meta' }); add(meta, (fam ? role(m, fam) + ' · ' : '') + progressText(r)); if (r && r.failed) { add(meta, ' · ', el('span', { 'class': 'warn' }, r.failed + ' step' + (r.failed > 1 ? 's' : '') + ' not assessed')); } add(div, meta); - var lead = r && (r.lead); + var lead = r && (closedRuns[m.sample] ? r.lead_closed || '' : r.lead); if (lead) { // Two sentences: the first often says only that nothing needs a doctor, the second names what differs. var one = U.firstSentence(lead), rest = lead.slice(one.length).trim(); @@ -139,8 +137,17 @@ // nothing needs a doctor. var screened = runs.filter(function (r) { return r.levels && r.screened; }).length; // A child of a family: results to act on stay closed on this page too (the report opens them). - function homeShut(r) { return !!r.gated; } - function closedTitle(r, f) { return r.gated ? U.closedText : f.title; } + closedRuns = {}; + var listed = {}; + var rolesUnknown = fams.some(function (f) { return !f.members.length; }); + fams.forEach(function (f) { + f.members.forEach(function (m) { + listed[m.sample] = true; + if (f.members.some(function (x) { return x.sample === m.father || x.sample === m.mother; })) { closedRuns[m.sample] = true; } + }); + }); + if (rolesUnknown) { runs.forEach(function (r) { if (!listed[r.name]) { closedRuns[r.name] = true; } }); } + function closedTitle(r, f) { return closedRuns[r.name] && f.level === 'act' ? 'A result to act on, closed until opened on the report' : f.title; } // The dateline and the headline sentence. var dl = el('div', { 'class': 'dateline' }); var today = new Date(); @@ -175,7 +182,7 @@ // One line per person and level: the first finding names the line, the rest are counted on it. runs.forEach(function (r) { var seen = {}; - homeTop(r).forEach(function (f) { + (r.top || []).forEach(function (f) { if (order[f.level] === undefined) { return; } if (seen[f.level]) { return; } // The rest of the level from its count, not from the first four findings the index names. @@ -185,8 +192,8 @@ }); items.sort(function (a, b) { return order[a.f.level] - order[b.f.level]; }); items.slice(0, 6).forEach(function (x) { - // A child's closed result links to the run's page, never to its own page: its id names its gene and variant. - var a = el('a', { href: x.r.url + (homeShut(x.r, x.f) ? '' : 'finding/' + encodeURIComponent(x.f.id) + '/') }); + // A child whose report the gate keeps closed (core.js) links to the run's page, never to a finding's own page. + var a = el('a', { href: x.r.url + (x.r.gated ? '' : 'finding/' + encodeURIComponent(x.f.id) + '/') }); var who = el('span', { 'class': 'who' }); var pc = el('span', { 'class': 'pc' }, x.r.name); if (colourOf[x.r.name]) { pc.style.setProperty('--pcol', colourOf[x.r.name]); } @@ -223,8 +230,8 @@ var c2 = el('div', {}); add(c2, el('div', { 'class': 'fig-label' }, 'Found so far')); var fs = el('ul', { 'class': 'fs' }); - homeTop(r).forEach(function (f) { add(fs, add(el('li', {}), U.findingLevel(f.level, ''), ' ', closedTitle(r, f))); }); - if (!fs.childNodes.length) { add(fs, el('li', {}, r.lead ? U.firstSentence(r.lead) : 'No section is written yet: the first ones appear as their steps end.')); } + (r.top || []).forEach(function (f) { add(fs, add(el('li', {}), U.findingLevel(f.level, ''), ' ', closedTitle(r, f))); }); + if (!fs.childNodes.length) { add(fs, el('li', {}, r.lead ? U.firstSentence(closedRuns[r.name] ? r.lead_closed || '' : r.lead) : 'No section is written yet: the first ones appear as their steps end.')); } add(c2, fs); var c3 = el('div', {}); add(c3, el('div', { 'class': 'fig-label' }, 'Timing')); @@ -305,5 +312,4 @@ }).catch(function () { root.setAttribute('data-pgp-status', 'offline'); setTimeout(start, 30000); }); } if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', start); } else { start(); } - }); }()); diff --git a/assets/live/map.js b/assets/live/map.js index 5f0b2f3de..3a4dfea4f 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -91,12 +91,11 @@ if (!(f.chrom[i] in by)) { return; } rows.push({ id: fid, level: f.level[i], gate: f.gate[i], chrom: f.chrom[i], pos: f.pos[i], s: f.s[i], e: f.e[i], gene: f.gene[i], title: f.title[i], href: f.href[i], phrase: f.phrase[i], veil: f.veil[i] >= 0 ? veilIds[f.veil[i]] : null }); - }); var gated = {}; rows.forEach(function (r) { if (r.gate) { gated[r.id] = r; } }); - // What can be closed: each veil tag, and each result to act on whose first tag is another result's (a row over - // two of them lists the first one first, and one over APOE lists APOE's first): its own id too. + // What can be closed: each veil tag, and each result to act on that carries another result's tag (a row + // over two of them carries only the first one's tag, and one over APOE carries APOE's): its own id too. var closable = veilIds.slice(); rows.forEach(function (r) { if (r.veil && r.level === 'act' && !r.gate && closable.indexOf(r.id) < 0) { closable.push(r.id); } }); return { doc: d, id: id, label: person.label || id, chroms: chroms, by: by, bands: bands, roh: roh, bins: bins, @@ -106,9 +105,7 @@ // --- the figure --------------------------------------------------------------------------------------- // create({url, veil(id, gated) -> 'open'|'closed'|'opened', now(id) -> shown at this moment, open(id), - // close(id), known(id) -> a finding of the page's results, closed() -> the ids of the results to act on the - // page keeps closed at this moment, stepDone(step) -> ended at this moment, href(id) -> the address of a finding - // the page opened (never its id on a child's page), fallback() -> rows}): one figure + // close(id), known(id) -> a finding of the page's results, stepDone(step) -> ended at this moment, fallback() -> rows}): one figure // per page. The page calls refresh(key) on each of its redraws; a new key // fetches the document again and keeps the camera. function create(opts) { @@ -156,8 +153,6 @@ var CW = 0, CH = 0, DPR = 1, phone = false; var cam = { ci: 0, c: 0, ls: 0, k: 0 }, tgt = { ci: 0, c: 0, ls: 0, k: 0 }; var hover = { row: -1, hit: null }, hits = [], raf = 0, settleT = 0, hashT = 0, confirming = null, touched = false; - // The results with no place on the map that its chip opened: each gets an "Opened in its section" chip while open. - var inSection = []; var cache = null, cacheKey = ''; var hatch = null; @@ -168,13 +163,10 @@ // A row that is a finding of the page (opts.known) follows the page's own veil only: the page already // hides every finding it keeps closed (opts.now), and its card shows the rest. Hiding such a row for // another result's tag would drop a finding whose card the page shows, and so place that closed result - // inside it. A map-owned gated row (APOE) follows its own gate for the same reason: opening it must not - // tell whether another, still-closed result overlaps it. Other rows retain the document's APOE warning tag. + // inside it. A row the page has no finding for (the map's APOE row) follows its tag too. function shownRow(r) { var own = M.closable.indexOf(r.id) < 0 || stateOf(r.id) !== 'closed'; - // Closed while ANY result the row lies over is closed: a row over two closed results never comes back with - // the first one opened. - var tag = !!r.gate || (opts.known && opts.known(r.id)) || !r.veil || stateOf(r.veil) !== 'closed'; + var tag = (opts.known && opts.known(r.id)) || !r.veil || stateOf(r.veil) !== 'closed'; return own && tag && (!opts.now || opts.now(r.id)); } // The runs of homozygosity, as this page shows them now: a replay before their step ended has none yet. @@ -605,16 +597,7 @@ function noFindingHere() { return closedAny() ? 'No open finding with a place on this chromosome.' : 'No finding with a place on this chromosome.'; } - function closedAny() { return closedActs().length > 0 || M.closable.some(function (id) { return stateOf(id) === 'closed'; }); } - // The results to act on this page keeps closed, in the order Open opens them: the page's own order (the - // order of its closed cards), with a place on the map or without. Its length is what the chip counts, so - // neither the count nor the order tells whether a closed result has a place. A page that passes no list - // closes only the map's own rows, in map order. - function closedActs() { - var page = opts.closed ? opts.closed() : null; - if (page) { return page.filter(function (id) { return !M.gated[id]; }); } - return M.closable.filter(function (id) { return !M.gated[id] && stateOf(id) === 'closed'; }); - } + function closedAny() { return M.closable.some(function (id) { return stateOf(id) === 'closed'; }); } function layerWords(lay) { if (lay.state === 'missing') { return 'Not on the map: ' + (lay.note || ('step ' + lay.step + ' wrote no ' + (lay.source || 'file') + '.')); } if (lay.state === 'unreadable') { return 'Not on the map: ' + (lay.source || '') + ' could not be read' + (lay.note ? ' (' + lay.note + ')' : '') + '.'; } @@ -882,7 +865,7 @@ function drawChips() { chips.textContent = ''; if (!M) { return; } - var acts = closedActs(), gates = [], opened = []; + var gates = [], opened = []; M.closable.forEach(function (id) { // A gated result (APOE) has a chip only once the page shows its card (a replay before its step has none). if (M.gated[id] && opts.now && !opts.now(id)) { return; } @@ -891,17 +874,11 @@ }); // Why it is closed: said beside the count, left out on a phone (the card says it too). function why() { return el('span', { 'class': 'gm-why' }, ' · Not drawn until opened on purpose'); } - if (acts.length) { - var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'act' }); - add(c, add(el('span', {}, acts.length === 1 ? '1 result closed' : acts.length + ' results closed'), why())); - add(c, confirming === 'act' ? confirmBox(acts[0], false) : button('Open', function () { confirming = 'act'; drawChips(); keyTo('.gm-yes'); })); - add(chips, c); - } gates.forEach(function (id) { var g = M.gated[id], name = g.gene || 'This result'; var c = el('div', { 'class': 'gm-chip closed', 'data-closed': 'gate' }); add(c, add(el('span', {}, name + ' closed'), why())); - add(c, confirming === id ? confirmBox(id, true) : button('Open ' + name, function () { confirming = id; drawChips(); keyTo('.gm-yes'); })); + add(c, confirming === id ? confirmBox(id) : button('Open ' + name, function () { confirming = id; drawChips(); keyTo('.gm-yes'); })); add(chips, c); }); opened.forEach(function (id) { @@ -911,27 +888,17 @@ add(c, button('Close it again', function () { confirming = null; opts.close(id); keyTo(); })); add(chips, c); }); - // A result the chip opened that has no place: named only once it is open, with a link to its card. - inSection = inSection.filter(function (id) { return stateOf(id) !== 'closed' && opts.title && opts.title(id); }); - inSection.forEach(function (id) { - var c = el('div', { 'class': 'gm-chip opened', 'data-opened': 'section' }); - add(c, add(el('span', {}, 'Opened in its section: '), el('a', { href: '#f/' + (opts.href ? opts.href(id) : id) }, opts.title(id)))); - add(c, button('Close it again', function () { confirming = null; opts.close(id); keyTo(); })); - add(chips, c); - }); } // Where the keyboard goes once a chip's button rebuilt the chips: the chips' first button, else the map. function keyTo(sel) { var b = chips.querySelector(sel || 'button'); (b || canvas).focus({ preventScroll: true }); } - function confirmBox(id, gated) { + function confirmBox(id) { var box = el('span', { 'class': 'gm-confirm' }); - add(box, el('span', {}, gated ? 'It is kept behind its warning on every page. Open it here and in its section?' - : 'A result to act on in adult life is usually not looked at before the child can choose. Open it here and in its section?')); + add(box, el('span', {}, 'It is kept behind its warning on every page. Open it here and in its section?')); add(box, button('Show the result', function () { confirming = null; - if (!gated && M.closable.indexOf(id) < 0 && inSection.indexOf(id) < 0) { inSection.push(id); } opts.open(id); // the page draws again, and the map's chips with it (refresh) keyTo(); }, 'gm-yes')); diff --git a/assets/live/person.js b/assets/live/person.js index 3ea338406..2eefb1dd8 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -104,12 +104,10 @@ var ms = (family && family.data.state && family.data.state.members) || []; return ms.filter(function (m) { return m.sample === personName(); })[0] || null; } - // A child of the family: a member whose father or mother is listed. Results to act on about a child - // stay closed until someone opens them on purpose. + // A child of the family: a member whose father or mother is listed. function isChild() { var m = me(); return !!(m && (m.father || m.mother)); } - // Results to act on stay closed for a child, and for anyone in a family whose roles are not known yet: - // the page cannot tell a child apart until family.json has answered. - // A family that answers without this person cannot say they are not a child either. + // A page whose results to act on were opened on purpose (core.js releases a child's report, or a person whose + // family roles are not known, only once each is opened): the map marks them opened, with a chip to close them. function closedPage() { return isChild() || familyStatus === 'pending' || familyStatus === 'failed' || (familyStatus === 'known' && !me()); } function memberUrl(s) { var r = family && family.data.runs && family.data.runs[s]; @@ -321,12 +319,8 @@ add(b, text); return b; } - // This renderer receives the full report only after the shared data gate releases. - function openTable(c) { return [c.table, 0]; } - function openFacts(c) { return [c.facts || [], c.widget, 0]; } - function handle(f) { return f.id; } - window.PGP.markClosed = function () { return false; }; - window.PGP.searchHidden = function () { return false; }; + // This renderer receives a child's report only once the shared data gate (core.js) has released it, so it + // closes nothing itself: no card row, chart mark or search hit. function card(c) { var g = gate(c); @@ -368,14 +362,13 @@ } else { (c.notes || []).forEach(function (n) { add(text, el('p', { 'class': 'note-stale' }, n)); }); if (c.headline) { add(text, el('p', { 'class': 'headline' }, c.headline)); } - var of = openFacts(c); - add(margin, U.facts(of[0])); - add(text, widget(of[1])); + add(margin, U.facts(c.facts)); + add(text, widget(c.widget)); Object.keys((results && results.charts) || {}).forEach(function (chart) { if (results.charts[chart] === c.id) { add(text, el('div', { 'class': 'chart-slot', 'data-chart-slot': chart })); } }); if (c.image) { add(text, el('img', { 'class': 'plot', src: c.image.src, alt: c.image.alt })); } - var ot = openTable(c), tb = U.table(ot[0]); + var tb = U.table(c.table); var lines = c.lines || []; if (tb) { add(text, tb); } else if (c.empty_text) { add(text, el('p', { 'class': 'empty' }, c.empty_text)); } add(text, lineList(lines)); @@ -471,44 +464,9 @@ add(box, d); return note('Caveat', box, 'caveat-note'); } - // The address never holds the id of a finding this page can close, closed or opened: #f/closed- while it is - // closed (its handle) and #f/opened- once opened on purpose (its place among the opened ones), so no history - // entry keeps the id. Any other #f/ on a page that keeps a result closed, a closed result's id or no id at all, - // lands the same way, on #f/closed-1: the page never picks a closed card by its id, so a guessed id tells nothing. - // Opening or closing a result moves the handles of the others: the address follows the finding it named. The - // finding behind a handle in the address is kept here only (hashId); a new address from the reader forgets it. - var hashId = null; - window.addEventListener('hashchange', function () { hashId = null; }); - function needsHandle(f) { return closedPage() && f.level === 'act'; } - function openedOrder() { - var out = []; - sectionsList().forEach(function (sec) { - (sec.findings || []).forEach(function (id) { var f = findingById(id); if (f && needsHandle(f) && out.indexOf(id) < 0) { out.push(id); } }); - }); - ((results && results.findings) || []).forEach(function (f) { if (needsHandle(f) && out.indexOf(f.id) < 0) { out.push(f.id); } }); - return out; - } - function addr(f) { return !needsHandle(f) ? f.id : 'opened-' + (openedOrder().indexOf(f.id) + 1); } - function hashFinding() { - var h; - try { h = decodeURIComponent((window.location.hash || '').slice(1)); } catch (e) { return null; } - if (h.indexOf('f/') !== 0) { return null; } - var x = h.slice(2), m = /^(closed|opened)-(\d+)$/.exec(x); - var f = (m && hashId && findingById(hashId)) || - (m ? findingById(openedOrder()[Number(m[2]) - 1]) : findingById(x)); - return f || null; - } - function syncHash() { - var f = hashFinding(); - if (!f || !window.history || !window.history.replaceState) { return; } - hashId = f.id; - var to = '#f/' + addr(f); - if (window.location.hash !== to) { window.history.replaceState(null, '', to); } - } - function keepHash(change) { var at = hashFinding(); if (at) { hashId = at.id; } change(); syncHash(); } - // The genome map's veil (map.js): 'closed' while this page keeps the result closed, 'opened' once it was - // opened on purpose, 'open' when this page never closes it. A gated result (APOE, behind its warning) is - // closed on every page, an adult's too, until opened; a tag naming no finding of these results stays closed. + // The genome map's veil (map.js): 'opened' for a result to act on this page shows because it was opened on + // purpose, 'open' when this page never closes it. A gated result (APOE, behind its warning) is closed on every + // page, an adult's too, until opened; a tag naming no finding of these results stays closed. function mapVeil(id, gated) { if (gated) { return shown[id] ? 'opened' : 'closed'; } if (!closedPage()) { return 'open'; } @@ -516,39 +474,28 @@ if (!f) { return 'closed'; } return f.level === 'act' ? 'opened' : 'open'; } - // One switch for a result, from the map's chip or the card's "Show the result" (openFinding): opening it in - // one place opens it in both. Closing it closes the findings that were opened with it. + // One switch for a result, from the map's chip or the APOE fold: opening it in one place opens it in both. + // Closing a child's result to act on hands the page back to the gate (core.js closeAct), which closes it again. function setShown(id, on) { if (!on && U.closeAct(id)) { return; } - var f = findingById(id); shown[id] = !!on; redraw(true); } // Whether the page shows a finding of the map at this moment: its card's steps are done (a replay draws // the run's end results from its first frame). A row with no finding of these results (the map's own APOE // row) follows the card that holds its fold (the traits card): a replay before that card shows has no APOE. - // A finding this page keeps closed is not shown either, whatever row tag it shares with another result. function mapNow(id) { var f = findingById(id); if (!f) { var t = cards().filter(function (c) { return c.hidden; })[0]; return !!(t && gate(t) === 'show'); } var c = cardById(f.card); return !!(c && gate(c) === 'show'); } - // The results to act on this page keeps closed at this moment, in the page's order: the map's chip counts - // them, whether or not they have a place on the map. - function mapClosed() { - return []; - } - // The title of a result once the page shows it, for the map's "Opened in its section" chip; never a closed one's. - // The address of a result the map opened, for its "Opened in its section" link: never its id (addr()). - function mapHref(id) { var f = findingById(id); return f ? addr(f) : id; } - function mapTitle(id) { var f = findingById(id); return f ? f.title : ''; } // Whether a row of the map is a finding of these results: then the page's own veil alone decides it (map.js). function mapKnown(id) { return !!findingById(id); } // Whether a step of the run has ended at this moment (a replay before it has not): the map's layers follow it. function stepDone(step) { var s = stepsBy()[step]; return !s || s.state === 'done'; } // Without the map (a browser with no DecompressionStream, or a map the server could not build): the findings - // with a place, as a table, each through this page's veil first. + // with a place, as a table. function mapFallback() { return findingsNow().filter(function (f) { return f.viewer; }).map(function (f) { return [U.LEVEL_WORDS[f.level] || f.level, f.title, f.viewer]; @@ -556,7 +503,7 @@ } function findingArticle(f, big) { var row = el('div', { 'class': 'row' }); - var art = el('div', { 'class': 'finding', id: 'f-' + handle(f), 'data-finding': handle(f), 'data-level': f.level }); + var art = el('div', { 'class': 'finding', id: 'f-' + f.id, 'data-finding': f.id, 'data-level': f.level }); var at = newAt(f); if (at) { art.classList.add('new'); add(art, el('span', { 'class': 'newband' }, 'New at ' + U.hm(at))); add(art, el('br', {})); } add(art, U.findingLevel(f.level, f.level_words)); @@ -579,14 +526,14 @@ var rel = (f.related || []).map(findingById).filter(function (r) { return r; }); if (rel.length) { var box = el('div', {}); - rel.slice(0, 4).forEach(function (r, i) { if (i) { add(box, el('br', {})); } add(box, el('a', { href: '#f/' + addr(r) }, r.title)); }); + rel.slice(0, 4).forEach(function (r, i) { if (i) { add(box, el('br', {})); } add(box, el('a', { href: '#f/' + r.id }, r.title)); }); add(side, note('Linked', box, 'linked-note')); } add(row, art, side); return row; } function findingLine(f) { - var li = el('li', { id: 'f-' + handle(f), 'data-finding': handle(f), 'data-level': f.level }); + var li = el('li', { id: 'f-' + f.id, 'data-finding': f.id, 'data-level': f.level }); add(li, el('a', { 'class': 't', href: findingHref(f) }, f.title), U.findingLevel(f.level, f.level_words)); if (f.detail) { add(li, el('span', { 'class': 'gene' }, f.detail)); } add(li, el('span', { 'class': 'x' }, f.text)); @@ -712,7 +659,7 @@ // One line for a card: what it found, or why it has nothing to show yet. function cardLine(c) { var g = gate(c); - return g === 'show' ? ((c.headline) || (openFacts(c)[0][0] ? openFacts(c)[0][0][0] + ': ' + openFacts(c)[0][0][1] : NOT_ALL_CLEAR)) + return g === 'show' ? (c.headline || ((c.facts || [])[0] ? c.facts[0][0] + ': ' + c.facts[0][1] : NOT_ALL_CLEAR)) : pending(g) ? 'Waiting for ' + stepWords(c) + '.' : g === 'failed' ? 'Not assessed: its step failed.' : 'Not assessed in this run.'; } // A card the report flags that has no finding of its own: on a phone it is the only place that flag @@ -811,7 +758,7 @@ var a = el('a', { href: '#s' + x[1], 'data-level': x[0] }); add(a, U.level(x[0], { act: 'Take to a doctor', medicine: 'Prescriber', review: 'Worth a look', carrier: 'Family planning', info: 'Information' }[x[0]])); add(a, el('span', { 'class': 'n' }, String(n[x[0]]))); - var words = x[0] === 'info' ? 'Origins, traits, scores' : x[0] === 'act' && closed && n.act ? 'Closed until opened' : + var words = x[0] === 'info' ? 'Origins, traits, scores' : fs.filter(function (f) { return f.level === x[0]; }).slice(0, 2).map(function (f) { return f.gene || f.title; }).join(', ') || (fs.some(function (f) { return f.level === x[0]; }) ? 'Closed until opened' : 'None in what was assessed'); add(a, el('span', { 'class': 'w' }, words)); @@ -824,12 +771,12 @@ add(trust, U.level('trust', 'Data quality', problems.length ? 'problem' : '')); var qc = cardById('qc'); if (qc && gate(qc) === 'show') { - openFacts(qc)[0].filter(function (f) { return /depth|contamination|sex from/i.test(f[0]); }).slice(0, 3).forEach(function (f, i) { + (qc.facts || []).filter(function (f) { return /depth|contamination|sex from/i.test(f[0]); }).slice(0, 3).forEach(function (f, i) { if (i) { add(trust, el('span', { 'class': 'sep' }, '·')); } add(trust, el('span', { title: f[0] + ': ' + f[1] }, String(f[0]).replace(/\s*\([^)]*\)/g, '') + ': ' + String(f[1]).split(/[.;(]\s/)[0].replace(/[.;]$/, '').trim())); }); } else { add(trust, el('span', {}, 'Waiting for the quality steps.')); } - openProblems.forEach(function (f) { add(trust, el('span', { 'class': 'sep' }, '·'), el('a', { href: '#f/' + addr(f) }, f.title)); }); + openProblems.forEach(function (f) { add(trust, el('span', { 'class': 'sep' }, '·'), el('a', { href: '#f/' + f.id }, f.title)); }); add(trust, el('span', { 'class': 'sep' }, '·'), el('a', { href: '#s5' }, 'Can I trust this?')); add(main, trust); var side = el('aside', {}); @@ -838,10 +785,10 @@ add(row, main, side); add(slot, row); // Figure 1: the genome map (map.js), one figure kept across redraws so its camera stays where the - // reader left it. It asks this page's veil for each closed row, and opens or closes a result through it. + // reader left it. It asks this page's veil for each row, and opens or closes a result through it. if (window.PGPMap) { if (!genomeMap) { - genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, closed: mapClosed, title: mapTitle, href: mapHref, stepDone: stepDone, open: function (id) { setShown(id, true); }, + genomeMap = window.PGPMap.create({ url: 'map.json', veil: mapVeil, now: mapNow, known: mapKnown, stepDone: stepDone, open: function (id) { setShown(id, true); }, close: function (id) { setShown(id, false); }, fallback: mapFallback }); } add(slot, genomeMap.node); @@ -1036,7 +983,7 @@ if (h.indexOf('s/') === 0 || (h && !/^(s\d+|f\/.*|q\/.*|progress)$/.test(h) && cardById(h))) { target = document.getElementById('card-' + (h.indexOf('s/') === 0 ? h.slice(2) : h)); isCard = true; - } else if (h.indexOf('f/') === 0) { var hf = hashFinding(); target = document.getElementById('f-' + (hf ? handle(hf) : h.slice(2))); } + } else if (h.indexOf('f/') === 0) { target = document.getElementById('f-' + h.slice(2)); } else if (h.indexOf('q/') === 0) { target = document.getElementById('s' + (CHAPTER_SECTION[h.slice(2)] || 1)); } else { target = document.getElementById(h); } return target ? { target: target, isCard: isCard } : null; @@ -1056,18 +1003,12 @@ } } function route() { - syncHash(); var t = routeTarget(); if (!t) { return; } var target = t.target; openAround(t); target.scrollIntoView({ block: 'start' }); if (target.focus) { target.focus({ preventScroll: true }); } - markToc(target); - } - // The contents mark the section the address names; a redraw marks it again, so the mark never depends on - // whether the page drew again after the address was read. - function markToc(target) { Array.prototype.forEach.call(rail.querySelectorAll('[data-toc]'), function (a) { var sec = target.closest ? target.closest('section.sec') : null; a.setAttribute('aria-current', sec && sec.id === 's' + a.getAttribute('data-toc') ? 'true' : 'false'); @@ -1143,9 +1084,6 @@ // A gated result's fold follows its switch, whatever fold state the reader left (setShown). Array.prototype.forEach.call(panel.querySelectorAll('details[data-gate]'), function (d) { d.open = !!shown[d.getAttribute('data-gate')]; }); window.scrollTo(0, y); - syncHash(); - var rt = routeTarget(); - if (rt) { markToc(rt.target); } document.getElementById('progress-q').textContent = running() ? 'Is it moving, when will it end, is anything wrong? One line per step.' : 'One line per step: when it ran, and what it found.'; // The step timeline closes the report, folded once the page has a report to show, while the run goes diff --git a/bin/owngenome_live_results.py b/bin/owngenome_live_results.py index e3698175b..4e6cf8610 100644 --- a/bin/owngenome_live_results.py +++ b/bin/owngenome_live_results.py @@ -5,7 +5,9 @@ facts, its rows, its caveats and its attention flag, and the Journal's findings, levels, lead and ten sections GET /runs//finding// one finding's page (assets/live/finding.html): 404 unless - is a finding of that run's results.json now + is a finding of that run's results.json now; for a + child (child_run) with a result to act on, the page for + any well-formed id The summary is the one the server already reads for the step headlines (Live.summary): the run's summary.json when step 24 wrote it after every done @@ -56,7 +58,10 @@ def results(req, m): def finding(req, m): """One finding's page. The page itself reads ../../results.json; this only checks that the id is - a finding of the run now, so a typed or stale address is a plain 404, never an empty page.""" + a finding of the run now, so a typed or stale address is a plain 404, never an empty page. A child with a + result to act on gets the page for any well-formed id: the page keeps the child's report closed + (assets/live/core.js) until every such result is opened, and a 404 would tell a guessed id apart from a + result the child has.""" live = req.live # A page: without a session it sends the browser to the login and back, as the viewer does. if live.password_hash and req.sid is None: @@ -70,7 +75,9 @@ def finding(req, m): try: d = run["dir"] or os.path.join(live.root, run["name"]) summary = live.summary(run["name"], d, live.base_state(run)) - ok = bool(summary) and any(f["id"] == fid for f in live_results_mod().build(run["name"], summary)["findings"]) + found = live_results_mod().build(run["name"], summary)["findings"] if summary else [] + ok = any(f["id"] == fid for f in found) or ( + any(f["level"] == "act" for f in found) and child_run(live, run["name"])) except Exception as e: # a reader bug is a 404 here, never a 500 print(f"owngenome_live: finding of {run['name']} not checked: {type(e).__name__}", file=sys.stderr) if not ok: @@ -87,6 +94,20 @@ def finding(req, m): app.route("GET", r"/runs/(%s)/finding/([^/]{1,200})/" % NAME, finding, auth=False) +def child_run(live, name): + """Whether the person page treats NAME as a child's run: a family lists it with a father or a mother, or a + family's members cannot be read and no readable family lists NAME.""" + fams = [r for r in live.runs().values() if r.get("kind") == "family"] + listed = False + for f in fams: + for m in f.get("members") or []: + if m.get("sample") == name: + listed = True + if m.get("father") or m.get("mother"): + return True + return not listed and any(not (f.get("members") or []) for f in fams) + + def live_results_mod(): import live_results return live_results From d6b6dc6c8be1179dbe26d5a87f373b3366af04da Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:13:10 +0100 Subject: [PATCH 065/255] test(live): check the gate against main's requests, order, addresses and slow reloads - gate_transparency compares DOM, address, canvas and every request, with and without the gate, for an adult, a redraw at a section address, a finding page, a no-act child, the home page and a fresh adult with an act and a same-gene ranking; controls read a file twice or close an adult. - gate_order: an unopened act earlier or later in the report leaves the page unchanged, and the first closed button's act follows the load's draw (control: producer order). - gate_addresses and gate_slow_release: same page for any finding id, #f/ after release, and release with a 6 s results.json. - gate_controls reads each probe's outcome before restoring the payload (the stale-control race), and two new probes make a view receiving a closed run's file and the family view receiving a gated child's rows observable. - The family test checks the server's finding route for a child and an adult. --- tests/test_live_demo.py | 562 +++++++++++++++++++--------- tests/test_owngenome_live_family.py | 59 ++- 2 files changed, 442 insertions(+), 179 deletions(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 5795a2f86..7846fa8ef 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -7,12 +7,9 @@ """ import base64 import copy -import difflib import gzip -import hashlib import json import os -import re import shutil import subprocess import sys @@ -173,6 +170,15 @@ def end_headers(self): self.send_header("Cache-Control", "no-store") super().end_headers() + # A path answered after a wait, once (the gate's slow-reload check): {path: seconds}. + DELAY = {} + + def do_GET(self): + wait = Static.DELAY.pop(self.path.split("?")[0], 0) + if wait: + time.sleep(wait) + super().do_GET() + def browser_checks(out): chrome = find_chrome() @@ -358,7 +364,7 @@ def widget_checks(b, base, out): keep = f.read() try: with open(js, "w") as f: - f.write(keep.replace("add(text, widget(of[1]));", "")) + f.write(keep.replace("add(text, widget(c.widget));", "")) control("person.js without its widget line", not widget_problems(widget_counts(b, base))) finally: with open(js, "w") as f: @@ -837,87 +843,6 @@ def map_now(b): return json.loads(b.eval(MAP_JS)) -# The page's closed results to act on and the map's chips, read at once (a chip's button rebuilds the chips at -# once: nothing to wait for): the act cards in page order (ids: each one's id, its handle while it is closed) and -# the places of the closed ones among them (closed: a closed card names no id, so a result is known by its place), the act chip's count, whether its confirm step -# shows, where the keyboard is (first: the chips' first button; canvas: the map), and the opened chips. -CHIP_JS = r"""JSON.stringify((function () { - var ids = [], closed = []; - Array.prototype.forEach.call(document.querySelectorAll('.finding[data-level=act]'), function (a, i) { - ids.push(a.dataset.finding); - if (a.querySelector('.veil')) { closed.push(i); } }); - var c = document.querySelector('.gm-chip[data-closed=act]'), f = document.activeElement; - var first = document.querySelector('.gm-chips button'); - return { closed: closed, ids: ids, buttons: !!first, chip: c ? c.firstChild.textContent : '', confirm: !!(c && c.querySelector('.gm-confirm')), - focus: f && f.classList.contains('gm-yes') ? 'gm-yes' : f === first ? 'first' : f === document.querySelector('.gm-canvas') ? 'canvas' - : (f ? f.tagName + '.' + f.className : ''), - opened: Array.prototype.map.call(document.querySelectorAll('.gm-chip.opened'), function (o) { var a = o.querySelector('a'); - var raw = a ? a.getAttribute('href') : ''; - return { kind: o.dataset.opened, text: o.textContent, raw: raw, href: opened(raw) }; }) }; - // The finding an "Opened in its section" link names: #f/opened- is the n-th finding the page can close and - // shows opened, in the order of the sections (person.js openedOrder()). - function opened(raw) { - var m = /^#f\/opened-(\d+)$/.exec(raw); - if (!m) { return raw; } - if (!window.__pgpResults) { - var x = new XMLHttpRequest(); x.open('GET', 'results.json', false); x.send(); window.__pgpResults = JSON.parse(x.responseText); - } - var r = window.__pgpResults, tags = (r.veil && r.veil.findings) || {}, byId = {}, order = []; - r.findings.forEach(function (f) { byId[f.id] = f; }); - function ok(f) { return f && (f.level === 'act' || (tags[f.id] && tags[f.id] !== f.id)) && !!document.getElementById('f-' + f.id); } - (r.journal || []).forEach(function (s) { (s.findings || []).forEach(function (id) { if (ok(byId[id]) && order.indexOf(id) < 0) { order.push(id); } }); }); - r.findings.forEach(function (f) { if (ok(f) && order.indexOf(f.id) < 0) { order.push(f.id); } }); - return '#f/' + (order[Number(m[1]) - 1] || raw); - } -}()))""" - - -def chips_now(b): - time.sleep(0.05) # a chip's button draws the page and the chips again before its click returns - return json.loads(b.eval(CHIP_JS)) - - -def closed_words(n): - return "" if n == 0 else "1 result closed" if n == 1 else f"{n} results closed" - - -def chip_open(b): - """The act chip's Open, then its Show the result, as a keyboard reader does it. Returns (the id it opened, its - place among the act cards, problems): one Open opens exactly one of the page's closed results, the count falls by one, the keyboard - goes to Show the result and then to the chips' first button (the map when no chip is left), and an - "Opened in its section" chip never names a result that is still closed.""" - before = chips_now(b) - b.eval("var o = document.querySelector('.gm-chip[data-closed=act] button'); o.focus(); o.click(); true") - mid = chips_now(b) - bad = [] - if not mid["confirm"] or mid["closed"] != before["closed"]: - bad.append(f"Open: confirm step {mid['confirm']}, closed {before['closed']} -> {mid['closed']}") - if mid["focus"] != "gm-yes": - bad.append(f"Open: the keyboard is on {mid['focus']!r}, not Show the result") - b.eval("document.querySelector('.gm-chip[data-closed=act] .gm-yes').click(); true") - after = chips_now(b) - gone = [x for x in before["closed"] if x not in after["closed"]] - n = len(before["closed"]) - 1 - if len(gone) != 1 or len(after["closed"]) != n: - bad.append(f"Show the result opened {gone}, not one: closed {before['closed']} -> {after['closed']}") - if not after["chip"].startswith(closed_words(n)) or bool(after["chip"]) != (n > 0): - bad.append(f"Show the result: the chip reads {after['chip']!r}, want {closed_words(n)!r}") - if after["focus"] != ("first" if after["buttons"] else "canvas"): - bad.append(f"Show the result: the keyboard is on {after['focus']!r}") - named = [o["href"] for o in after["opened"] if o["kind"] == "section" and o["href"][3:] not in after["ids"]] - if named: - bad.append(f"an Opened in its section chip names a result that is not open: {named}") - raw = [o["raw"] for o in after["opened"] if o["kind"] == "section" and not re.match(r"^#f/opened-\d+$", o["raw"])] - if raw: - bad.append(f"an Opened in its section chip links a result by its id: {raw}") - leaked = [after["ids"][i] for i in after["closed"] if not re.match(r"^closed-\d+$", after["ids"][i] or "")] - if leaked: - bad.append(f"a closed card carries an id, not a handle: {leaked}") - if len(gone) != 1: - return None, None, bad - return after["ids"][gone[0]], gone[0], bad - - def parity_problems(b, base, rows): """The karyogram's marks the map does not place with the same level and chromosome within 0.01 Mb.""" bad = [] @@ -1098,8 +1023,7 @@ def missing_layer_problems(b, base, out): def fallback_problems(b, base, out): - """A browser with no DecompressionStream: the sentence and the findings with a place, each through the veil.""" - son = OTHERS[1] + """A browser with no DecompressionStream: the sentence and the findings with a place.""" b.call("Page.enable", session=b.session) # a script for each new document needs the Page domain on sid = b.call("Page.addScriptToEvaluateOnNewDocument", {"source": "delete window.DecompressionStream;"}, b.session)["identifier"] try: @@ -1254,6 +1178,46 @@ def map_checks(b, base, out): print("map screenshots: " + ", ".join(map_screens(b, base, os.environ["MAP_SCREENS"]))) +# Draws the same numbers in every load, so the order of the closed controls is the same in two loads that are +# compared; the order property (gate_order) runs without it. +SEEDED = "crypto.getRandomValues = function (a) { for (var i = 0; i < a.length; i++) { a[i] = %s; } return a; };" + + +def open_act(b, title, scope=''): + """Open the act whose own record has TITLE. A closed control opens an act the reader cannot choose, so each is + tried in turn and a wrong one is closed again. Returns the number of clicks.""" + closed, clicks = json.dumps(scope + '[data-opening^=closed]'), 0 + for k in range(10): + n = b.eval("document.querySelectorAll(%s).length" % closed) + assert n > k, 'no closed control opens ' + title + m = b.eval("document.querySelectorAll(%s).length" % json.dumps(scope + '[data-opening^=opened]')) + 1 + b.eval("document.querySelectorAll(%s)[%d].click(); true" % (closed, k)) + clicks += 1 + if n == 1: + return clicks + got = json.dumps(scope + '[data-finding=opened-%d] .opened-act h2' % m) + assert b.wait("!!document.querySelector(%s)" % got), 'an opening never showed' + if b.eval("document.querySelector(%s).textContent" % got) == title: + return clicks + b.eval("document.querySelector(%s).click(); true" % json.dumps(scope + '[data-opening=opened-%d]' % m)) + clicks += 1 + assert b.wait("document.querySelectorAll(%s).length === %d" % (closed, n)), 'closing again never settled' + raise AssertionError('no closed control opens ' + title) + + +# How far a run is, never what it found: the home and family pages show it for a gated child too (core.js PROGRESS). +PROGRESS = ['name', 'kind', 'url', 'state', 'started_utc', 'last_heard_utc', 'finished_utc', 'planned', 'done', 'running', 'failed'] + + +def progress(run): + return {k: run[k] for k in PROGRESS if k in run} + + +def resources(b, name): + return b.eval("performance.getEntriesByType('resource').filter(function (r) { return r.name.endsWith(%s); }).length" + % json.dumps(name)) + + def gate_checks(b, base, out, family_only=False): """A single ownership property over every payload, including fields added after this test.""" import itertools @@ -1267,8 +1231,10 @@ def gate_checks(b, base, out, family_only=False): result_path = os.path.join(out, 'runs', son, 'results.json') original = json.loads(saved[result_path]) acts = [f for f in original['findings'] if f['level'] == 'act'] - # Include placed, unplaced and multi-gene acts with stable identities. Other records are not controls. - acts = (acts + [dict(acts[0], id='gate-third', gene='HBA1', genes=['HBA1', 'HBA2'], viewer=None)])[:3] + # Placed, unplaced and multi-gene acts with stable identities and their own titles. Other records are not controls. + acts = (acts + [dict(acts[0], id='gate-third', gene='HBA1', genes=['HBA1', 'HBA2'], viewer=None, + title='Gate fixture result in HBA1', detail='HBA1 and HBA2')])[:3] + assert len({f['title'] for f in acts}) == 3, 'the acts need their own titles' def write(p, d): if p.endswith('/map.json'): @@ -1315,7 +1281,9 @@ def payloads(count, changed=False, keep_first=False): elif changed and p.endswith('/runs.json'): for i, run in enumerate(d['runs']): if run['name'] == son: - d['runs'][i] = dict(walk(run), name=son, url=run['url']) + d['runs'][i] = dict(walk(run), **dict(progress(run), name=son, url=run['url'])) + # How many results to act on is what the closed page may say. + d['runs'][i]['levels']['act'] = run['levels']['act'] elif changed and p.endswith('/family.json'): # One generic walk, then restore the other open owners and immutable routing controls. changed_family = walk(d) @@ -1329,7 +1297,11 @@ def payloads(count, changed=False, keep_first=False): for key in ('trio_files', 'trio_file_state'): if key in d: d[key] = changed_family[key] - d['state']['plan'] = changed_family['state']['plan'] + # Plan rows about the parents alone are theirs; a row naming the child is the child's. + d['state']['plan'] = [dict(changed_family['state']['plan'][j], members=r['members']) + if son in str(r.get('members', '')).split(',') else r + for j, r in enumerate(d['state']['plan'])] + d['state']['plan'].append(dict(analysis='future_child_analysis', members=son, decision='runs', reason='new')) d['state']['reports'] = changed_family['state']['reports'] for key in ('segments', 'trio_status'): if son in d.get(key, {}): @@ -1350,7 +1322,7 @@ def payloads(count, changed=False, keep_first=False): sb['rows'].append(copy.deepcopy(sb['rows'][0])) sb['rows'].reverse() if son in d.get('runs', {}): - d['runs'][son] = dict(changed_family['runs'][son], url=d['runs'][son]['url']) + d['runs'][son] = dict(changed_family['runs'][son], **dict(progress(d['runs'][son]), url=d['runs'][son]['url'])) d['future_child_payload'] = changed_family['future_payload_field'] d = {k: d[k] for k in reversed(d)} write(p, d) @@ -1388,6 +1360,8 @@ def snap(): f.write(raw.replace(b'', b'', 1)) b.call('Network.enable', {}, b.session) b.call('Network.setCacheDisabled', {'cacheDisabled': True}, b.session) + b.call('Page.enable', session=b.session) # scripts for each new document need the Page domain + seed = b.call('Page.addScriptToEvaluateOnNewDocument', {'source': SEEDED % '0'}, b.session)['identifier'] bad = [] try: b.eval(helper) @@ -1402,19 +1376,22 @@ def snap(): snapshots, partials = [], [] for changed in (False, True): payloads(3, changed); visit(route); snapshots.append(snap()) - if route != '/': + if route != '/' and '#' not in route: payloads(3, changed, keep_first=True); visit(route) - completed = b.eval("performance.getEntriesByType('resource').filter(function(r){return r.name.endsWith('/"+son+"/results.json');}).length") - b.eval("document.querySelector('[data-opening^=closed]').click(); true") - assert b.wait("!!document.querySelector('[data-opening=opened-1]')"), 'partial opening never settled' - if route == '/runs/'+FAM+'/': - assert b.wait("performance.getEntriesByType('resource').filter(function(r){return r.name.endsWith('/"+son+"/results.json');}).length > "+str(completed)), 'family projection refresh never settled' + scope = '[data-gate-child=%s] ' % son if route == '/runs/'+FAM+'/' else '' + completed = resources(b, '/'+son+'/results.json') + clicks = open_act(b, acts[0]['title'], scope) + if scope: + # Each click asks for the family again, and the gate for the child's results. + assert b.wait("performance.getEntriesByType('resource').filter(function(r){return r.name.endsWith('/"+son+"/results.json');}).length >= "+str(completed + clicks)), 'family projection refresh never settled' partials.append(snap()) - if snapshots[0] != snapshots[1]: - keys = [k for k in snapshots[0] if snapshots[0][k] != snapshots[1][k]] - bad.append((width, media, route, keys, [v['requests'] for v in snapshots] if 'requests' in keys else [])) - if partials and partials[0] != partials[1]: - bad.append((width, media, route, 'partially opened', [k for k in partials[0] if partials[0][k] != partials[1][k]])) + for what, pair in (('closed', snapshots), ('partially opened', partials)): + if pair and pair[0] != pair[1]: + keys = [k for k in pair[0] if pair[0][k] != pair[1][k]] + x, y = pair[0]['dom'], pair[1]['dom'] + at = next((i for i, (m, n) in enumerate(zip(x, y)) if m != n), min(len(x), len(y))) + bad.append((width, media, route, what, keys, x[max(0, at-60):at+60], y[max(0, at-60):at+60], + [v['requests'] for v in pair] if 'requests' in keys else [])) print(' gate matrix time: %.2f s' % (time.monotonic()-start)) check('gate property: every mutable payload produces identical DOM, AX text, address, pixels, SVG, storage and requests', not bad, bad) if not family_only: @@ -1426,15 +1403,16 @@ def snap(): for order in itertools.permutations(range(count)): visit('/runs/'+son+'/') for stage, i in enumerate(order): - before = snap() - b.eval("document.querySelectorAll('[data-opening]')[%d].click()" % i) if stage < count - 1: + before = snap() + open_act(b, acts[i]['title']) assert b.wait("document.documentElement.dataset.pgpGated === 'true'"), 'released early' - b.eval("document.querySelectorAll('[data-opening]')[%d].click()" % i) + b.eval("document.querySelector('[data-opening=opened-%d]').click(); true" % (stage + 1)) if snap() != before: transitions.append((count, order, stage, 'close-again differs')) - b.eval("document.querySelectorAll('[data-opening]')[%d].click()" % i) + open_act(b, acts[i]['title']) else: + b.eval("document.querySelector('[data-opening^=closed]').click(); true") assert b.wait("!document.documentElement.dataset.pgpGated && !!document.querySelector('#results .finding')"), 'last opening never released' # Search something while open; closing must replace this whole document and its caches. b.wait("document.querySelector('input[type=search]') && !document.querySelector('input[type=search]').disabled") @@ -1465,8 +1443,10 @@ def snap(): assert b.wait("location.hash === '#f/closed-3'") b.eval("document.querySelector('[data-opening=closed-1]').click(); true") check('gate handles follow their act after another act opens', b.eval("location.hash") == '#f/closed-2') - + gate_addresses(b, base, out, acts, visit, snap) + gate_slow_release(b, base, acts, visit, payloads) finally: + b.call('Page.removeScriptToEvaluateOnNewDocument', {'identifier': seed}, b.session) for p, raw in dict(saved, **html_saved).items(): with open(p, 'wb') as f: f.write(raw) @@ -1475,11 +1455,126 @@ def snap(): if family_only: gate_family_independence(b, base, out) if not family_only: + gate_order(b, base, out) gate_controls(b, base, out) gate_transparency(b, base, out) print('gate browser time: %.2f s' % (time.monotonic()-start)) +def gate_addresses(b, base, out, acts, visit, snap): + """A gated child's finding address answers the same page for any id, and keeps no id in the address; once the + report opens, its addresses are the full report's (#f/).""" + son = OTHERS[1] + src = os.path.join(out, 'runs', son, 'finding', acts[0]['id'], 'index.html') + page = open(src).read() + made, seen = [], [] + try: + # The live server answers every well-formed id of a gated child with this page (bin/owngenome_live_results.py). + for fid in ('two-hits-cftr', 'zz-not-a-finding'): + d = os.path.join(out, 'runs', son, 'finding', fid) + os.makedirs(d) + made.append(d) + with open(os.path.join(d, 'index.html'), 'w') as f: + f.write(page.replace(acts[0]['id'], fid)) + for fid in (acts[0]['id'], 'two-hits-cftr', 'zz-not-a-finding'): + visit('/runs/'+son+'/finding/'+fid+'/') + s = snap() + seen.append({k: s[k] for k in ('dom', 'text', 'title', 'address', 'accessibility')}) + finally: + for d in made: + shutil.rmtree(d) + check('gate addresses: a gated finding page is the same for an act, a guessed act and nonsense, with no id in the address', + all(x == seen[0] for x in seen) and acts[0]['id'] not in seen[0]['address'], [x['address'] for x in seen]) + visit('/runs/'+son+'/') + open_gate(b) + assert b.wait("!!document.querySelector('#results .finding')"), 'full report never loaded' + fid = acts[0]['id'] + got = b.eval("[!!document.querySelector('a[href=\"#f/%s\"]'), !!document.querySelector('a[href^=\"#f/opened-\"],a[href^=\"#f/closed-\"]')]" % fid) + b.eval("location.hash = 'f/%s'; true" % fid) + kept = b.wait("location.hash === '#f/%s' && !!document.getElementById('f-%s')" % (fid, fid), timeout=5) + check('gate addresses: the opened report links and keeps #f/, as the full report does', got == [True, False] and kept, (got, kept)) + + +def gate_slow_release(b, base, acts, visit, payloads): + """Opening the last act releases the report however slowly the reloaded page's results arrive.""" + son = OTHERS[1] + payloads(2) + visit('/runs/'+son+'/') + open_act(b, acts[0]['title']) + Static.DELAY['/runs/'+son+'/results.json'] = 6 + try: + b.eval("document.querySelector('[data-opening^=closed]').click(); true") + released = b.wait("!document.documentElement.dataset.pgpGated && !!document.querySelector('#results .finding')", timeout=25) + finally: + Static.DELAY.clear() + check('gate release: a reload whose results.json takes 6 s still releases the report', released, released) + + +def gate_order(b, base, out): + """Only an unopened act changes, its rule and gene and so its place in the producer's order; the opened act is the + same. The pages must not differ, and which act a closed control opens comes from a draw of this page load.""" + son = OTHERS[1] + rj = os.path.join(out, 'runs', son, 'results.json') + raw = open(rj).read() + data = json.loads(raw) + acts = [f for f in data['findings'] if f['level'] == 'act'] + others = [f for f in data['findings'] if f['level'] != 'act'] + opened, closed = acts[0], acts[1] + tp53 = dict(closed, id='cpsr-tp53-c-743g-a', gene='TP53', genes=['TP53'], title='Cancer predisposition (TP53)', + detail='TP53 c.743G>A', viewer='chr17:7674220') + # The producer lists findings by rule and gene: the KCNH2 act before a later act, a TP53 CPSR act before it. + fixtures = {'later': [opened, closed], 'earlier': [tp53, opened]} + pages = {} + try: + for name, order in fixtures.items(): + with open(rj, 'w') as f: + json.dump(dict(data, findings=order + others, veil=dict(data.get('veil') or {}, acts=[x['id'] for x in order])), f) + for route, scope in (('/runs/'+son+'/', ''), ('/runs/'+FAM+'/', '[data-gate-child=%s] ' % son)): + b.go('about:blank'); assert b.wait("location.href === 'about:blank'") + b.go(base + route) + assert b.wait("!!document.querySelector(%s)" % json.dumps(scope + '[data-opening^=closed]')), route + ' not gated' + completed = resources(b, '/'+son+'/results.json') + clicks = open_act(b, opened['title'], scope) + assert b.wait("!!document.querySelector(%s)" % json.dumps(scope + '.opened-act')), 'opening never settled' + if scope: + assert b.wait("performance.getEntriesByType('resource').filter(function(r){return r.name.endsWith('/"+son+"/results.json');}).length >= "+str(completed + clicks)), 'family refresh never settled' + time.sleep(0.3) + pages[(name, route)] = b.eval("[document.documentElement.outerHTML, document.body.innerText, location.href, document.title]") + bad = [r for r in ('/runs/'+son+'/', '/runs/'+FAM+'/') if pages[('later', r)] != pages[('earlier', r)]] + check('gate order: an unopened act earlier or later in the report leaves the page with an opened act unchanged', not bad, bad) + + def first_opens(draw): + seed = b.call('Page.addScriptToEvaluateOnNewDocument', {'source': SEEDED % draw}, b.session)['identifier'] + try: + b.go('about:blank'); assert b.wait("location.href === 'about:blank'") + b.go(base + '/runs/'+son+'/') + assert b.wait("!!document.querySelector('[data-opening=closed-1]')"), 'not gated' + b.eval("document.querySelector('[data-opening=closed-1]').click(); true") + assert b.wait("!!document.querySelector('[data-finding=opened-1] .opened-act h2')") + return b.eval("document.querySelector('[data-finding=opened-1] .opened-act h2').textContent") + finally: + b.call('Page.removeScriptToEvaluateOnNewDocument', {'identifier': seed}, b.session) + + def draws(): + return {first_opens('0'), first_opens('4294967295')} + got = draws() + check('gate order: which act the first closed control opens comes from the page load, not the report', len(got) == 2, got) + js = os.path.join(out, 'assets', 'core.js') + source = open(js).read() + anchor = ': shuffled(ids);' + assert source.count(anchor) == 1, 'mutation anchor moved: shuffled' + try: + with open(js, 'w') as f: + f.write(source.replace(anchor, ': ids.slice();')) + control('gate order: closed controls in the producer order', len(draws()) == 2) + finally: + with open(js, 'w') as f: + f.write(source) + finally: + with open(rj, 'w') as f: + f.write(raw) + + def gate_controls(b, base, out): """Each gate guard must fail when its production implementation is deliberately broken.""" son = OTHERS[1] @@ -1489,16 +1584,20 @@ def gate_controls(b, base, out): data = json.loads(raw); data['future_payload_field'] = 'FUTURE_GATE_SENTINEL' with open(rj, 'w') as f: json.dump(data, f) + seen = {} - def visit(): + def visit(route=None): b.go('about:blank'); assert b.wait("location.href === 'about:blank'") - b.go(base+'/runs/'+son+'/') - assert b.wait("document.documentElement.dataset.pgpGated === 'true'"), 'fixture never gated' + b.go(base+(route or '/runs/'+son+'/')) + if not route: + assert b.wait("document.documentElement.dataset.pgpGated === 'true'"), 'fixture never gated' def partial(): visit() b.eval("document.querySelector('[data-opening^=closed]').click(); true") - return bool(b.wait("document.documentElement.dataset.pgpGated === 'true' && document.querySelectorAll('[data-opening^=closed]').length === 1", timeout=2)) + ok = bool(b.wait("document.documentElement.dataset.pgpGated === 'true' && document.querySelectorAll('[data-opening^=closed]').length === 1", timeout=2)) + seen['released'] = bool(b.wait("!!document.documentElement.dataset.pgpResults && !document.documentElement.dataset.pgpGated", timeout=10)) if not ok else False + return ok def raw_view(): visit() @@ -1508,13 +1607,37 @@ def raw_view(): assert b.wait("document.documentElement.dataset.gateRawRead === 'done'", timeout=2), 'raw probe did not settle' return not b.eval("document.documentElement.outerHTML.includes('FUTURE_GATE_SENTINEL')") + def handoff(): + # A view that asks a closed run for its files gets no answer at all, whatever it reads the body with. + visit() + b.eval("window.gateRaw = null; fetch('results.json').then(function (r) { return r.text(); }).then(function (t) { window.gateRaw = t.length; }); true") + got = b.wait("window.gateRaw", timeout=2) + seen['handed'] = bool(got) + return not got + + def family_rows(): + # The family view never receives a gated child's trio rows, even ones it would not draw. + visit('/runs/'+FAM+'/') + assert b.wait("!!document.querySelector('[data-gate-child=%s]')" % son), 'family never gated' + b.eval("window.gateRows = null; PGPUI.getJSON('family.json').then(function (d) { window.gateRows = " + "(d.de_novo || []).concat(d.compound_het || []).filter(function (r) { return r.child === %s; }).length; }); true" + % json.dumps(son)) + assert b.wait("window.gateRows !== null"), 'family read never settled' + n = b.eval("window.gateRows") + seen['rows'] = n + return n == 0 + def cleanup(): visit(); open_gate(b) assert b.wait("!!document.querySelector('#results .finding')"), 'full report never loaded' assert b.wait("document.querySelector('input[type=search]') && !document.querySelector('input[type=search]').disabled"), 'search never loaded' b.eval("var s=document.querySelector('input[type=search]'); s.value='KCNH2'; s.dispatchEvent(new Event('input')); true") fid = next(f['id'] for f in data['findings'] if f['level']=='act') - return b.eval('PGPUI.closeAct('+json.dumps(fid)+"); PGPUI.gateOwnsPage() && !document.querySelector('input[type=search],.search-results,.search-status,canvas,svg,img')") + # Read in the same turn as the close: the document replacement that follows would hide a kept query. + got = b.eval('PGPUI.closeAct('+json.dumps(fid)+"); var s = document.querySelector('input[type=search]');" + "[PGPUI.gateOwnsPage() && !document.querySelector('input[type=search],.search-results,.search-status,canvas,svg,img'), s ? s.value : '']") + seen['query'] = got[1] + return got[0] def address(): b.go('about:blank'); assert b.wait("location.href === 'about:blank'") @@ -1523,18 +1646,25 @@ def address(): return b.eval("location.search === '' && location.hash === '#f/closed-1'") def stale_control(): - b.go('about:blank'); assert b.wait("location.href === 'about:blank'") - b.go(base+'/runs/'+FAM+'/') - assert b.wait("!!document.querySelector('[data-gate-child=DEMO-003] [data-opening^=closed]')") - b.eval("window.gateOldControl=document.querySelector('[data-gate-child=DEMO-003] [data-opening^=closed]'); true") + visit('/runs/'+FAM+'/') + assert b.wait("!!document.querySelector('[data-gate-child=%s] [data-opening^=closed]')" % son) + b.eval("window.gateOldControl=document.querySelector('[data-gate-child=%s] [data-opening^=closed]'); true" % son) changed = copy.deepcopy(data) - next(f for f in changed['findings'] if f['level']=='act')['text'] += ' REPLACEMENT_ACT_RECORD' + for f in changed['findings']: + if f['level'] == 'act': + f['text'] += ' REPLACEMENT_ACT_RECORD' try: with open(rj, 'w') as f: json.dump(changed, f) b.eval("window.gateRefreshDone=false; PGPUI.getJSON('family.json').then(function(){window.gateRefreshDone=true;}); true") assert b.wait('window.gateRefreshDone'), 'family refresh never settled' - return b.eval("gateOldControl.click(); !document.querySelector('[data-gate-child=DEMO-003] .opened-act')") + b.eval("gateOldControl.click(); true") + # Read the outcome while the replacement is still what the server answers: the redraw the click starts + # reads results.json again. + opened = b.wait("!!document.querySelector('[data-gate-child=%s] .opened-act')" % son, timeout=1.5) + seen['replacement'] = bool(opened) and b.eval( + "document.querySelector('[data-gate-child=%s] .opened-act').textContent.includes('REPLACEMENT_ACT_RECORD')" % son) + return not opened finally: with open(rj, 'w') as f: json.dump(data, f) @@ -1551,36 +1681,48 @@ def refresh_cleanup(): json.dump(changed, f) b.eval("PGPUI.getJSON('results.json'); true") assert b.wait('window.gateTransferStarted'), 'refresh never reached document replacement' - return b.eval("PGPUI.gateOwnsPage() && !document.querySelector('#results,canvas')") + ok = b.eval("PGPUI.gateOwnsPage() && !document.querySelector('#results,canvas')") + seen['report'] = b.eval("!!document.querySelector('#results .finding')") + return ok finally: with open(rj, 'w') as f: json.dump(data, f) + gate_open = "if (ownsPage || closedRun(requestURL)) { return new Promise(function () {}); }" + gate_late = "return got.then(function (response) {\n if (ownsPage) { return new Promise(function () {}); }" mutations = [ - ('stale control opens a replacement act', 'if (!f || JSON.stringify(f) !== JSON.stringify(rec.acts[i]))', 'if (!f)', stale_control), - ('refresh leaves the old report visible', 'if (shut(current)) { ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer();', 'if (shut(current)) { transfer();', refresh_cleanup), - ('keep a map query in the closed address', 'location.pathname + to', 'location.pathname + location.search + to', address), - ('release one act early', "rec.acts.some(function (f) { return !rec.opened[f.id]; })", "rec.acts.filter(function (f) { return !rec.opened[f.id]; }).length > 1", partial), - ('one view reads the raw payload', 'add(document.body, wrap);', "add(document.body, wrap); jsonAt(current.url + 'results.json').then(function (r) { add(wrap, el('p', {}, JSON.stringify(r))); document.documentElement.setAttribute('data-gate-raw-read', 'done'); });", raw_view), - ('skip cleanup after closing', 'delete current.opened[id]; ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return true;', 'delete current.opened[id]; return true;', cleanup)] + ('stale control opens a replacement act', ['if (!same(byId(active, id), byId(rec, id))) { changed(); return; }'], [''], stale_control, + lambda: seen.get('replacement')), + ('refresh leaves the old report visible', ['if (shut(current)) { ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer();'], + ['if (shut(current)) { transfer();'], refresh_cleanup, lambda: seen.get('report')), + ('keep a map query in the closed address', ['location.pathname + to'], ['location.pathname + location.search + to'], address, lambda: True), + ('release one act early', ["rec.acts.some(function (f) { return !rec.opened[f.id]; })"], + ["rec.acts.filter(function (f) { return !rec.opened[f.id]; }).length > 1"], partial, lambda: seen.get('released')), + ('one view reads the raw payload', ['add(document.body, wrap);'], + ["add(document.body, wrap); jsonAt(current.url + 'results.json').then(function (r) { add(wrap, el('p', {}, JSON.stringify(r))); document.documentElement.setAttribute('data-gate-raw-read', 'done'); });"], + raw_view, lambda: True), + ('a view receives a closed run file', [gate_open, gate_late], ['', 'return got.then(function (response) {'], handoff, + lambda: seen.get('handed')), + ('the family view receives a gated child\'s trio rows', ['copy((raw[key] || []).filter(function (r) { return !gated(r.child); }))'], + ['copy(raw[key] || [])'], family_rows, lambda: seen.get('rows')), + ('skip cleanup after closing', ['ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return true;'], + ['return true;'], cleanup, lambda: seen.get('query') == 'KCNH2')] try: - for desc, before, after, probe in mutations: - assert source.count(before)==1, 'mutation anchor moved: '+desc + for desc, befores, afters, probe, broke in mutations: + for before in befores: + assert source.count(before) == 1, 'mutation anchor moved: '+desc + seen.clear() check('gate guard: '+desc, probe()) + planted = source + for before, after in zip(befores, afters): + planted = planted.replace(before, after) with open(js, 'w') as f: - f.write(source.replace(before, after)) + f.write(planted) + seen.clear() still_passes = probe() control('gate: '+desc, still_passes) - if desc == 'stale control opens a replacement act': - assert b.eval("document.querySelector('[data-gate-child=DEMO-003] .opened-act').textContent.includes('REPLACEMENT_ACT_RECORD')"), 'control failed without opening the replacement' - elif desc == 'refresh leaves the old report visible': - assert b.eval("!!document.querySelector('#results .finding')"), 'control failed without retaining the report' - elif desc == 'release one act early': - assert b.eval("!!document.documentElement.dataset.pgpResults && !document.documentElement.dataset.pgpGated"), 'control failed without releasing the report' - elif desc == 'skip cleanup after closing': - assert b.eval("document.querySelector('input[type=search]').value === 'KCNH2'"), 'control failed without retaining the query' + assert still_passes or broke(), 'control failed without the broken outcome: '+desc print(' observed guard failure: '+desc) - with open(js, 'w') as f: f.write(source) finally: @@ -1590,64 +1732,136 @@ def refresh_cleanup(): f.write(raw) +# A summary that gives an adult a result to act on (BRCA1) and two symptom rankings, one of the same gene: the +# home page's top entries and the person page come from a fresh producer run, not from the demo's files. +SYNTH = {"sections": { + "cpsr": {"state": "ok", "data": {"plp": [{"gene": "BRCA1", "variant": "c.68_69del", "final": "Pathogenic", + "clinvar": "Pathogenic"}]}}, + "exomiser": {"state": "ok", "data": {"genes": [{"gene": "BRCA1", "rank": 1, "exomiser_score": 0.9, "moi": "AD"}, + {"gene": "TTN", "rank": 2, "exomiser_score": 0.8, "moi": "AR"}]}}, + "clinvar": {"state": "ok", "data": {"hits": []}}, + "coverage": {"state": "ok", "data": {"mean_depth": 31.2}}}} + + def gate_transparency(b, base, out): - """The gate must leave adult and no-act documents byte-equal to the unmediated full renderer.""" + """The gate leaves adult and no-act pages equal to the renderer without it: DOM, attributes, address, canvas and + the requests, in order and counted.""" + son = OTHERS[1] js = os.path.join(out, 'assets', 'core.js') - rj = os.path.join(out, 'runs', OTHERS[1], 'results.json') - source = open(js).read(); original = open(rj).read() + files = [os.path.join(out, 'runs', S, 'results.json'), os.path.join(out, 'runs', son, 'results.json'), + os.path.join(out, 'runs.json')] + source = open(js).read() + saved = {p: open(p).read() for p in files} # The same current renderer with the loader absent is the transparent reference. - bare = source[:source.index(' // The report gate owns raw payloads.')] + '''\n PGPUI.ready = Promise.resolve(); + marker = ' // --- the report gate' + assert source.count(marker) == 1, 'gate marker moved' + bare = source[:source.index(marker)] + ''' + PGPUI.ready = Promise.resolve(); PGPUI.gateOwnsPage = function () { return false; }; PGPUI.openedActs = function () { return {}; }; PGPUI.closeAct = function () { return false; }; }()); ''' - helper = open(os.path.join(REPO, 'tests', 'demo', 'gate-property.js')).read() - def capture(run, timeout=20): + def capture(route, apoe=False, timeout=20): b.go('about:blank'); assert b.wait("location.href === 'about:blank'") - b.go(base+'/runs/'+run+'/') - assert b.wait("document.documentElement.dataset.pgpResults && !!document.querySelector('[data-pgp-map=drawn]')", timeout=timeout), 'open report not drawn' - assert b.wait("!!document.querySelector('#rail a[href*=DEMO-FAMILY]') && !!document.documentElement.dataset.pgpCharts"), 'family or charts never settled' - time.sleep(0.2) - b.eval(helper) - return b.eval("""(function () { + b.go(base + route) + if route == '/': + assert b.wait("document.documentElement.dataset.pgpStatus === 'ready' && !!document.documentElement.dataset.pgpPeople", timeout=timeout), 'home never drawn' + elif '/finding/' in route: + assert b.wait("document.documentElement.dataset.pgpStatus === 'ready' && !!document.querySelector('h1')", timeout=timeout), 'finding never drawn' + else: + assert b.wait("document.documentElement.dataset.pgpResults && !!document.querySelector('[data-pgp-map=drawn]')", timeout=timeout), 'open report not drawn' + assert b.wait("!!document.querySelector('#rail a[href*=DEMO-FAMILY]') && !!document.documentElement.dataset.pgpCharts"), 'family or charts never settled' + if apoe: + # A redraw while the address names a place: the contents rail must come back as it did. + b.eval("document.querySelector('details[data-gate=apoe] summary').click(); true") + time.sleep(0.5) + got = b.eval("""(function () { var clone=document.documentElement.cloneNode(true); clone.querySelectorAll('[data-pgp-map-paint-ms],[data-pgp-map-parse-ms]').forEach(function (n) { n.removeAttribute('data-pgp-map-paint-ms'); n.removeAttribute('data-pgp-map-parse-ms'); }); - return [clone.outerHTML, location.hash, Array.from(document.querySelectorAll('canvas')).map(function(c){return c.toDataURL();})]; + return [clone.outerHTML, location.hash, Array.from(document.querySelectorAll('canvas')).map(function(c){return c.toDataURL();}), + performance.getEntriesByType('resource').map(function (r) { return new URL(r.name).pathname; })]; }())""") + # Scripts and the views' own files load in parallel, in any order, with or without the gate: count every + # request, and keep the order of the files the gate reads first. + got[3] = [sorted(got[3]), [p for p in got[3] if p.endswith(('/state.json', '/frames.json', '/family.json', '/results.json'))]] + return got + + def write(p, d): + with open(p, 'w') as f: + json.dump(d, f) + + def no_act(): + d = json.loads(saved[files[1]]); d['findings'] = [f for f in d['findings'] if f['level'] != 'act'] + d['levels']['act'] = 0 + write(files[1], d) + idx = json.loads(saved[files[2]]) + for r in idx['runs']: + if r['name'] == son: + r['levels']['act'] = 0 + r['top'] = [t for t in r['top'] if t['level'] != 'act'] + r['lead'] = r['lead_closed'] = 'No finding in what these steps assessed needs a doctor.' + write(files[2], idx) + + def synthetic(): + no_act() + write(files[0], live_results.build(S, SYNTH)) + o = live_results.overview(SYNTH) + idx = json.load(open(files[2])) + for r in idx['runs']: + if r['name'] == S: + r.update(attention=o['attention'], levels=o['levels'], top=o['top'], lead=o['lead'], + lead_closed=o['lead_closed'], screened=o['screened']) + write(files[2], idx) + + first = next(f['id'] for f in json.loads(saved[files[0]])['findings'] if f['level'] != 'act') + cases = [('adult', None, '/runs/'+S+'/', False), ('adult, a redraw at a section address', None, '/runs/'+S+'/#s3', True), + ('adult finding page', None, '/runs/'+S+'/finding/'+first+'/', False), + ('child with no act', no_act, '/runs/'+son+'/', False), ('home with no gated child', no_act, '/', False), + ('fresh adult with an act and a same-gene ranking', synthetic, '/runs/'+S+'/', False)] try: - for label, run in (('adult', S), ('child with no act', OTHERS[1])): - if run == OTHERS[1]: - data=json.loads(original); data['findings']=[f for f in data['findings'] if f['level'] != 'act']; data['veil']={'acts':[]} - with open(rj, 'w') as f: - json.dump(data, f) - pair=[] + for label, prep, route, apoe in cases: + for p in files: + with open(p, 'w') as f: + f.write(saved[p]) + if prep: + prep() + pair = [] for text in (bare, source): with open(js, 'w') as f: f.write(text) - pair.append(capture(run)) - detail = [i for i in range(3) if pair[0][i] != pair[1][i]] - if 0 in detail: - at = next((i for i,(a,z) in enumerate(zip(pair[0][0],pair[1][0])) if a!=z),0) - detail += [pair[0][0][max(0,at-80):at+120], pair[1][0][max(0,at-80):at+120]] - check('gate transparency: '+label+' has identical DOM, attributes, address and canvas', pair[0] == pair[1], detail) - assert source.count('ownsPage = shut(current);') == 1 - with open(js, 'w') as f: - f.write(source.replace('ownsPage = shut(current);', 'ownsPage = true;')) - try: - still_equal = capture(OTHERS[1], timeout=2) == pair[1] - except AssertionError: - still_equal = False - control('gate closing an adult or no-act report', still_equal) - + pair.append(capture(route, apoe)) + detail = [('dom', 'hash', 'canvas', 'requests')[i] for i in range(4) if pair[0][i] != pair[1][i]] + if pair[0][3] != pair[1][3]: + detail += [pair[0][3], pair[1][3]] + if pair[0][0] != pair[1][0]: + at = next((i for i, (a, z) in enumerate(zip(pair[0][0], pair[1][0])) if a != z), 0) + detail += [pair[0][0][max(0, at-80):at+120], pair[1][0][max(0, at-80):at+120]] + check('gate transparency: '+label+' has identical DOM, attributes, address, canvas and requests', pair[0] == pair[1], detail) + for p in files: + with open(p, 'w') as f: + f.write(saved[p]) + no_act() + reference = capture('/runs/'+son+'/') + for desc, before, after in (('closing an adult or no-act report', 'ownsPage = shut(current);', 'ownsPage = true;'), + ('reading a file twice', 'var kept = shared[requestURL];', 'var kept = null;')): + assert source.count(before) == 1, 'mutation anchor moved: '+desc + with open(js, 'w') as f: + f.write(source.replace(before, after)) + try: + still_equal = capture('/runs/'+son+'/', timeout=4) == reference + except AssertionError: + still_equal = False + control('gate transparency: '+desc, still_equal) finally: with open(js, 'w') as f: f.write(source) - with open(rj, 'w') as f: - f.write(original) + for p in files: + with open(p, 'w') as f: + f.write(saved[p]) def gate_family_independence(b, base, out): diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index e013613f5..f7f419ba4 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -27,8 +27,10 @@ drawing, a missing trio file and one linking out of the folder are worded apart (the link's target never shown), and the side by side lists every blood group system with a stale value marked in its cell; - 7. the demo family's veil checks and every mutation control from test_live_demo.py, - run here in Guard shard 0 so the person-page suite in shard 2 stays below its time limit. + 7. the demo family's report-gate property and two children opening on their own (test_live_demo.py's + gate_checks on the family page), here so that browser time is not added to test_live_demo.py's; + 8. the real server answers every finding id of a child with a result to act on with the same page, and keeps + the 404 for an adult's id that is not a finding. Checks 2 and 4 are run again on a copy of the fixture with two parents swapped and must fail there: a check that cannot fail is no check. @@ -349,6 +351,7 @@ def get(path): check("the real server does not serve notes.txt", code == 404, code) code, _, body = get("/runs/family_a/") check("the family page loads family.js", code == 200 and b"/assets/plugins/family.js" in body, code) + finding_route_checks(live, get, root) browser_checks(port, os.path.dirname(root)) finally: web.shutdown() @@ -356,6 +359,49 @@ def get(path): web.server_close() +# A summary with one result to act on (BRCA1): what child_a and loner (an adult in no family) both get below. +ACT_SUMMARY = {"sections": {"cpsr": {"state": "ok", "data": {"plp": [ + {"gene": "BRCA1", "variant": "c.68_69del", "final": "Pathogenic", "clinvar": "Pathogenic"}]}}, + "clinvar": {"state": "ok", "data": {"hits": []}}}} + + +def finding_route_checks(live, get, root): + """A child with a result to act on: every well-formed finding id answers the same page, so a guessed address + cannot tell whether the child has that result. An adult keeps the plain 404 for an id that is not a finding.""" + import live_results + status = os.path.join(root, "loner", "logs", "run_status.tsv") + with open(status, "w") as f: + f.write("meta\tstarted_epoch\t1767347940\nstep\t06\tplanned\n") + real = live.summary + live.summary = lambda name, d, st: ACT_SUMMARY if name in ("child_a", "loner") else real(name, d, st) + act = next(f["id"] for f in live_results.build("child_a", ACT_SUMMARY)["findings"] if f["level"] == "act") + + def answers(run): + # The page names the id it was asked for (data-finding), as the address does: the rest is the same. + return [(lambda r: (r[0], r[2].replace(x.encode(), b"ID")))(get("/runs/%s/finding/%s/" % (run, x))) + for x in (act, "two-hits-cftr", "zz-not-a-finding")] + try: + child, adult = answers("child_a"), answers("loner") + check("a child's finding route answers the same page for its act, a guessed act and nonsense", + [c[0] for c in child] == [200] * 3 and len({c[1] for c in child}) == 1, [c[0] for c in child]) + check("an adult's finding route: the page for its finding, 404 for any other id", + [c[0] for c in adult] == [200, 404, 404], [c[0] for c in adult]) + # The server loads its plugins itself: patch every loaded copy of the module. + mods = [m for m in list(sys.modules.values()) if getattr(m, "child_run", None) is not None and hasattr(m, "register")] + keep = [m.child_run for m in mods] + for m in mods: + m.child_run = lambda *a: False + try: + control("a server that tells a child's guessed finding id apart", + bool(mods) and [c[0] for c in answers("child_a")] == [200] * 3) + finally: + for m, k in zip(mods, keep): + m.child_run = k + finally: + live.summary = real + os.remove(status) + + def browser_checks(port, work): sys.path.insert(0, os.path.join(REPO, "tests")) import test_owngenome_live as tpl @@ -376,9 +422,12 @@ def browser_checks(port, work): loaded = b.wait("var i = document.querySelector('img.pedigree'); i && i.complete && i.naturalWidth > 0 && " "i.naturalWidth", 10) check("browser: the pedigree image loads", loaded, loaded) - # Unreadable child reports cannot be opened a place at a time under the data gate. - check("browser: unreadable child reports keep the couple screen and chromosome image closed", - b.eval("!!document.querySelector('.report-gate') && !document.querySelector('#ch-couples table') && !document.querySelector('img.segments')")) + # child_a has a run whose results cannot be read: closed whole, chromosome image included. Its parents have + # no run here, so nothing of theirs is closed: their couple screen shows. + got = b.wait("var g = document.querySelector('[data-gate-child=child_a]'); " + "[!!g, !!document.querySelector('img.segments'), !!document.querySelector('#ch-couples table')].join()", 10) + check("browser: child_a's unreadable report stays closed whole; its parents' couple screen shows", + got == "true,false,true", got) blocked = b.eval("document.documentElement.dataset.pgpBlocked || ''") check("browser: the policy blocked nothing on the family page", not blocked, blocked) depth_checks(b, base, os.path.join(work, "genome")) From b172b7ee136f21d9a059c82c6d7ae9aadc19a9bf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:13:10 +0100 Subject: [PATCH 066/255] docs(live): describe the report gate and restore the unrelated paragraphs The live-site page is main's again apart from the gate: what a gated child sees, how buttons and addresses work, the opening ticket, the cost, the home and family pages, the fetch and EventSource boundary and the static files. The stopped-run, running-band and runs.json/results.json paragraphs are back. --- CHANGELOG.md | 4 ++-- docs/live-site.md | 40 ++++++++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd642a3bb..73b131764 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,7 +125,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Each output is written whole or not at all: a run that stops leaves no part-written table behind. A read-only `family_plan.tsv` is replaced; a folder that cannot be written still stops the plan. - Step 38, step 39 with a families file, `run-all.sh` with `FAMILIES` and the family workflow now need `og` (`scripts/setup.sh --og`). -- A child's live report now has one gate at the data loader. While any act result is unopened, the page shows fixed text, opaque opening controls and the own record of each act opened on purpose. It shows none of the rest of the report until every act opens. This replaces per-element tags, fixed-panel lists and exemption registries, which repeatedly missed copied or new fields. Home summaries use fixed text. Family cells close for that child while parents retain their values. Closing an act replaces the document and clears view caches and search results. Reload closes the acts again. Map and section addresses return to the default view after the final opening. Adults and children with no act result keep the full report. APOE retains its separate warning. The boundary remains browser-only: static JSON files remain downloadable. `results.veil.acts` replaces the element metadata introduced during this unreleased work. The map retains its first-tag `veil` column and drops the unreleased multi-tag `veils` column. No output file is renamed. +- A child's live report now has one gate where the page reads its data, instead of a veil on each place that could name a result. While any of a child's results to act on is unopened, the page shows a fixed explanation, the number of such results, one "Show the result" button each, and the own record of each result opened on purpose; nothing else of the child's report reaches the page, and the page is the same whichever result is closed. Which result a button opens is drawn at random on each load, and the address names buttons (`#f/closed-1`), never a result. The server answers every finding address of such a child with the same page. Opening the last result shows the full report with its own addresses; closing one again, a reload or Back closes them all. The cost: such a child sees none of the rest of their report until every result to act on is opened. The home page shows a gated child's tile with fixed text, the count and the run's progress; the family page shows the child's buttons and leaves out the child's rows, stretches, couple screen and cells (the side by side reads "closed"), while the parents' values stay. Adults and children with no result to act on see the same pages as before, with the same requests. APOE keeps its own warning. No output file, field or option changes; `results.json` keeps its `veil` field, which the pages no longer read. The gate guards the page in the browser: the static files still reach a browser that asks for them. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. @@ -180,7 +180,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The CYP2D6 depth check and the RFC1 motif count, ported to the Rust `og` binary: `og pgx cyp2d6-bed`, `og pgx cyp2d6-measure` and `og pgx cyp2d6-check` (from `bin/cyp2d6_depth_check.py`) and `og str motif` (from `bin/rfc1_motif.py`) read BAMs with htslib, without pysam. On 67 cases they write what the Python writes, byte for byte, with one fix: a mosdepth regions file that lists one region twice with two depths is now refused (exit 2); the Python averaged the two. Steps 21, 32 and 9b run them (see above); no option, output file or format changes. -- Genome map, first pieces, in the Rust `og` binary. `og map build` writes one person's map document: the chromosome bands, the runs of homozygosity, and every finding with a place on the genome (its own position, or its gene's span). A result the page keeps closed (a finding to act on, and APOE on every page) is tagged with its id on every row that could reveal it, so the page's own closed-result rule hides those rows too. `og map genes` builds the gene and exon files once per GENCODE release, offline. No pipeline output, option or page changes yet; the live page starts drawing the map in a later change. The format is in `rust/crates/og-map/README.md`. +- Genome map, first pieces, in the Rust `og` binary. `og map build` writes one person's map document: the chromosome bands, the runs of homozygosity, and every finding with a place on the genome (its own position, or its gene's span). A result the page keeps closed (a finding to act on, and APOE on every page) is tagged with its id on every row that could reveal it, so the page can keep APOE's rows behind its warning and mark a result opened on purpose. `og map genes` builds the gene and exon files once per GENCODE release, offline. No pipeline output, option or page changes yet; the live page starts drawing the map in a later change. The format is in `rust/crates/og-map/README.md`. - The run-log commands are now `og run state`, `og run eta`, `og run headroom` and `og run delta`, with the same options and the same output. The command lines of `bin/run_state.py`, `bin/run_eta.py`, `bin/run_headroom.py` and `bin/run_delta.py` are gone: run as commands they exit 1 and name the og command; the modules stay for the live site, its alerts, the database re-check and the summary, which import them. `scripts/generate-report.sh` writes the delta with `og run delta --newest-history `, so og picks the previous run by the one rule the summary uses too, and the report now needs og in `bin/.og/` (`scripts/setup.sh --og`), as steps 04, 05 and 07 do. On macOS, `bin/og` runs in a Linux container; docs/run-state.md says how to run the commands there. - `og run delta` copied every summary section once per section it compared and looked ClinVar hits up one by one. On a synthetic summary with 50,000 hits it took 24 s and 770 MB, against 0.3 s for the Python. It now borrows the sections and hashes the keys: 0.3 s and 310 MB, with the same three files byte for byte. diff --git a/docs/live-site.md b/docs/live-site.md index af13d80d7..feb14b64e 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -56,17 +56,31 @@ Every result carries one of six levels, with a word, a shape and a colour, so th 9. **What it cannot see**: what the run does not assess and the steps that did not run. 10. **Sources and files**: the database versions, the database re-check, and the report and its files. -A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. A child's report has one gate at the data loader. While any result to act on is unopened, the page shows fixed explanatory text, the number of act results and one opaque opening control for each. An act opened on purpose shows only its own explanation, plan, evidence and caveat. It has no related list, next finding, map mark or chart. Nothing else from that child's report reaches a view: no cards, tables, level counts, images, file links, search box, search status or progress headlines. +A card has its rows (the first ones, then a button for all), its own sentences and its charts at reading width, and in a margin beside them its numbers, its caveat (the first sentence, then the rest behind "Full caveat") and its links; below 1100 px the margin goes under the text. A card with more than six lines keeps the ones that say something in view and folds the rest (an allele absent, a call not determined) behind one line that counts them. A section waiting for its step says "being written" or "waiting", one whose step failed says so, and a newer database release shows on the cards it would change. Some cards draw their numbers as well: the repeat expansions as one gauge per locus (the count against Stranger's normal and pathologic thresholds, with its 95% interval; an allele not sized from short reads is drawn hollow and never called normal), Lp(a) as the KIV-2 count with its 95% interval, CHIP as each call's VAF against the detection limits at this depth (hatched below every gene's limit, shaded where some genes see a clone and others do not, and each call named against its own gene's limit), the copy numbers of SMN1, SMN2, HBA1, HBA2 and the RCCX module as dots (Parascopy's SMN1 and SMN2 at exon 7 too, from short reads, hollow with its reason when the count is not resolved), and the Y haplogroup as its path from the root, with the marker counts of each level. Each drawing has the same numbers as text beside it. APOE stays behind its warning in the traits card; the polygenic scores show only once they are placed against the ancestry panel. APOE is closed on the genome map on every page, an adult's too, until it is opened with its own chip or with the fold of its warning in the traits card, which is the same switch; the chip and the switch appear only once the traits card shows, so a replay before its step has none. A child's page has the report gate below. -The cost is that a child with an unopened act result sees none of the rest of the report until every act result is opened. Opening the last act releases the full report. Closing an act from the map replaces the document with the closed view. That replacement discards search results, view caches and in-flight view callbacks. A short-lived, one-use opening ticket contains only opening flags and a digest, and is consumed immediately. Reloading closes the acts again. Map and section addresses use the closed controls while the gate is shut. After the last opening, the page returns to its default view. +A stopped run's page says so under the name and in the summary line, with the day it was last heard from, and warns that the report may be incomplete; a section still waiting for its step says the run stopped before that step ended. -The controls use ordinal handles such as `closed-1` and `opened-1`. Guessed finding hashes use the same first closed handle. Opening or closing a control preserves the addressed act as the ordinals change. A finding address for an opened act shows that act alone while the gate is shut. Other finding addresses show the fixed explanation and controls. Adults and children with no act result use the full report. Unknown family roles and a child's unreadable results stay closed. +While the run goes, the page still opens on the report being written. A short band under the name says how far the run is ("23 of 27 are done, about 8–13 hours remain"), with the steps done as a number and a bar, the time elapsed, the time to go from the usual step times and the retries, and links to the step timeline. Each section and each line of the contents says whether it is written, being written or waiting. The run itself, below, closes the page after section 10, folded until opened (or reached through `#progress`, the band or the contents); a page with no results yet keeps it open. -The home tile and attention row use fixed text for a child whose gate is shut. The family loader removes that child's report objects before rendering. Every retained side-by-side row has a fixed closed marker in that child's cell. Rows supplied only by that child are absent. Parents keep their values. Each child opens independently. The family tree remains visible because it comes from the declared pedigree, not the child's report. +**A child's report gate.** A child's report stays closed while any of the child's results to act on in adult life (the findings of level `act`) is unopened. A child is a person their family lists with a father or a mother. A person whose family roles cannot be read is treated as a child, and so is a child whose results cannot be read. While the gate is shut, the page shows only: -This replaces the per-element veil. There is no fixed-panel list, gene-matching rule or prose exemption registry. `results.veil.acts` lists the act identities. The loader derives the controls from their own records. The map retains its separate APOE warning on every full report, including adults' reports. +- a fixed explanation and the number of results to act on ("1 result to act on", "2 results to act on"); +- one "Show the result" button per closed result, all in one block; +- for each result opened on purpose, its own record (title, detail, explanation, plan, evidence and caveat) with a button to close it again, in the order they were opened. -This is a browser-only boundary. Static result, map, chart, search and family JSON files remain downloadable and contain their complete data. The gate protects the rendered document, attributes, accessibility text, title, address, canvas, SVG, browser storage and requests made by views. It is not an access-control boundary for those static files. +Nothing else of the child's report reaches the page: no cards, tables, lead, counts by level, genome map, charts, images, file links, search box or progress view. The title is fixed. The page shows the same thing whichever result is closed. + +Which result a button opens is drawn at random on each page load, so neither a button's place nor what it opens follows the report's order. The buttons have opaque names in the address, `#f/closed-1` and `#f/opened-1`. Any other `#f/` and any query land on `#f/closed-1`. On a finding address of a gated child the server answers the same page for every well-formed id, and the page then shows the run's own address and the same closed page, so a guessed address cannot tell whether the child has that result. + +Opening the last result loads the page again, and the full report shows, as an adult's does, with its own addresses (`#f/`). A map or section address returns to the default view. Only which results were opened crosses that load: a one-use ticket in session storage with a digest of the results' records, bound to that one load by a random value in the window's name. The page reads and removes it as it starts, before any network read, so a slow server does not lose the opening. Closing a result again (the genome map's "Close it again") clears the page at once and loads it again closed, without its search, caches or views. A result to act on that appears or changes while the report is open closes it the same way. A reload, or a page shown again with Back, closes every result again. + +The cost: a child with an unopened result to act on sees none of the rest of their report until every one is opened. Adults and children with no result to act on never meet the gate. The gate reads the files their page reads anyway and hands each view its first read from that copy, so each file is fetched as often, and in the same order, as without it. + +The home page decides from `/runs.json` alone. A gated child is a child of a readable family, or, while a family's members cannot be read, a run no readable family lists, whose entry counts or names a result to act on. Its tile and attention rows show the fixed text and the count of results to act on, and link to the run's page. The run's progress (its state, the steps done, planned, running and failed, and its times) stays, because it says how far the run is, never what it found. + +On the family page, chapter IV shows a gated child's buttons only, and each child opens on its own. The family view gets the family without that child's report data: no de novo or compound-het rows, stretches, trio status or trio files of the child, no relationship or shared-segment row with the child, and no couple screen of a couple the child is part of. The side by side keeps a row only when an open member has a value in it, in a fixed order, and the child's cell reads "closed". Plan rows that name the child are left out. The parents' other plan rows and the couple reports of couples with no gated partner stay. Other reports of the family folder are left out while a child is gated, because they can hold the child's data. A member with no run here has no report, so the gate does not close anything of theirs. The pedigree and the members' progress stay. + +The gate holds the page's `fetch` and `EventSource`. A view or plugin reads a run's files through them. `XMLHttpRequest`, an `` and `import()` are not held, so the views use them only on what the gate has released. The gate guards the page in the browser: its document, attributes, accessibility text, title, address, canvas and drawings, storage and the requests its views make. The static files (`results.json`, `map.json`, the charts, the search index and `family.json`) still reach a browser that asks for them whole: the gate is not an access control for them. - **How the run went.** One line per step the run planned, on a time axis: a bar from its start to its end, or to now while it runs, with an outline to its usual hours. The colour is the step's state from `bin/run_state.py` ([Where a Run Stands](run-state.md)): waiting, running, done, failed, skipped, not finished. Under it the tasks running now against their usual time, a step's detail, the wire of what was found so far and the plugins' cards (run health, the database re-check). - **A step's detail.** Click a step: its state and reason, start and end, the usual hours, each process with its tasks waiting, running, done, from the cache or failed, and the headline of its sections with a link into the report. @@ -74,15 +88,15 @@ This is a browser-only boundary. Static result, map, chart, search and family JS - **The report.** Once written, `report.html` opens the run's `_report.html`, and the files it links to beside it. - **The genome viewer.** `/runs//viewer/` shows the run's reads and variants in igv.js, as [`scripts/owngenome-view.sh`](viewer.md) does on one computer. The report's Viewer links, the charts' "See the reads" links and a finding page's reads open it at their position. -The header search appears only on a full report. It finds a gene or rsID across the results and opens the card that names it. A result row with a field of the wrong type produces the finding "A row of this section could not be read". It takes the highest level its rule can give. If that level is act, it keeps a child's gate shut until someone opens it. On a full report, the card says which table cannot be drawn and the other cards remain available. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. +The header search finds a gene or rsID across the results and opens the card that names it; a child's page has none while its report gate is shut (above). One row of the results that the page cannot read (a field of the wrong type) costs that row only: it shows as a finding on its card, "A row of this section could not be read", at the highest level its rule can give (to act on for a rule that can give one, so a child's report gate stays shut until that finding is opened too), the card whose table cannot be drawn says so, and every other card and finding is on the page. `#s` opens a section, `#s/
` a card (the report's own anchors), `#f/` a finding, `#q/` the section a report chapter falls in, `#progress` the timeline. **Figure 1, the genome map.** The 24 chromosomes as rows, to scale (23 for a person with no Y), each with its bands, the centromere as a waist and the stretches few reads can map hatched (centromeres, heterochromatin, the acrocentric arms). Each finding with a place sits on its chromosome as its level's shape, with its gene beside it when the name fits; runs of homozygosity of 5 Mb or more are a line under the row, shorter ones a faint band. A finding that names a gene but no position (a medicine gene, Lp(a)) sits on that gene's span; a finding with neither stays in its section only. Click a chromosome (or, with the map focused, press `[` or `]`) and it opens in place: an overview with the part in view, a ruler in Mb, the bands with their names, a findings lane with each gene and its level's words, and a lane for the runs of homozygosity, each run with its length. Drag to move along it, scroll or press the + and − buttons beside the zoom stops to zoom in (as far as about 10 Mb across a wide screen; closer views come later), and press Esc (or click Genome) to go back. Back in the browser returns to the view before, the whole genome when the address had no place. A dashed line marks the centre, and the margin says what is under it: the chromosome, the position and the band, the findings in view and the runs. The findings count, the chromosome sentences, the runs sentence and each finding's phrase come from the map document; the page adds its own short lines: a lane or stretch with no finding, the runs in view, how to move, and why a layer is not on the map. A replay draws the runs of homozygosity, and says the sentences about them, only once their step has ended. The address follows the map: `#genome`, `#chr7`, or `#chr7:117.4-137.4M`, and a link to one of them opens that view; a place the page cannot read shows a short note and leaves the map as it was. The map comes from `/runs//map.json`, which the map plugin (`bin/owngenome_live_map.py`) builds with `og map build` from the run folder and the results the page gets (`rust/crates/og-map/README.md` describes the document). It needs og on the host (`scripts/setup.sh --og`); without it, or in a browser that cannot unpack the file (no `DecompressionStream`), the figure says so and lists the findings with a place in a table instead. A map that fails to load again after it was drawn says so the same way and draws nothing more from the map it had. A layer whose file is missing or cannot be read says so in words, never as a blank lane. -**A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. +**A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. For a child with a result to act on, the server answers every well-formed id with the page, which stays closed while the report gate is shut. -**A family** (`/runs//`). Five chapters show the pedigree, measured relationships, couple carrier screens, each child's inheritance and the members side by side. The shared data gate above applies before the family view receives child report data. Gated children have fixed opening controls and closed cell markers. Parent values and independently opened children remain available. The full family report returns for a child after that child's final act opens. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. While a child's report gate is shut, chapter IV shows that child's buttons only (the report gate above). V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. While a child's report gate is shut, the child's cell reads "closed" in every row, and a row stays only when an open member has a value in it. On a phone the grid becomes a list of pairs. -On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). The report gate above applies at every width. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. +On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). The report gate is the same at every width. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. ![The person page of DEMO-002 on a 390 px phone: the lead, the counts one line per level, the two medicine results and the nine results worth a look open, every other card one line with a plus, sections 5 to 10 each folded to one line that names what they hold, and the tab bar at the bottom](images/demo-live-phone.png){ loading=lazy width="390" } @@ -94,7 +108,9 @@ Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with it The reads cross the network to the browser that logged in: only the byte ranges igv.js asks for, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS); without it the password and the reads cross the network in clear. Opening the viewer writes one access-log row; the byte ranges igv.js reads write none. -`GET /` lists the runs. The home page reads `/runs.json`, then resolves family membership and each child's acts before projecting the run summaries. A person's loader reads `results.json` and the family routing information. Full reports contain the chapters, findings, journal, lead and source metadata built by `bin/live_results.py`. The gate passes none of that report to views while an act remains closed, apart from explicitly opened acts' own records. The family loader reads `family.json` and each child's results, and refreshes every 30 seconds. A changed act record closes again. An unchanged record keeps its opening for the current page. +`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. For a gated child the page replaces `levels`, `top` and `lead` with fixed text before the home page draws them (the report gate above). + +The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (the rows outside a result's own card that name a result to act on, and `spans`, the results to act on with their gene and position; the file keeps it, and the pages no longer read it: the report gate reads the findings of level `act`). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. ## Privacy and what is stored @@ -190,4 +206,4 @@ The core files (`bin/owngenome_live.py`, `assets/live/index.html`, `home.html`, The page has a replay mode for a demo with no server: `` reads `frames.json` (`{"interval_ms": 1000, "frames": [{"t": ..., "state": }]}`) and plays it with a play button and a slider, through the same drawing code. A replay whose last frame is a finished run opens at that frame, on the finished report, until you press Play, which starts again from the first frame; with `"autoplay": true` it plays from the start at once. The bar shows each frame's moment as day, month and time in UTC ("2 Jan 10:05"), the full time on hover. -`tests/test_owngenome_live.py` checks the server, login, message storage, plugins and browser. `tests/test_live_results.py` checks card caveats, wording and login boundaries. `tests/test_live_journal.py` checks finding levels, leads and routes. `tests/test_live_demo.py` checks the demo, maps, phone layouts and report gate. `tests/test_owngenome_live_family.py` runs family checks and its browser property in a separate CI shard. `tests/test_live_post.sh` and `tests/fake-docker/run-all-live.sh` check progress messages from the launcher. +`tests/test_owngenome_live.py` checks the server, the login, the stored messages, the plugin interface and the page in a headless Chrome; `tests/test_live_results.py` checks `results.json` (every card's caveat word for word, no wording that reads as an all-clear, `results.json` behind the login); `tests/test_live_journal.py` checks the levels, the lead paragraph and the finding page's route; `tests/test_live_demo.py` builds the demo and checks every page, phone width included; `tests/test_live_post.sh` and `tests/fake-docker/run-all-live.sh` check what `run-all.sh` posts. From fe1542c5908c452e42f8cc9d7a3ab197820a662a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:23:42 +0100 Subject: [PATCH 067/255] fix(ancestry): keep the previous archaic outputs when a run stops, and refuse bad report inputs og archaic genotypes now stages samples.txt, every genotype file and every mask, and publishes them only when the whole run has passed. A chromosome with no row deletes its old genotype file only on success. The VCF order check compares each record with the last record handed to the join, so a skipped record of another chromosome cannot hide an unsorted return. og ancient report stops on a non-finite eigenvalue or person PC, and on an ancient row with fewer PCs than the distance uses. The parity goldens of the four failing genotypes cases now expect the previous outputs to stay (no new file). --- CHANGELOG.md | 5 +- rust/crates/og-ancestry/src/ancient.rs | 120 +++++++++- rust/crates/og-ancestry/src/archaic.rs | 209 +++++++++++++++--- .../bad-code/gt/samples.txt.absent | 0 .../comes-back/gt/chr21.Altai.mask.bed.absent | 0 .../gt/chr21.Denisova.mask.bed.absent | 0 .../gt/chr21.Vindija.mask.bed.absent | 0 .../comes-back/gt/chr21.gt.gz.absent | 0 .../comes-back/gt/chr22.Altai.mask.bed.absent | 0 .../gt/chr22.Denisova.mask.bed.absent | 0 .../gt/chr22.Vindija.mask.bed.absent | 0 .../comes-back/gt/chr22.gt.gz.absent | 0 .../comes-back/gt/samples.txt.absent | 0 .../no-table-chrom/gt/samples.txt.absent | 0 .../no-table-chrom/gt/spans.tsv.absent | 0 .../two-samples/gt/samples.txt.absent | 0 16 files changed, 302 insertions(+), 32 deletions(-) create mode 100644 tests/parity/golden/archaic-genotypes/bad-code/gt/samples.txt.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Altai.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Denisova.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Vindija.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.gt.gz.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Altai.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Denisova.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Vindija.mask.bed.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.gt.gz.absent create mode 100644 tests/parity/golden/archaic-genotypes/comes-back/gt/samples.txt.absent create mode 100644 tests/parity/golden/archaic-genotypes/no-table-chrom/gt/samples.txt.absent create mode 100644 tests/parity/golden/archaic-genotypes/no-table-chrom/gt/spans.tsv.absent create mode 100644 tests/parity/golden/archaic-genotypes/two-samples/gt/samples.txt.absent diff --git a/CHANGELOG.md b/CHANGELOG.md index e2a01e5d8..e30598b54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,12 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Steps 43 (archaic ancestry) and 44 (ancient similarity) now run in the Rust `og` binary: `og archaic table`, `strict`, `genotypes` and `report`, and `og ancient select`, `lift`, `person-geno`, `align` and `report`. They replace `bin/archaic_report.py` and `bin/ancient_report.py`, which are gone, in the step scripts, the Nextflow modules, `setup.sh --archaic` and `setup.sh --ancient-panel`. og is static, so each part still runs in the image it ran in. `og ancient lift` reads UCSC's chain file with a new chain reader in `og-intervals`. The outputs are the same, byte for byte (the gzip genotype files by content), on the 49 parity cases, except where a line below says otherwise. No option, output file or format changes. On the real archaic table, `og archaic table` merges chromosome 1 (4.1 million sites) in 18 s and 190 MB where the Python took 76 s and 1.25 GB; bcftools and Picard still take most of each chromosome's setup. On a synthetic input of real size, step 44's panel column over 1,150,106 sites and 819 people takes 2.5 s and 342 MB instead of 4.2 s and 524 MB. What changes on purpose: - The archaic table's setup stamp now holds a format id (`ARCHAIC_TABLE_FORMAT` in `setup.sh`) instead of the sha256 of the Python's table code. A table the last Python built has the same bytes (chromosomes 1 to 6 and 21 of the real table, rebuilt from the same public files, are the same after decompression), so its stamp is taken as current once and rewritten: no install downloads the 150 GB of archaic genomes again for this change. - - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. + - `og archaic genotypes` streams the sorted VCF and table together with one duplicate-position window, and writes mask runs as they close. On a synthetic chromosome with 3 million table rows and 1 million VCF records, peak memory falls from the initial port's 166 MB to 4 MB (3.79 s before, 3.91 s after). A position that goes backwards within a VCF chromosome or in the table stops with exit 2, also when a skipped record (an indel, ALT `.`) of another chromosome sits between the two. Duplicate calls, clashes, masked sites, mask runs and span counts keep their meaning. - `og archaic strict` reads the lifted rows in position order, as Picard writes them, and stops with exit 2 on a row that goes back. The Python kept every position of a chromosome in memory to drop repeats. - - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. + - Every output is written to a temporary file and renamed, so a step that stops leaves no half-written file. The Python wrote the archaic report, the panel lists, the stats and the ancient text report in place, and left a partial table or genotype file behind when it stopped. `og archaic genotypes` stages `samples.txt`, every genotype file and every mask, and publishes them only when the whole run has passed: a run that stops leaves the previous set as it was, where the Python had already rewritten `samples.txt`, earlier chromosomes and, for a chromosome with no row, deleted the old genotype file. The renames are one at a time, so the set is not one transaction. - A gzip input is told by its first bytes, not by a `.gz` name, and a `.fai` that does not match its FASTA stops `og ancient lift`. - Malformed input stops with one `ERROR:` line, not a Python traceback. + - `og ancient report` stops when an eigenvalue or the person's PCs in the distance are not finite (NaN or infinity), and when an ancient group's row holds fewer PCs than the distance uses. The Python wrote NaN distances, or ranked that group on fewer PCs. - `sites_genotyped` in step 44's stats still counts a position twice when the AADR holds two SNPs there, as the Python did; the genotypes themselves are right. - CI evidence rejects FASTA offset overflow and offsets outside the file before reading or publishing. Guard tests this in both debug and release builds. Every new dev verb fails on stdout or stderr write or flush errors. A truth VCF header after the first data record fails without replacing the destination. Additional deliberate refusals are documented and tested: zero-byte gzip inputs, zero-width unused FAI rows, non-ASCII or out-of-range annotation limits, non-writable staging parents, and gzip FASTA files with long FNAME headers. Publication replaces a file and leaves other hard links on the old inode. diff --git a/rust/crates/og-ancestry/src/ancient.rs b/rust/crates/og-ancestry/src/ancient.rs index 6a3b6fc52..3693dec13 100644 --- a/rust/crates/og-ancestry/src/ancient.rs +++ b/rust/crates/og-ancestry/src/ancient.rs @@ -1063,9 +1063,10 @@ fn nearest_populations( let mut rows = Vec::with_capacity(order.len()); for g in order { let members = &groups[g]; - let k = members[0].0.len().min(n_pcs); - let mut centre = Vec::with_capacity(k); - for i in 0..k { + // Every member must hold all n_pcs PCs: a shorter row would rank + // its group on fewer PCs than the person's distance claims. + let mut centre = Vec::with_capacity(n_pcs); + for i in 0..n_pcs { let mut xs = Vec::with_capacity(members.len()); for (pcs, _) in members { xs.push(*pcs.get(i).ok_or_else(|| { @@ -1348,6 +1349,19 @@ fn cmd_report(a: &Args, out: &mut dyn Write) -> Result<()> { let n_pcs = (DISTANCE_PCS as usize) .min(panel.evals.len()) .min(pcs.len()); + // NaN or infinity in the eigenvalues or the person's PCs would make + // every distance and plotted point meaningless: stop instead. + let not_finite = |path: &str, what: &str| { + Fail(format!( + "{path}: {what} holds a value that is not a finite number" + )) + }; + if panel.evals.iter().take(n_pcs).any(|v| !v.is_finite()) { + return Err(not_finite(pp, "the #eigvals: line")); + } + if pcs.iter().take(n_pcs.max(2)).any(|v| !v.is_finite()) { + return Err(not_finite(rp, &format!("the row of {sample}"))); + } let signs = axis_signs(&panel, pp, &run, rp, n_pcs.max(2))?; let person: Vec = signs.iter().zip(pcs).map(|(s, v)| s * v).collect(); let ancients = read_ancients(a.req("--ancients"))?; @@ -1569,4 +1583,104 @@ mod tests { assert_eq!(clean(" a\t b "), "a b"); assert_eq!(clean(" \t "), "."); } + + fn evec(evals: &[f64], rows: &[(&str, &str, &[f64])]) -> Evec { + let mut e = Evec { + evals: evals.to_vec(), + rows: Vec::new(), + index: HashMap::new(), + }; + for (id, label, pcs) in rows { + e.index.insert((*id).to_owned(), e.rows.len()); + e.rows + .push(((*id).to_owned(), (*label).to_owned(), pcs.to_vec())); + } + e + } + + fn ancient(group: &str) -> AncientRow { + AncientRow { + group: group.to_owned(), + date_bp: None, + country: None, + locality: None, + } + } + + #[test] + fn every_ancient_member_carries_the_distance_pcs() { + // A group whose rows hold 2 PCs must not be ranked on 2 of 4. + let panel = evec( + &[1.0; 4], + &[ + ("a1", ANCIENT_LABEL, &[0.0, 0.0]), + ("a2", ANCIENT_LABEL, &[0.0, 0.0, 9.0, 0.0]), + ], + ); + let ancients = vec![ + ("a1".to_owned(), ancient("Short")), + ("a2".to_owned(), ancient("Full")), + ]; + let Err(Fail(msg)) = nearest_populations(&[0.0, 0.0, 10.0, 0.0], &panel, &ancients, 4) + else { + panic!("a group short of PCs was ranked"); + }; + assert!(msg.contains("ancient group Short has no PC3"), "{msg}"); + } + + #[test] + fn a_person_or_eigenvalue_that_is_not_finite_stops_the_report() { + let d = std::env::temp_dir().join(format!("og-ancient-finite-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + let modern: String = (1..=12) + .map(|i| format!("M{i} {}.0 {}.5 {} {} Pop\n", i, 13 - i, i % 3, i % 5)) + .collect(); + let path = |n: &str| d.join(n).to_str().unwrap().to_owned(); + for (person, evals) in [ + ("S1 nan 0.1 0.2 0.3 Person\n", "4 3 2 1"), + ("S1 0.1 0.2 0.3 0.4 Person\n", "4 inf 2 1"), + ] { + std::fs::write( + d.join("panel.evec"), + format!("#eigvals: {evals}\n{modern}A1 1 2 3 4 Ancient\n"), + ) + .unwrap(); + std::fs::write( + d.join("run.evec"), + format!("#eigvals: 4 3 2 1\n{modern}{person}"), + ) + .unwrap(); + std::fs::write(d.join("anc.tsv"), "id\tgroup\nA1\tG\n").unwrap(); + let args: Vec = [ + "report", + "--panel-evec", + &path("panel.evec"), + "--run-evec", + &path("run.evec"), + "--ancients", + &path("anc.tsv"), + "--sample", + "S1", + "--release", + "v1", + "--out-tsv", + &path("o.tsv"), + "--out-info", + &path("o.info"), + "--out-txt", + &path("o.txt"), + "--out-svg", + &path("o.svg"), + ] + .map(str::to_owned) + .to_vec(); + let mut err = Vec::new(); + assert_eq!(cli(&args, &mut Vec::new(), &mut err), 1, "{person}{evals}"); + let err = String::from_utf8(err).unwrap(); + assert!(err.contains("not a finite number"), "{err}"); + assert!(!d.join("o.tsv").exists()); + } + let _ = std::fs::remove_dir_all(&d); + } } diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 4468fc815..6f668b698 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -723,7 +723,9 @@ struct VcfRec { /// The biallelic SNVs (and the other records, as code 9) of a one-sample /// VCF, in file order. A record whose FILTER is not PASS or `.` gives /// code 9 (not read), so a DeepVariant RefCall is neither your allele nor -/// the reference. +/// the reference. Every record is checked against the last record handed +/// out (`previous`), so a record that is skipped cannot hide a chromosome +/// that comes back at a lower position. struct Vcf { text: Text, previous: Option<(String, i64)>, @@ -772,11 +774,11 @@ impl Vcf { self.text.path.display() )); } - self.previous = Some((c[0].to_owned(), pos)); if c[3].chars().count() != 1 || c[4] == "." { continue; } if c[4].contains(',') || c[4].chars().count() != 1 { + self.previous = Some((c[0].to_owned(), pos)); return Ok(Some(VcfRec { chrom: c[0].to_owned(), pos, @@ -796,6 +798,7 @@ impl Vcf { if c[6] != "PASS" && c[6] != "." { code = 9; } + self.previous = Some((c[0].to_owned(), pos)); return Ok(Some(VcfRec { chrom: c[0].to_owned(), pos, @@ -922,7 +925,16 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { samples += &format!("{id}\n"); } head.push('\n'); - put(&Path::new(out_dir).join("samples.txt"), samples.as_bytes())?; + // Every output is staged and published only when the whole run has + // passed, so a run that stops leaves the previous set as it was. + let samples_path = Path::new(out_dir).join("samples.txt"); + let mut samples_file = + AtomicFile::create(&samples_path).map_err(|e| cannot_write(&samples_path, &e))?; + samples_file + .write_all(samples.as_bytes()) + .map_err(|e| cannot_write(&samples_path, &e))?; + let mut staged: Vec<(PathBuf, AtomicFile)> = vec![(samples_path, samples_file)]; + let mut stale: Vec = Vec::new(); let vcf_path = a.req("--vcf"); let mut calls = Calls { @@ -942,7 +954,7 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { done.push(chrom.clone()); let table = tables.iter().find(|(c, _)| *c == chrom).map(|(_, t)| t); if let Some(table) = table { - if let Some(span) = write_chrom( + match write_chrom( &chrom, table, &mut calls, @@ -953,7 +965,11 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { two_n, out_dir, )? { - spans.push(span); + Written::Rows(span, files) => { + spans.push(span); + staged.extend(files); + } + Written::Empty(path) => stale.push(path), } } else { while pending.as_ref().is_some_and(|r| r.chrom == chrom) { @@ -961,12 +977,6 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { } } } - let mut text = - String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); - for (c, f, l, n, carried, dropped) in &spans { - text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); - } - put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; if spans.is_empty() { let names: Vec<&str> = tables.iter().map(|(c, _)| c.as_str()).collect(); let mut sorted = names.clone(); @@ -976,6 +986,25 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { sorted.join(", ") )); } + // Published one rename at a time: the set is not one transaction, and + // a rename that fails part way leaves part of the new set in place. + for (path, file) in staged { + file.commit().map_err(|e| cannot_write(&path, &e))?; + } + for path in stale { + match std::fs::remove_file(&path) { + Err(e) if e.kind() != io::ErrorKind::NotFound => { + return Err(Stop::from(cannot_write(&path, &e))); + } + _ => {} + } + } + let mut text = + String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); + for (c, f, l, n, carried, dropped) in &spans { + text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); + } + put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; let sites: u64 = spans.iter().map(|s| s.3).sum(); let _ = writeln!(err, "{} chromosomes, {sites} sites", spans.len()); Ok(()) @@ -985,8 +1014,18 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { /// position, sites, sites with your ALT, sites with another allele. type Span = (String, i64, i64, u64, u64, u64); +/// What one chromosome leaves to publish when the whole run has passed. +enum Written { + /// Rows were written: the span row, and the staged genotype file and + /// masks with their final paths. + Rows(Span, Vec<(PathBuf, AtomicFile)>), + /// No row was written: a genotype file of this name from an earlier + /// run is stale and goes. + Empty(PathBuf), +} + /// IBDmix's genotype file of one chromosome, and each archaic genome's -/// mask; the chromosome's span row, or `None` when no row was written. +/// mask, staged under temporary names. #[allow(clippy::too_many_arguments)] fn write_chrom( chrom: &str, @@ -998,7 +1037,7 @@ fn write_chrom( af_col: usize, two_n: f64, out_dir: &str, -) -> Out> { +) -> Out { let num = &chrom[3..]; let path = Path::new(out_dir).join(format!("{chrom}.gt.gz")); let file = AtomicFile::create(&path).map_err(|e| cannot_write(&path, &e))?; @@ -1124,26 +1163,22 @@ fn write_chrom( } if n == 0 { drop(gz); - match std::fs::remove_file(&path) { - Err(e) if e.kind() != io::ErrorKind::NotFound => { - return Err(Stop::from(cannot_write(&path, &e))); - } - _ => {} - } - return Ok(None); + return Ok(Written::Empty(path)); } - gz.finish().and_then(AtomicFile::commit).map_err(werr)?; + let gz = gz.finish().map_err(werr)?; + let mut files = vec![(path, gz)]; // IBDmix's mask (-r): the chromosome as in the genotype file, and a // line covers the positions start + 1 to end, as in a BED. - for (k, mut mask) in masks.into_iter().enumerate() { + for ((k, mut mask), mask_path) in masks.into_iter().enumerate().zip(mask_paths) { if let Some((s, e)) = open_run[k] { - writeln!(mask, "{num}\t{}\t{e}", s - 1) - .map_err(|e| cannot_write(&mask_paths[k], &e))?; + writeln!(mask, "{num}\t{}\t{e}", s - 1).map_err(|e| cannot_write(&mask_path, &e))?; } - mask.commit() - .map_err(|e| cannot_write(&mask_paths[k], &e))?; + files.push((mask_path, mask)); } - Ok(Some((chrom.to_owned(), first, last, n, carried, dropped))) + Ok(Written::Rows( + (chrom.to_owned(), first, last, n, carried, dropped), + files, + )) } // --- report (step 43) ------------------------------------------------------------- @@ -1752,6 +1787,126 @@ mod tests { assert!(error.1.contains("not sorted")); } + #[test] + fn an_ignored_record_cannot_hide_a_chromosome_return() { + // chr22 comes back after an ignored chrX record (ALT "."), at a + // lower position: the join would write 0 at chr22:1000 and lose + // the call, so it must stop as unsorted. + let text = "chr22\t3000\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\n\ + chrX\t1\t.\tA\t.\t.\tPASS\t.\tGT\t0/0\n\ + chr22\t1000\t.\tC\tT\t.\tPASS\t.\tGT\t1/1\n"; + let mut vcf = Vcf::new(Text { + path: "sample.vcf".into(), + lines: Lines::new( + Box::new(io::Cursor::new(text.as_bytes().to_vec())), + Utf8::Strict, + ), + line: String::new(), + }); + assert_eq!(vcf.next().unwrap().unwrap().pos, 3000); + let error = vcf.next().err().expect("chr22 goes back to 1000"); + assert_eq!(error.0, 2); + assert!( + error.1.contains("chr22:1000 comes after 3000"), + "{}", + error.1 + ); + } + + /// A scratch folder of its own under the system temp folder. + fn scratch(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-archaic-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(d.join("table")).unwrap(); + d + } + + fn gzip_to(path: &Path, text: &str) { + let mut gz = GzEncoder::new(std::fs::File::create(path).unwrap(), Compression::new(1)); + gz.write_all(text.as_bytes()).unwrap(); + gz.finish().unwrap(); + } + + /// `og archaic genotypes` on the folder's table into `gt`, with a VCF + /// of these lines: the exit code. + fn genotypes(d: &Path, vcf_lines: &str, pseudo: &str) -> u8 { + let vcf = d.join("s.vcf"); + std::fs::write( + &vcf, + format!("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n{vcf_lines}"), + ) + .unwrap(); + let args: Vec = [ + "genotypes", + "--table-dir", + d.join("table").to_str().unwrap(), + "--vcf", + vcf.to_str().unwrap(), + "--pseudo", + pseudo, + "--out-dir", + d.join("gt").to_str().unwrap(), + ] + .map(str::to_owned) + .to_vec(); + cli(&args, &mut Vec::new(), &mut Vec::new()) + } + + fn snapshot(d: &Path) -> Vec<(String, Vec)> { + let mut files: Vec<(String, Vec)> = std::fs::read_dir(d.join("gt")) + .unwrap() + .map(|e| { + let e = e.unwrap(); + ( + e.file_name().into_string().unwrap(), + std::fs::read(e.path()).unwrap(), + ) + }) + .collect(); + files.sort(); + files + } + + #[test] + fn a_failed_run_leaves_every_previous_output() { + let d = scratch("failed-run"); + let head = TABLE_HEADER.join("\t"); + let row = |c: &str, p: i64, r: &str, a: &str| { + format!("{c}\t{p}\t{r}\t{a}\t0\t1\t12\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n") + }; + gzip_to( + &d.join("table/chr21.tsv.gz"), + &format!( + "{head}\n{}{}", + row("chr21", 100, "A", "G"), + row("chr21", 200, "C", "T") + ), + ); + gzip_to( + &d.join("table/chr22.tsv.gz"), + &format!("{head}\n{}", row("chr22", 100, "G", "A")), + ); + let good = + "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\nchr22\t100\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n"; + assert_eq!(genotypes(&d, good, "2"), 0); + let before = snapshot(&d); + assert!(before.iter().any(|(n, _)| n == "chr22.gt.gz")); + // chr21 is written, then chr22 is found unsorted: nothing changes, + // not samples.txt for the new --pseudo nor chr21's files. + let unsorted = "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t1/1\n\ + chr22\t300\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n\ + chr22\t400\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n\ + chr22\t100\t.\tG\tA\t.\tPASS\t.\tGT\t1/1\n"; + assert_eq!(genotypes(&d, unsorted, "3"), 2); + assert_eq!(snapshot(&d), before); + // No row written on any chromosome (chr22's only site holds + // another allele): the run fails and chr22's file stays. + let other = "chr22\t100\t.\tG\tC\t.\tPASS\t.\tGT\t1/1\n"; + assert_ne!(genotypes(&d, other, "3"), 0); + assert_eq!(snapshot(&d), before); + let _ = std::fs::remove_dir_all(&d); + } + #[test] fn pseudo_columns_carry_the_count() { let p = pseudo_columns(2); diff --git a/tests/parity/golden/archaic-genotypes/bad-code/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/bad-code/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Altai.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Altai.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Denisova.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Denisova.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Vindija.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.Vindija.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.gt.gz.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr21.gt.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Altai.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Altai.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Denisova.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Denisova.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Vindija.mask.bed.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.Vindija.mask.bed.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.gt.gz.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/chr22.gt.gz.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/comes-back/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/comes-back/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/spans.tsv.absent b/tests/parity/golden/archaic-genotypes/no-table-chrom/gt/spans.tsv.absent new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/archaic-genotypes/two-samples/gt/samples.txt.absent b/tests/parity/golden/archaic-genotypes/two-samples/gt/samples.txt.absent new file mode 100644 index 000000000..e69de29bb From 21d6a76e13c2d35e55c65318238cb9192861e600 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:32:43 +0100 Subject: [PATCH 068/255] test(live): order the transparency check's requests by when they started The browser lists a request when it ends, so two files read at once could swap places in CI. Sorting by start time compares the order the pages ask. --- tests/test_live_demo.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 7846fa8ef..c187af9ca 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1783,7 +1783,9 @@ def capture(route, apoe=False, timeout=20): n.removeAttribute('data-pgp-map-paint-ms'); n.removeAttribute('data-pgp-map-parse-ms'); }); return [clone.outerHTML, location.hash, Array.from(document.querySelectorAll('canvas')).map(function(c){return c.toDataURL();}), - performance.getEntriesByType('resource').map(function (r) { return new URL(r.name).pathname; })]; + // The browser lists a request when it ends: sort by when each one started. + performance.getEntriesByType('resource').slice().sort(function (a, z) { return a.startTime - z.startTime; }) + .map(function (r) { return new URL(r.name).pathname; })]; }())""") # Scripts and the views' own files load in parallel, in any order, with or without the gate: count every # request, and keep the order of the files the gate reads first. From fa7a6571f57ecb4e8ecc43c93a5b59221e47216a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 11:41:42 +0100 Subject: [PATCH 069/255] fix(ancestry): remove a stale genotype file only after every new archaic output is in place spans.tsv is now staged with the other outputs, and a chromosome's old genotype file is removed last. A failed publication keeps it. --- rust/crates/og-ancestry/src/archaic.rs | 47 ++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 6f668b698..0dbd7db0c 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -986,8 +986,21 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { sorted.join(", ") )); } + let mut text = + String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); + for (c, f, l, n, carried, dropped) in &spans { + text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); + } + let spans_path = Path::new(out_dir).join("spans.tsv"); + let mut spans_file = + AtomicFile::create(&spans_path).map_err(|e| cannot_write(&spans_path, &e))?; + spans_file + .write_all(text.as_bytes()) + .map_err(|e| cannot_write(&spans_path, &e))?; + staged.push((spans_path, spans_file)); // Published one rename at a time: the set is not one transaction, and // a rename that fails part way leaves part of the new set in place. + // Stale genotype files go last, once every new file is in place. for (path, file) in staged { file.commit().map_err(|e| cannot_write(&path, &e))?; } @@ -999,12 +1012,6 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { _ => {} } } - let mut text = - String::from("chrom\tfirst\tlast\tsites\tsites_with_your_alt\tsites_other_allele\n"); - for (c, f, l, n, carried, dropped) in &spans { - text += &format!("{c}\t{f}\t{l}\t{n}\t{carried}\t{dropped}\n"); - } - put(&Path::new(out_dir).join("spans.tsv"), text.as_bytes())?; let sites: u64 = spans.iter().map(|s| s.3).sum(); let _ = writeln!(err, "{} chromosomes, {sites} sites", spans.len()); Ok(()) @@ -1907,6 +1914,34 @@ mod tests { let _ = std::fs::remove_dir_all(&d); } + #[test] + fn a_failed_publication_keeps_a_stale_genotype_file() { + let d = scratch("failed-publication"); + let head = TABLE_HEADER.join("\t"); + let row = + |c: &str, p: i64| format!("{c}\t{p}\tA\tG\t0\t1\t2\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n"); + gzip_to( + &d.join("table/chr21.tsv.gz"), + &format!("{head}\n{}", row("chr21", 100)), + ); + gzip_to( + &d.join("table/chr22.tsv.gz"), + &format!("{head}\n{}", row("chr22", 100)), + ); + let both = + "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\nchr22\t100\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\n"; + assert_eq!(genotypes(&d, both, "2"), 0); + // spans.tsv cannot be replaced: chr22 now has no row, and its old + // genotype file must outlive the failed publication. + std::fs::remove_file(d.join("gt/spans.tsv")).unwrap(); + std::fs::create_dir_all(d.join("gt/spans.tsv/full")).unwrap(); + let chr21_only = + "chr21\t100\t.\tA\tG\t.\tPASS\t.\tGT\t1/1\nchr22\t100\t.\tA\tC\t.\tPASS\t.\tGT\t1/1\n"; + assert_ne!(genotypes(&d, chr21_only, "2"), 0); + assert!(d.join("gt/chr22.gt.gz").exists()); + let _ = std::fs::remove_dir_all(&d); + } + #[test] fn pseudo_columns_carry_the_count() { let p = pseudo_columns(2); From c67f4404e120d5455ded8de7d31f9ba9a098ff2f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:57:41 +0100 Subject: [PATCH 070/255] perf(report): write summary.json with og summary collect render_report.py (step 24, HTML_REPORT and generate-report.sh) now runs og summary collect instead of the Python collector. generate-report.sh renders the text report from the summary step 24 wrote when nothing it reads has changed since, so a run collects each sample once. The folder walk leaves out the Nextflow launch folder run-all.sh keeps in the sample folder. collect_summary.py keeps a collect() that runs og, and only the names other Python still imports. Bead: pgp-bs9u.45 --- bin/collect_summary.py | 4099 +------------------- bin/render_report.py | 12 +- modules/local/html_report/main.nf | 13 +- rust/crates/og-summary/src/collect/scan.rs | 28 +- scripts/24-html-report.sh | 8 +- scripts/generate-report.sh | 47 +- 6 files changed, 235 insertions(+), 3972 deletions(-) diff --git a/bin/collect_summary.py b/bin/collect_summary.py index 054d9a684..a492746d8 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -1,67 +1,127 @@ #!/usr/bin/env python3 -"""collect_summary.py: read one sample's step outputs into one JSON summary. - -Both reports (scripts/24-html-report.sh and scripts/generate-report.sh) are -rendered from this summary by bin/render_report.py, so they cannot disagree -about a count. Standard library only (it runs in the plain python image). - - collect_summary.py --sample S --sample-dir DIR [--out S/summary.json] - -DIR is a bash output folder (GENOME_DIR/S) or a Nextflow one (outdir/S): both -publish names are read (vcf/ or roh/ for ROH, mosdepth/ or coverage/ for -depth, hla_t1k/ or hla/ for HLA, vcf/ or pharmcat/ for PharmCAT). - -Each section has a state: - ok read from its file - missing no file: the step has not run - failed no file, and its step failed in the latest run-all.sh run - (logs/run_status.tsv) - stale the file is older than the latest run-all.sh run and its step was - not ok in that run (logs/run_status.tsv); the values are shown - with the file's date - unreadable the file exists but could not be read (the reason is in note) - -The run manifest (S/run_manifest.tsv, bin/write_manifest.sh) is copied in for -the report footer, with the ClinVar release date and the HLA database release. -Two optional keys, for the report's run facts and badges: recheck, the last -database re-check (S/recheck/state.json, bin/owngenome_recheck.py), and new_since, -the date of the previous run kept in S/history/ and the sections whose results -changed since it (bin/run_delta.py's comparison). - -The step tools that once lived here are og verbs now (rust/crates/og-summary): -`og qc sample` writes step 33's identity and contamination table, `og prs -table` step 25's PRS table, and `og mito annotate|deletions|copies` step 20's -chrM tables. This file reads what they write. +# remnant: kept for the reports, the couple screen, the search index and the live site; deleted in lane R6b +"""collect_summary.py: what other Python still reads of the summary collector. + +The collector itself is `og summary collect` (rust/crates/og-summary): it +reads one sample folder's step outputs into summary.json, the one file both +reports, the live site and the run delta read. collect() below runs it, for +bin/render_report.py (steps 24, HTML_REPORT and generate-report.sh) and for +the live site's fallback while a run has no summary.json yet. The rest are +the names other Python imports until its own port: the section keys, titles +and steps (render_report.py), ClinVar hit rows and the gene tables' allele +cells (couple_report.py), the CSQ rule and the table readers +(results_index.py), and the HLA quality floor (render_report.py). `og dev +remnants` (tests/parity/remnants.tsv) checks that nothing else comes back. +Standard library only. """ -import argparse -import base64 import csv -import glob import gzip import json -import math import os -import re -import struct -import sys +import subprocess +import tempfile import zlib -from datetime import datetime, timezone -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import pgx_outside_calls # noqa: E402 step 36's rule for a CYP2D6 call and an HLA allele -import ancestry_fine_report # noqa: E402 step 26b's group names and painting colours -import paraphase_report # noqa: E402 step 35b's tables and its SMN1 rule beside Parascopy -import clip_check # noqa: E402 CPSR's variant ids and class names, as step 17's read check reads them -import annotsv_postfilter # noqa: E402 step 05's class 4-5 checks and the step 22 support join -import chip_report # noqa: E402 step 49's detection limit at the median depth -import run_delta # noqa: E402 the comparison with the previous run, for the "New since" badges -import traits_report # noqa: E402 step 42's genotype-rules columns and the rules the traits table holds -import genotype_rules_indels # noqa: E402 the APOL1 G2 deletion beside step 42's APOL1 rule +HERE = os.path.dirname(os.path.abspath(__file__)) +# The repository's assets/ folder beside bin/: og summary collect reads the +# ACMG SF rules, the carrier list and the other gene tables there. The +# containers of step 24 and HTML_REPORT mount bin/ alone, so render_report.py +# --assets names the folder instead. +ASSETS_DIR = os.path.join(os.path.dirname(HERE), "assets") + +# The summary's sections in report order: key, title, step. og summary +# collect writes the same keys, titles and steps (rust/crates/og-summary). +SECTIONS = [ + ("coverage", "Coverage (mosdepth)", "16b"), + ("callability", "Gene callability (mosdepth)", "16b"), + ("sex_check", "Sex check (indexcov)", "16"), + ("sample_qc", "Sample identity and contamination (somalier, VerifyBamID2)", "33"), + ("mosaic_sex_loss", "Mosaic loss of chromosome Y or X", "33"), + ("read_count", "Reads kept by alignment", "02"), + ("variants", "Variant calling", "03"), + ("clinvar", "ClinVar screen", "06"), + ("couple", "Couple carrier screen", "39"), + ("carrier", "Carrier list", "06"), + ("pharmcat", "PharmCAT", "07"), + ("cpic", "CPIC drug-gene recommendations", "27"), + ("pypgx", "pypgx", "32"), + ("cyrius", "Cyrius (CYP2D6)", "21"), + ("parascopy", "SMN1/SMN2 copy number (Parascopy, short reads)", "35"), + ("paralogs", "Paralog genes (Paraphase, PacBio HiFi)", "35b"), + ("hla", "HLA typing (T1K)", "08"), + ("prs", "Polygenic risk scores", "25"), + ("ancestry_fine", "Fine-grained ancestry (ADMIXTURE, RFMix)", "26b"), + ("manta", "Structural variants (Manta)", "04"), + ("delly", "Structural variants (Delly)", "19"), + ("cnvpytor", "Copy number (CNVpytor)", "18"), + ("sv_consensus", "SV consensus merge", "22"), + ("annotsv", "Structural variants, class 4 and 5 (AnnotSV)", "05"), + ("expansions", "Repeat expansions (ExpansionHunter)", "09"), + ("telomere", "Telomere content (TelomereHunter)", "10"), + ("roh", "Runs of homozygosity", "11"), + ("haplogroup", "Mitochondrial haplogroup", "12"), + ("y_haplogroup", "Y-chromosome haplogroup (Yleaf)", "37"), + ("y_tree", "Y placement on the current public tree", "37"), + ("y_str", "Y-STR markers (HipSTR)", "45"), + ("consumer_export", "Raw-data export (23andMe v5 layout)", "40"), + ("bloodtype", "Blood groups (predicted)", "41"), + ("lpa", "Lp(a): LPA KIV-2 repeat count", "46"), + ("vntyper", "MUC1 VNTR frameshifts (VNtyper)", "52"), + ("locityper", "Locityper second opinion at hard loci", "53"), + ("mito", "Mitochondrial variants", "20"), + ("traits", "Traits panel", "42"), + ("archaic", "Archaic ancestry (IBDmix)", "43"), + ("ancient", "Ancient populations (AADR PCA)", "44"), + ("viral", "Viral DNA in the unmapped reads", "47"), + ("exomiser", "Symptom-driven ranking (Exomiser)", "48"), + ("chip", "Clonal haematopoiesis (CHIP)", "49"), + ("mei", "Mobile element insertions (MEGAnE)", "50"), + ("mosaic", "Mosaic chromosomal alterations (MoChA)", "51"), + ("cpsr", "Cancer predisposition (CPSR)", "17"), + ("clinical", "Clinical variant filter", "23"), + ("slivar", "Variant prioritization (slivar)", "31"), +] + + +def og(): + """The og binary: OG when set (the tests), else bin/og beside this file.""" + return os.environ.get("OG") or os.path.join(HERE, "og") + + +def collect(sample, sample_dir, declared_sex=None, out=None): + """One sample folder's summary, as og summary collect writes it. With OUT + the summary is also kept in that file; else it goes through a temporary + file, removed after. og's messages go to stderr. RuntimeError when og + writes no summary (the folder cannot be read, or the file cannot be + written).""" + tmp = None + if out is None: + fd, tmp = tempfile.mkstemp(prefix="summary.", suffix=".json") + os.close(fd) + out = tmp + cmd = [og(), "summary", "collect", "--sample", sample, "--sample-dir", sample_dir, + "--assets", ASSETS_DIR, "--out", out] + if declared_sex: + cmd += ["--declared-sex", declared_sex] + try: + try: + # og reads bin/hla_alleles.tsv and bin/str_intermediate_ranges.tsv + # beside this file, wherever the og binary sits. + env = dict(os.environ, OG_BIN_TABLES=os.environ.get("OG_BIN_TABLES", HERE)) + rc = subprocess.run(cmd, stdout=subprocess.DEVNULL, env=env, check=False).returncode + except OSError as e: + raise RuntimeError(f"og summary collect did not start ({cmd[0]}): {e}") from e + if rc != 0: + raise RuntimeError(f"og summary collect wrote no summary of {sample} (exit {rc})") + with open(out, encoding="utf-8") as f: + return json.load(f) + finally: + if tmp: + try: + os.unlink(tmp) + except OSError: + pass -# 2: cyp2d6.consensus, Cyrius's Filter applied, HLA quality per allele, -# Stranger's status on the repeat loci, the CPIC unclassified count, and the -# section state 'failed'. -SCHEMA_VERSION = 2 # ClinVar review status -> stars. og clinvar hits (rust/crates/og-clinvar) holds the same table # (tests/test_collect_summary.py checks that the two agree). @@ -74,20 +134,17 @@ "criteria_provided,_conflicting_interpretations": 1, } + # The call floor of step 6 (scripts/06-clinvar-screen.sh writes a floor other # than this default into the hits VCF as ##PGP_CALL_FLOOR=GQ=..,DP=..): a P/LP # hit below either is a low-confidence call, listed but counted apart. A hits # file without the line used this default, or was written before the floor. CALL_FLOOR = {"GQ": 30, "DP": 10} + + # gnomAD allele frequency above which a P/LP hit is marked common. COMMON_AF = 0.01 -# The genotype check of the clinical filter (bin/csq_rank.awk, step 23 and the -# CLINICAL_FILTER module): step 6's GQ and DP floor, and the alt read fraction -# (AD) a heterozygote must lie within and a homozygous alt call must reach. -# bin/csq_rank.awk writes the floors it used into the clinical VCF as -# ##PGP_CALL_FLOOR=GQ=..,DP=..,HET_AB=lo-hi,HOM_AB=..; these are its defaults. -AB_FLOOR = {"HET_AB": (0.2, 0.8), "HOM_AB": 0.8} -IMPACT_RANK = {"HIGH": 4, "MODERATE": 3, "LOW": 2, "MODIFIER": 1} + # Steps run-all.sh runs and records in logs/run_status.tsv: the rows of # assets/steps.tsv with recorded=yes (tests/test_run_state.py keeps the two @@ -99,50 +156,7 @@ "26b", "27", "28", "29", "30", "31", "32", "33", "35", "35b", "37", "38", "39", "40", "41", "42", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53"} -# What this pipeline does not assess, whatever ran. -NOT_ASSESSED = [ - "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", - "Methylation and imprinting", - "Repeat expansions outside the ExpansionHunter catalog", - "Phase: two variants in one gene are compound-het candidates, not confirmed", - "Variants of uncertain significance (not interpreted)", -] - -# Added to NOT_ASSESSED unless step 35's depth count (og cn call --locus HBA, -# short reads) or step 35b (Paraphase, PacBio HiFi, both genes) made a call. -HBA_NOT_ASSESSED = ("HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count " - "(short reads) nor step 35b (Paraphase, PacBio HiFi) made a call") - -# Added to NOT_ASSESSED when step 50 (opt-in, MEGAnE) did not run; a failed -# step 50 is listed under not run, and unreadable tables under their section. -MEI_NOT_ASSESSED = ("Mobile element insertions (Alu, L1 and SVA, the FKTN, TAF1 and BBS1 SVA insertions among them): " - "not looked for (step 50, MEGAnE, is opt-in)") - -# Added to NOT_ASSESSED unless step 51 (MoChA) assessed the sample. -MCA_NOT_ASSESSED = ("Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share " - "of the blood cells): step 51 (MoChA) did not assess this sample") - -# Added to NOT_ASSESSED unless step 52 (VNtyper) gave a positive, possible or -# negative reading: no other step can see a frameshift inside the MUC1 VNTR. -VNTYPER_NOT_ASSESSED = ("MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; " - "no standard caller sees them") -# Added to NOT_ASSESSED unless step 25 ran with an ancestry reference panel. -PRS_NOT_ADJUSTED = "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums" - -EH_LOCI = ["HTT", "FMR1", "C9ORF72", "ATXN1", "DMPK"] - -# ACMG SF v3.3 (Lee et al., Genet Med 2025;27(8):101454): the 84 genes in which -# the ACMG recommends reporting pathogenic and likely pathogenic variants found -# by chance, and the reporting rule of each gene (Table 1), are the rows of -# assets/acmg_sf.tsv. A new list version means new rows there and a new -# ACMG_SF_VERSION. The sample's carrier list reads assets/carrier_genes.tsv. -# Both are read from ASSETS_DIR: the repository's assets/ folder beside bin/, -# or the folder render_report.py --assets names (the containers of step 24 and -# of HTML_REPORT mount bin/ only). -ACMG_SF_VERSION = "ACMG SF v3.3" -ASSETS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "assets") -SF_REPORT_IF = ("any", "biallelic", "hom_variant", "hemizygous", "het_female_too") SEVERITY_KINDS = ("hypomorph", "pseudodeficiency") @@ -150,29 +164,12 @@ class Unreadable(Exception): pass -def iso(ts): - return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") - - def open_text(path): if path.endswith((".gz", ".bgz")): return gzip.open(path, "rt", errors="replace") return open(path, errors="replace") -def first_existing(base, rels): - for rel in rels: - for p in sorted(glob.glob(os.path.join(base, rel), recursive=True)): - if os.path.isfile(p): - return p - return None - - -def newest(base, rels): - found = [p for rel in rels for p in glob.glob(os.path.join(base, rel), recursive=True) if os.path.isfile(p)] - return max(found, key=os.path.getmtime) if found else None - - def vcf_records(path): """Yield the split data lines of a VCF (plain or gzip).""" try: @@ -198,27 +195,6 @@ def read_tsv(path): return list(csv.DictReader(f, delimiter="\t")) -# --- sections ------------------------------------------------------------------- - -def sec_variants(d, s): - p = first_existing(d, [f"vcf/{s}.vcf.gz"]) - if not p: - return None, {} - total = passed = snps = indels = 0 - for r in vcf_records(p): - if len(r) < 7: - continue - total += 1 - if r[6] == "PASS": - passed += 1 - alts = [a for a in r[4].split(",") if a not in (".", "*", "")] - if any(len(a) == len(r[3]) == 1 for a in alts): - snps += 1 - if any(len(a) != len(r[3]) and not a.startswith("<") for a in alts): - indels += 1 - return p, {"total": total, "pass": passed, "snps": snps, "indels": indels} - - def clinvar_variants(v, sep): """'chr1:10:C:T|chr1:15:G:A' -> 'chr1:10 C>Tchr1:15 G>A'.""" out = [] @@ -247,22 +223,6 @@ def clinvar_combined_note(info, gt=""): return "." -def call_floor(path): - """Step 6's call floor from its hits VCF header, CALL_FLOOR without the line.""" - floor = dict(CALL_FLOOR) - try: - with open_text(path) as f: - for line in f: - if not line.startswith("##"): - break - if line.startswith("##PGP_CALL_FLOOR="): - for k, v in re.findall(r"(GQ|DP)=(\d+)", line): - floor[k] = int(v) - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(path)}: {e}") from e - return floor - - def low_confidence(dp, gq, floor): """True when the call's GQ or DP is known and below the floor.""" return (gq is not None and gq < floor["GQ"]) or (dp is not None and dp < floor["DP"]) @@ -304,753 +264,6 @@ def clinvar_row(r, floor=None): } -def clinvar_cdna_checks(d, s, p=None): - """Step 06's cDNA read check (bin/clip_check.py), keyed by (chrom, pos, - ref, alt), or the table at p. None when the file is absent: the Nextflow - pipeline does not write it.""" - p = p or first_existing(d, [f"clinvar/{s}_clinvar_cdna_check.tsv"]) - if not p: - return None - try: - rows = read_tsv(p) - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(p)}: {e}") from e - - def num(v): - return int(v) if (v or "").isdigit() else None - out = {} - for r in rows: - if not (r.get("pos") or "").isdigit(): - continue - out[(r.get("chrom"), int(r["pos"]), r.get("ref"), r.get("alt"))] = { - "check": r.get("cdna_check") or "not_checked", "note": r.get("cdna_note") or "", - "alt_reads": num(r.get("alt_reads")), "alt_clipped": num(r.get("alt_clipped")), - "ref_reads": num(r.get("ref_reads")), "ref_clipped": num(r.get("ref_clipped")), - "construct": r.get("construct_site") or None} - return out - - -def clinvar_cdna_scan(d, s): - """Step 06's genome-wide cDNA scan (bin/cdna_scan.py): {"note", "loci"}, - or None when the table is absent (a run before the scan existed).""" - p = first_existing(d, [f"clinvar/{s}_clinvar_cdna_scan.tsv"]) - if not p: - return None - try: - with open_text(p) as f: - lines = f.read().splitlines() - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(p)}: {e}") from e - note = lines[0][len("# cdna_scan: "):] if lines and lines[0].startswith("# cdna_scan: ") else "" - body = [ln for ln in lines if not ln.startswith("#")] - loci = [] - for r in csv.DictReader(body, delimiter="\t"): - if not (r.get("start") or "").isdigit(): - continue - loci.append({"chrom": r.get("chrom"), "start": int(r["start"]), "end": int(r.get("end") or 0), - "junctions": int(r.get("junctions") or 0), "reads": int(r.get("reads") or 0), - "junction_list": r.get("junction_list") or ""}) - return {"note": note, "loci": loci} - - -def clinvar_deep_intronic(d, s): - """Step 06's always-report list (the known deep intronic sites): one row - per site with the sample's call, or None when the table is absent.""" - p = first_existing(d, [f"clinvar/{s}_clinvar_deep_intronic.tsv"]) - if not p: - return None - try: - rows = read_tsv(p) - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(p)}: {e}") from e - return [{"chrom": r.get("chrom"), "pos": int(r["pos"]), "ref": r.get("ref"), "alt": r.get("alt"), - "gene": r.get("gene") or ".", "hgvs": r.get("hgvs") or "", "clinvar_id": r.get("clinvar_id") or "", - "call": r.get("call") or "no call", "detail": r.get("detail") or ""} - for r in rows if (r.get("pos") or "").isdigit()] - - -def tabix_linear_index(path): - """{contig: linear index} of a bgzipped VCF's .tbi: for each 16 kb window, - the virtual file offset of the first record that can overlap it (standard - library only, as the rest of this script).""" - with gzip.open(path + ".tbi", "rb") as f: - b = f.read() - if b[:4] != b"TBI\x01": - raise ValueError(f"{os.path.basename(path)}.tbi is not a tabix index") - n_ref = struct.unpack_from("> 14, len(ioff) - 1)] - if voff == 0xFFFFFFFFFFFFFFFF: - voff = 0 - out, seen = [], False - with open(path, "rb") as raw: - raw.seek(voff >> 16) - g = gzip.GzipFile(fileobj=raw) - g.read(voff & 0xFFFF) - for line in g: - if line.startswith(b"#"): - continue - f = line.split(b"\t", 2) - if len(f) < 3 or not f[1].isdigit(): - continue - if f[0].decode() != chrom: - if seen: - break - continue - seen = True - if int(f[1]) > pos: - break - if int(f[1]) == pos: - out.append(line.decode(errors="replace").rstrip("\n").split("\t")) - return out - - -def vep_gnomad_af(vep, hits): - """gnomAD exome and genome AF and the rsIDs of each hit from step 13's VEP - output, as {(chrom, pos, ref, alt): (exome, genome, rsids)}; a hit with - no record is left out. The fields are VEP's gnomADe_AF, gnomADg_AF and - Existing_variation (--everything); rsids keeps the rs ids of - Existing_variation ("rs123&COSV1" gives ["rs123"]), sorted, [] without - the field.""" - fmt = None - with open_text(vep) as f: - for line in f: - if not line.startswith("#"): - break - if line.startswith("##INFO= 7 else [] - if r[3] != ref or alt not in alts: - continue - # VEP's Allele drops the base an indel's alleles share. - snv = len(ref) == 1 and all(len(a) == 1 for a in alts) - allele = alt if snv or any(a[:1] != ref[:1] for a in alts) else (alt[1:] or "-") - got, ids = [None, None], set() - for tr in info_map(r[7]).get("CSQ", "").split(","): - v = tr.split("|") - if len(v) != len(fmt) or (len(alts) > 1 and v[fmt.index("Allele")] != allele): - continue - for i, c in enumerate(cols): - if c is not None and got[i] is None: - got[i] = num(v[c]) - if ev is not None: - ids.update(x.lower() for x in v[ev].split("&") if re.fullmatch(r"(?i)rs\d+", x)) - out[key] = (got[0], got[1], sorted(ids, key=lambda x: int(x[2:]))) - return out - - -def clinvar_frequencies(d, s, rows): - """Put the gnomAD AF and the rsIDs (rsids, [] when unknown) from step - 13's VEP output on each ClinVar hit, and say where they came from or why - there are none. The hits VCF's ID column is ClinVar's own variation id, - not an rsID, so VEP is where a hit's rsID comes from.""" - vep = first_existing(d, [f"vep/{s}_vep.vcf.gz"]) - note = None - if not vep: - note = "no VEP output (step 13) in this folder" - elif not os.path.exists(vep + ".tbi"): - note = f"{os.path.relpath(vep, d)} has no .tbi index" - found = {} - if note is None: - try: - found = vep_gnomad_af(vep, rows) - except (OSError, EOFError, ValueError, struct.error, gzip.BadGzipFile, zlib.error) as e: - note = f"{os.path.relpath(vep, d)} could not be read: {e}" - for x in rows: - e, g, ids = found.get((x["chrom"], x["pos"], x["ref"], x["alt"]), (None, None, [])) - known = [v for v in (e, g) if v is not None] - x["gnomad_exome_af"], x["gnomad_genome_af"] = e, g - x["rsids"] = ids - x["af"] = max(known) if known else None - x["common"] = x["af"] is not None and x["af"] > COMMON_AF - return {"source": os.path.relpath(vep, d) if note is None else None, "note": note, "common_af": COMMON_AF} - - -def sec_clinvar(d, s): - p = first_existing(d, [f"clinvar/{s}_clinvar_hits.vcf"]) - if not p: - return None, {} - floor = call_floor(p) - rows = [clinvar_row(r, floor) for r in vcf_records(p) if len(r) >= 8] - rows.sort(key=lambda x: (-x["stars"], x["chrom"], x["pos"])) - # The headline count and the stars leave the low-confidence calls out; - # they stay in the hits, marked, and are counted on their own. - sure = [x for x in rows if not x["low_confidence"]] - by = {str(k): sum(1 for x in sure if x["stars"] == k) for k in (4, 3, 2, 1, 0)} - data = {"count": len(sure), "by_stars": by, "hits": rows, "low_confidence": len(rows) - len(sure), - "call_floor": floor, "frequency": clinvar_frequencies(d, s, rows)} - # The cDNA read check, on each hit when step 06 wrote it. - cdna = clinvar_cdna_checks(d, s) - if cdna is not None: - for x in rows: - x["cdna"] = cdna.get((x["chrom"], x["pos"], x["ref"], x["alt"])) or { - "check": "not_checked", "note": "not in the read-check table", "alt_reads": None, - "alt_clipped": None, "ref_reads": None, "ref_clipped": None} - data["cdna_flagged"] = sum(1 for x in rows if x["cdna"]["check"] == "flagged") - data["construct_sites"] = sum(1 for x in rows if x["cdna"].get("construct")) - scan = clinvar_cdna_scan(d, s) - if scan is not None: - data["cdna_scan"] = scan - deep = clinvar_deep_intronic(d, s) - if deep is not None: - data["deep_intronic"] = deep - return p, data - - -def pharmcat_json(d, s): - return newest(d, [f"pharmcat/{s}.report.json", "pharmcat/*.report.json", - f"vcf/{s}.report.json", "vcf/*.report.json"]) - - -def sec_pharmcat(d, s): - p = pharmcat_json(d, s) - if not p: - return None, {} - try: - with open(p) as f: - data = json.load(f) - except (OSError, ValueError) as e: - raise Unreadable(f"{os.path.basename(p)}: {e}") - html = os.path.isfile(p[: -len(".json")] + ".html") - return p, {"version": str(data.get("pharmcatVersion", "")) if isinstance(data, dict) else "", - "html_report": html} - - -def sec_cpic(d, s): - p = first_existing(d, [f"cpic/{s}_phenotypes.tsv"]) - if not p: - return None, {} - genes = [] - for row in read_tsv(p): - genes.append({"gene": row.get("Gene", ""), "diplotype": row.get("Diplotype", ""), - "phenotype": row.get("Phenotype", ""), "status": row.get("Status", "")}) - rec = os.path.join(os.path.dirname(p), f"{s}_cpic_recommendations.txt") - warnings, failed = [], False - if os.path.isfile(rec): - with open(rec, errors="replace") as f: - for line in f: - if "PARSING FAILED" in line: - failed = True - if line.strip().startswith("WARNING:"): - warnings.append(line.strip()[len("WARNING:"):].strip()) - return p, {"genes": genes, - "non_normal": sum(1 for g in genes if g["status"] == "non-normal"), - "ambiguous": sum(1 for g in genes if g["status"] == "ambiguous"), - "not_called": sum(1 for g in genes if g["status"] == "not called"), - "unclassified": sum(1 for g in genes if g["status"] == "unclassified"), - "parse_failed": failed or not genes, - "warnings": warnings} - - -# The rows of the CYP2D6 depth check (bin/cyp2d6_depth_check.py) the reports -# show: its verdict and the MAPQ shares beside it. -DEPTH_CHECK_KEYS = ("status", "gene_mapq1_share", "gene_mapq20_share", "body_mapq1_share", "body_mapq20_share") - - -def sec_pypgx(d, s): - p = first_existing(d, [f"pypgx/{s}_pypgx_summary.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - # A no-call by step 36's list (Indeterminate, FAILED, N/A...), in any case - called = [r for r in rows if (r.get("Diplotype") or "").strip().lower() not in pgx_outside_calls.NO_CALL] - cyp_row = next((r for r in rows if r.get("Gene") == "CYP2D6"), None) - cyp = (cyp_row or {}).get("Diplotype", "") - out = {"genes_total": len(rows), "genes_called": len(called), "cyp2d6": cyp or "not called"} - # pypgx's copy-number call from read depth, one entry per BAM-based gene - # (a VCF gene has none). A gene not callable on this reference gets its - # own line below instead; one without a call ('N/A': pypgx failed, or - # CYP2D6 after a failed depth check) says so, so no gene drops out. - out["cnv_call"] = {} - for r in rows: - status = (r.get("Status") or "").strip() - if (r.get("Source") or "").strip().upper() != "BAM" or status == pgx_outside_calls.NOT_CALLABLE: - continue - v = (r.get("CNV_call") or "").strip() - if v not in ("", ".") and not v.startswith("N/A"): - out["cnv_call"][r["Gene"]] = v - elif status.lower() == "failed" or (r.get("Diplotype") or "").strip().upper() == "FAILED": - out["cnv_call"][r["Gene"]] = "not called: pypgx failed" - elif "depth check" in status or "depth check" in v: - out["cnv_call"][r["Gene"]] = "not called: the CYP2D6 depth check failed" - else: - out["cnv_call"][r["Gene"]] = "not called: pypgx gave no copy-number call" - out["not_callable"] = [r["Gene"] for r in rows - if (r.get("Status") or "").strip() == pgx_outside_calls.NOT_CALLABLE] - out["low_gq"] = [{"gene": r["Gene"], "diplotype": (r.get("Diplotype") or "").strip(), - "min_gq": int(r["Min_GQ"]) if (r.get("Min_GQ") or "").strip().isdigit() else None, - "weakest": (r.get("Weakest_variant") or "").strip()} - for r in rows if (r.get("Quality") or "").strip() == pgx_outside_calls.LOW_GQ] - if cyp_row is not None: - out["cyp2d6_evidence"] = {"phenotype": (cyp_row.get("Phenotype") or "").strip(), - "defining_filter": (cyp_row.get("Defining_filter") or "").strip(), - "quality": (cyp_row.get("Quality") or "").strip()} - dc = os.path.join(os.path.dirname(p), f"{s}_cyp2d6_depth_check.tsv") - if os.path.isfile(dc): - with open(dc, errors="replace") as f: - kv = dict(line.rstrip("\n").split("\t", 1) for line in f if "\t" in line) - out["depth_check"] = {k: kv[k] for k in DEPTH_CHECK_KEYS if k in kv} - comp = os.path.join(os.path.dirname(p), f"{s}_pharmcat_comparison.tsv") - if os.path.isfile(comp): - crow = read_tsv(comp) - out["comparison"] = { - "conflicts": sum(1 for r in crow if r.get("Match") == "No"), - "one_tool_only": sum(1 for r in crow if r.get("Match") in ("pypgx only", "PharmCAT only")), - "concordant": sum(1 for r in crow if r.get("Match") == "Yes"), - } - return p, out - - -def sec_cyrius(d, s): - p = first_existing(d, [f"cyrius/{s}_cyp2d6.tsv", "cyrius/*_cyp2d6.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - r = rows[0] if rows else {} - return p, {"genotype": r.get("Genotype", "") or "none", "filter": r.get("Filter", "")} - - -def paralog_regions(): - """The Paraphase regions the report shows gene by gene, in the order of - assets/paraphase_genes.tsv; the others are counted.""" - out = [] - for r in asset_rows("paraphase_genes.tsv"): - if r["region"] not in out: - out.append(r["region"]) - return out - - -def sec_paralogs(d, s): - """Step 35b (Paraphase, PacBio HiFi): the genes of the regions of - assets/paraphase_genes.tsv, a count of the other regions, the variant rows - per region, and SMN1 beside step 35's Parascopy call when that table is - there too.""" - p = first_existing(d, [f"paralogs/{s}_paraphase_copy_number.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - if rows and not {"region", "gene", "copy_number", "status"} <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} lacks the region, gene, copy_number or status column") - PARALOG_REGIONS = paralog_regions() - genes = [{"region": r["region"], "gene": r["gene"], "copy_number": r["copy_number"], - "status": r["status"], "detail": r.get("detail") or "."} - for r in rows if r["region"] in PARALOG_REGIONS] - genes.sort(key=lambda g: PARALOG_REGIONS.index(g["region"])) - others = [r for r in rows if r["region"] not in PARALOG_REGIONS] - variants = {} - vp = os.path.join(os.path.dirname(p), f"{s}_paraphase_variants.tsv") - if os.path.isfile(vp): - for r in read_tsv(vp): - variants[r.get("region", "")] = variants.get(r.get("region", ""), 0) + 1 - smn1 = next((g["copy_number"] for g in genes if g["gene"] == "SMN1"), None) - smn1 = int(smn1) if smn1 and smn1.isdigit() else None - parascopy = None - pp = os.path.join(os.path.dirname(p), f"{s}_smn_copy_number.tsv") - if os.path.isfile(pp): - parascopy = paraphase_report.parascopy_smn1(pp) - return p, {"genes": genes, - "other_regions": len(others), - "other_regions_called": sum(1 for r in others if r["status"] == "called"), - "variants": variants, - "smn1": {"paraphase": smn1, "parascopy": parascopy, - "agree": paraphase_report.compare_smn1(smn1, parascopy)}} - - -# SMN2 exon 7, c.840 (the base that tells SMN2 from SMN1), GRCh38 1-based; -# SMN1's is paraphase_report.SMN1_EXON7. -SMN2_EXON7 = ("chr5", 70076526) - - -def parascopy_cn_at(rows, chrom, pos): - """The copy number at one position from step 35's table: the - paralog-specific value (psCN) of the row, or of its homologous region, - that holds the position, with the row's aggregate value (agCN). As - paraphase_report.parascopy_smn1: cn is None unless psCN's filter is PASS - and its quality at least 20 (Parascopy's threshold).""" - for r in rows: - try: - regions = [(r["chrom"], int(r["start"]) + 1, int(r["end"]))] - except (KeyError, ValueError, TypeError): # TypeError: a short row (None fields) - continue - for h in (r.get("homologous_regions") or "*").split(","): - m = re.match(r"^([^:]+):(\d+)-(\d+)", h.strip()) - if m: - regions.append((m.group(1), int(m.group(2)), int(m.group(3)))) - idx = next((i for i, (c, b, e) in enumerate(regions) if c == chrom and b <= pos <= e), None) - if idx is None: - continue - cns = (r.get("psCN") or "").split(",") - quals = (r.get("psCN_qual") or "").split(",") - cn_txt = cns[idx] if idx < len(cns) else "?" - q_txt = quals[idx] if idx < len(quals) else "*" - filt = r.get("psCN_filter") or "." - try: - q = float(q_txt) - except ValueError: - q = None - ok = filt == "PASS" and q is not None and q >= 20 and cn_txt.isdigit() - return {"cn": int(cn_txt) if ok else None, - "text": f"{cn_txt} (quality {q_txt}, filter {filt})" + ("" if ok else ", not resolved"), - "agcn": r.get("agCN") or ".", "agcn_qual": r.get("agCN_qual") or ".", - "agcn_filter": r.get("agCN_filter") or "."} - return {"cn": None, "text": "no row covers exon 7", "agcn": ".", "agcn_qual": ".", "agcn_filter": "."} - - -MARKER_FIELDS = ("marker", "rsid", "status", "alt_reads", "depth", "fraction", "copies", "message") - - -def read_marker_table(p, step, key): - """A copy-aware marker table (metric, value; bin/traits_report.py MARKERS): - step 46's KIV-2 4925G>A beside the count or step 35's SMN1 g.27134T>G - beside the psCN. A table that is not there or has no known status reads - as not assessed, with the reason in its message, so the report line - never goes missing beside the count.""" - m = traits_report.MARKERS[key] - what = m["name"] - out = dict.fromkeys(MARKER_FIELDS, ".") - out.update(marker=what, rsid=m["dbsnp"][0], status="not assessed") - if not os.path.isfile(p): - out["message"] = (f"{what}: not assessed, step {step} wrote no {os.path.basename(p)} (a run from before " - "this reader, or the check did not run).") - return out - try: - kv = dict(read_tsv_pairs(p)) - except OSError as e: - kv = {"message": f"{os.path.basename(p)} could not be read: {e}"} - if kv.get("status") not in traits_report.MARKER_STATUSES: - out["message"] = (f"{what}: not assessed, {os.path.basename(p)} has no status row " - f"({', '.join(traits_report.MARKER_STATUSES)}).") - return out - out.update({k: kv[k] for k in MARKER_FIELDS if kv.get(k)}) - return out - - -def smn_reading(smn1): - """What an SMN1 copy number means for SMA, in one line.""" - if smn1 is None: - return "SMN1 not resolved (quality under 20 or filter not PASS): no carrier reading" - if smn1 == 0: - return "no SMN1 copy: the pattern of spinal muscular atrophy (SMA) itself" - if smn1 == 1: - return "one SMN1 copy: the pattern of an SMA carrier" - if smn1 == 2: - return ("two SMN1 copies: not a carrier by copy number; a 2+0 carrier (both copies on one chromosome) " - "cannot be ruled out") - return f"{smn1} SMN1 copies: not a carrier by copy number" - - -# Tables in paralogs/ that other parts of steps 35 and 35b write, not catalog rows. -NOT_CATALOG_TABLES = ("hba", "paraphase") -# agCN_filter and agCN_qual too: without them a catalog locus would read as -# "not resolved" instead of unreadable. -PARASCOPY_COLUMNS = {"chrom", "start", "end", "agCN", "agCN_filter", "agCN_qual", "psCN", "psCN_qual", "psCN_filter"} - - -def tsv_columns(path): - """The header of a tab-separated table as a set ({} for an empty file), so - a table with a header and no row is checked for its columns too.""" - with open_text(path) as f: - return set(next(csv.reader(f, delimiter="\t"), [])) - - -def parascopy_tables(d, s): - """{table: path} of step 35's catalog tables (__copy_number.tsv).""" - out = {} - for p in sorted(glob.glob(os.path.join(glob.escape(os.path.join(d, "paralogs", s)) + "_*_copy_number.tsv"))): - name = os.path.basename(p)[len(s) + 1:-len("_copy_number.tsv")] - if name not in NOT_CATALOG_TABLES and os.path.isfile(p): - out[name] = p - return out - - -def parascopy_locus(name, path, row): - """One catalog table that is not SMN1's: the locus's aggregate copy number - (agCN) per region of its profile, the catalog's genes, tag and reading. - psCN is not read here: the catalog marks it reliable only for SMN1.""" - out = {"table": name, "genes": (row or {}).get("genes") or name, "tag": (row or {}).get("tag") or ".", - "reading": (row or {}).get("reading") or "."} - rows = read_tsv(path) - if not PARASCOPY_COLUMNS <= tsv_columns(path): - return dict(out, status="unreadable", cn=None, - text=f"unreadable: {os.path.basename(path)} lacks the copy-number columns") - na = next((r["agCN_filter"] for r in rows if (r.get("agCN_filter") or "").startswith("not assessed")), None) - if na: - return dict(out, status="not assessed", cn=None, text=na) - regions = [] - for r in rows: - try: - q = float(r.get("agCN_qual") or "") - except ValueError: - q = None - cn = r.get("agCN") or "." - ok = cn.isdigit() and r.get("agCN_filter") == "PASS" and q is not None and q >= 20 - regions.append((r.get("chrom"), r.get("start"), r.get("end"), cn, r.get("agCN_qual") or ".", - r.get("agCN_filter") or ".", ok)) - if not regions: - return dict(out, status="not assessed", cn=None, text="not assessed: the table has no region row") - values = {x[3] for x in regions if x[6]} - if len(values) == 1 and all(x[6] for x in regions): - cn = int(values.pop()) - quals = [float(x[4]) for x in regions] - text = f"{cn} ({len(regions)} region(s), quality {min(quals):g} or more, filter PASS)" - return dict(out, status="called", cn=cn, text=text) - # every region, so a LowQual or discordant one is never hidden; no single - # number, so no catalog reading either (the reading follows a call) - text = "; ".join(f"{c}:{int(b) + 1 if str(b).isdigit() else b}-{e} {cn} (quality {q}, filter {f})" - for c, b, e, cn, q, f, _ in regions) - called = any(x[6] for x in regions) - return dict(out, status="called" if called else "not resolved", cn=None, reading=".", - text=text + ("" if called else ": no region with a whole copy number, filter PASS and quality 20 or more")) - - -NO_SMN_TABLE = "no SMN1 table (step 35 wrote no SMN copy-number table)" - - -def sec_parascopy(d, s): - """Step 35 (Parascopy, short reads): SMN1 and SMN2 copy number at exon 7, - the aggregate copy number of the pair there and the SMA reading; the HBA - count from depth; and every other catalog table, one locus each. Each part - is read on its own, so a catalog without the SMN1 row still shows HBA and - the other loci.""" - tables = parascopy_tables(d, s) - hp = os.path.join(d, "paralogs", f"{s}_hba_copy_number.tsv") - ip = os.path.join(d, "paralogs", f"{s}_cn_intervals.tsv") - cp = os.path.join(d, "paralogs", f"{s}_cyp21a2_psv.tsv") - dp = os.path.join(d, "paralogs", f"{s}_cn_dosage.tsv") - p = (tables.get("smn") or next(iter(tables.values()), None) - or next((x for x in (hp, ip, cp, dp) if os.path.isfile(x)), None)) - if not p: - return None, {} - if "smn" in tables: - rows = read_tsv(tables["smn"]) - if not PARASCOPY_COLUMNS <= tsv_columns(tables["smn"]): - raise Unreadable(f"{os.path.basename(tables['smn'])} lacks the chrom, start, end, agCN, agCN_filter, " - "agCN_qual, psCN, psCN_qual or psCN_filter column") - smn1 = parascopy_cn_at(rows, *paraphase_report.SMN1_EXON7) - smn2 = parascopy_cn_at(rows, *SMN2_EXON7) - # Step 35 writes one row whose agCN_filter is "not assessed: " - # when the locus was not run (no read over it, no model, Parascopy failed). - na = next((r["agCN_filter"] for r in rows if (r.get("agCN_filter") or "").startswith("not assessed")), None) - reading = (f"SMN1 and SMN2 {na}: no carrier reading" if na - else "no row of step 35's table covers SMN1 exon 7: no carrier reading" - if smn1["text"] == "no row covers exon 7" else smn_reading(smn1["cn"])) - else: - rows = [] - smn1 = {"cn": None, "text": NO_SMN_TABLE, "agcn": ".", "agcn_qual": ".", "agcn_filter": "."} - smn2 = {"cn": None, "text": NO_SMN_TABLE} - reading = f"{NO_SMN_TABLE}: no carrier reading" - others = {n: t for n, t in tables.items() if n != "smn"} - catalog = {r["table"]: r for r in asset_rows("paralog_loci.tsv")} if others else {} - return p, {"smn1": smn1, "smn2": {"cn": smn2["cn"], "text": smn2["text"]}, - "agcn": smn1["agcn"], "agcn_qual": smn1["agcn_qual"], "agcn_filter": smn1["agcn_filter"], - # SMN1 g.27134T>G beside the psCN (step 35, _smn1_marker.tsv) - "smn1_marker": read_marker_table(os.path.join(d, "paralogs", f"{s}_smn1_marker.tsv"), "35", - "SMN1_27134"), - "rows": len(rows), - # HBA1/HBA2 from depth (og cn call --locus HBA); None when - # step 35 wrote no HBA table (a run from before the check) - "hba": read_hba(hp) if os.path.isfile(hp) else None, - # the interval rows of bin/cn_loci.tsv (recurrent CNV regions, - # carrier deletions, AZF); None without step 35's table - "cn_intervals": read_cn_intervals(ip) if os.path.isfile(ip) else None, - # CYP21A2 from depth and PSV sites (og cn call, the - # CYP21A2 row): copies, module total, finding; None without it - "cyp21a2": read_hba(cp) if os.path.isfile(cp) else None, - # the gene-dosage rows (HP, LCE3BC, CCL3L1): total copies from - # depth, one entry per locus; None without step 35's table - "dosage": read_cn_dosage(dp) if os.path.isfile(dp) else None, - "reading": reading, - # every other catalog table, in the catalog's order (a table the - # catalog does not name, from a PARALOG_LOCI of the user's, last) - "loci": [parascopy_locus(n, others[n], catalog.get(n)) - for n in sorted(others, key=lambda n: (list(catalog).index(n) if n in catalog - else len(catalog), n))]} - - -def read_cn_intervals(path): - """Step 35's interval table (og cn intervals): the rows with - a deletion or a duplication, and a count of the rows by status.""" - rows = read_tsv(path) - cols = ("locus", "name", "chrom", "start", "end", "status", "copies", "expected", "change") - if not set(cols) <= tsv_columns(path): - raise Unreadable(f"{os.path.basename(path)} lacks the locus, name, status, copies, expected or change column") - short_row(path, rows, cols) - counts = {} - for r in rows: - counts[r["status"]] = counts.get(r["status"], 0) + 1 - return {"rows": len(rows), "by_status": counts, - "changes": [{k: r[k] for k in ("locus", "name", "chrom", "start", "end", "copies", "expected", "change")} - for r in rows if r["change"] in ("deletion", "duplication")]} - - -DOSAGE_COLUMNS = ("locus", "status", "copies", "estimate", "reason", "reading") - - -def read_cn_dosage(path): - """Step 35's gene-dosage table (HP, LCE3BC, CCL3L1): one entry per locus, - in the table's order.""" - if not set(DOSAGE_COLUMNS) <= tsv_columns(path): - raise Unreadable(f"{os.path.basename(path)} lacks the {', '.join(DOSAGE_COLUMNS[:-1])} or reading column") - rows = read_tsv(path) - short_row(path, rows, DOSAGE_COLUMNS) - return [{k: r.get(k) or "." for k in DOSAGE_COLUMNS} for r in rows] - - -def short_row(path, rows, cols): - """A row cut short of a column (csv gives None there) makes the table - unreadable, so it never reads as a row of '.' values.""" - bad = next((i for i, r in enumerate(rows, 2) if any(r.get(k) is None for k in cols)), None) - if bad: - raise Unreadable(f"{os.path.basename(path)} line {bad} lacks a field") - - -def read_hba(path): - """Step 35's HBA table (metric, value) as a dict, and its CYP21A2 table - the same way; a table without a status row reads as not assessed, with - that reason.""" - out = {} - with open(path, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t", 1) - if len(c) == 2 and c[0] != "metric": - out[c[0]] = c[1] - if out.get("status") not in ("called", "not assessed"): - out = {"status": "not assessed", "message": f"not assessed: {os.path.basename(path)} has no status row"} - return out - - -def sec_hla(d, s): - p = first_existing(d, [f"hla_t1k/{s}_hla_genotype.tsv", f"hla/{s}_hla_genotype.tsv", "hla/*_hla_genotype.tsv"]) - if not p: - return None, {} - loci = [] - with open(p, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 3 or c[0].startswith("#"): - continue - # T1K: allele, abundance, quality, twice; '.', 0, -1 when there is no - # second allele. Each quality stays with its allele. - alleles, quals = [], [] - for i in (2, 5): - a = c[i] if len(c) > i else "" - if a and a != ".": - alleles.append(a) - quals.append(c[i + 2] if len(c) > i + 2 else "") - # Step 36's rule (bin/pgx_outside_calls.py): an allele quality of 0 or - # below withholds the whole gene; only HLA-A and HLA-B reach PharmCAT. - low = any(hla_quality(q) <= 0 for q in quals) - loci.append(hla_locus_extras({"gene": c[0], "alleles": alleles, "quality": quals, "low_confidence": low, - "withheld_from_pharmcat": low and c[0] in pgx_outside_calls.HLA_GENES}, - c[1] if len(c) > 1 else "")) - out = {"loci": loci, "derived": hla_derived(loci)} - ot, kir = hla_optitype(d, s), kir_genotypes(d, s) - if ot is not None: - out["optitype"] = ot - if kir is not None: - out["kir"] = kir - return p, out - - -def first_comment(path): - """The first '#' line of a file step 08 wrote instead of a result, without the '#'.""" - with open(path, errors="replace") as f: - for line in f: - if line.startswith("#"): - return line.lstrip("#").strip() - return "" - - -def hla_optitype(d, s): - """Step 08's second HLA caller (HLA_QC=true): per class I gene the row of - its concordance table (pgx_outside_calls.py hla-concordance), or, when - there is no table, why OptiType's result file says it did not run. None - when OptiType was not asked for.""" - conc = first_existing(d, [f"hla_optitype/{s}_hla_concordance.tsv"]) - res = first_existing(d, [f"hla_optitype/{s}_optitype_result.tsv"]) - if not conc and not res: - return None - if not conc: - return {"genes": [], "reason": first_comment(res) or "no concordance table: step 08 did not compare the callers"} - genes = [] - with open(conc, errors="replace") as f: - for r in csv.DictReader(f, delimiter="\t"): - if (r.get("Gene") or "").strip(): - genes.append({"gene": r["Gene"].strip(), "t1k": (r.get("T1K") or "").strip(), - "optitype": (r.get("OptiType") or "").strip(), "agree": (r.get("Agree") or "").strip(), - "note": (r.get("Note") or "").strip()}) - return {"genes": genes, "reason": "" if genes else "the concordance table has no rows"} - - -def kir_genotypes(d, s): - """Step 08's KIR pass (KIR=true): the genes T1K gave an allele, with each - allele's quality, and the genes it gave none; or the line step 08 wrote - for a BAM with too few KIR reads. None when KIR was not typed.""" - p = first_existing(d, [f"kir_t1k/{s}_kir_genotype.tsv", f"kir/{s}_kir_genotype.tsv"]) - if not p: - return None - typed, not_called = [], [] - with open(p, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 3 or c[0].startswith("#"): - continue - alleles, quals = [], [] - for i in (2, 5): - if len(c) > i and c[i] not in ("", "."): - alleles.append(c[i]) - quals.append(c[i + 2] if len(c) > i + 2 else "") - if alleles: - typed.append({"gene": c[0], "alleles": alleles, "quality": quals}) - else: - not_called.append(c[0]) - reason = "" if typed or not_called else (first_comment(p) or "the KIR genotype file has no gene rows") - return {"typed": typed, "not_called": not_called, "reason": reason} - - def hla_quality(q): """T1K's quality as a number; one that is not a number counts as -1, as in step 36.""" try: @@ -1059,2000 +272,49 @@ def hla_quality(q): return -1.0 -# --- HLA: candidate caps, the HFE label, the DQ haplotypes and the allele table --- -# Rules in docs/08-hla-typing.md "What the report derives". The allele table is -# bin/hla_alleles.tsv (in bin/ because the report containers mount only bin/). - -HLA_ALLELES_TSV = os.path.join(os.path.dirname(os.path.abspath(__file__)), "hla_alleles.tsv") HLA_MAX_CANDIDATES = 3 # a quality-0 allele with more candidates is "no confident call" -HLA_HFE_LABEL = ("T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the " - "variant calls (ClinVar section)") - - -def hla_name(x): - """'HLA-DQB1*02:01:01' or 'DQB1*02:01:01' -> 'DQB1*02:01:01'; 'HLA-DQB1' -> 'DQB1'.""" - return x.strip()[4:] if x.strip().startswith("HLA-") else x.strip() - - -def hla_candidates(allele): - """T1K's allele field: one allele, or equally likely candidates separated by commas.""" - return [a for a in (hla_name(x) for x in allele.split(",")) if a] - - -def hla_locus_extras(locus, n_diff): - """Cap a quality-0 candidate list to HLA_MAX_CANDIDATES (the full list stays - in the TSV), count the candidates, mark one homozygous call and label the - HFE row.""" - counts, kept = [], [] - for a, q in zip(locus["alleles"], locus["quality"] + [""] * len(locus["alleles"])): - cands = hla_candidates(a) - counts.append(len(cands)) - kept.append(",".join(cands[:HLA_MAX_CANDIDATES]) if hla_quality(q) <= 0 and len(cands) > HLA_MAX_CANDIDATES - else a) - locus["alleles"] = kept - locus["candidates"] = counts - # T1K writes one allele and '. 0 -1' for a homozygous gene (or a second allele it did not see). - locus["one_allele_type"] = n_diff.strip() == "1" and len(kept) == 1 - if hla_name(locus["gene"]) == "HFE": - locus["label"] = HLA_HFE_LABEL - return locus - - -def hla_match(allele, group): - """'yes' when every candidate of the allele is in the group (B*57:01 holds - B*57:01:01:02), 'no' when none is, 'mixed' when only some are.""" - hits = [c == group or c.startswith(group + ":") for c in hla_candidates(allele)] - return "yes" if hits and all(hits) else ("mixed" if any(hits) else "no") - - -def hla_read_table(path=HLA_ALLELES_TSV): - """bin/hla_alleles.tsv as a list of dicts (tests/test_hla_report.py checks its schema).""" - rows, head = [], None - with open(path) as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if head is None: - head = c - continue - rows.append(dict(zip(head, c))) - return rows - - -def hla_derived(loci, table=None): - """The DQ haplotypes, the DQB1*02 dose and each table allele present or absent.""" - by = {} - for l in loci: - by.setdefault(hla_name(l["gene"]), l) - - def usable(gene): - l = by.get(gene) - if not l or not l["alleles"]: - return None, f"HLA-{gene} not typed" - if l["low_confidence"]: - return None, f"HLA-{gene} low confidence" - return l, "" - - def carries(l, group): - """'yes', 'no' or 'mixed' over both alleles of a locus.""" - m = [hla_match(a, group) for a in l["alleles"]] - return "yes" if "yes" in m else ("mixed" if "mixed" in m else "no") - - def both(a, b): - return "present" if a == b == "yes" else ("absent" if "no" in (a, b) else "uncertain") - - dqa, ra = usable("DQA1") - dqb, rb = usable("DQB1") - coeliac = {"status": "not_determined", "reason": "; ".join(r for r in (ra, rb) if r), - "dq2_5": None, "dq2_5_phase": None, "dq2_2": None, "dq8": None, "dqb1_02_dose": None} - if dqb: - m = [hla_match(a, "DQB1*02") for a in dqb["alleles"]] - if "mixed" not in m: - n = m.count("yes") - coeliac["dqb1_02_dose"] = 2 * n if dqb["one_allele_type"] else n - if dqa and dqb: - coeliac.update(status="derived", reason="") - coeliac["dq2_5"] = both(carries(dqa, "DQA1*05"), carries(dqb, "DQB1*02")) - coeliac["dq2_2"] = both(carries(dqa, "DQA1*02:01"), carries(dqb, "DQB1*02:02")) - coeliac["dq8"] = both(carries(dqa, "DQA1*03"), carries(dqb, "DQB1*03:02")) - if coeliac["dq2_5"] == "present": - drb1, why = usable("DRB1") - dr3 = carries(drb1, "DRB1*03") if drb1 else None - coeliac["dq2_5_phase"] = ("likely cis (DRB1*03 present)" if dr3 == "yes" - else "possible trans (no DRB1*03)" if dr3 == "no" - else f"phase unknown ({why or 'DRB1*03 uncertain'})") - alleles = [] - for r in (hla_read_table() if table is None else table): - gene = r["allele"].split("*")[0] - l, why = usable(gene) - state = carries(l, r["allele"]) if l else None - alleles.append({"allele": r["allele"], "kind": r["kind"], "association": r["association"], - "pmids": r["pmids"], "citation": r["citation"], - "state": {"yes": "present", "no": "absent", "mixed": "uncertain", None: "not_determined"}[state], - "reason": why}) - return {"coeliac": coeliac, "alleles": alleles} - - -def sec_prs(d, s): - p = first_existing(d, [f"prs/{s}_prs_summary.tsv"]) - if not p: - return None, {} - rows = [] - for r in read_tsv(p): - pct = r.get("Percentile", "") or "" - rows.append({"condition": r.get("Condition", ""), "pgs_id": r.get("PGS_ID", ""), - "score": r.get("Score_SUM", ""), "matched": r.get("Variants_Matched", ""), - "total": r.get("Variants_Total", ""), - "chrx": r.get("ChrX", ""), - # "12 dropped", "12 kept", "12 dropped, strand check failed", "0"; empty in older tables - "ambiguous": r.get("Ambiguous", ""), - # the sex a sex-specific score applies to, and the check against the declared sex - "applies_to": r.get("Applies_To", ""), "sex_check": r.get("Sex_Check", ""), - "percentile": "" if pct == "NA" else pct, - "group": "" if r.get("Ancestry_Group", "NA") == "NA" else r.get("Ancestry_Group", ""), - # the groups the PGS Catalog has an OR or HR of the score for, and the - # one of the sample's group ("HR 1.53 (EUR, PPM000591)"); empty in older tables - # None when the table has no Evaluated_In column (a score list without - # evaluations, or an older table): nothing is known, so nothing is said - "evaluated_in": ([x for x in r["Evaluated_In"].split(";") if x] - if r.get("Evaluated_In") is not None else None), - "effect": r.get("Effect", "") or ""}) - data = {"scores": rows, - # Percentiles exist only when step 25 ran pgsc_calc with an ancestry panel. - "adjusted": any(r["percentile"] for r in rows)} - a = first_existing(d, [f"ancestry/{s}_ancestry.tsv", f"prs/{s}_ancestry.tsv"]) - if a: - kv = {r.get("key"): r.get("value", "") for r in read_tsv(a) if r.get("key")} - data["ancestry"] = {"population": kv.get("population", ""), "panel": kv.get("reference_panel", ""), - "low_confidence": kv.get("population_low_confidence", "")} - return p, data - - -AUTOSOMES = frozenset(str(i) for i in range(1, 23)) - - -def count_vcf(p): - total = passed = 0 - for r in vcf_records(p): - total += 1 - if len(r) > 6 and r[6] == "PASS": - passed += 1 - return total, passed - - -def sec_ancestry_fine(d, s): - """Step 26b: _ancestry_fine.tsv (level, code, group, fraction, se, low, - high; levels continent, region and painting, plus left_out for a group the - model leaves out and not_painted for a chromosome RFMix failed on) and - the painting picture.""" - p = first_existing(d, [f"ancestry_fine/{s}_ancestry_fine.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - need = {"level", "code", "group", "fraction", "se", "low", "high"} - if not rows or not need <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {sorted(need)}") - out = {"continent": [], "region": [], "painting": [], "left_out": [], "not_painted": [], - "min_references": ancestry_fine_report.MIN_REFERENCES} - for r in rows: - if r["level"] in out: - out[r["level"]].append({"code": r["code"], "group": r["group"], "fraction": num(r["fraction"]), - "low": num(r["low"]), "high": num(r["high"])}) - # The painting picture travels in the summary, so the HTML report shows it - # from the summary alone (render_report.py --summary). - png = first_existing(d, [f"ancestry_fine/{s}_painting.png"]) - out["painting_png"] = None - out["legend"] = [{"code": c, "group": ancestry_fine_report.label("continent", c), - "colour": "#%02x%02x%02x" % rgb} for c, rgb in ancestry_fine_report.COLOURS.items()] - if png and os.path.getsize(png) < 2_000_000: - with open(png, "rb") as f: - out["painting_png"] = base64.b64encode(f.read()).decode() - return p, out - - -def sec_ancient(d, s): - """Step 44: ancient/_ancient.tsv (the nearest ancient groups on the AADR - panel's PCs), _ancient_info.tsv (release, sites used, people, the - sample's PC1 and PC2) and the plot, which travels in the summary so the - HTML report shows it from the summary alone.""" - p = first_existing(d, [f"ancient/{s}_ancient.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - need = {"rank", "group", "n", "distance", "dates", "country", "locality"} - if not rows or not need <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {sorted(need)}") - info = {} - ip = first_existing(d, [f"ancient/{s}_ancient_info.tsv"]) - if ip: - info = {r.get("key"): r.get("value") for r in read_tsv(ip)} - out = {"nearest": [{k: r[k] for k in ("rank", "group", "n", "distance", "dates", "country", "locality")} for r in rows], - "release": info.get("release") or ".", "sites_used": info.get("sites_used") or ".", - "panel_sites": info.get("panel_sites") or ".", "ancient_people": info.get("ancient_people") or ".", - "ancient_groups": info.get("ancient_groups") or ".", "present_day_people": info.get("present_day_people") or ".", - "plot_svg": None} - svg = first_existing(d, [f"ancient/{s}_ancient_pca.svg"]) - if svg and os.path.getsize(svg) < 3_000_000: - with open(svg, "rb") as f: - out["plot_svg"] = base64.b64encode(f.read()).decode() - return p, out - - -def sec_manta(d, s): - p = first_existing(d, ["manta/results/variants/diploidSV.vcf.gz", "manta/*diploidSV.vcf.gz", - "manta/**/diploidSV.vcf.gz"]) - if not p: - return None, {} - t, ps = count_vcf(p) - return p, {"total": t, "pass": ps} - - -def sec_delly(d, s): - p = first_existing(d, [f"delly/{s}_sv.vcf.gz"]) - if not p: - return None, {} - t, ps = count_vcf(p) - return p, {"total": t, "pass": ps} - - -# A CNVpytor call is kept in the filtered count when fewer than half of its -# reads have mapping quality 0 (q0, column 9) and fewer than half of its bases -# are N (pN, column 10), it touches no centromere or heterochromatin band -# (MASKED_BANDS), and it is not on chrY of a declared female (collect() takes -# those out, once the declared sex is known). -CNV_MAX_Q0 = CNV_MAX_PN = 0.5 - - -def sec_cnvpytor(d, s): - p = first_existing(d, [f"cnvpytor/{s}_cnvs.txt"]) - if not p: - return None, {} - bands = masked_bands() - total = dels = dups = sig = kept = kept_y = 0 - dropped = {"q0": 0, "gap": 0, "band": 0} - with open(p, errors="replace") as f: - for line in f: - c = line.split() - if not c: - continue - total += 1 - dels += c[0] == "deletion" - dups += c[0] == "duplication" - m = re.fullmatch(r"(\S+):(\d+)-(\d+)", c[1]) if len(c) > 4 else None - try: - if float(c[4]) >= 0.01: - continue - except (IndexError, ValueError): - continue - sig += 1 - q0, pn = (num(c[8]), num(c[9])) if len(c) > 9 else (None, None) - if q0 is not None and q0 >= CNV_MAX_Q0: - dropped["q0"] += 1 - elif pn is not None and pn >= CNV_MAX_PN: - dropped["gap"] += 1 - elif m and masked_overlap(bands, m.group(1), int(m.group(2)), int(m.group(3))): - dropped["band"] += 1 - else: - kept += 1 - kept_y += bool(m) and m.group(1) in ("chrY", "Y") - return p, {"total": total, "deletions": dels, "duplications": dups, "significant": sig, - "significant_filtered": kept, "filtered_chry": kept_y, "dropped": dropped} - - -def cnvpytor_sex_filter(data, declared_sex): - """Take the chrY calls out of the filtered count of a declared female; - with no declared sex they stay, and the data says so.""" - if "significant_filtered" not in data: - return - data["declared_sex"] = declared_sex or None - if declared_sex == "female": - data["dropped"]["chry_female"] = data["filtered_chry"] - data["significant_filtered"] -= data["filtered_chry"] - -def sec_sv_consensus(d, s): - p = first_existing(d, [f"sv_merged/{s}_sv_consensus.vcf.gz"]) - if not p: - return None, {} - t, _ = count_vcf(p) - return p, {"consensus": t} - -def sec_annotsv(d, s): - """Step 05's class 4 and 5 SVs (annotsv/_sv_class4_5_checked.tsv, - bin/annotsv_postfilter.py), each with how many callers support it from - step 22's consensus (sv_merged/_sv_consensus.vcf.gz), joined here - because step 22 runs after step 05 (and beside ANNOTSV in Nextflow).""" - p = first_existing(d, [f"annotsv/{s}_sv_class4_5_checked.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) if os.path.getsize(p) else [] - need = {"SV_chrom", "SV_start", "SV_end", "SV_type", "ACMG_class", "Caller", "Exon_edge", "Population_AF", "Common"} - if rows and not need <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {sorted(need)}") - cons = first_existing(d, [f"sv_merged/{s}_sv_consensus.vcf.gz"]) - cons_note = None - try: - support = annotsv_postfilter.consensus_support(rows, cons) - except (OSError, EOFError, zlib.error, UnicodeDecodeError) as e: - support = annotsv_postfilter.consensus_support(rows, None) - cons_note = f"{os.path.basename(cons)} could not be read: {e}" - cons = None - out = [] - for r, sp in zip(rows, support): - try: - af = float(r["Population_AF"]) if r["Population_AF"] else None - except ValueError: - af = None - genes = [g for g in (r.get("Gene_name") or "").split(";") if g] - fmt = annotsv_postfilter.sample_format(r) - out.append({"sv": f"{r['SV_chrom']}:{r['SV_start']}-{r['SV_end']}", "type": r["SV_type"], - # the call's genotype and phase set (PS), for the two-hit rule (two_hits) - "gt": fmt.get("GT") or None, "ps": fmt.get("PS") or None, - "length": r.get("SV_length") or "", "acmg_class": r["ACMG_class"], "genes": genes, - "caller": r["Caller"], "support": sp["Support"], - "support_callers": sp["Support_callers"].split(",") if sp["Support_callers"] else [], - "exon_edge": r["Exon_edge"], "exon_edge_detail": r.get("Exon_edge_detail") or "", - "population_af": af, "population_source": r.get("Population_source") or "", - "common": r["Common"] == "yes", "annotsv_class": r.get("AnnotSV_class") or r["ACMG_class"], - "rule": r.get("Rule") or "", "rule_detail": r.get("Rule_detail") or ""}) - # A call step 05's rules moved from AnnotSV's class 4 or 5 to class 3 - # (bin/annotsv_postfilter.py: a coding tandem repeat, an IG/TR locus, the - # MHC, a pseudogene alias, no read-depth support) is listed apart with - # the reason, and the class 4-5 counts leave it out. - moved = [x for x in out if str(x["acmg_class"]) not in ("4", "5")] - out = [x for x in out if str(x["acmg_class"]) in ("4", "5")] - return p, {"count": len(out), "rows": out, "downgraded": moved, - "consensus": os.path.relpath(cons, d) if cons else None, "consensus_note": cons_note, - "single_caller": sum(x["support"] == 1 for x in out), - "exon_edge": sum(x["exon_edge"] == "yes" for x in out), - "exon_edge_checked": not any(x["exon_edge"] == "not checked" for x in out), - "common": sum(x["common"] for x in out)} - - -STR_RANGES_TSV = os.path.join(os.path.dirname(os.path.abspath(__file__)), "str_intermediate_ranges.tsv") -# How well ExpansionHunter sized an allele, from its FORMAT SO, ADSP and -# REPCI fields, worst first; a locus takes the worst of its alleles. -STR_SIZING = ["no_call", "flanking_only", "in_repeat", "low_confidence", "sized", "unknown"] -STR_MIN_SPANNING = 2 - - -def str_ranges(path=STR_RANGES_TSV): - """bin/str_intermediate_ranges.tsv by locus (tests/test_str_ranges.py checks it).""" - rows, head = {}, None - with open(path) as f: - for line in f: - if line.startswith("#") or not line.strip(): - continue - c = line.rstrip("\n").split("\t") - if head is None: - head = c - continue - r = dict(zip(head, c)) - rows[r["locus"]] = r - return rows - - -def str_int(x): - try: - return int(x) - except (TypeError, ValueError): - return None - - -def str_allele_sizing(so, adsp, ci, cn, path_min): - """One allele's sizing: sized, low_confidence (fewer than - STR_MIN_SPANNING spanning reads, or a confidence interval that reaches - the catalog's pathogenic minimum from below it), in_repeat (in-repeat - reads only: longer than a read), flanking_only, no_call, or unknown - (no SO field).""" - if str_int(cn) is None: - return "no_call" - if not so or so == ".": - return "unknown" - if so == "FLANKING": - return "flanking_only" - if so == "INREPEAT": - return "in_repeat" - hi = str_int(ci.split("-")[-1]) if ci else None - if (str_int(adsp) is not None and str_int(adsp) < STR_MIN_SPANNING) or ( - path_min is not None and hi is not None and str_int(cn) < path_min <= hi): - return "low_confidence" - return "sized" - - -def str_field(fmt, val, key): - return val[fmt.index(key)] if key in fmt and fmt.index(key) < len(val) else "" - - -def str_split(x, n): - """A per-allele FORMAT value as n parts; one value serves every allele.""" - parts = x.split("/") if x else [] - return parts if len(parts) == n else (parts * n if len(parts) == 1 else [""] * n) - - -def rfc1_motif_row(d, s, after): - """Step 9b's RFC1 motif count (bin/rfc1_motif.py), unless it is older than - the Stranger file it was made from.""" - p = first_existing(d, [f"expansion_hunter/{s}_rfc1_motif.tsv"]) - if not p or (after and os.path.getmtime(p) < os.path.getmtime(after)): - return None - rows = read_tsv(p) - if not rows: - return None - r = rows[0] - return {k: r.get(k, ".") for k in ("verdict", "note", "dominant_motif", "dominant_class", - "dominant_share", "motif_shares", "repeat_reads")} - - -def str_motif_rows(d, s, after): - """Step 9b's motif count of the other flagged motif loci (the FAME loci, - SCA31, SCA37; expansion_hunter/_str_motifs.tsv, bin/rfc1_motif.py - --flagged) by locus, unless the table is older than the Stranger file it - was made from.""" - p = first_existing(d, [f"expansion_hunter/{s}_str_motifs.tsv"]) - if not p or (after and os.path.getmtime(p) < os.path.getmtime(after)): - return {} - return {r["locus"]: {k: r.get(k, ".") for k in ("verdict", "note", "dominant_motif", "dominant_class", - "dominant_share", "motif_shares", "repeat_reads")} - for r in read_tsv(p) if r.get("locus")} - - -def str_not_graded(d, s): - """{locus: reason} of the loci the catalog ExpansionHunter ran on gives no - thresholds (its NoThresholds field, og str catalog: MUC1, CEL, - TYMS and PLIN4 among them), from the copy step 09 keeps beside its VCF - (expansion_hunter/_eh_catalog.json). Empty without the copy, or with - one that is not a catalog (Stranger's bundled catalog has no such loci).""" - p = first_existing(d, [f"expansion_hunter/{s}_eh_catalog.json"]) - if not p: - return {} - try: - with open(p, encoding="utf-8") as f: - cat = json.load(f) - except (OSError, ValueError): - return {} - if not isinstance(cat, list): - return {} - return {e["LocusId"]: str(e["NoThresholds"]) for e in cat - if isinstance(e, dict) and e.get("LocusId") and e.get("NoThresholds")} - - -# The repeats STRchive leaves out (assets/str_extra_loci.json), read from -# their counts when step 09 sized them (STR_EXTRA_LOCI=yes; docs/09-str-expansions.md). -STR_EXTRA_SETUP = "STR_EXTRA_LOCI=yes ./scripts/setup.sh --strchive-data " -# What a section state other than ok says about a table another step wrote. -OTHER_STEP_STATE = {"stale": "from an earlier run", "failed": "its step failed in the last run", - "unreadable": "unreadable"} -# CFTR intron 9 (legacy IVS8): ExpansionHunter writes the TG and the poly-T -# tract as two records, so which TG count sits on the copy of which T count -# is known only when one of the two is the same on both copies. R117H -# (rs78655421, chr7:117530975 G>A) is step 42's traits row, 17.6 kb from the -# tract, farther than a read pair spans: with both heterozygous its phase is -# not known. Deignan et al. 2020 (PMID 32404922). -CFTR_R117H = "rs78655421" -CFTR_SOURCE = "Deignan et al. 2020, PMID 32404922" -# UGT1A1 promoter: ExpansionHunter counts the TA units from the first T of -# A(TA)6TAA, so its count is one more than the allele's TA number. -UGT1A1_ALLELES = {6: "*36", 7: "*1", 8: "*28", 9: "*37"} -UGT1A1_DECREASED = ("*28", "*37") -UGT1A1_SOURCE = "Gammal et al. 2016, PMID 26417955" -HTTLPR_READING = ("not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found " - "that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would " - "carry no recommendation.") - - -def str_counts(item): - """The two repeat counts of a sized record, sorted, or None.""" - if not item or item["sizing"] not in ("sized", "unknown"): - return None - c = [str_int(x) for x in item["repeat_count"].split("/")] - if len(c) == 1 and c[0] is not None: - c = c * 2 # a one-copy call (a haploid contig): not expected on chr7 or chr2 - return sorted(c) if len(c) == 2 and None not in c else None +# Pages that show more of a called branch, free and with no upload. The URL +# patterns were checked live on 2026-10-25 (docs/37-y-haplogroup.md, +# "Explore your branch further"): FamilyTreeDNA Discover answers its own +# HaplogroupSearch for R-M269 and H5a7, and YFull's tree pages answer 200 for +# a branch it has and 404 for one it does not. +EXPLORE_DOCS = "https://github.com/GeiserX/OwnGenome/blob/main/docs/" -def trait_genotype(d, s, rsid): - """(status, genotype) of a traits table row of step 42 (traits/_traits.tsv), or None.""" - p = first_existing(d, [f"traits/{s}_traits.tsv"]) - if not p: - return None - try: - row = next((r for r in read_tsv(p) if r.get("rsid") == rsid), None) - except (OSError, csv.Error): - return None - return (row.get("status") or "", row.get("genotype") or "") if row else None +def csq_rank(fmt, csq): + """The rule of bin/csq_rank.awk on one CSQ value: (class, entries, freq). + class is MANE (entries of the MANE Select transcript), else CANONICAL, + else ANY (every entry); freq is rare, common, unknown (no value) or none + (the CSQ has no frequency field): MAX_AF, else gnomADe_AF and gnomADg_AF, + the highest over the entries.""" + ix = {k: i for i, k in enumerate(fmt)} + entries = [v for v in (t.split("|") for t in csq.split(",") if t) if len(v) == len(fmt)] + def get(v, k): + return v[ix[k]] if k in ix else "" + mane = [v for v in entries if get(v, "MANE_SELECT")] + canon = [v for v in entries if get(v, "CANONICAL") == "YES"] + cls, chosen = ("MANE", mane) if mane else ("CANONICAL", canon) if canon else ("ANY", entries) + fields = ["MAX_AF"] if "MAX_AF" in ix else [f for f in ("gnomADe_AF", "gnomADg_AF") if f in ix] + afs = [] + for v in entries: + for f in fields: + for a in get(v, f).split("&"): + try: + afs.append(float(a)) + except ValueError: + pass + freq = "none" if not fields else "unknown" if not afs else "rare" if max(afs) < COMMON_AF else "common" + return cls, [{"impact": get(v, "IMPACT"), "gene": get(v, "SYMBOL"), + "consequence": get(v, "Consequence").replace("&", ",")} for v in chosen], freq -def r117h_genotype(d, s, state): - """(genotype or None, why not read, note) of R117H in step 42's traits - table, given the traits section's state: a table of a failed or - unreadable step is not read, one from an earlier run is read and said so.""" - if state in ("failed", "unreadable"): - return None, f"step 42's traits table is {OTHER_STEP_STATE[state]}", "" - r = trait_genotype(d, s, CFTR_R117H) - if not r or r[0] != "called": - return None, "no step 42 traits row for rs78655421 was called", "" - return r[1], "", (" (R117H from step 42's traits table of an earlier run)" if state == "stale" else "") - -def cftr_reading(by, d, s, traits_state="ok"): - tg, t = by.get("CFTR_TG"), by.get("CFTR_T") - out = {"locus": "CFTR", "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", "source": CFTR_SOURCE} - if not tg or not t: - return dict(out, status="not_sized", genotype=".", phase=".", - reading=f"not in step 09's output: {STR_EXTRA_SETUP} adds the tract to the catalog " - "(docs/09-str-expansions.md).") - a, b = str_counts(tg), str_counts(t) - if not a or not b: - return dict(out, status="not_sized", genotype=".", phase=".", - reading=f"ExpansionHunter did not size both tracts (TG {tg['repeat_count']}, {tg['sizing']}; " - f"poly-T {t['repeat_count']}, {t['sizing']}): not read.") - if a[0] == a[1] or b[0] == b[1]: - pairs = [f"TG{a[0]}-{b[0]}T/TG{a[1]}-{b[1]}T"] - else: - pairs = [f"TG{a[0]}-{b[0]}T/TG{a[1]}-{b[1]}T", f"TG{a[1]}-{b[0]}T/TG{a[0]}-{b[1]}T"] - genotype = pairs[0] if len(pairs) == 1 else f"TG{a[0]}/TG{a[1]} {b[0]}T/{b[1]}T" - out.update(status="read", genotype=genotype, pairings=pairs, tg=a, poly_t=b) - five = b.count(5) - other = sorted({n for n in b if n not in (5, 7, 9)}) - odd = (f" A poly-T length of {', '.join(f'{n}T' for n in other)} is none of the usual 5T, 7T and 9T." - if other else "") - r117, why, note = r117h_genotype(d, s, traits_state) - if not five: - # No 5T risk of its own, but the poly-T still sets how R117H reads: - # with no 5T copy, an R117H copy sits on a 7T or 9T (or other) copy. - line = (f"No 5T copy ({b[0]}T/{b[1]}T): the tract adds no 5T allele of its own. The poly-T length still " - "matters beside R117H, which is milder and of variable penetrance on a 7T or 9T copy and CF-causing " - "with 5T in cis.") - if r117 is None: - line += f" R117H was not read ({why})." - elif r117.split("/").count("A") == 2: - copies = f"both {b[0]}T copies" if b[0] == b[1] else f"both copies ({b[0]}T and {b[1]}T)" - line += f" R117H ({r117}) is read: it sits on {copies}.{note}" - elif r117.split("/").count("A"): - copies = f"{b[0]}T" if b[0] == b[1] else f"{b[0]}T or {b[1]}T" - line += f" R117H ({r117}) is read: with no 5T copy it sits on a {copies} copy.{note}" - else: - line += f" No R117H copy.{note}" - return dict(out, five_t=False, phase="not_needed", r117h=r117 or ".", reading=line + odd) - if len(pairs) == 1: - # One TG count on both copies, or 5T on both copies with their two TG counts. - tgs = [a[0]] if a[0] == a[1] else a - tg_text = " and ".join(f"TG{x}" for x in tgs) + (" on the 5T copy" if five == 1 else " on the 5T copies") - else: - tg_text = f"TG{a[0]} or TG{a[1]} on the 5T copy, not told by the counts ({' or '.join(pairs)})" - line = (f"5T: {five} copy{'s' if five > 1 else ''} with the 5T poly-T tract, {tg_text}. 5T lowers the splicing " - "of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often " - "causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant.") - if r117 is None: - phase = "not_read" - text = f" R117H was not read ({why}), so whether it sits on the 5T copy is not known." - else: - n = r117.split("/").count("A") - if n == 0: - phase = "not_needed" - text = " No R117H copy: the 5T is read on its own." - elif n == 2 or five == 2: - phase = "in_cis" - text = (f" R117H ({r117}) and 5T share a copy: R117H with 5T in cis is a CF-causing allele of " - "variable penetrance.") - else: - phase = "unknown" - text = (" One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther " - "than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable " - "penetrance) or not (in trans) is not read; testing a parent tells them apart.") - text += note - return dict(out, five_t=True, phase=phase, r117h=r117 or ".", reading=line + text + odd) - - -def ugt1a1_reading(by, d, s, cpic_state="ok"): - item = by.get("UGT1A1") - out = {"locus": "UGT1A1", "repeat": "promoter TA repeat", "source": UGT1A1_SOURCE} - if not item: - return dict(out, status="not_sized", genotype=".", pharmcat=".", agreement="not_compared", - reading=f"not in step 09's output: {STR_EXTRA_SETUP} adds the repeat to the catalog " - "(docs/09-str-expansions.md). PharmCAT (step 7) still reads UGT1A1 *28.") - c = str_counts(item) - if not c: - return dict(out, status="not_sized", genotype=".", pharmcat=".", agreement="not_compared", - reading=f"ExpansionHunter did not size the repeat ({item['repeat_count']}, {item['sizing']}): " - "not read.") - names = [UGT1A1_ALLELES.get(n, f"TA{n - 1}") for n in c] - genotype = "/".join(names) - dec = sum(n in UGT1A1_DECREASED for n in names) - ta = "/".join(f"TA{n - 1}" for n in c) - out.update(status="read", genotype=genotype, ta_units=c) - reading = f"{genotype} ({ta}, {c[0]}/{c[1]} TA units as ExpansionHunter counts them): " - if not all(n in UGT1A1_ALLELES for n in c): - # A length with no named allele has no function to read, and so no - # count of decreased-function copies to set beside PharmCAT's. - return dict(out, pharmcat=".", agreement="not_compared", - reading=reading + "a TA length outside TA5 to TA8 is not a named allele here, so its function " - "is not read and it is not compared with PharmCAT (step 7).") - reading += ("no decreased-function TA allele." if not dec else - f"{dec} decreased-function allele{'s' if dec > 1 else ''} (*28 and *37 are decreased function).") - # PharmCAT's UGT1A1 call (step 27's table, with the reference-block fill - # of bin/pgx_refblock_fill.py): its count of *28 and *37 copies. - pc, agreement = ".", "not_compared" - p = first_existing(d, [f"cpic/{s}_phenotypes.tsv"]) if cpic_state not in ("failed", "unreadable") else None - row = None - if p: - try: - row = next((r for r in read_tsv(p) if r.get("Gene") == "UGT1A1"), None) - except (OSError, csv.Error): - row = None - if row: - pc = (row.get("Diplotype") or "").strip() or "." - alleles = pc.split("/") - if len(alleles) == 2 and "possible" not in pc and (row.get("Status") or "") != "not called": - # A combination allele reads "*80+*28" or "[*80 + *28]". - n = sum(any(x in re.split(r"[+ \[\]]", a) for x in UGT1A1_DECREASED) for a in alleles) - agreement = "agrees" if n == dec else "differs" - old = " (step 27's table is from an earlier run)" if cpic_state == "stale" and agreement != "not_compared" else "" - if agreement == "agrees": - reading += f" PharmCAT (step 7) called {pc}: the same number of *28 and *37 copies{old}." - elif agreement == "differs": - reading += (f" PharmCAT (step 7) called {pc}, a different number of *28 and *37 copies{old}: read the reads " - "at chr2:233760234 before using either.") - elif cpic_state in ("failed", "unreadable"): - reading += f" Not compared with PharmCAT (step 27's table is {OTHER_STEP_STATE[cpic_state]})." - else: - reading += " Not compared with PharmCAT (no unambiguous UGT1A1 call from step 27)." - return dict(out, pharmcat=pc, agreement=agreement, reading=reading) - - -def str_readings(by, d, s, states=None): - """What the counts of the extra repeats mean, one entry per locus, and the - 5-HTTLPR line that no step can test. states: the traits and cpic section - states, once collect() knows them (a stale, failed or unreadable table is - said so, not read as current).""" - states = states or {} - return [cftr_reading(by, d, s, states.get("traits", "ok")), ugt1a1_reading(by, d, s, states.get("cpic", "ok")), - {"locus": "SLC6A4", "repeat": "5-HTTLPR promoter length polymorphism", "status": "not_tested", - "genotype": ".", "reading": HTTLPR_READING, "source": "Bousman et al. 2023, PMID 37032427"}] - - -def sec_expansions(d, s): - eh = first_existing(d, [f"expansion_hunter/{s}_eh.vcf", f"expansion_hunter/{s}.vcf", - "expansion_hunter/*_eh.vcf"]) - st = first_existing(d, [f"expansion_hunter/{s}_eh_stranger.vcf", "expansion_hunter/*_eh_stranger.vcf"]) - # Stranger (step 9b) copies ExpansionHunter's records and adds STR_STATUS. - # Its file is read unless it is older than the ExpansionHunter file it - # would have been made from; then the report says to rerun step 9b. - outdated = bool(st and eh and os.path.getmtime(st) < os.path.getmtime(eh)) - if outdated: - st = None - p = st or eh - if not p: - return None, {} - ranges = str_ranges() - rfc1 = rfc1_motif_row(d, s, st) if st else None - motifs = str_motif_rows(d, s, st) if st else {} - ungraded = str_not_graded(d, s) if st else {} - loci, tested, flagged, no_status, not_graded = [], 0, [], 0, [] - for r in vcf_records(p): - if len(r) < 10: - continue - tested += 1 - info = info_map(r[7]) - # A locus of several repeats (CFTR: CFTR_TG and CFTR_T) writes one - # record per repeat, each with the locus as REPID and its own VARID: - # the record is named by its VARID and looked up by that name in the - # ranges and motif tables (the pathogenic repeat of a multi-repeat - # locus carries the locus name), and in the catalog's NoThresholds by - # its locus (REPID). A one-repeat locus has VARID = REPID. - lid = info.get("REPID") or info.get("VARID") or f"{r[0]}:{r[1]}" - rep = info.get("VARID") or lid - fmt = r[8].split(":") - val = r[9].split(":") - repcn = str_field(fmt, val, "REPCN") - counts = repcn.split("/") if repcn and repcn != "." else [] - n = max(len(counts), 1) - path_min = str_int(info.get("STR_PATHOLOGIC_MIN")) - sizing = [str_allele_sizing(so, ad, ci, cn, path_min) for so, ad, ci, cn in - zip(str_split(str_field(fmt, val, "SO"), n), str_split(str_field(fmt, val, "ADSP"), n), - str_split(str_field(fmt, val, "REPCI"), n), counts or ["."])] - status = [x for x in info.get("STR_STATUS", "").split(",") if x and x != "."] if st else [] - # Stranger writes '.' where its catalog has no value. - item = {"locus": rep, "repeat_count": repcn or ".", "status": ",".join(status), - "normal_max": info.get("STR_NORMAL_MAX", "").strip("."), - "pathologic_min": info.get("STR_PATHOLOGIC_MIN", "").strip("."), - "support": str_field(fmt, val, "SO"), "adsp": str_field(fmt, val, "ADSP"), - "ci": str_field(fmt, val, "REPCI"), "sizing": min(sizing, key=STR_SIZING.index), - "allele_sizing": "/".join(sizing), "zone": "", "zone_label": "", "zone_citation": ""} - rng = ranges.get(rep) - sized = [str_int(c) for c, z in zip(counts, sizing) if z in ("sized", "unknown") and str_int(c) is not None] - if rng and any(int(rng["intermediate_min"]) <= c <= int(rng["intermediate_max"]) for c in sized): - item.update(zone=f"{rng['intermediate_min']}-{rng['intermediate_max']}", zone_label=rng["label"], - zone_citation=rng["citation"]) - if rep == "RFC1" and st and any(x != "normal" for x in status): - item["rfc1_motif"] = rfc1 or {"verdict": "not_checked", - "note": "no motif count beside Stranger's output (step 9b runs it " - "with the sample's BAM): confirm the flag with repeat-primed PCR"} - if rep != "RFC1" and rep in motifs and any(x != "normal" for x in status): - item["motif"] = motifs[rep] - if st and not status and lid in ungraded: - item["not_graded"] = ungraded[lid] - loci.append(item) - if st: - # One value per record, the most severe of normal, pre_mutation and - # full_mutation; a locus outside Stranger's catalog has none. - if not status and "not_graded" in item: - if lid not in not_graded: - not_graded.append(lid) - elif not status: - no_status += 1 - elif any(x != "normal" for x in status): - f = {"locus": item["locus"], "repeat_count": item["repeat_count"], "status": item["status"], - "sizing": item["sizing"]} - if "rfc1_motif" in item: - f["rfc1_motif"] = item["rfc1_motif"] - if "motif" in item: - f["motif"] = item["motif"] - flagged.append(f) - by = {l["locus"]: l for l in loci} - # Every locus, the five key loci first, then by name. - loci.sort(key=lambda l: (EH_LOCI.index(l["locus"]) if l["locus"] in EH_LOCI else len(EH_LOCI), l["locus"])) - out = {"records": tested, "key_loci": [{"locus": k, "repeat_count": by[k]["repeat_count"] if k in by - else "not in output"} for k in EH_LOCI], - "loci": loci, "stranger": bool(st), "stranger_outdated": outdated, - "unsized": [l["locus"] for l in loci if l["sizing"] not in ("sized", "unknown")], - "intermediate": [l["locus"] for l in loci if l["zone"]], - "readings": str_readings(by, d, s)} - if st: - # not_graded: {locus: why} of the loci the catalog gives no thresholds, - # so Stranger gives them no status: not graded, never normal. - out.update(flagged=flagged, no_status=no_status, not_graded={k: ungraded[k] for k in not_graded}) - return p, out - - -# TelomereHunter's estimate rests on its intratelomeric reads. Below this many -# per 30x of depth, about 30 times fewer than a typical 30x short-read genome -# has, the value is too thin to compare (a library or platform problem, or -# reads too short for its repeat threshold). The floor scales with the depth -# TelomereHunter itself counted (total_reads x read length / GENOME_BASES). -TELOMERE_FLOOR_PER_30X = 2000 -GENOME_BASES = 3.1e9 - - -def sec_telomere(d, s): - p = first_existing(d, [f"telomere/{s}/{s}/{s}_summary.tsv", f"telomere/{s}/{s}_summary.tsv", - "telomere/**/*_summary.tsv"]) - if not p: - return None, {} - with open(p, errors="replace") as f: - lines = [l.rstrip("\n").split("\t") for l in f if l.strip()] - if len(lines) < 2: - return p, {"tel_content": "."} - head, row = lines[0], lines[1] - # By header name; older TelomereHunter versions put it in column 11. - i = head.index("tel_content") if "tel_content" in head else 10 - out = {"tel_content": row[i] if i < len(row) else "."} - - def col(*names): - for n in names: - if n in head and head.index(n) < len(row): - return row[head.index(n)] - return None - # TelomereHunter 1.1.0 names it intratel_reads. - intratel = num(col("intratel_reads", "intratelomeric_reads")) - total = num(col("total_reads")) - lengths = [num(x) for x in re.split(r"[,;| ]+", col("read_lengths", "read_length") or "")] - lengths = [x for x in lengths if x] - if intratel is None: - return p, dict(out, read_check="no intratelomeric read count in the summary") - out["intratelomeric_reads"] = int(intratel) - if not total or not lengths: - return p, dict(out, read_check="depth unknown (no total_reads or read_lengths in the summary): not checked") - depth = total * max(lengths) / GENOME_BASES - floor = TELOMERE_FLOOR_PER_30X * depth / 30 - out.update(depth=round(depth, 1), read_floor=int(round(floor)), low_reads=intratel < floor) - return p, out - - -# GRCh38's centromeres and heterochromatin: the acen, gvar and stalk rows of -# UCSC's cytoBand.hg38.txt (the file setup.sh installs as -# reference/cytoBand.hg38.txt, sha256 e514e48b... in scripts/lib/common.sh), -# neighbouring rows merged, as chrom:start-end with a 0-based start and an -# exclusive end. Reads there map badly, so a run of homozygosity or a copy -# number call over them is mostly an artefact. Regenerate with -# gzip -cd cytoBand.txt.gz | awk '$1 ~ /^chr([0-9]+|X|Y)$/ && $5 ~ /^(acen|gvar|stalk)$/' \ -# | sort -k1,1V -k2,2n | awk '{ if ($1 == c && $2 == e) e = $3; else { if (c != "") printf "%s:%d-%d ", c, s, e; -# c = $1; s = $2; e = $3 } } END { print c ":" s "-" e }' -# scripts/11-roh-analysis.sh and modules/local/roh/main.nf carry the same -# line (tests/test_collect_summary.py checks that the three agree). -MASKED_BANDS = ("chr1:121700000-143200000 chr2:91800000-96000000 chr3:87800000-98600000 chr4:48200000-51800000 " - "chr5:46100000-51400000 chr6:58500000-62600000 chr7:58100000-62100000 chr8:43200000-47200000 " - "chr9:42200000-61500000 chr10:38000000-41600000 chr11:51000000-55800000 chr12:33200000-37800000 " - "chr13:0-18900000 chr14:0-18200000 chr15:0-20500000 chr16:35300000-47000000 " - "chr17:22700000-27400000 chr18:15400000-21500000 chr19:19900000-31900000 chr20:25700000-30400000 " - "chr21:0-13000000 chr22:0-17400000 chrX:58100000-63800000 chrY:10300000-10600000 " - "chrY:26600000-57227415") - - -def masked_bands(text=MASKED_BANDS): - """{chrom: [(start, end), ...]} of MASKED_BANDS, 0-based, end exclusive.""" - out = {} - for x in text.split(): - c, _, se = x.partition(":") - a, _, b = se.partition("-") - out.setdefault(c, []).append((int(a), int(b))) - return out - - -def masked_overlap(bands, chrom, start, end): - """Bases of the 1-based, inclusive span start-end inside the masked bands.""" - return sum(max(0, min(end, b) - max(start - 1, a)) for a, b in bands.get(chrom, ())) - - -# GRCh38's chr1 to chr22 (UCSC hg38.chrom.sizes), and the part of them outside -# MASKED_BANDS: the length F_ROH is a share of, since a run's part in a band is -# not counted either. -AUTOSOME_BP = 2875001522 -ROH_MIN_BP = 5e6 # the segments F_ROH and the one-chromosome flag count (docs/11-roh-analysis.md) -# One chromosome carrying most of the long runs: at least UPD_MIN_BP of it in -# segments of 5 Mb or more, and more than UPD_SHARE of the length of all such -# autosomal runs. -# Our thresholds: a run over 20 Mb on one chromosome is the long-standing sign -# of isodisomy, and related parents spread their runs over many chromosomes. -UPD_MIN_BP = 20e6 -UPD_SHARE = 0.5 -# The rows of docs/11-roh-analysis.md's table: F of a child of the parents. -# A value is placed at the row whose F is nearest on a log scale (the -# boundaries are the geometric means of neighbouring rows), and under 1/128 -# (half the second-cousin F) as no sign of related parents. -ROH_BANDS = [(1 / 128, "no sign of related parents"), - ((1 / 64 * 1 / 32) ** 0.5, "second cousins (F 1/64)"), - ((1 / 32 * 1 / 16) ** 0.5, "first cousins once removed (F 1/32)"), - ((1 / 16 * 1 / 8) ** 0.5, "first cousins (F 1/16)"), - ((1 / 8 * 1 / 4) ** 0.5, "half siblings, uncle and niece, double first cousins (F 1/8)"), - (float("inf"), "parent and child, full siblings (F 1/4)")] - -# assets/imprinted_regions.tsv, the same rows (tests/test_collect_summary.py -# checks that the two agree). Held here too because the Nextflow report task -# sees bin/ but not assets/. (chrom, start, end, region, maternal UPD, -# paternal UPD, PMIDs); start and end None for the whole chromosome. -IMPRINTED_REGIONS = [ - ("chr6", None, None, "6q24 (PLAGL1, HYMAI)", "-", "Transient neonatal diabetes mellitus (6q24-related)", - "20301706"), - ("chr7", None, None, "7p12 (GRB10), 7q32 (MEST)", "Silver-Russell syndrome", "-", "27585961"), - ("chr11", 0, 22000000, "11p15 (H19/IGF2, KCNQ1OT1)", "Silver-Russell syndrome", "Beckwith-Wiedemann syndrome", - "29377879;27585961"), - ("chr14", None, None, "14q32.2 (DLK1-MEG3)", "Temple syndrome", "Kagami-Ogata syndrome", "24891339;26377239"), - ("chr15", 20500000, 33400000, "15q11-q13 (SNRPN, UBE3A)", "Prader-Willi syndrome", "Angelman syndrome", - "22237428;27615419"), - ("chr20", None, None, "20q13.32 (GNAS)", "Mulchandani-Bhoj-Conlin syndrome (growth failure)", - "Pseudohypoparathyroidism type 1B", "26248010;29959430"), -] - - -def autosome_unmasked_bp(bands=None): - """AUTOSOME_BP without the autosomal MASKED_BANDS.""" - bands = masked_bands() if bands is None else bands - return AUTOSOME_BP - sum(b - a for c, spans in bands.items() if c[3:] in AUTOSOMES for a, b in spans) - - -def roh_band(f): - """The row of docs/11-roh-analysis.md's table an F_ROH is nearest to.""" - return next(label for top, label in ROH_BANDS if f < top) - - -def imprinted_rows(chrom, segs): - """The IMPRINTED_REGIONS rows a flagged chromosome joins: its own - chromosome's rows, and of a row with a region only when one of the runs - (1-based, inclusive start-end) overlaps it.""" - out = [] - for c, a, b, region, mat, pat, pmid in IMPRINTED_REGIONS: - if c != chrom: - continue - if a is not None and not any(masked_overlap({c: [(a, b)]}, c, s, e) for s, e in segs): - continue - out.append({"region": region, "maternal_upd": mat, "paternal_upd": pat, "pmid": pmid}) - return out - - -def sec_roh(d, s): - """Step 11's bcftools roh output. Segments on chrX and chrY are left out - (a man's single X is one long run), segments on any other contig that is - not chr1 to chr22 (chrM, chrEBV, chrUn_*) are not counted, and so is the - part of a segment that lies in a centromere or heterochromatin band - (MASKED_BANDS), where few markers map. Total, Largest and the list of segments over 5 Mb are of the - autosomal length that is left.""" - p = first_existing(d, [f"vcf/{s}_roh.txt", f"roh/{s}_roh.txt", "roh/*_roh.txt"]) - if not p: - return None, {} - bands = masked_bands() - total = largest = masked = 0.0 - segs, sex_segs, big = 0, 0, [] - per_chrom = {} # chrom -> [long bp, [(start, end), ...]] of the segments over ROH_MIN_BP - with open(p, errors="replace") as f: - for line in f: - c = line.split() - if len(c) < 6 or c[0] != "RG": - continue - try: - start, end, length = int(c[3]), int(c[4]), float(c[5]) - except ValueError: - continue - if not math.isfinite(length): - raise Unreadable(f"{os.path.basename(p)} has a run of length {c[5]}") - if c[2] in ("chrX", "chrY", "X", "Y"): - sex_segs += 1 - continue - if not (c[2].startswith("chr") and c[2][3:] in AUTOSOMES): - continue - inside = min(length, masked_overlap(bands, c[2], start, end)) - masked += inside - length -= inside - if length <= 0: - continue - segs += 1 - total += length - largest = max(largest, length) - if length >= ROH_MIN_BP: - pc = per_chrom.setdefault(c[2], [0.0, []]) - pc[0] += length - pc[1].append((start, end)) - row = {"region": f"{c[2]}:{c[3]}-{c[4]}", "mb": round(length / 1e6, 1)} - if inside: - row["masked_mb"] = round(inside / 1e6, 1) - big.append(row) - long_bp = sum(v[0] for v in per_chrom.values()) - f_roh = long_bp / autosome_unmasked_bp(bands) - upd = None - if per_chrom: - chrom, (bp, spans) = max(per_chrom.items(), key=lambda kv: kv[1][0]) - if bp >= UPD_MIN_BP and bp > UPD_SHARE * long_bp: - upd = {"chrom": chrom, "mb": round(bp / 1e6, 1), "share": round(bp / long_bp, 2), - "imprinted": imprinted_rows(chrom, spans)} - return p, {"segments": segs, "total_mb": round(total / 1e6, 1), "largest_mb": round(largest / 1e6, 1), - "autosomal_over_5mb": big, "masked_mb": round(masked / 1e6, 1), "sex_chromosome_segments": sex_segs, - "long_mb": round(long_bp / 1e6, 1), "f_roh": round(f_roh, 4), - # a chromosome that may come from one parent fills F_ROH by itself, so no band then - "relationship": None if upd else roh_band(f_roh), "upd_flag": upd} - - -# Pages that show more of a called branch, free and with no upload. The URL -# patterns were checked live on 2026-10-25 (docs/37-y-haplogroup.md, -# "Explore your branch further"): FamilyTreeDNA Discover answers its own -# HaplogroupSearch for R-M269 and H5a7, and YFull's tree pages answer 200 for -# a branch it has and 404 for one it does not. -EXPLORE_DOCS = "https://github.com/GeiserX/OwnGenome/blob/main/docs/" - - -def haplogroup_explore(kind, hg): - """[{name, url}] for a called branch: kind 'y' (Yleaf, e.g. R-M269 or - R-M269*(xR-L23)) or 'mt' (haplogrep3, e.g. H5a7). Empty when there is no call.""" - from urllib.parse import quote - hg = (hg or "").strip() - if kind == "y": - hg = hg.split("*", 1)[0] # Yleaf's paragroup mark: the branch is the name before '*' - if not hg or hg in (".", "NA", "insufficient markers"): - return [] - q = quote(hg, safe="") - if kind == "y": - return [{"name": "FamilyTreeDNA Discover (free, no upload)", "url": f"https://discover.familytreedna.com/y-dna/{q}"}, - {"name": "YFull YTree", "url": f"https://www.yfull.com/tree/{q}/"}, - {"name": "More routes, and the files to share", "url": EXPLORE_DOCS + "37-y-haplogroup.md#explore-your-branch-further"}] - return [{"name": "FamilyTreeDNA Discover (free, no upload)", "url": f"https://discover.familytreedna.com/mtdna/{q}"}, - {"name": "YFull MTree", "url": f"https://www.yfull.com/mtree/{q}/"}, - {"name": "More routes, and the files to share", "url": EXPLORE_DOCS + "12-mito-haplogroup.md#explore-your-branch-further"}] - - -def sec_haplogroup(d, s): - p = first_existing(d, [f"mito/{s}_haplogroup.txt"]) - if not p: - return None, {} - rows = read_tsv(p) - r = rows[0] if rows else {} - hg = (r.get("Haplogroup") or (list(r.values())[1] if len(r) > 1 else "") or "").strip('"') - out = {"haplogroup": hg or ".", "explore": haplogroup_explore("mt", hg)} - # haplocheck (step 12 with step 20's Mutect2 calls): a second haplogroup - # in the allele fractions is another person's DNA in the sample. - hc = first_existing(d, [f"mito/{s}_haplocheck.txt"]) - if hc: - rows = [{k.strip('"'): (v or "").strip('"') for k, v in row.items() if k} for row in read_tsv(hc)] - if not rows or "Contamination Status" not in rows[0]: - raise Unreadable(f"{os.path.basename(hc)} has no Contamination Status column") - out["contamination_status"] = rows[0]["Contamination Status"] or "." - out["contamination_level"] = rows[0].get("Contamination Level") or "." - return p, out - - -def sec_y_haplogroup(d, s): - p = first_existing(d, [f"y_haplogroup/{s}_y_haplogroup.txt"]) - if not p: - return None, {} - rows = read_tsv(p) - if not rows or "Hg" not in rows[0]: - raise Unreadable(f"{os.path.basename(p)} has no Hg column or no sample row") - r = rows[0] - hg = (r.get("Hg") or "").strip() - return p, {"haplogroup": hg if hg not in ("", "NA") else "insufficient markers", - "valid_markers": r.get("Valid_markers") or ".", "qc_score": r.get("QC-score") or ".", - "explore": haplogroup_explore("y", hg)} - - -def sec_y_tree(d, s): - """Step 37's placement on the current public tree (y_haplogroup/_y_tree.txt, - og y tree-place): the terminal branch, how deep it sits, the SNP - calls along the path, and the tree release the names come from. The - per-level table beside it gives each branch of that path.""" - p = first_existing(d, [f"y_haplogroup/{s}_y_tree.txt"]) - if not p: - return None, {} - rows = read_tsv(p) - need = ("Hg", "Level", "Tree") - if not rows or any(k not in rows[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {', '.join(need)}") - r = rows[0] - hg = (r.get("Hg") or "").strip() - out = {"haplogroup": hg if hg not in ("", "NA") else "no branch has a derived call", - "level": r.get("Level") or ".", "tree": r.get("Tree") or ".", - "path_derived": r.get("Path_derived") or ".", "path_ancestral": r.get("Path_ancestral") or ".", - "path_no_call": r.get("Path_no_call") or ".", - "ancestral_in_terminal": r.get("Ancestral_in_terminal") or ".", - "outvoted_ancestral": r.get("Outvoted_ancestral") or ".", - "snps_called": r.get("SNPs_called") or ".", "min_depth": r.get("Min_depth") or ".", - "explore": haplogroup_explore("y", hg)} - lvl = first_existing(d, [f"y_haplogroup/{s}_y_tree_path.tsv"]) - if lvl: - # og y tree-place writes the header alone when no branch is derived: - # an empty path, not an unreadable file. - with open_text(lvl) as f: - rd = csv.DictReader(f, delimiter="\t") - prows = list(rd) - header = rd.fieldnames or [] - if "Branch" not in header: - raise Unreadable(f"{os.path.basename(lvl)} has no Branch column") - out["path"] = [{k: (row.get(k) or "") for k in - ("Level", "Branch", "SNPs", "On_tables", "Derived", "Ancestral", "No_call", - "Ancestral_SNPs")} for row in prows] - return p, out - - -def sec_y_str(d, s): - """Step 45's table (y_str/_y_str.tsv, og y str-report): one row per - FamilyTreeDNA marker with its count and confidence, and the counts per - confidence.""" - p = first_existing(d, [f"y_str/{s}_y_str.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - need = ("order", "panel", "marker", "count", "confidence") - if not rows or any(k not in rows[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {', '.join(need)}") - keep = [{k: r.get(k) or "" for k in need} for r in rows] - counts = {c: sum(1 for r in keep if r["confidence"] == c) for c in ("high", "low", "no call", "not callable")} - if sum(counts.values()) != len(keep): - raise Unreadable(f"{os.path.basename(p)} has a confidence other than high, low, no call or not callable") - return p, {"markers": keep, **{k.replace(" ", "_"): v for k, v in counts.items()}} - - -def sec_consumer_export(d, s): - """Step 40's summary table (key, value): how many chip sites got a genotype.""" - p = first_existing(d, [f"consumer_export/{s}_consumer_export.tsv"]) - if not p: - return None, {} - data = {r["key"]: r.get("value", "") for r in read_tsv(p) if r.get("key")} - for k in ("sites", "called", "no_call"): - if not (data.get(k) or "").isdigit(): - raise Unreadable(f"{os.path.basename(p)} has no whole-number '{k}' row") - reasons = {} - for k, v in data.items(): - if k.startswith("no_call: "): - if not (v or "").isdigit(): - raise Unreadable(f"{os.path.basename(p)} has a '{k}' row without a whole-number count") - reasons[k[len("no_call: "):]] = int(v) - return p, {"sites": int(data["sites"]), "called": int(data["called"]), "no_call": int(data["no_call"]), - "from_reference_blocks": int(data.get("from_reference_blocks") or 0), - "template_sites": data.get("template_sites") or ".", - "file_23andme": data.get("file_23andme") or ".", - "file_ancestrydna": data.get("file_ancestrydna") or ".", - "no_call_reasons": reasons} - - -def sec_bloodtype(d, s): - """Step 41's table: one row per blood group system, predicted from the genome.""" - p = first_existing(d, [f"bloodtype/{s}_bloodtype.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - if not rows or not {"system", "phenotype", "confidence"} <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} has no system, phenotype and confidence columns, or no row") - systems = [{"system": r["system"], "phenotype": r["phenotype"] or ".", "genotype": r.get("genotype") or ".", - "confidence": r["confidence"] or "."} for r in rows] - by = {x["system"]: x["phenotype"] for x in systems} - return p, {"systems": systems, "abo": by.get("ABO", "."), "rhd": by.get("RhD", ".")} - - -def sec_archaic(d, s): - """Step 43's summary (archaic/_archaic.tsv, bin/archaic_report.py): the - Neanderthal and Denisovan shares and segment counts, the analysed span and - the ten longest segments.""" - p = first_existing(d, [f"archaic/{s}_archaic.tsv"]) - if not p: - return None, {} - data = {r["key"]: r.get("value") or "" for r in read_tsv(p) if r.get("key")} # a short row: None - out = {"population": data.get("population", "."), - "longest": [data[k] for k in sorted((k for k in data if re.fullmatch(r"longest_[0-9]+", k)), - key=lambda k: int(k.split("_")[1]))]} - for k in ("neanderthal_segments", "neanderthal_bp", "denisovan_segments", "denisovan_bp", "analysed_bp"): - if not re.fullmatch(r"[0-9]+", data.get(k, "")): - raise Unreadable(f"{os.path.basename(p)} has no whole number for {k}") - out[k] = int(data[k]) - for k in ("neanderthal_percent", "denisovan_percent"): - if not re.fullmatch(r"[0-9]+\.[0-9]+", data.get(k, "")): - raise Unreadable(f"{os.path.basename(p)} has no percentage for {k}") - out[k] = data[k] - return p, out - - -# The two LPA risk variants of assets/traits.tsv (rsid -> its alt, the allele -# that goes with a high Lp(a)); tests/test_lpa_kiv2.py checks they agree. -LPA_RISK_ALLELES = {"rs10455872": "G", "rs3798220": "C"} - - -def sec_lpa(d, s): - """Step 46's KIV-2 count (lpa/_lpa_kiv2.tsv, og cn call --locus LPA_KIV2), with - step 42's two LPA risk variants when the traits table is there, and - whether either points to a genetically high Lp(a).""" - p = first_existing(d, [f"lpa/{s}_lpa_kiv2.tsv"]) - if not p: - return None, {} - kv = {r.get("metric"): r.get("value") for r in read_tsv(p)} - if kv.get("status") not in ("estimated", "too little depth", "not assessed"): - raise Unreadable(f"{os.path.basename(p)} has no status row (estimated, too little depth or not assessed)") - data = {k: kv.get(k) or "." for k in ("status", "kiv2_copies", "ci95_low", "ci95_high", "class", - "kiv2_depth", "flank_depth", "mapq0_share", "message", "advice")} - # KIV-2 4925G>A beside the count (step 46, lpa/_lpa_kiv2_4925.tsv) - data["null_allele"] = read_marker_table(os.path.join(os.path.dirname(p), f"{s}_lpa_kiv2_4925.tsv"), "46", - "LPA_KIV2_4925") - variants = [] - t = first_existing(d, [f"traits/{s}_traits.tsv"]) - for r in (read_tsv(t) if t else []): - alt = LPA_RISK_ALLELES.get(r.get("rsid")) - if not alt: - continue - called = r.get("status") == "called" - n = re.split(r"[/|]", r.get("genotype") or "").count(alt) if called else None - variants.append({"rsid": r["rsid"], "genotype": r.get("genotype") or ".", "status": r.get("status") or ".", - "risk_alleles": n}) - risk = sum(v["risk_alleles"] for v in variants if v["risk_alleles"] is not None) - reasons = ([f"{risk} risk allele{'s' if risk != 1 else ''} at rs10455872 and rs3798220"] if risk else []) \ - + (["a low KIV-2 repeat count"] if data["class"] == "low" else []) - read = data["status"] == "estimated" or any(v["risk_alleles"] is not None for v in variants) - data.update(variants=variants, risk_alleles=risk, flag="high" if reasons else ".", - flag_text=("Genetic signs of a high Lp(a): " + " and ".join(reasons) + ". Measure Lp(a) in blood." - if reasons else - "No genetic sign of a high Lp(a) from these markers. They explain only part of it: " - "the KIV-2 count explained 21% to 27% of the variation in a Danish population study." - if read else - "Not determined: neither the KIV-2 count nor the two risk variants could be read. " - "Measure Lp(a) in blood.")) - return p, data - - -VNTYPER_STATUSES = ("positive", "possible", "negative", "not assessed") - - -def sec_vntyper(d, s): - """Step 52's table (vntyper/_vntyper.tsv, og vntyper report): - VNtyper's MUC1 VNTR reading with its depth and sensitivity line.""" - p = first_existing(d, [f"vntyper/{s}_vntyper.tsv"]) - if not p: - return None, {} - kv = {r.get("metric"): r.get("value") for r in read_tsv(p)} - if kv.get("status") not in VNTYPER_STATUSES: - raise Unreadable(f"{os.path.basename(p)} has no status row ({', '.join(VNTYPER_STATUSES)})") - return p, {k: kv.get(k) or "." for k in ("status", "confidence", "variant", "flag", "alt_depth", - "vntr_mean_depth", "coverage_qc", "sensitivity", "message", "advice")} - - -LOCITYPER_COLUMNS = ("locus", "region", "status", "genotype", "haplotype_1", "haplotype_2", "quality", - "total_reads", "unexplained_reads", "weighted_distance", "warnings", "filter", "reason") - - -def sec_locityper(d, s): - """Step 53's table (locityper/_locityper.tsv, og locityper report): - one row per locus of assets/locityper_loci.bed, the two pangenome - haplotypes Locityper picked with its quality and the authors' filter. - collect() adds each locus's primary call (locityper_primary).""" - p = first_existing(d, [f"locityper/{s}_locityper.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - if not rows or not set(LOCITYPER_COLUMNS) <= set(rows[0]): - raise Unreadable(f"{os.path.basename(p)} lacks a row or one of the columns {', '.join(LOCITYPER_COLUMNS)}") - loci = [{k: (r.get(k) or ".") for k in LOCITYPER_COLUMNS} for r in rows] - return p, {"loci": loci, "called": sum(x["status"] == "called" for x in loci), - "passed": sum(x["filter"] == "pass" for x in loci)} - - -def locityper_primary(sections): - """The primary caller's call at each Locityper locus, as text, so the - report shows the two side by side: T1K (step 08) for HLA-DRB1 and - HLA-DQB1, T1K's KIR pass (KIR=true) for KIR, pypgx (step 32) and Cyrius - (step 21) for CYP2D6. FCGR has no primary caller here. Locityper names - pangenome haplotypes, not alleles, so the two are shown, not compared.""" - def old(key): # a table older than the last run-all.sh run says so - return " (stale: from an earlier run)" if sections[key]["state"] == "stale" else "" - - hla = sections["hla"]["data"] if sections["hla"]["state"] in ("ok", "stale") else {} - by_gene = {x["gene"]: x for x in hla.get("loci", [])} - out = {} - for g in ("HLA-DRB1", "HLA-DQB1"): - x = by_gene.get(g) or by_gene.get(g.replace("HLA-", "")) - out[g] = ("T1K (step 08): " + ("/".join(x["alleles"]) or "no allele") - + (" (low confidence)" if x.get("low_confidence") else "") + old("hla")) if x else "T1K (step 08) did not run" - kir = hla.get("kir") - out["KIR"] = (f"T1K KIR pass (step 08): {len(kir['typed'])} genes typed{old('hla')}" if kir and kir.get("typed") - else "T1K's KIR pass (step 08, KIR=true) gave no call") - cyp = [] - if sections["pypgx"]["state"] in ("ok", "stale"): - cyp.append(f"pypgx (step 32): {sections['pypgx']['data'].get('cyp2d6', 'not called')}{old('pypgx')}") - if sections["cyrius"]["state"] in ("ok", "stale"): - cy = sections["cyrius"]["data"] - usable = "" if cy.get("filter") == "PASS" else f" (not usable: Filter {cy.get('filter') or 'missing'})" - cyp.append(f"Cyrius (step 21): {cy.get('genotype', 'none')}{usable}{old('cyrius')}") - out["CYP2D6"] = "; ".join(cyp) or "pypgx (step 32) and Cyrius (step 21) did not run" - out["FCGR"] = "no other step calls FCGR" - return out - - -def sec_viral(d, s): - """Step 47's table (viral/_viral.tsv, og viral report): one row per - viral genome; the HHV-6 line and the genomes with reads (phiX included).""" - p = first_existing(d, [f"viral/{s}_viral.tsv"]) - if not p: - return None, {} - rows = read_tsv(p) - need = ("virus", "accession", "reads", "breadth", "relative_depth", "copies_per_cell", "call") - if not rows or any(k not in rows[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {', '.join(need)}") - keep = [{k: r[k] or "." for k in need + ("human_depth", "note")} for r in rows] - ici = [r["virus"] for r in keep if r["call"] == "inherited integrated HHV-6 likely"] - # No read on chr1 to chr22 leaves no human depth, so integration was never assessed. - hhv6 = (f"inherited integrated {ici[0]} likely" if ici - else "not assessed (no human depth)" if keep[0]["human_depth"] == "." - else "no integrated HHV-6") - return p, {"hhv6": hhv6, - "human_depth": keep[0]["human_depth"], - "found": [r for r in keep if r["reads"].isdigit() and int(r["reads"]) > 0], "genomes": len(keep)} - - -def sec_chip(d, s): - """Step 49's tables (chip/_chip.tsv and chip/_chip_depth.tsv, bin/chip_report.py): - the CHIP calls, how many other protein changes were listed and not counted, - and the depth and detection limit per gene with their median over the - covered genes. The limit is the one at the median depth, as step 49 prints it.""" - p = first_existing(d, [f"chip/{s}_chip.tsv"]) - dp = first_existing(d, [f"chip/{s}_chip_depth.tsv"]) - if not p or not dp: - return None, {} - calls, depth = read_tsv(p), read_tsv(dp) - need = ("gene", "protein_change", "vaf", "alt_reads", "depth", "status", "note") - with open(p, encoding="utf-8", errors="replace") as f: - head = f.readline().rstrip("\n").split("\t") - if any(k not in head for k in need): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {', '.join(need)}") - if not depth or not {"gene", "mean_depth", "detection_limit_vaf", "status"} <= set(depth[0]): - raise Unreadable(f"{os.path.basename(dp)} has no gene, mean_depth, detection_limit_vaf and status columns, or no row") - chip = [{k: r.get(k) or "." for k in need} for r in calls if r.get("status") == "chip"] - covered = sorted(float(r["mean_depth"]) for r in depth if r["status"] == "covered") - med = None - if covered: - n = len(covered) - med = covered[n // 2] if n % 2 else (covered[n // 2 - 1] + covered[n // 2]) / 2 - return p, {"calls": chip, "other": len(calls) - len(chip), - "genes": [{k: r.get(k) or "." for k in ("gene", "mean_depth", "detection_limit_vaf", "status")} - for r in depth], - "median_depth": med, "median_limit": chip_report.detection_limit(med) if med is not None else None, - "not_covered": [r["gene"] for r in depth if r["status"] != "covered"]} - - -def sec_mei(d, s): - """Step 50's tables (mei/_mei.tsv and _mei_sites.tsv, og mei report): - the insertions MEGAnE calls, counted per element, and every row of the - known-site catalog with seen, not seen or not callable.""" - p = first_existing(d, [f"mei/{s}_mei_sites.tsv"]) - if not p: - return None, {} - sites = read_tsv(p) - need = ("gene", "element", "condition", "inheritance", "status", "mean_depth", "call") - if not sites or any(k not in sites[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {', '.join(need)}") - cp = os.path.join(os.path.dirname(p), f"{s}_mei.tsv") - if not os.path.isfile(cp): - raise Unreadable(f"{os.path.basename(cp)} is missing beside {os.path.basename(p)}") - with open_text(cp) as f: # the header alone: a run with no insertion writes no row - r = csv.DictReader(f, delimiter="\t") - head, calls = r.fieldnames or [], list(r) - if any(k not in head for k in ("chrom", "pos", "element", "copies")): - raise Unreadable(f"{os.path.basename(cp)} lacks a column of chrom, pos, element, copies") - counts = {} - for c in calls: - counts[c["element"]] = counts.get(c["element"], 0) + 1 - return p, {"calls": len(calls), "by_element": dict(sorted(counts.items())), - "sites": [{k: r[k] or "." for k in need} for r in sites]} - - -MOCHA_SUMMARY_KEYS = ("status", "message", "depth", "het_sites", "het_per_10mb", "chromosomes", "floor_read", - "floor_min", "floor", "calls_mocha", "calls_reported", "phasing") -MOCHA_CALL_KEYS = ("chrom", "start", "end", "length_mb", "arms", "type", "cell_fraction", "n_hets", "lod_baf_phase", - "reported", "reason") - - -def sec_mocha(d, s): - """Step 51's tables (mosaic/_mocha_summary.tsv and _mocha.tsv, - og mocha report): the status, the floor, and the calls reported.""" - p = first_existing(d, [f"mosaic/{s}_mocha_summary.tsv"]) - if not p: - return None, {} - kv = {r.get("metric"): r.get("value") for r in read_tsv(p)} - if kv.get("status") not in ("assessed", "not assessed"): - raise Unreadable(f"{os.path.basename(p)} has no status row (assessed or not assessed)") - data = {k: kv.get(k) or "." for k in MOCHA_SUMMARY_KEYS} - c = first_existing(d, [f"mosaic/{s}_mocha.tsv"]) - if not c: - raise Unreadable(f"{os.path.basename(p)} has no {s}_mocha.tsv beside it") - with open_text(c) as f: - head = f.readline().rstrip("\r\n").split("\t") - if any(k not in head for k in MOCHA_CALL_KEYS): - raise Unreadable(f"{os.path.basename(c)} lacks a column of {', '.join(MOCHA_CALL_KEYS)}") - rows = read_tsv(c) - data["calls"] = [{k: r[k] or "." for k in MOCHA_CALL_KEYS} for r in rows if r.get("reported") == "yes"] - data["not_reported"] = sum(1 for r in rows if r.get("reported") != "yes") - if data["calls_reported"] != "." and data["calls_reported"] != str(len(data["calls"])): - raise Unreadable(f"{os.path.basename(c)} has {len(data['calls'])} reported calls, " - f"{os.path.basename(p)} says {data['calls_reported']}") - return p, data - - -def sec_traits(d, s): - """Step 42's traits table (traits/_traits.tsv, bin/traits_report.py): - every row with its genotype, status and reading, the counts per status, - and the APOE line on its own (the reports fold it away).""" - p = first_existing(d, [f"traits/{s}_traits.tsv"]) - if not p: - return None, {} - with open(p, encoding="utf-8", errors="replace") as f: - rows = list(csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE)) - need = ("rsid", "gene", "trait", "genotype", "status", "reading", "ancestry", "note", "pmids") - if not rows or any(k not in rows[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} has no rows or lacks a column of {', '.join(need)}") - # A row cut short (DictReader fills the missing fields with None) or with - # extra fields (kept under the key None) is a damaged table, never a - # called trait without its reading. - bad = next((i for i, r in enumerate(rows, 2) if None in r or any(r.get(k) is None for k in need)), None) - if bad: - raise Unreadable(f"{os.path.basename(p)} line {bad} does not have one field per column") - keep = [{k: r[k] for k in need} for r in rows] - apoe = [r for r in keep if r["gene"] == "APOE"] - counts = {st: sum(1 for r in keep if r["status"] == st and r["gene"] != "APOE") - for st in ("called", "not_covered", "other_allele")} - return p, {"rows": [r for r in keep if r["gene"] != "APOE"], "apoe": apoe, **counts, - "rules": genotype_rules(d, s)} - - -def genotype_rules(d, s): - """Step 42's genotype rules (traits/_genotype_rules.tsv): one line per - rule that reads several sites together (TAS2R38's haplotypes), or None - when the table is not there (a run from before the rules). The rules - whose line the traits table already holds (APOE) are left out, so the - reports keep folding APOE away.""" - p = first_existing(d, [f"traits/{s}_genotype_rules.tsv"]) - if not p: - return None - with open(p, encoding="utf-8", errors="replace") as f: - rows = list(csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE)) - need = traits_report.RULE_OUT_COLUMNS - if rows and any(k not in rows[0] for k in need): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {', '.join(need)}") - bad = next((i for i, r in enumerate(rows, 2) if None in r or any(r.get(k) is None for k in need)), None) - if bad: - raise Unreadable(f"{os.path.basename(p)} line {bad} does not have one field per column") - out = [{k: r[k] for k in need} for r in rows if r["rule"] not in traits_report.TRAITS_TABLE_RULES] - for r in out: - if r["rule"] == genotype_rules_indels.G2["rule"]: - g2 = apol1_g2(d, s) - r["g2"] = dict(g2, **genotype_rules_indels.combine(r, g2)) - return out - - -def vcf_window(path, chrom, start, end): - """The split data lines of a VCF on chrom that can cover start..end: read - from the tabix linear index's offset for start when the .tbi is there, - else the whole file (stopping past the window).""" - out = [] - if os.path.exists(path + ".tbi"): - ioff = tabix_linear_index(path).get(chrom) - if not ioff: - return out - voff = ioff[min((start - 1) >> 14, len(ioff) - 1)] - voff = 0 if voff == 0xFFFFFFFFFFFFFFFF else voff - with open(path, "rb") as raw: - raw.seek(voff >> 16) - g = gzip.GzipFile(fileobj=raw) - g.read(voff & 0xFFFF) - lines = (x.decode(errors="replace").rstrip("\n").split("\t") for x in g if not x.startswith(b"#")) - seen = False - for r in lines: - if len(r) < 2 or not r[1].isdigit(): - continue - if r[0] != chrom: - if seen: - break - continue - seen = True - if int(r[1]) > end: - break - out.append(r) - return out - seen = False - for r in vcf_records(path): - if len(r) < 2 or not r[1].isdigit(): - continue - if r[0] != chrom: - if seen: - break - continue - seen = True - if int(r[1]) > end: - break - out.append(r) - return out - - -def apol1_g2(d, s): - """APOL1 G2 (rs71785313, a 6-base deletion step 42's rule table cannot - read) from the sample's gVCF when it is there (vcf/.g.vcf.gz, whose - reference blocks tell a reference base from no reads), else its VCF - (vcf/.vcf.gz): bin/genotype_rules_indels.py's read_g2, plus source.""" - m = genotype_rules_indels - for name, kind in ((f"vcf/{s}.g.vcf.gz", "gvcf"), (f"vcf/{s}.vcf.gz", "vcf")): - p = os.path.join(d, name) - if not os.path.isfile(p): - continue - try: - recs = vcf_window(p, m.G2["chrom"], m.CONTEXT_START, m.SPAN[1]) - except (OSError, EOFError, ValueError, struct.error, gzip.BadGzipFile, zlib.error, Unreadable) as e: - return {"status": "not_read", "copies": None, "gq": ".", "source": name, - "text": f"G2 (rs71785313) not read: {name} could not be read ({e})."} - return dict(m.read_g2(recs, kind), source=name) - return {"status": "not_read", "copies": None, "gq": ".", "source": None, - "text": "G2 (rs71785313) not read: no VCF of the sample in this folder."} - - -def sec_carrier(d, s): - """The sample's own carrier list (carrier_list) from step 06's ClinVar - hits; its own section, so the ClinVar section reads the same whether or - not the assets folder is there.""" - p = first_existing(d, [f"clinvar/{s}_clinvar_hits.vcf"]) - if not p: - return None, {} - return p, carrier_list_or_note(p, d, s) - - -def sec_exomiser(d, s): - """Step 48's ranking (exomiser/_exomiser_top.tsv, og exomiser report): - the HPO terms searched, and each rank with its gene, mode of inheritance, - scores and contributing variants. A table without rows means no gene - passed Exomiser's filters.""" - p = first_existing(d, [f"exomiser/{s}_exomiser_top.tsv"]) - if not p: - return None, {} - with open(p, encoding="utf-8", errors="replace") as f: - rows = list(csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE)) - f.seek(0) - header = f.readline().rstrip("\n").split("\t") - need = ("rank", "gene", "moi", "exomiser_score", "phenotype_score", "variant", "genotype", "consequence", - "hgvs", "acmg", "disease") - if any(k not in header for k in need): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {', '.join(need)}") - if any(None in r or any(r.get(k) is None for k in need) for r in rows): - raise Unreadable(f"{os.path.basename(p)} has a line that does not have one field per column") - terms = [] - tp = os.path.join(os.path.dirname(p), f"{s}_exomiser_terms.tsv") - if os.path.isfile(tp): - with open(tp, encoding="utf-8", errors="replace") as f: - terms = [{"id": r.get("hpo_id", ""), "label": r.get("label", "")} - for r in csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE)] - genes = [] - for r in rows: - if not genes or genes[-1]["rank"] != r["rank"]: - genes.append({k: r[k] for k in ("rank", "gene", "moi", "exomiser_score", "phenotype_score")}) - genes[-1]["variants"] = [] - genes[-1]["variants"].append({k: r[k] for k in ("variant", "genotype", "consequence", "hgvs", "acmg", "disease")}) - return p, {"terms": terms, "genes": genes} - - -def sec_couple(d, s): - """Step 39's couple table (couple/_couple_carrier.tsv, bin/couple_report.py): - the partner, the shared and X-linked genes with the chance for a child, and - how many genes only one partner carries a change in. A table without rows - (no finding in either partner) names no partner.""" - p = first_existing(d, [f"couple/{s}_couple_carrier.tsv"]) - if not p: - return None, {} - with open(p, encoding="utf-8", errors="replace") as f: - reader = csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE) - rows = list(reader) - cols = reader.fieldnames or [] - need = ("gene", "condition", "result", "offspring_risk", "partner_a", "findings_a", "partner_b", - "findings_b", "explanation") - if any(k not in cols for k in need): - raise Unreadable(f"{os.path.basename(p)} lacks a column of {', '.join(need)}") - bad = next((i for i, r in enumerate(rows, 2) if None in r or any(r.get(k) is None for k in need) - or s not in (r["partner_a"], r["partner_b"]) or r["result"] not in ("shared", "x_linked", "one_partner")), - None) - if bad: - raise Unreadable(f"{os.path.basename(p)} line {bad} is not a row of {s}'s couple table") - - def side(r): - own = "a" if r["partner_a"] == s else "b" - other = "b" if own == "a" else "a" - return {"gene": r["gene"], "condition": r["condition"], "risk": r["offspring_risk"], - "own": r["findings_" + own], "partner_findings": r["findings_" + other], - "explanation": r["explanation"]} - partner = next((r["partner_b"] if r["partner_a"] == s else r["partner_a"] for r in rows), None) - return p, {"partner": partner, - "shared": [side(r) for r in rows if r["result"] == "shared"], - "x_linked": [side(r) for r in rows if r["result"] == "x_linked"], - "one_partner": sum(1 for r in rows if r["result"] == "one_partner")} - - -# The D-loop poly-C tracts (rCRS m.303-315 and m.16184-16193). Length -# variants there, an extra or missing C, are found in most people, often as a -# mix of lengths that reads as heteroplasmy; they are counted apart and left -# out of the heteroplasmic count. -POLYC_TRACTS = ((303, 315), (16184, 16193)) - - -def mito_span(pos, ref, alt): - """The rCRS positions a chrM allele changes: an SNV or MNV its own bases, - an indel the bases after its anchor base (Mutect2 writes the anchor).""" - if len(ref) == len(alt): - return pos, pos + len(ref) - 1 - return pos + 1, pos + max(len(ref) - 1, 1) - - -def in_polyc(pos, ref, alt): - lo, hi = mito_span(pos, ref, alt) - return any(lo <= b and hi >= a for a, b in POLYC_TRACTS) - - -def min_allele(pos, ref, alt): - """(pos, ref, alt) without the bases ref and alt share at the end and then - at the start, one base always kept: Mutect2's A>AC of a two-ALT record - AC>A,ACC becomes the same key as ClinVar's and gnomAD's.""" - while len(ref) > 1 and len(alt) > 1 and ref[-1] == alt[-1]: - ref, alt = ref[:-1], alt[:-1] - while len(ref) > 1 and len(alt) > 1 and ref[0] == alt[0]: - ref, alt, pos = ref[1:], alt[1:], pos + 1 - return pos, ref, alt - - -MITO_ANNOT_COLS = ["chrom", "pos", "ref", "alt", "filter", "af", "clinvar_id", "clinvar_significance", - "clinvar_review", "clinvar_condition", "gnomad_af_hom", "gnomad_af_het", "gnomad_max_hl"] -# ClinVar classes: P/LP are counted; a record whose every class is benign or -# carries no classification is not listed unless the call is heteroplasmic. -CLINVAR_PLP = ("Pathogenic", "Likely_pathogenic", "Pathogenic/Likely_pathogenic") -CLINVAR_BENIGN = ("Benign", "Likely_benign", "Benign/Likely_benign", "not_provided") - - -def clinvar_terms(sig): - return [t for t in re.split(r"[|,]", sig or "") if t and t != "."] - - -# --- step 20: large mtDNA deletions (eKLIPse) and the mtDNA copy number ---------- - -# _chrM_deletions.tsv (og mito deletions): eKLIPse's deletions, one row per -# deletion. bkp5 and bkp3 are its 5' and 3' breakpoints as it writes them: the -# last rCRS base kept before the deletion and the first one kept after it, so -# the deleted bases are bkp5+1 to bkp3-1 (across the origin, m.16569 to m.1, -# when bkp3 is the smaller). level is the share of the mtDNA -# molecules with the deletion (in %), the mean of the forward and reverse -# strand estimates. -# A deletion of assets/mito_confirmed.tsv is named when both breakpoints lie -# within this many bases of its own: a deletion between two direct repeats -# can be written anywhere along them (the common one's repeats are 13 bp). -MITO_DEL_TOLERANCE = 50 -# The report lists every named deletion and any other at this level or more; -# the rest are counted. -MITO_DEL_LIST_PCT = 1.0 -# The mtDNA copy number: chrM depth over autosomal depth, from the read counts -# of the BAM index (samtools idxstats), times 2 for the two nuclear copies. -# Under this many reads per kb of autosome (about 3x at 150 bp) the BAM is a -# slice, a panel or a shallow genome, and the ratio would mean nothing. An -# exome can pass the floor and is not told apart from a genome here: the -# number is meant for whole-genome BAMs (docs/20-mtoolbox.md). -MITO_CN_COLS = ["status", "reason", "chrm_reads", "chrm_length", "autosomal_reads", "autosomal_length", - "depth_ratio", "mtdna_copies"] - - -def mito_confirmed(): - """(SNVs {(pos, ref, alt): row}, deletions [row]) of assets/mito_confirmed.tsv.""" - snv, dels = {}, [] - for r in asset_rows("mito_confirmed.tsv"): - if r["kind"] == "snv": - snv[(int(r["pos"]), r["ref"], r["alt"])] = r - elif r["kind"] == "deletion": - dels.append(r) - return snv, dels - - -def known_deletion(row, dels): - """The deletion of assets/mito_confirmed.tsv a called deletion is, or None.""" - for k in dels: - if abs(row["bkp5"] + 1 - int(k["pos"])) <= MITO_DEL_TOLERANCE \ - and abs(row["bkp3"] - 1 - int(k["end"])) <= MITO_DEL_TOLERANCE: - return k - return None - - -def _confirmed_fields(k): - return {"hgvs": k["hgvs"], "gene": k["gene"], "disease": k["disease"], "clinvar_id": k["clinvar_id"], - "pmids": k["pmids"], "note": k["heteroplasmy_note"]} - - -def mito_extended(d, s, vcf): - """Step 20's confirmed variants (assets/mito_confirmed.tsv) among the chrM - calls, its large deletions and the mtDNA copy number.""" - out = {} - try: - snv, dels = mito_confirmed() - except Unreadable as e: - snv, dels = None, [] - out["confirmed_note"] = str(e) - confirmed = [] - for r in vcf_records(vcf) if snv is not None else (): - if len(r) < 8: - continue - fmt = r[8].split(":") if len(r) > 9 else [] - val = r[9].split(":") if len(r) > 9 else [] - afs = val[fmt.index("AF")].split(",") if "AF" in fmt and fmt.index("AF") < len(val) else [] - for i, alt in enumerate(r[4].split(",")): - k = snv.get(min_allele(int(r[1]), r[3], alt)) - if k: - confirmed.append(_confirmed_fields(k) | {"pos": int(k["pos"]), "filter": r[6], - "af": afs[i] if i < len(afs) else "."}) - if snv is not None: - out["confirmed"] = confirmed - p = first_existing(d, [f"mito/{s}_chrM_deletions.tsv"]) - if not p: - out["deletions_note"] = ("no deletion table (mito/_chrM_deletions.tsv): step 20 ran before it looked " - "for large deletions, eKLIPse wrote no table (see the step's log), or the Nextflow " - "MITO_DELETIONS task failed (see the Nextflow log)") - else: - with open(p, errors="replace") as f: - rows = list(csv.DictReader(f, delimiter="\t")) - listed = [] - for r in rows: - row = {"bkp5": int(r["bkp5"]), "bkp3": int(r["bkp3"]), - "length": None if r["length"] == "." else int(r["length"]), "level_pct": float(r["level_pct"])} - k = known_deletion(row, dels) - if k or row["level_pct"] >= MITO_DEL_LIST_PCT: - listed.append(row | ({"known": _confirmed_fields(k)} if k else {})) - out["deletions"] = {"source": os.path.relpath(p, d), "total": len(rows), "listed": listed, - "list_floor_pct": MITO_DEL_LIST_PCT} - p = first_existing(d, [f"mito/{s}_mtdna_copies.tsv"]) - if p: - m = dict(read_tsv_pairs(p)) - out["copies"] = {k: (None if m.get(k) in (None, ".") else m[k]) for k in MITO_CN_COLS} - else: - out["copies_note"] = ("no copy-number table (mito/_mtdna_copies.tsv): step 20 ran before it counted " - "mtDNA copies, or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)") - return out - - -def read_tsv_pairs(path): - """[(metric, value)] of a two-column table with a header line.""" - with open(path, errors="replace") as f: - lines = [ln.rstrip("\n").split("\t") for ln in f if ln.strip()] - return [(c[0], c[1]) for c in lines[1:] if len(c) >= 2] - - -def mito_annotation(d, s, vcf): - """Step 20's annotation table, as the report lists it, or a note on why - there is none.""" - p = first_existing(d, [f"mito/{s}_chrM_annotated.tsv"]) - if not p: - return {"note": "no annotation table (mito/_chrM_annotated.tsv): step 20 ran before it annotated, " - "or the Nextflow MITO_VARIANTS_ANNOTATE task failed (see the Nextflow log)"} - if os.path.getmtime(p) < os.path.getmtime(vcf): - return {"note": f"{os.path.relpath(p, d)} is older than the chrM calls: rerun step 20"} - sources, lines = {}, [] - with open(p, errors="replace") as f: - for line in f: - if line.startswith("# "): - c = line[2:].rstrip("\n").split("\t") - if len(c) >= 3: - sources[c[0]] = (f"not annotated: {c[2]}" if c[1] == "not annotated" - else c[1] + (f" ({c[2]})" if c[2] != "-" else "")) - elif line.strip(): - lines.append(line) - rows = [r for r in csv.DictReader(lines, delimiter="\t") if r.get("filter") == "PASS"] - return {"source": os.path.relpath(p, d), "clinvar": sources.get("clinvar"), "gnomad": sources.get("gnomad"), - "rows": rows} - - -def sec_mito(d, s): - p = first_existing(d, [f"mito/{s}_chrM_filtered.vcf.gz"]) - if not p: - return None, {} - passed = het = 0 - polyc, sites = [], [] - for r in vcf_records(p): - if len(r) < 10 or r[6] != "PASS": - continue - passed += 1 - fmt = r[8].split(":") - val = r[9].split(":") - if "AF" not in fmt or fmt.index("AF") >= len(val): - continue - try: - afs = [float(x) for x in val[fmt.index("AF")].split(",") if x not in ("", ".")] - af = max(afs) - except ValueError: - continue - # Heteroplasmic: 5% to 95%. Below 5% NUMT reads and noise dominate. - if not 0.05 <= af < 0.95: - continue - alts = r[4].split(",") - if any(in_polyc(int(r[1]), r[3], a) for a in alts): - polyc.append({"variant": f"{r[0]}:{r[1]} {r[3]}>{r[4]}", "af": round(af, 3)}) - else: - het += 1 - # Each counted site, for the report's chart (bin/report_charts.py). - sites.append({"pos": int(r[1]), "ref": r[3], "alt": r[4], "af": round(af, 3)}) - ann = mito_annotation(d, s, p) - listed, plp = [], 0 - for r in ann.pop("rows", []): - terms = clinvar_terms(r.get("clinvar_significance")) - is_plp = any(t in CLINVAR_PLP for t in terms) - plp += is_plp - try: - het_row = 0.05 <= float(r.get("af")) < 0.95 - except (TypeError, ValueError): - het_row = False - if is_plp or het_row or (terms and not all(t in CLINVAR_BENIGN for t in terms)): - listed.append({k: r.get(k) for k in MITO_ANNOT_COLS if k not in ("chrom", "filter")} - | {"polyc": in_polyc(int(r["pos"]), r["ref"], r["alt"])}) - if "source" in ann: - ann.update(variants=listed, clinvar_plp=plp) - return p, {**mito_extended(d, s, p), "pass": passed, "heteroplasmic": het, "heteroplasmic_sites": sites, "heteroplasmy_floor": 0.05, - "polyc_heteroplasmic": polyc, - "polyc_tracts": [f"m.{a}-{b}" for a, b in POLYC_TRACTS], "annotation": ann} - - -def cpsr_cdna_checks(d, s): - """Step 17's cDNA read check of CPSR's P/LP rows (bin/clip_check.py - --cpsr), keyed like step 06's; None when step 17 did not write it.""" - p = first_existing(d, [f"cpsr/{s}_cpsr_cdna_check.tsv"]) - return clinvar_cdna_checks(d, s, p) if p else None - - -def sec_cpsr(d, s): - html = first_existing(d, [f"cpsr/{s}.cpsr.grch38.html", "cpsr/*.cpsr.*.html"]) - tsv = first_existing(d, [f"cpsr/{s}.cpsr.grch38.classification.tsv.gz", "cpsr/*.cpsr.*.classification.tsv.gz"]) - p = tsv or html - if not p: - return None, {} - out = {"html_report": bool(html)} - if tsv: - try: - with gzip.open(tsv, "rt", errors="replace") as f: - rows = list(csv.DictReader(f, delimiter="\t")) - cols = rows[0].keys() if rows else [] - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(tsv)}: {e}") from e - col = next((c for c in ("CLASSIFICATION", "FINAL_CLASSIFICATION", "CPSR_CLASSIFICATION") if c in cols), None) - if col: - counts = {} - for r in rows: - counts[r.get(col) or "."] = counts.get(r.get(col) or ".", 0) + 1 - out["classification_column"] = col - out["classification"] = dict(sorted(counts.items(), key=lambda kv: -kv[1])) - out.update(cpsr_compare(d, s, rows, cols, col)) - else: - out["classification_column"] = None - return p, out - - -def cpsr_compare(d, s, rows, cols, col): - """CPSR's own class beside the final class and ClinVar's. - - The final class (CLASSIFICATION) is ClinVar's whenever ClinVar has a - trusted record, so it hides CPSR's own call (CPSR_CLASSIFICATION). This - lists the P/LP rows (final or own class), counts the final P/LP rows that - CPSR's own rules class benign or likely benign, and lists every row where - ClinVar and CPSR's own class disagree on P/LP. ClinVar's class is - CLINVAR_CLASSIFICATION, or the final class where ASSERTION_AUTHORITY is - ClinVar. A table with one class column has no disagreement to show. - Low-confidence: DP_CONTROL below step 6's depth floor (CPSR writes no - GQ). The cDNA read check of step 17 goes on each P/LP row.""" - own = "CPSR_CLASSIFICATION" if "CPSR_CLASSIFICATION" in cols and col != "CPSR_CLASSIFICATION" else None - hits = first_existing(d, [f"clinvar/{s}_clinvar_hits.vcf"]) - floor = call_floor(hits) if hits else dict(CALL_FLOOR) - cdna = cpsr_cdna_checks(d, s) - norm = clip_check.norm_class - plp, disagree, own_benign = [], [], 0 - for r in rows: - final, mine = norm(r.get(col)), norm(r.get(own)) if own else "" - authority = (r.get("ASSERTION_AUTHORITY") or "").strip() - clinvar_raw = ((r.get("CLINVAR_CLASSIFICATION") or "").strip() if "CLINVAR_CLASSIFICATION" in cols - else (r.get(col) or "").strip() if authority.lower() == "clinvar" else "") - clinvar = norm(clinvar_raw) - if final in clip_check.PLP and mine in ("benign", "likely benign"): - own_benign += 1 - is_plp = final in clip_check.PLP or mine in clip_check.PLP - differ = bool(own and clinvar and mine) and (clinvar in clip_check.PLP) != (mine in clip_check.PLP) - if not (is_plp or differ): - continue - v = clip_check.cpsr_variant(r) - dp = (r.get("DP_CONTROL") or "").strip() - dp = int(dp) if dp.isdigit() else None - x = {"variant": f"{v[0]}:{v[1]} {v[2]}>{v[3]}" if v else (r.get("GENOMIC_CHANGE") or "."), - "gene": r.get("SYMBOL") or ".", "final": r.get(col) or ".", - "cpsr": (r.get(own) or ".") if own else None, "clinvar": clinvar_raw if clinvar else None, - "authority": authority or None, "acmg_codes": (r.get("ACMG_CODE") or "").strip() or None, - "dp": dp, "low_confidence": dp is not None and dp < floor["DP"]} - if cdna is not None and is_plp: - x["cdna"] = (cdna.get((v[0], v[1], v[2], v[3])) if v else None) or { - "check": "not_checked", "note": "not in the read-check table", "alt_reads": None, - "alt_clipped": None, "ref_reads": None, "ref_clipped": None} - if is_plp: - plp.append(x) - if differ: - disagree.append(x) - out = {"plp": plp, "disagreements": disagree, "final_plp_own_benign": own_benign, - "own_class_column": own, "low_confidence": sum(1 for x in plp if x["low_confidence"]), - "call_floor_dp": floor["DP"]} - if cdna is not None: - out["cdna_flagged"] = sum(1 for x in plp if x["cdna"]["check"] == "flagged") - return out - - -def csq_rank(fmt, csq): - """The rule of bin/csq_rank.awk on one CSQ value: (class, entries, freq). - class is MANE (entries of the MANE Select transcript), else CANONICAL, - else ANY (every entry); freq is rare, common, unknown (no value) or none - (the CSQ has no frequency field): MAX_AF, else gnomADe_AF and gnomADg_AF, - the highest over the entries.""" - ix = {k: i for i, k in enumerate(fmt)} - entries = [v for v in (t.split("|") for t in csq.split(",") if t) if len(v) == len(fmt)] - - def get(v, k): - return v[ix[k]] if k in ix else "" - mane = [v for v in entries if get(v, "MANE_SELECT")] - canon = [v for v in entries if get(v, "CANONICAL") == "YES"] - cls, chosen = ("MANE", mane) if mane else ("CANONICAL", canon) if canon else ("ANY", entries) - fields = ["MAX_AF"] if "MAX_AF" in ix else [f for f in ("gnomADe_AF", "gnomADg_AF") if f in ix] - afs = [] - for v in entries: - for f in fields: - for a in get(v, f).split("&"): - try: - afs.append(float(a)) - except ValueError: - pass - freq = "none" if not fields else "unknown" if not afs else "rare" if max(afs) < COMMON_AF else "common" - return cls, [{"impact": get(v, "IMPACT"), "gene": get(v, "SYMBOL"), - "consequence": get(v, "Consequence").replace("&", ",")} for v in chosen], freq - - -def call_check(fmt, sample, floor): - """bin/csq_rank.awk's genotype check of one sample: (weak, reasons, gq, - dp, ad). Weak: GQ or DP below the floor, a heterozygote's alt read - fraction outside HET_AB, a homozygous alt call's below HOM_AB.""" - f = dict(zip(fmt.split(":"), sample.split(":"))) - gq, dp = (int(f[k]) if f.get(k, "").isdigit() else None for k in ("GQ", "DP")) - ad = f.get("AD", "") - why = [] - if gq is not None and gq < floor["GQ"]: - why.append(f"GQ {gq} < {floor['GQ']}") - if dp is not None and dp < floor["DP"]: - why.append(f"DP {dp} < {floor['DP']}") - al = f.get("GT", "").replace("|", "/").split("/") - depths = ad.split(",") - if len(al) == 2 and all(a.isdigit() for a in al) and len(depths) > 1 and all(x.isdigit() for x in depths): - tot = sum(int(x) for x in depths) - if tot: - ab = sum(int(x) for x in depths[1:]) / tot - lo, hi = floor["HET_AB"] - if (al[0] == "0") != (al[1] == "0") and not lo <= ab <= hi: - why.append(f"alt reads {ab:.2f} of the depth on a heterozygote") - elif al[0] == al[1] != "0" and ab < floor["HOM_AB"]: - why.append(f"alt reads {ab:.2f} of the depth on a homozygous call") - return bool(why), why, gq, dp, ad or None - - -def clinical_call_floor(path): - """The floors bin/csq_rank.awk wrote into the clinical VCF, the defaults without the line.""" - floor = dict(CALL_FLOOR, **AB_FLOOR) - try: - with open_text(path) as f: - for line in f: - if not line.startswith("##"): - break - if line.startswith("##PGP_CALL_FLOOR="): - for k, v in re.findall(r"\b(GQ|DP)=(\d+)", line): - floor[k] = int(v) - m = re.search(r"HET_AB=([0-9.]+)-([0-9.]+)", line) - if m: - floor["HET_AB"] = (float(m.group(1)), float(m.group(2))) - m = re.search(r"HOM_AB=([0-9.]+)", line) - if m: - floor["HOM_AB"] = float(m.group(1)) - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(path)}: {e}") from e - return floor - - -def csq_high_impact_genes(path): - """{(chrom, pos, ref, alt): record} of the records whose ranked CSQ entries - (bin/csq_rank.awk's rule: the MANE Select transcript, else the canonical - one, else any) hold a HIGH-impact consequence. record: genotype, genes - ({gene: consequence}), transcript (MANE, CANONICAL or ANY), frequency - (rare, common, unknown or none), weak (the call check failed), why, gq, - dp and ad. A HIGH consequence on a transcript other than the MANE Select - one does not count when the record has a MANE Select transcript.""" - floor = clinical_call_floor(path) - fmt, out = None, {} - try: - with open_text(path) as f: - for line in f: - if line.startswith("##INFO= 9 else "" - weak, why, gq, dp, ad = call_check(r[8], r[9], floor) if len(r) > 9 else (False, [], None, None, None) - out[(r[0], int(r[1]), r[3], r[4])] = { - "genotype": gt, "ps": gt_field(r, "PS"), "genes": genes, "transcript": cls, "frequency": freq, - "weak": weak, "why": why, "gq": gq, "dp": dp, "ad": ad} - except (OSError, EOFError, gzip.BadGzipFile, zlib.error) as e: - raise Unreadable(f"{os.path.basename(path)}: {e}") from e - return out - - -# --- the ACMG SF rules and the carrier list (assets/) -------------------------- - -_ASSETS = {} - - -def asset_rows(name): - """The rows of assets/ (comment lines left out), read once.""" - p = os.path.join(ASSETS_DIR, name) - if p in _ASSETS: - return _ASSETS[p] - if not os.path.isfile(p): - raise Unreadable(f"{name} is not in {ASSETS_DIR}: the report needs the repository's assets/ folder " - "(render_report.py --assets)") - with open(p, encoding="utf-8") as f: - rows = list(csv.DictReader((x for x in f if not x.startswith("#")), delimiter="\t", quoting=csv.QUOTE_NONE)) - _ASSETS[p] = rows - return rows - - -def allele_key(locus): - """'chr6:26092913:G:A' -> ('chr6', 26092913, 'G', 'A').""" - c, pos, ref, alt = locus.split(":") - return (c, int(pos), ref, alt) +def allele_key(locus): + """'chr6:26092913:G:A' -> ('chr6', 26092913, 'G', 'A').""" + c, pos, ref, alt = locus.split(":") + return (c, int(pos), ref, alt) def parse_alleles(text, kinds=False): @@ -3072,1056 +334,3 @@ def parse_alleles(text, kinds=False): raise ValueError(f"severity '{kind}' is not one of {', '.join(SEVERITY_KINDS)}: {e}") out[allele_key(parts[0])] = {"kind": kind, "name": parts[1] if len(parts) > 1 else parts[0]} return out - - -def acmg_sf_rules(): - """{gene: rule} of assets/acmg_sf.tsv.""" - rules = {} - for n, r in enumerate(asset_rows("acmg_sf.tsv"), 1): - try: - g = r["gene"] - if r["report_if"] not in SF_REPORT_IF or r["sv_cn"] not in ("yes", "no"): - raise ValueError(f"report_if must be one of {', '.join(SF_REPORT_IF)}, sv_cn yes or no") - if r["sv_cn"] == "yes" and r["report_if"] not in ("any", "het_female_too"): - raise ValueError("sv_cn yes needs a rule that one variant meets (any, het_female_too)") - regions = [] - for e in (r["sv_regions"] or ".").split(";"): - e = e.strip() - if e not in ("", "."): - loc, _, name = e.partition(" ") - c, _, span = loc.partition(":") - a, _, b = span.partition("-") - regions.append((c, int(a), int(b), name or loc)) - variants = parse_alleles(r["variants"]) - if r["report_if"] == "hom_variant" and not variants: - raise ValueError("report_if hom_variant needs its variants") - if g in rules: - raise ValueError("the gene is on two rows") - rules[g] = {"inheritance": r["inheritance"], "report_if": r["report_if"], "variants": variants, - "sv_cn": r["sv_cn"] == "yes", "sv_regions": regions, - "severity": parse_alleles(r["severity"], kinds=True), - "table_text": r["table_text"], "note": None if r["note"] in ("", ".") else r["note"]} - except (KeyError, ValueError, TypeError) as e: - raise Unreadable(f"acmg_sf.tsv row {n}: {e}") from e - if not rules: - raise Unreadable("acmg_sf.tsv has no gene") - return rules - - -def carrier_genes(): - """{gene: row} of assets/carrier_genes.tsv, the severity cell parsed.""" - out = {} - for n, r in enumerate(asset_rows("carrier_genes.tsv"), 1): - try: - out[r["gene"]] = {"inheritance": r["inheritance"], "lists": r["lists"], "condition": r["condition"], - "severity": parse_alleles(r.get("severity"), kinds=True), - "span": (r["chrom"], int(r["start"]), int(r["end"]))} - except (KeyError, ValueError, TypeError) as e: - raise Unreadable(f"carrier_genes.tsv row {n}: {e}") from e - return out - - -def allele_copies(gt): - """(alt copies, ploidy) of a GT: '0/1' (1, 2), '1/1' (2, 2), '1' (1, 1), - '1/.' (1, 2): a missing allele is a copy not called, not a missing copy, - so the half of a split 1/2 call is not read as hemizygous.""" - a = [x for x in re.split(r"[/|]", gt or "") if x != ""] - return sum(1 for x in a if x not in ("0", ".")), len(a) - - -def zygosity_word(copies, ploidy): - if not copies: - return "unknown" - if ploidy == 1: - return "hemizygous" - return "hom" if copies >= 2 else "het" - - -# --- two hits in one gene across callers: a deletion plus a small variant -------- -# -# Small variants alone miss a gene hit twice when one hit is a deletion: an -# exon deletion of one copy of MUTYH plus a pathogenic variant on the other, an -# HBA deletion plus an HBA point variant, one SMN1 copy plus an SMN1 variant, a -# 1q21.1 deletion plus the RBM8A low-expression allele (TAR syndrome: Albers et -# al., Nat Genet 2012;44(4):435-439, PMID 22366785). The deletions come from the -# engines that call them: AnnotSV's class 4 and 5 deletions (step 05), step 35's -# interval table (bin/cn_loci.tsv), its HBA count and Parascopy's SMN1 count. -# Two hits are in trans (biallelic) only when the reads show it: the variant sits -# inside the deletion and is called homozygous (only one copy is left, and it -# carries the variant), or both calls carry a read-backed phase (GT with "|" and -# the same PS) on different copies. Otherwise they are possibly in trans; on the -# same copy (cis) they are no second hit. -TWO_HIT_ORDER = ("biallelic", "possibly biallelic") - - -def gene_deletions(d, s, spans): - """The genes a called deletion reaches: ({gene: [deletion]}, engines read, - engines not read). spans: {gene: (chrom, start, end)}, 1-based inclusive - (assets/carrier_genes.tsv). A deletion: chrom, start, end (1-based, - inclusive), text, inside (a variant inside its span is on the one copy left; - False for the HBA count, which cannot say which gene of the pair is gone), - gt and ps (its phase, AnnotSV only).""" - out, read, unread = {}, [], [] - - def add(g, x): - out.setdefault(g, []).append(x) - - def over(chrom, a, b): - return [g for g, (c, gs, ge) in spans.items() if c == chrom and gs <= b and ge >= a] - try: - p, data = sec_annotsv(d, s) - except (Unreadable, OSError, ValueError, KeyError) as e: - p, data = None, {} - unread.append(f"AnnotSV (step 05): {e}") - if p: - read.append("AnnotSV class 4 and 5 deletions (step 05)") - for x in data["rows"]: - chrom, _, span = x["sv"].partition(":") - a, _, b = span.partition("-") - if x["type"] != "DEL" or x["common"] or not (a.isdigit() and b.isdigit()): - continue - # A call genotyped 0/0 (or 0|0) is no deletion in this sample: it pairs with nothing. - if x.get("gt") and set(x["gt"].replace("|", "/").split("/")) == {"0"}: - continue - chrom = chrom if chrom.startswith("chr") else "chr" + chrom - # SV_start is the base before the deleted bases (annotsv_postfilter.sv_span): - # they are SV_start+1 to SV_end, 1-based. The text keeps AnnotSV's coordinates. - first = int(a) + 1 - hit = set(over(chrom, first, int(b))) - # A listed gene AnnotSV names counts only when its span reaches the deleted bases: one - # that ends at the padding base loses none. A gene with no span here is taken as named. - named = {g for g in x["genes"] if g not in spans} - for g in sorted(hit | named): - add(g, {"chrom": chrom, "start": first, "end": int(b), "inside": True, - "text": f"DEL {chrom}:{a}-{b} (AnnotSV class {x['acmg_class']}, {x['caller']})", - "gt": x.get("gt"), "ps": x.get("ps")}) - elif not unread: - unread.append("AnnotSV (step 05): no table") - try: - p, data = sec_parascopy(d, s) - why = "none" - except (Unreadable, OSError, ValueError, KeyError) as e: - p, data, why = None, {}, str(e) - if not p: - unread.append(f"step 35's copy-number tables: {why}") - return out, read, unread - iv = data.get("cn_intervals") - if iv is not None: - read.append("step 35's interval table (read depth)") - for r in iv["changes"]: - if r["change"] != "deletion" or not (r["start"].isdigit() and r["end"].isdigit()): - continue - a, b = int(r["start"]) + 1, int(r["end"]) # the table is BED: 0-based start - for g in over(r["chrom"], a, b): - add(g, {"chrom": r["chrom"], "start": a, "end": b, "inside": True, "gt": None, "ps": None, - "text": f"{r['locus']} deletion, {r['copies']} of {r['expected']} copies " - "(read depth, step 35)"}) - else: - unread.append("step 35's interval table: none") - hba = data.get("hba") or {} - n = hba.get("alpha_globin_genes", "") - if hba.get("status") == "called" and n.isdigit(): - read.append("step 35's HBA1/HBA2 count") - # 0 genes: no alpha-globin gene is left to carry a variant, so a hit there pairs with nothing. - if 1 <= int(n) < 4: - for g in ("HBA1", "HBA2"): - if g in spans: - c, a, b = spans[g] - add(g, {"chrom": c, "start": a, "end": b, "inside": False, "gt": None, "ps": None, - "text": f"{n} of 4 alpha-globin genes (read depth, step 35)"}) - else: - unread.append(f"step 35's HBA1/HBA2 count: {hba.get('message') or 'not called'}") - smn1 = (data.get("smn1") or {}).get("cn") - if smn1 is not None: - read.append("Parascopy's SMN1 count (step 35)") - if smn1 == 1 and "SMN1" in spans: - c, a, b = spans["SMN1"] - add("SMN1", {"chrom": c, "start": a, "end": b, "inside": True, "gt": None, "ps": None, - "text": "one SMN1 copy (Parascopy, step 35)"}) - else: - unread.append("Parascopy's SMN1 count: not resolved") - return out, read, unread - - -def phase_haps(gt): - """The copies (haplotype indexes) a phased GT puts the alt allele on, or - None when the GT is not phased or a copy is not called: '0|1' -> {1}, - '1|0' -> {0}, '0/1' -> None, '.|1' -> None (the missing copy could carry - the other allele, so no cis or trans can be read from it).""" - if not gt or "|" not in gt: - return None - alleles = gt.split("|") - if "." in alleles: - return None - return frozenset(i for i, a in enumerate(alleles) if a != "0") - - -def pair_phase(dl, v): - """(verdict, why) of a deletion and a small variant in one gene: verdict - biallelic, possibly biallelic, or cis (no second hit).""" - inside = dl["inside"] and dl["chrom"] == v["chrom"] and dl["start"] <= v["pos"] <= dl["end"] - # Read-backed phase first: on the same copy (cis) is no second hit, wherever the variant sits. - hd, hv = phase_haps(dl.get("gt")), phase_haps(v.get("gt")) - phased = None - if hd and hv and dl.get("ps") and dl.get("ps") == v.get("ps"): - if hd == hv: - return "cis", f"in cis: read-backed phase puts both on the same copy (phase set {v['ps']})" - phased = (f"in trans: read-backed phase puts them on different copies (phase set {v['ps']})" if not hd & hv - else f"read-backed phase puts the variant on both copies (phase set {v['ps']}); check the reads") - if inside and v["copies"] >= 2: - return "biallelic", ("the variant is called homozygous inside the deletion: it is hemizygous, on the one " - "copy the deletion leaves") - if inside and v["ploidy"] == 2: - return "possibly biallelic", ("a heterozygous call inside a called deletion: one of the two calls may be " - "wrong; check the reads") - if phased: - return ("biallelic" if phased.startswith("in trans") else "possibly biallelic"), phased - return "possibly biallelic", "phase unknown, possibly in trans (no read-backed phase links the two calls)" - - -def two_hits(variants, dels): - """The genes hit twice: ({gene: entry}, {(gene, key): deletion text} of - the variants called homozygous inside a deletion, shown as hemizygous). - variants: [{gene, key, chrom, pos, text, genotype, copies, ploidy, gt, ps, - allele}] (allele: the severity name of a hypomorph, or None). An entry is - the best pair of the gene: verdict, why, deletion, variant, genotype, and - the allele name; a gene whose every pair is in cis has none. pairs: - {(gene, key): " with : "}, the best pair of each - variant that has one.""" - entries, hemi, pairs = {}, {}, {} - for v in variants: - for dl in dels.get(v["gene"], []): - verdict, why = pair_phase(dl, v) - if verdict == "biallelic" and why.startswith("the variant is called homozygous"): - hemi[(v["gene"], v["key"])] = dl["text"] - if verdict == "cis": - continue - e = {"gene": v["gene"], "verdict": verdict, "why": why, "deletion": dl["text"], "variant": v["text"], - "genotype": "hemizygous" if (v["gene"], v["key"]) in hemi else v["genotype"], - "allele": v.get("allele")} - old = entries.get(v["gene"]) - if old is None or TWO_HIT_ORDER.index(verdict) < TWO_HIT_ORDER.index(old["verdict"]): - entries[v["gene"]] = e - k = (v["gene"], v["key"]) - if k not in pairs or TWO_HIT_ORDER.index(verdict) < TWO_HIT_ORDER.index(pairs[k][0]): - pairs[k] = (verdict, f"{verdict} with {dl['text']}: {why}") - return entries, hemi, {k: t for k, (_, t) in pairs.items()} - - -def sf_verdicts(gene, rule, alleles): - """Which of a gene's variants the ACMG rule reports. alleles: {key: - (copies, ploidy)} of the gene's distinct small variants, structural ones - keyed by their span. {key: None (reported) or the reason it is not}, and - the partial-deficiency note of a reported gene with a hypomorph allele.""" - sev = rule["severity"] - out, kept = {}, {} - for k, cp in alleles.items(): - if sev.get(k, {}).get("kind") == "pseudodeficiency": - out[k] = f"{sev[k]['name']}: a pseudodeficiency allele, not counted" - else: - kept[k] = cp - n = sum(c for c, _ in kept.values()) - ri = rule["report_if"] - if ri in ("any", "het_female_too"): - ok = {k: True for k in kept} - elif ri == "biallelic": - ok = {k: n >= 2 for k in kept} - elif ri == "hemizygous": - hemi = any(p == 1 for _, p in kept.values()) - ok = {k: hemi or n >= 2 for k in kept} - else: # hom_variant - ok = {k: k in rule["variants"] and c >= 2 and p == 2 for k, (c, p) in kept.items()} - for k, good in ok.items(): - if good: - out[k] = None - elif ri == "hom_variant": - out[k] = f"{gene} is reported only for: {rule['table_text']}" - elif ri == "hemizygous": - out[k] = f"one heterozygous variant; {gene} is reported for {rule['table_text']}" - else: - out[k] = f"one heterozygous variant; {gene} is reported with {rule['table_text']}" - partial = None - hypo = [sev[k]["name"] for k, why in out.items() if why is None and sev.get(k, {}).get("kind") == "hypomorph"] - if hypo: - partial = (f"partial, not profound: {'; '.join(hypo)} is a hypomorph allele, so the deficiency it causes " - "is partial") - return out, partial - - -def sf_structural(d, s, rules): - """Step 05's class 4 and 5 SVs (AnnotSV) in genes whose rule allows them - (sv_cn yes), and the class 4 or 5 deletions over a gene's sv_regions (the - 3' end of EPCAM for MSH2). Common SVs are left out. (rows, checked, note).""" - try: - p, data = sec_annotsv(d, s) - except (Unreadable, OSError, ValueError, KeyError) as e: - return [], False, f"the AnnotSV table could not be read: {e}" - if not p: - return [], False, None - out = [] - for x in data["rows"]: - if x["common"] or str(x["acmg_class"]) not in ("4", "5"): - continue - chrom, _, span = x["sv"].partition(":") - a, _, b = span.partition("-") - chrom = chrom if chrom.startswith("chr") else "chr" + chrom - hits = {} - for g in x["genes"]: - if g in rules and rules[g]["sv_cn"]: - hits[g] = None - for g, rule in rules.items(): - for c, rs, re_, name in rule["sv_regions"]: - if (x["type"] == "DEL" and c == chrom and a.isdigit() and b.isdigit() - and int(a) <= re_ and int(b) >= rs and g not in hits): - hits[g] = name - for g, via in sorted(hits.items()): - out.append({"gene": g, "variant": f"{x['type']} {chrom}:{a}-{b}", "acmg_class": str(x["acmg_class"]), - "caller": x["caller"], "support": x["support"], "via": via, - "key": ("sv", chrom, a, b, x["type"])}) - return out, True, None - - -def acmg_sf_tier(d, s, clinical_vcf): - """The variants that are ACMG secondary findings by the rule of their gene - (assets/acmg_sf.tsv): the ClinVar P/LP hits of step 6, the clinical - filter's rare HIGH-impact records, and step 05's class 4 and 5 SVs in the - genes whose rule allows them. A variant in an SF gene that its rule does - not report (one MUTYH variant, a heterozygous HFE p.C282Y) is listed - under not_reported with the reason.""" - rules = acmg_sf_rules() - clinvar, high = [], [] - hits = first_existing(d, [f"clinvar/{s}_clinvar_hits.vcf"]) - if hits: - floor = call_floor(hits) - for r in vcf_records(hits): - if len(r) < 8: - continue - row = clinvar_row(r, floor) - copies, ploidy = allele_copies(r[9].split(":")[0] if len(r) > 9 else "") - for g in row["gene"].split(","): - if g in rules: - clinvar.append({"gene": g, "variant": f"{row['chrom']}:{row['pos']} {row['ref']}>{row['alt']}", - "genotype": row["genotype"], "significance": row["significance"], - "stars": row["stars"], "low_confidence": row["low_confidence"], - "key": (row["chrom"], row["pos"], row["ref"], row["alt"]), - "copies": copies, "ploidy": ploidy, - "gt": r[9].split(":")[0] if len(r) > 9 else "", "ps": gt_field(r, "PS")}) - if clinical_vcf: - # A common record is in the clinical VCF through the ClinVar tier, - # not as a rare HIGH impact; one of unknown frequency is listed and - # marked, never counted as rare; a weak call is listed and marked. - for (c, pos, ref, alt), x in sorted(csq_high_impact_genes(clinical_vcf).items()): - if x["frequency"] == "common": - continue - for g, cons in sorted(x["genes"].items()): - if g in rules: - gt = x["genotype"] - copies, ploidy = allele_copies(gt) - zyg = {"0/1": "het", "1/0": "het", "0|1": "het", "1|0": "het", "1/1": "hom", "1|1": "hom"}.get(gt, gt) - high.append({"gene": g, "variant": f"{c}:{pos} {ref}>{alt}", "genotype": zyg, "consequence": cons, - "transcript": x["transcript"], "frequency": x["frequency"], - "frequency_unknown": x["frequency"] == "unknown", - "weak_call": x["weak"], "call_note": "; ".join(x["why"]) or None, - "gq": x["gq"], "dp": x["dp"], "ad": x["ad"], - "key": (c, pos, ref, alt), "copies": copies, "ploidy": ploidy, - "gt": gt, "ps": x.get("ps")}) - svs, sv_checked, sv_note = sf_structural(d, s, rules) - # One verdict per gene, over its distinct variants: a ClinVar hit that is - # also a HIGH-impact record is one variant. - alleles = {} - for x in clinvar + high: - alleles.setdefault(x["gene"], {}).setdefault(x["key"], (max(x["copies"], 1), x["ploidy"] or 2)) - for x in svs: - alleles.setdefault(x["gene"], {}).setdefault(x["key"], (1, 2)) - # A called deletion of a gene whose rule needs two variants counts as one - # of them (two_hits), unless read-backed phase puts it on the variant's copy. - two_hit_note = None - try: - spans = {g: x["span"] for g, x in carrier_genes().items()} - except Unreadable as e: - spans = {} - two_hit_note = (f"A deletion as the second hit: only AnnotSV's gene names were read; step 35's deletions " - f"need the gene spans of {e}") - dels, _, _ = gene_deletions(d, s, spans) - seen, variants = set(), [] - for x in clinvar + high: - pseudo = rules[x["gene"]]["severity"].get(x["key"], {}).get("kind") == "pseudodeficiency" - if (x["gene"], x["key"]) not in seen and not pseudo: - seen.add((x["gene"], x["key"])) - variants.append({"gene": x["gene"], "key": x["key"], "chrom": x["key"][0], "pos": x["key"][1], - "text": x["variant"], "genotype": x["genotype"], "copies": x["copies"], - "ploidy": x["ploidy"] or 2, "gt": x["gt"], "ps": x["ps"], "allele": None}) - entries, hemi, pairs = two_hits(variants, dels) - two = {} - for g in entries: - if rules[g]["report_if"] in ("biallelic", "hemizygous"): - alleles[g][("deletion", g)] = (1, 2) - two.update({k: t for k, t in pairs.items() if k[0] == g}) - verdict = {g: sf_verdicts(g, rules[g], a) for g, a in alleles.items()} - reported = {"clinvar": [], "high": [], "sv": []} - not_reported = [] - for kind, rows, source in (("clinvar", clinvar, "ClinVar"), ("high", high, "HIGH impact"), ("sv", svs, "AnnotSV")): - for x in rows: - why, partial = verdict[x["gene"]][0][x["key"]], verdict[x["gene"]][1] - rule = rules[x["gene"]] - key = x["key"] - if (x["gene"], key) in hemi: - x["genotype"] = "hemizygous" - for k in ("key", "copies", "ploidy", "gt", "ps"): - x.pop(k, None) - if why is None: - x.update(rule=rule["table_text"], rule_note=rule["note"], severity=partial) - if kind != "sv" and (x["gene"], key) in two: - x["two_hit"] = two[(x["gene"], key)] - reported[kind].append(x) - else: - not_reported.append({"gene": x["gene"], "variant": x["variant"], "genotype": x.get("genotype"), - "source": source, "reason": why}) - high = reported["high"] - sure = [x for x in high if not (x["weak_call"] or x["frequency_unknown"])] - return {"version": ACMG_SF_VERSION, "genes_on_list": len(rules), "rules": "assets/acmg_sf.tsv", - "clinvar_hits": reported["clinvar"], "clinvar_checked": bool(hits), - "high_impact": high, "high_impact_checked": bool(clinical_vcf), - "high_impact_rare": len(sure), - "high_impact_weak": sum(1 for x in high if x["weak_call"]), - "high_impact_frequency_unknown": sum(1 for x in high if x["frequency_unknown"]), - "structural": reported["sv"], "structural_checked": sv_checked, "structural_note": sv_note, - "not_reported": not_reported, "two_hit_note": two_hit_note} - - -def gt_field(r, key): - """One FORMAT field of a VCF record's first sample, or None.""" - if len(r) < 10: - return None - v = dict(zip(r[8].split(":"), r[9].split(":"))).get(key) - return None if v in (None, "", ".") else v - - -def carrier_list(p, d=None, s=None): - """The sample's own carrier list: the ClinVar hits of step 6 (p) in the - genes of assets/carrier_genes.tsv, one person, no partner needed. An - allele the list marks hypomorph or pseudodeficiency is set aside, never a - carrier finding on its own. With the sample folder (d, s), the genes of - autosomal recessive conditions that a called deletion and a hit both reach - (two_hits), and a hit called homozygous inside a deletion is shown as - hemizygous.""" - genes = carrier_genes() - carriers, aside, variants = [], [], [] - for r in vcf_records(p): - if len(r) < 8: - continue - row = clinvar_row(r) - gt = r[9].split(":")[0] if len(r) > 9 else "" - copies, ploidy = allele_copies(gt) - key = (row["chrom"], row["pos"], row["ref"], row["alt"]) - for g in row["gene"].split(","): - if g not in genes: - continue - x = genes[g] - item = {"gene": g, "variant": f"{row['chrom']}:{row['pos']} {row['ref']}>{row['alt']}", - "genotype": row["genotype"] if half_call(gt) else zygosity_word(copies, ploidy), - "significance": row["significance"], - "stars": row["stars"], "inheritance": x["inheritance"], "condition": x["condition"]} - sev = x["severity"].get(key) - if sev: - item.update(kind=sev["kind"], name=sev["name"]) - aside.append(item) - else: - carriers.append(item) - if not sev or sev["kind"] == "hypomorph": - variants.append({"gene": g, "key": key, "chrom": row["chrom"], "pos": row["pos"], - "text": item["variant"], "genotype": item["genotype"], "copies": copies, - "ploidy": ploidy, "gt": gt, "ps": gt_field(r, "PS"), - "allele": sev["name"] if sev else None, "item": item}) - out = {"genes_on_list": len(genes), "list": "assets/carrier_genes.tsv", - "carriers": carriers, "set_aside": aside} - if d is not None: - dels, read, unread = gene_deletions(d, s, {g: x["span"] for g, x in genes.items()}) - entries, hemi, _ = two_hits(variants, dels) - for v in variants: - if (v["gene"], v["key"]) in hemi: - v["item"].update(genotype="hemizygous", - hemizygous=f"called homozygous inside {hemi[(v['gene'], v['key'])]}") - rows = [] - for g, e in sorted(entries.items()): - if genes[g]["inheritance"] == "AR": - e.update(condition=genes[g]["condition"], inheritance="AR") - rows.append(e) - out["two_hits"] = {"genes": rows, "deletions_read": read, "deletions_not_read": unread} - return out - - -def acmg_sf_or_note(d, s, clinical_vcf): - """acmg_sf_tier, or the reason it could not run (a missing or broken - assets/acmg_sf.tsv), which both reports print: the clinical counts stay.""" - try: - return acmg_sf_tier(d, s, clinical_vcf) - except Unreadable as e: - return {"version": ACMG_SF_VERSION, "unavailable": str(e), "clinvar_hits": [], "high_impact": [], - "structural": [], "not_reported": []} - - -def carrier_list_or_note(p, d=None, s=None): - """carrier_list, or the reason it could not run, which both reports print: - the ClinVar hits stay.""" - try: - return carrier_list(p, d, s) - except Unreadable as e: - return {"unavailable": str(e)} - - -def sec_clinical(d, s): - p = first_existing(d, [f"clinical/{s}_clinical_summary.tsv"]) - v = first_existing(d, [f"clinical/{s}_clinical.vcf.gz"]) - if not p: - # The Nextflow report gets the clinical VCF, not the summary table: - # the count only (one record per variant, as in the table). - if not v: - return None, {} - return v, {"variants": sum(1 for _ in vcf_records(v)), "acmg_sf": acmg_sf_or_note(d, s, v)} - rows = read_tsv(p) - by = {} - for r in rows: - by[r.get("IMPACT") or "."] = by.get(r.get("IMPACT") or ".", 0) + 1 - genes = sorted({r.get("GENE") for r in rows if r.get("GENE") not in (None, "", ".")}) - out = {"variants": len(rows), "by_impact": by, "genes": len(genes), "acmg_sf": acmg_sf_or_note(d, s, v)} - # Step 23's tier and call check columns; a table written before them has neither. - if rows and "FREQ_TIER" in rows[0]: - out["frequency_unknown"] = sum(1 for r in rows if r.get("FREQ_TIER") == "unknown") - if rows and "CALL_CHECK" in rows[0]: - out["weak_calls"] = sum(1 for r in rows if r.get("CALL_CHECK") == "weak") - return p, out - - -def sec_slivar(d, s): - p = first_existing(d, [f"slivar/{s}_slivar_summary.tsv"]) - if not p: - # The Nextflow report gets the prioritized VCF: the count, plus the - # phase when COMPHET_PHASE's pairs table sits beside it (HTML_REPORT - # stages it there). - v = first_existing(d, [f"slivar/{s}_prioritized.vcf.gz"]) - if not v: - return None, {} - out = {"prioritized": sum(1 for _ in vcf_records(v))} - out.update(comphet_phase(os.path.join(os.path.dirname(v), f"{s}_compound_het_pairs.tsv"))) - return v, out - rows = read_tsv(p) - out = {"prioritized": len(rows)} - # Step 31's frequency tier: prioritized for the compound-het search, not rare. - if rows and "FREQ_TIER" in rows[0]: - out["frequency_unknown"] = sum(1 for r in rows if r.get("FREQ_TIER") == "unknown") - ch = os.path.join(os.path.dirname(p), f"{s}_compound_hets.tsv") - if os.path.isfile(ch): - crow = read_tsv(ch) - out["compound_het_variants"] = len(crow) - out["compound_het_genes"] = len({r.get("GENE") for r in crow if r.get("GENE")}) - out.update(comphet_phase(os.path.join(os.path.dirname(p), f"{s}_compound_het_pairs.tsv"))) - return p, out - - -# The trans pairs the report lists by name; the rest are counted. -COMPHET_TRANS_SHOWN = 20 - - -def comphet_phase(path): - """Step 31's pairs table (bin/comphet_phase.py): the pairs by phase from - the reads, and the trans ones (both copies of the gene hit) by name. A run - from before the table has none of these keys.""" - if not os.path.isfile(path): - return {} - rows = read_tsv(path) - phase = {k: sum(1 for r in rows if r.get("PHASE") == k) for k in ("trans", "cis", "unknown")} - - def n(x): - return int(x) if (x or "").isdigit() else 0 - trans = [{"gene": r.get("GENE", ""), - "variants": f"{r.get('CHROM')}:{r.get('POS_1')} {r.get('REF_1')}>{r.get('ALT_1')} + " - f"{r.get('CHROM')}:{r.get('POS_2')} {r.get('REF_2')}>{r.get('ALT_2')}", - "reads": f"{r.get('TRANS_READS')} of {n(r.get('TRANS_READS')) + n(r.get('CIS_READS'))}"} - for r in rows if r.get("PHASE") == "trans"] - return {"compound_het_pairs": len(rows), "compound_het_phase": phase, - "compound_het_trans": trans[:COMPHET_TRANS_SHOWN]} - - -def sec_coverage(d, s): - p = first_existing(d, [f"mosdepth/{s}.mosdepth.summary.txt", f"coverage/{s}.mosdepth.summary.txt"]) - if not p: - return None, {} - mean = None - for r in read_tsv(p): - if r.get("chrom") == "total": - mean = r.get("mean") - return p, {"mean_depth": float(mean) if mean not in (None, "") else None} - - -# A gene of assets/callability_genes.tsv counts as read when at least this -# share of its bases has the depth step 16b counts (its thresholds column, -# 10x of reads with MAPQ 20 or more). -CALLABLE_SHARE = 0.9 - - -def callability(catalog, table): - """The reading of every gene of the catalog rows from step 16b's per-gene - table (mosdepth --thresholds over the catalog's genes: chrom, start, end, - gene, bases at the threshold), as the section's data.""" - with open_text(table) as f: - lines = [x.rstrip("\n").split("\t") for x in f if x.strip()] - if not lines or not lines[0][0].startswith("#") or len(lines[0]) != 5 \ - or not re.fullmatch(r"[0-9]+X", lines[0][4]): - raise Unreadable(f"{os.path.basename(table)} is not mosdepth's thresholds table of one depth") - min_depth = int(lines[0][4][:-1]) - bases = {} - for c in lines[1:]: - if len(c) != 5: - raise Unreadable(f"{os.path.basename(table)}: a row has {len(c)} columns, not 5") - bases[c[3]] = (int(c[2]) - int(c[1]), int(c[4])) - if not bases: - raise Unreadable(f"{os.path.basename(table)} has no gene rows") - # mosdepth writes no row on a contig the BAM lacks: a chromosome with no row at all is - # not on the sample's reference; a gene missing from a chromosome that has rows means - # the table was written from another catalog. - contigs = {c[0] for c in lines[1:]} - genes, unowned, owned, not_on_reference = [], [], [], [] - for r in catalog: - g = {"gene": r["gene"], "hgnc_id": r["hgnc_id"], "reasons": r["reasons"].split(","), - "owner": None if r["owner"] == "." else r["owner"], "share": None} - if r["start"] == "." or r["chrom"] not in contigs: - g["status"] = "not on this reference" - not_on_reference.append(g["gene"]) - else: - if r["gene"] not in bases: - raise Unreadable(f"{os.path.basename(table)} has no row for {r['gene']}: it was written from " - "another assets/callability_genes.tsv; run step 16b again") - n, ok = bases[r["gene"]] - g["share"] = round(ok / n, 4) if n else 0.0 - g["status"] = "read" if g["share"] >= CALLABLE_SHARE else "not read" - genes.append(g) - if g["status"] != "read": - (owned if g["owner"] else unowned).append(g) - return {"min_depth": min_depth, "min_mapq": 20, "min_share": CALLABLE_SHARE, "catalog_genes": len(genes), - "read": sum(1 for g in genes if g["status"] == "read"), - "not_read": [{k: g[k] for k in ("gene", "hgnc_id", "reasons", "share", "status")} for g in unowned], - "not_read_owned": [{k: g[k] for k in ("gene", "owner", "share", "status")} for g in owned], - "not_on_reference": not_on_reference} - - -def sec_callability(d, s): - """Step 16b's gene callability (the MOSDEPTH task's in a Nextflow folder).""" - p = first_existing(d, [f"mosdepth/{s}_callability.bed.gz", f"coverage/{s}_callability.bed.gz"]) - if not p: - return None, {} - return p, callability(asset_rows("callability_genes.tsv"), p) - - -# How far a chrX or chrY copy number may sit from a whole number and still be -# read as that number. Further off (a mosaic, a mixed sample, a noisy run) -# names no karyotype. -KARYOTYPE_TOL = 0.3 - - -# The sex-chromosome aneuploidies named, by (chrX, chrY) copies. Any other pair of -# whole numbers (chrY 4, chrX 6, a degenerate 100000) names nothing: in a run it is -# far more often depth noise or a broken input than a karyotype. -KARYOTYPES = {(1, 0): "45,X", (2, 1): "47,XXY", (1, 2): "47,XYY", (3, 0): "47,XXX", - (2, 2): "48,XXYY", (3, 1): "48,XXXY", (4, 0): "48,XXXX", - (4, 1): "49,XXXXY", (5, 0): "49,XXXXX"} - - -def karyotype(cn_x, cn_y, tol=KARYOTYPE_TOL): - """The sex-chromosome karyotype the two copy numbers name, one of - KARYOTYPES; None for 46,XX and 46,XY, for a missing value, for a value - between whole numbers, and for any pair of copies not in KARYOTYPES.""" - x, y = num(cn_x), num(cn_y) - if x is None or y is None or not (-1 < x < 10 and -1 < y < 10): # also keeps inf from round() - return None - rx, ry = int(round(x)), int(round(y)) - if abs(x - rx) > tol or abs(y - ry) > tol: - return None - return KARYOTYPES.get((rx, ry)) - - -def sec_sex_check(d, s): - p = first_existing(d, ["indexcov/indexcov-indexcov.ped", "indexcov/*-indexcov.ped"]) - if not p: - return None, {} - with open(p, errors="replace") as f: - lines = [l.rstrip("\n").split("\t") for l in f if l.strip()] - head = [h.lstrip("#") for h in lines[0]] if lines else [] - if "sex" not in head or len(lines) < 2: - raise Unreadable(f"{os.path.basename(p)} has no sex column or no sample row") - row = dict(zip(head, lines[-1])) - sex = {"1": "male", "2": "female"}.get(row.get("sex", ""), "unknown") - return p, {"inferred_sex": sex, "cn_chrX": row.get("CNchrX", ""), "cn_chrY": row.get("CNchrY", ""), - "karyotype": karyotype(row.get("CNchrX"), row.get("CNchrY"))} - - -# --- sample identity and contamination (step 33) ---------------------------------- - -def num(x): - try: - v = float(x) - except (TypeError, ValueError): - return None - return v if v == v else None # NaN is no value - - -def sec_sample_qc(d, s): - p = first_existing(d, [f"qc/{s}_sample_qc.tsv"]) - if not p: - return None, {} - data = {} - for r in read_tsv(p): - if r.get("key"): - data[r["key"]] = r.get("value", "") - if "inferred_sex" not in data: - raise Unreadable(f"{os.path.basename(p)} has no inferred_sex row") - # somalier's chrX and chrY depth over the autosomal depth: one copy reads 0.5. - # Named only from more than 10 chrX sites, the floor somalier's own sex - # call needs (og qc sample); a thinner ratio is noise. - xr, yr = num(data.get("x_depth_ratio")), num(data.get("y_depth_ratio")) - xs = num(data.get("x_sites")) - data["karyotype"] = (karyotype(2 * xr, 2 * yr) - if xr is not None and yr is not None and xs is not None and math.isfinite(xs) and xs > 10 - else None) - return p, data - - -# --- mosaic loss of chrY or of one chrX (step 33, og qc sex-loss) ------------------- - -MOSAIC_CALLS = ("detected", "not_detected", "gain", "not_estimated") -MOSAIC_NUMBERS = ("depth_ratio", "depth_ratio_expected", "loss_depth", "loss_depth_low", "loss_depth_high", - "loss_af", "loss_af_low", "loss_af_high", "floor") - - -def sec_mosaic_sex_loss(d, s): - """The mosaic_* rows of the step 33 table, without the prefix; none in a - table written before the estimate existed.""" - p = first_existing(d, [f"qc/{s}_sample_qc.tsv"]) - if not p: - return None, {} - data = {r["key"][len("mosaic_"):]: r.get("value") or "" for r in read_tsv(p) - if (r.get("key") or "").startswith("mosaic_")} - if not data: - return None, {} - if data.get("call") not in MOSAIC_CALLS: - raise Unreadable(f"{os.path.basename(p)} has no mosaic_call row ({', '.join(MOSAIC_CALLS)})") - for k in MOSAIC_NUMBERS: - v = num(data.get(k)) - data[k] = v if v is not None and abs(v) != float("inf") else None # inf is no value either - data["x_het_sites"] = int(data["x_het_sites"]) if (data.get("x_het_sites") or "").isdigit() else None - return p, data - - -def sec_read_count(d, s): - """Step 02's read count (and the ALIGN_MINIMAP2 task's): the reads that - went into the aligner and the primary records of the BAM it wrote.""" - p = first_existing(d, [f"qc/{s}_read_count.tsv"]) - if not p: - return None, {} - data = {} - for r in read_tsv(p): - if r.get("key"): - data[r["key"]] = r.get("value", "") - for k in ("input_reads", "primary_records", "missing_reads"): - if not re.fullmatch(r"-?[0-9]+", data.get(k, "")): - raise Unreadable(f"{os.path.basename(p)} has no whole number for {k}") - data[k] = int(data[k]) - if data.get("read_count_check") not in ("ok", "mismatch"): - raise Unreadable(f"{os.path.basename(p)} has no read_count_check row (ok or mismatch)") - return p, data - - -# Sections that are not a step of their own, or not one of one person (the -# couple screen needs two): no "not run" line without them. -NOT_A_STEP = {"read_count", "couple", "carrier"} - -SECTIONS = [ - # key, title, step, reader - ("coverage", "Coverage (mosdepth)", "16b", sec_coverage), - ("callability", "Gene callability (mosdepth)", "16b", sec_callability), - ("sex_check", "Sex check (indexcov)", "16", sec_sex_check), - ("sample_qc", "Sample identity and contamination (somalier, VerifyBamID2)", "33", sec_sample_qc), - ("mosaic_sex_loss", "Mosaic loss of chromosome Y or X", "33", sec_mosaic_sex_loss), - ("read_count", "Reads kept by alignment", "02", sec_read_count), - ("variants", "Variant calling", "03", sec_variants), - ("clinvar", "ClinVar screen", "06", sec_clinvar), - ("couple", "Couple carrier screen", "39", sec_couple), - ("carrier", "Carrier list", "06", sec_carrier), - ("pharmcat", "PharmCAT", "07", sec_pharmcat), - ("cpic", "CPIC drug-gene recommendations", "27", sec_cpic), - ("pypgx", "pypgx", "32", sec_pypgx), - ("cyrius", "Cyrius (CYP2D6)", "21", sec_cyrius), - ("parascopy", "SMN1/SMN2 copy number (Parascopy, short reads)", "35", sec_parascopy), - ("paralogs", "Paralog genes (Paraphase, PacBio HiFi)", "35b", sec_paralogs), - ("hla", "HLA typing (T1K)", "08", sec_hla), - ("prs", "Polygenic risk scores", "25", sec_prs), - ("ancestry_fine", "Fine-grained ancestry (ADMIXTURE, RFMix)", "26b", sec_ancestry_fine), - ("manta", "Structural variants (Manta)", "04", sec_manta), - ("delly", "Structural variants (Delly)", "19", sec_delly), - ("cnvpytor", "Copy number (CNVpytor)", "18", sec_cnvpytor), - ("sv_consensus", "SV consensus merge", "22", sec_sv_consensus), - ("annotsv", "Structural variants, class 4 and 5 (AnnotSV)", "05", sec_annotsv), - ("expansions", "Repeat expansions (ExpansionHunter)", "09", sec_expansions), - ("telomere", "Telomere content (TelomereHunter)", "10", sec_telomere), - ("roh", "Runs of homozygosity", "11", sec_roh), - ("haplogroup", "Mitochondrial haplogroup", "12", sec_haplogroup), - ("y_haplogroup", "Y-chromosome haplogroup (Yleaf)", "37", sec_y_haplogroup), - ("y_tree", "Y placement on the current public tree", "37", sec_y_tree), - ("y_str", "Y-STR markers (HipSTR)", "45", sec_y_str), - ("consumer_export", "Raw-data export (23andMe v5 layout)", "40", sec_consumer_export), - ("bloodtype", "Blood groups (predicted)", "41", sec_bloodtype), - ("lpa", "Lp(a): LPA KIV-2 repeat count", "46", sec_lpa), - ("vntyper", "MUC1 VNTR frameshifts (VNtyper)", "52", sec_vntyper), - ("locityper", "Locityper second opinion at hard loci", "53", sec_locityper), - ("mito", "Mitochondrial variants", "20", sec_mito), - ("traits", "Traits panel", "42", sec_traits), - ("archaic", "Archaic ancestry (IBDmix)", "43", sec_archaic), - ("ancient", "Ancient populations (AADR PCA)", "44", sec_ancient), - ("viral", "Viral DNA in the unmapped reads", "47", sec_viral), - ("exomiser", "Symptom-driven ranking (Exomiser)", "48", sec_exomiser), - ("chip", "Clonal haematopoiesis (CHIP)", "49", sec_chip), - ("mei", "Mobile element insertions (MEGAnE)", "50", sec_mei), - ("mosaic", "Mosaic chromosomal alterations (MoChA)", "51", sec_mocha), - ("cpsr", "Cancer predisposition (CPSR)", "17", sec_cpsr), - ("clinical", "Clinical variant filter", "23", sec_clinical), - ("slivar", "Variant prioritization (slivar)", "31", sec_slivar), -] - - -# --- run status and manifest ----------------------------------------------------- - -def read_run_status(d): - p = os.path.join(d, "logs", "run_status.tsv") - if not os.path.isfile(p): - return None - st = {"started_epoch": None, "started_utc": None, "declared_sex": None, "steps": {}} - with open(p, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) < 3 or c[0].startswith("#"): - continue - if c[0] == "meta" and c[1] == "started_epoch": - try: - st["started_epoch"] = float(c[2]) - except ValueError: - pass - elif c[0] == "meta": - st[c[1]] = c[2] - elif c[0] == "step": - st["steps"][c[1]] = c[2] - if st["started_epoch"] is not None: - st["started_utc"] = iso(st["started_epoch"]) - return st - - -def read_manifest(d): - p = os.path.join(d, "run_manifest.tsv") - if not os.path.isfile(p): - return None - rows = [] - with open(p, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) >= 3 and not c[0].startswith("#"): - rows.append([c[0], c[1], "\t".join(c[2:])]) - return rows - - -def manifest_value(rows, section, key): - for r in rows or []: - if r[0] == section and r[1] == key: - return r[2] - return None - - -def read_consensus(d, s, sections): - """Step 36's CYP2D6 row (pgx_consensus/_pgx_consensus.tsv), or None - when the table is missing or older than a caller's result it was made from.""" - p = first_existing(d, [f"pgx_consensus/{s}_pgx_consensus.tsv"]) - if not p: - return None - for k in ("pypgx", "cyrius"): - src = sections[k]["source"] - if src and os.path.getmtime(os.path.join(d, src)) > os.path.getmtime(p): - return None - row = next((r for r in read_tsv(p) if (r.get("Gene") or "").strip() == "CYP2D6"), None) - if row is None: - return None - return {"result": row.get("Result") or "", "passed_to_pharmcat": row.get("Outside_call") == "yes", - "reason": row.get("Reason") or "", "source": os.path.relpath(p, d)} - - -def cyp2d6_block(sample, sample_dir, sections): - """CYP2D6 from each caller, side by side, judged by step 36's rule - (bin/pgx_outside_calls.py): a no-call in any case (Indeterminate, None, - '.'...) is not a call, and Cyrius counts only with Filter PASS. PharmCAT - calls no CYP2D6 from a VCF: its column is what step 36 passed to it, so - agreement is between pypgx and Cyrius. With step 36's table its verdict - is kept as well.""" - pc = next((g for g in sections["cpic"]["data"].get("genes", []) if g["gene"] == "CYP2D6"), None) - pg = sections["pypgx"]["data"].get("cyp2d6") if sections["pypgx"]["state"] == "ok" else None - cy = sections["cyrius"]["data"] if sections["cyrius"]["state"] == "ok" else None - # A caller whose step failed says so, not 'not run'. - failed = {k: f"failed (step {sections[k]['step']})" for k in ("cpic", "pypgx", "cyrius") - if sections[k]["state"] == "failed"} - pg_ok = pg is not None and pgx_outside_calls.is_call(pg) - cy_ok = cy is not None and pgx_outside_calls.is_call(cy.get("genotype")) and cy.get("filter") == "PASS" - cy_txt = None - if cy is not None: - cy_txt = cy.get("genotype") or "none" - if pgx_outside_calls.is_call(cy_txt) and not cy_ok: - cy_txt += f" (not usable: Filter {cy.get('filter') or 'missing'})" - calls = { - "PharmCAT": (pc["diplotype"] if pc["status"] not in ("not called", "ambiguous") else pc["status"]) if pc - else failed.get("cpic"), - "pypgx": failed.get("pypgx", pg), - "Cyrius": failed.get("cyrius", cy_txt), - } - agree = pgx_outside_calls.same_diplotype(pg, cy["genotype"]) if pg_ok and cy_ok else None - try: - consensus = read_consensus(sample_dir, sample, sections) - except (OSError, csv.Error) as e: - consensus = {"result": "unreadable", "passed_to_pharmcat": False, "reason": str(e), "source": None} - # pypgx's call alone, labelled, when step 36's rule allows it - # (pgx_outside_calls.probable_cyp2d6); it never reaches PharmCAT. - probable = None - if pg_ok: - pd = sections["pypgx"]["data"] - ev = pd.get("cyp2d6_evidence") or {} - probable = pgx_outside_calls.probable_cyp2d6( - {"Diplotype": pg, "Phenotype": ev.get("phenotype"), "Defining_filter": ev.get("defining_filter"), - "Quality": ev.get("quality")}, - (cy.get("genotype"), cy.get("filter")) if cy is not None else None, - (pd.get("depth_check") or {}).get("status")) - return {"calls": calls, "agree": agree, "consensus": consensus, "probable": probable} - - -def collect(sample, sample_dir, declared_sex=None): - status = read_run_status(sample_dir) - manifest = read_manifest(sample_dir) - sections = {} - for key, title, step, reader in SECTIONS: - sec = {"title": title, "step": step, "state": "missing", "source": None, - "file_date": None, "note": None, "data": {}} - try: - path, data = reader(sample_dir, sample) - except Unreadable as e: - sec["state"], sec["note"] = "unreadable", str(e) - sections[key] = sec - continue - except (OSError, ValueError, KeyError, IndexError, EOFError, csv.Error, zlib.error) as e: - sec["state"], sec["note"] = "unreadable", f"{type(e).__name__}: {e}" - sections[key] = sec - continue - if not path and status and (status["steps"].get(step) or "").startswith("failed"): - # run-all.sh records 'failed' for a step it ran that exited non-zero - sec["state"] = "failed" - sec["note"] = (f"step {step} failed in the run of {status['started_utc'] or 'unknown date'}; " - "its log is in logs/") - if path: - mtime = os.path.getmtime(path) - sec.update(state="ok", source=os.path.relpath(path, sample_dir), file_date=iso(mtime), data=data) - if status and status["started_epoch"] is not None and step in RUN_ALL_STEPS: - result = status["steps"].get(step) - if result != "ok" and mtime < status["started_epoch"]: - sec["state"] = "stale" - sec["note"] = (f"from an earlier run ({iso(mtime)}); step {step} was " - f"{result or 'not run'} in the run of {status['started_utc']}") - sections[key] = sec - - cyp2d6 = cyp2d6_block(sample, sample_dir, sections) - # The repeat readings lean on the traits (R117H) and CPIC (UGT1A1) - # tables, whose sections come after the expansions one: read them again - # with those sections' states. - ex = sections["expansions"] - if ex["state"] in ("ok", "stale"): - ex["data"]["readings"] = str_readings({l["locus"]: l for l in ex["data"]["loci"]}, sample_dir, sample, - {k: sections[k]["state"] for k in ("traits", "cpic")}) - if sections["locityper"]["state"] in ("ok", "stale"): - primary = locityper_primary(sections) - for x in sections["locityper"]["data"]["loci"]: - x["primary"] = primary.get(x["locus"], "no other step calls this locus") - declared = (declared_sex or (status or {}).get("declared_sex") or manifest_value(manifest, "run", "declared_sex") - or sections["sample_qc"]["data"].get("declared_sex") or None) - cnvpytor_sex_filter(sections["cnvpytor"]["data"], declared) - - clinvar_date = manifest_value(manifest, "data", "clinvar_file_date") - summary = { - "schema_version": SCHEMA_VERSION, - "sample": sample, - "generated_utc": iso(datetime.now(tz=timezone.utc).timestamp()), - "run": { - "status_file": status is not None, - "started_utc": status["started_utc"] if status else None, - "declared_sex": declared, - }, - "databases": { - "clinvar_release": clinvar_date, - "cpsr_clinvar_release": manifest_value(manifest, "data", "cpsr_clinvar_release"), - "hla_database": manifest_value(manifest, "data", "hla_database"), - }, - "manifest": manifest or [], - "sections": sections, - "cyp2d6": cyp2d6, - # The read count is written by alignment, which a run from a BAM or - # a VCF never does, so it is no step left undone. - "not_run": [sections[k]["title"] + (f" (step {sections[k]['step']} failed)" if sections[k]["state"] == "failed" else "") - for k, *_ in SECTIONS if sections[k]["state"] in ("missing", "failed") and k not in NOT_A_STEP], - "not_assessed": NOT_ASSESSED + ([] if sections["prs"]["data"].get("adjusted") else [PRS_NOT_ADJUSTED]) - # Step 35b on a region subset (--paraphase_regions) may leave hba out - + ([] if {"HBA1", "HBA2"} <= {g["gene"] for g in sections["paralogs"]["data"].get("genes", []) - if g["region"] == "hba" and g["status"] == "called"} - or (sections["parascopy"]["data"].get("hba") or {}).get("status") == "called" - else [HBA_NOT_ASSESSED]) - + ([MEI_NOT_ASSESSED] if sections["mei"]["state"] == "missing" else []) - + ([] if sections["vntyper"]["state"] == "ok" - and sections["vntyper"]["data"].get("status") in ("positive", "possible", "negative") - else [VNTYPER_NOT_ASSESSED]) - + ([] if sections["mosaic"]["data"].get("status") == "assessed" else [MCA_NOT_ASSESSED]), - } - recheck = run_delta.read_recheck(os.path.join(sample_dir, "recheck", "state.json")) - if recheck: - summary["recheck"] = recheck - since = new_since(sample_dir, summary) - if since: - summary["new_since"] = since - return summary - - -def new_since(sample_dir, summary): - """{previous_utc, sections}: what changed since the newest run kept in history/ (generate-report.sh - keeps one per run), or None without one. A folder that cannot be read counts as none.""" - prev_dir = run_delta.newest_history(sample_dir) - if not prev_dir: - return None - try: - with open(os.path.join(prev_dir, "summary.json")) as f: - prev = json.load(f) - changes = run_delta.compare(prev, run_delta.read_manifest(os.path.join(prev_dir, "run_manifest.tsv")), - summary, run_delta.read_manifest(os.path.join(sample_dir, "run_manifest.tsv"))) - except (OSError, ValueError, AttributeError, TypeError, KeyError): - return None - return {"previous_utc": prev.get("generated_utc"), "sections": run_delta.changed_sections(changes)} - - -def main(argv=None): - argv = sys.argv[1:] if argv is None else argv - ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - ap.add_argument("--sample", required=True) - ap.add_argument("--sample-dir", required=True) - ap.add_argument("--declared-sex", help="the samplesheet's sex, when no run status or manifest records it") - ap.add_argument("--out", help="summary JSON (default: SAMPLE_DIR/summary.json)") - a = ap.parse_args(argv) - summary = collect(a.sample, a.sample_dir, a.declared_sex) - out = a.out or os.path.join(a.sample_dir, "summary.json") - tmp = out + ".tmp" - with open(tmp, "w") as f: - json.dump(summary, f, indent=1) - f.write("\n") - os.replace(tmp, out) - print(f"Summary written: {out}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bin/render_report.py b/bin/render_report.py index 84be25703..eac22b4f4 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -10,7 +10,8 @@ sections that name them (bin/results_index.py); the HTML report closes with the same list. -With --sample-dir it first collects the summary (bin/collect_summary.py) and +With --sample-dir it first collects the summary with og summary collect +(rust/crates/og-summary, through bin/collect_summary.py's collect()), which writes it to --json (default DIR/summary.json). Every number in both reports comes from that summary, so the two cannot disagree. A section whose result is from an earlier run (state "stale") is shown with that date and a STALE mark, @@ -3223,7 +3224,7 @@ def charts_json(s): # --- html page: chapters, contents, section frames ----------------------------- CSP = "default-src 'none'; img-src data:; style-src 'unsafe-inline'" -STEP_OF = {k: step for k, _, step, _ in collect_summary.SECTIONS} +STEP_OF = {k: step for k, _, step in collect_summary.SECTIONS} # A section's state from the states of its keys. Never green: a section that # ran is not a finding, and the ClinVar section must show no green at all. STATE_BADGE = {"ok": ("Complete", "state"), "stale": ("Stale", "yellow"), "failed": ("Failed", "red"), @@ -3484,8 +3485,11 @@ def main(argv=None): elif a.sample and a.sample_dir: if a.assets: collect_summary.ASSETS_DIR = a.assets - s = collect_summary.collect(a.sample, a.sample_dir, a.declared_sex) - write(a.json or os.path.join(a.sample_dir, "summary.json"), json.dumps(s, indent=1) + "\n") + try: + s = collect_summary.collect(a.sample, a.sample_dir, a.declared_sex, + out=a.json or os.path.join(a.sample_dir, "summary.json")) + except RuntimeError as e: + sys.exit(f"ERROR: {e}") else: ap.error("give --summary, or --sample and --sample-dir") # Only the HTML report and --search-index read the index: a text-only run diff --git a/modules/local/html_report/main.nf b/modules/local/html_report/main.nf index 845f470f6..2734c6ba1 100644 --- a/modules/local/html_report/main.nf +++ b/modules/local/html_report/main.nf @@ -2,14 +2,15 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ HTML_REPORT — The sample's HTML report, rendered by bin/render_report.py ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The same code scripts/24-html-report.sh runs: bin/collect_summary.py - reads the step outputs into summary.json (the bash step's name for it), + The same code scripts/24-html-report.sh runs: og summary collect + (rust/crates/og-summary, run by render_report.py through bin/og) reads + the step outputs into summary.json (the bash step's name for it), and render_report.py writes the report from it, so a number on the Nextflow report is the number the bash report shows for the same file. The inputs are the outputs of the steps that ran for this sample, as one list (main.nf joins them, so the report waits for each). They are linked - into the folder layout collect_summary.py reads (clinvar/, pharmcat/, + into the folder layout og summary collect reads (clinvar/, pharmcat/, cpic/, vep/, clinical/, slivar/, cpsr/, qc/, roh/, mito/, telomere/, hla_t1k/, hla_optitype/, kir/, pypgx/, annotsv/, traits/, chip/, couple/, ...); a step that did @@ -18,7 +19,7 @@ name is the user's. indexcov's folder becomes indexcov/ itself (the sex check reads indexcov/indexcov-indexcov.ped), and CPIC_LOOKUP's PharmCAT comparison goes beside the pypgx summary, where - collect_summary.py looks for it. assets/acmg_sf.tsv (the ACMG SF rules) + og summary collect looks for it. assets/acmg_sf.tsv (the ACMG SF rules) and assets/carrier_genes.tsv (the carrier list) and assets/callability_genes.tsv (the hard-to-read genes) come in the same list and go to assets/, which --assets names, as do assets/mito_confirmed.tsv @@ -69,7 +70,7 @@ process HTML_REPORT { def gvcf_suffix = gvcf_index.endsWith('.csi') ? 'csi' : 'tbi' def release = workflow.manifest.version ?: '' """ - # Where collect_summary.py looks for each output (bin/collect_summary.py) + # Where og summary collect looks for each output (rust/crates/og-summary) S=layout/${id} mkdir -p "\$S" place() { mkdir -p "\$S/\$1" && ln -s "\$(readlink -f "\$2")" "\$S/\$1/\$(basename "\$2")"; } @@ -153,7 +154,7 @@ process HTML_REPORT { --search-index ${id}_search_index.json \\ --charts-json ${id}_charts.json - printf '"%s":\\n python: %s\\n' "${task.process}" "${task.container.replaceFirst(/^[^:@]+[:@]/, '')}" > versions.yml + printf '"%s":\\n python: %s\\n og: %s\\n' "${task.process}" "${task.container.replaceFirst(/^[^:@]+[:@]/, '')}" "\$(og --version)" > versions.yml """ stub: diff --git a/rust/crates/og-summary/src/collect/scan.rs b/rust/crates/og-summary/src/collect/scan.rs index d50bd251a..2a0cb9695 100644 --- a/rust/crates/og-summary/src/collect/scan.rs +++ b/rust/crates/og-summary/src/collect/scan.rs @@ -13,10 +13,16 @@ use std::fs; use std::path::{Path, PathBuf}; /// How deep the walk goes below the sample folder. The deepest file a -/// section reads today is three folders down; Nextflow's work folders are -/// not inside a sample folder. +/// section reads today is three folders down. pub const MAX_DEPTH: usize = 6; +/// The top-level folder the walk leaves out: `run-all.sh` launches +/// Nextflow in `/nextflow`, whose `work/` holds every task's +/// files and staged links (a whole VEP cache among them). No section +/// reads a file there, and the Python collector, which globbed each +/// section's own folder, never listed it. +pub const SKIP_TOP: &str = "nextflow"; + /// Every regular file of a sample folder, by its path relative to the /// folder (`/` between parts). Symlinks are followed, so the links the /// HTML_REPORT task stages count as the files they point to; the depth cap @@ -54,7 +60,7 @@ impl FolderIndex { }; if meta.is_file() { files.insert(rel); - } else if meta.is_dir() && depth + 1 < MAX_DEPTH { + } else if meta.is_dir() && depth + 1 < MAX_DEPTH && !(depth == 0 && rel == SKIP_TOP) { stack.push((path, rel, depth + 1)); } } @@ -210,6 +216,22 @@ mod tests { assert_eq!(ix.first(&pats(&["vcf/T.vcf.gz"])), None); } + #[test] + fn the_nextflow_launch_folder_is_left_out() { + let d = Dir::new("scan-nextflow"); + d.put("nextflow/work/ab/cd/vcf/S.vcf.gz", "x"); + d.put("nextflow/S_sample_qc.tsv", "x"); + d.put("qc/nextflow/S_sample_qc.tsv", "x"); + let ix = FolderIndex::walk(&d.0); + assert_eq!(ix.first(&pats(&["nextflow/work/ab/cd/vcf/S.vcf.gz"])), None); + assert_eq!(ix.first(&pats(&["nextflow/*"])), None); + // Only the top-level folder: a deeper one of that name is read. + assert_eq!( + ix.first(&pats(&["qc/nextflow/S_sample_qc.tsv"])), + Some("qc/nextflow/S_sample_qc.tsv") + ); + } + #[test] fn a_missing_folder_is_empty() { let ix = FolderIndex::walk(Path::new("/nonexistent/og-summary-scan")); diff --git a/scripts/24-html-report.sh b/scripts/24-html-report.sh index 234a5c92b..1d7f0c3f9 100755 --- a/scripts/24-html-report.sh +++ b/scripts/24-html-report.sh @@ -2,10 +2,10 @@ # 24-html-report.sh — Generate a self-contained HTML report of all pipeline results # Usage: ./scripts/24-html-report.sh # -# Reads every step's output once into ${SAMPLE}/summary.json -# (bin/collect_summary.py) and renders the HTML report from it -# (bin/render_report.py). generate-report.sh renders the text report from the -# same summary, so the two reports show the same values. A result from an +# Reads every step's output once into ${SAMPLE}/summary.json (og summary +# collect, run by bin/render_report.py) and renders the HTML report from it. +# generate-report.sh renders the text report from the same summary when +# nothing has changed since, so the two reports show the same values. A result from an # earlier run whose step was skipped or failed in the latest run-all.sh run is # marked stale with its date. No internet connection is needed to view it. # ${SAMPLE}_search_index.json lists every gene and rsID the results mention diff --git a/scripts/generate-report.sh b/scripts/generate-report.sh index b6e3081de..5e5d3161d 100755 --- a/scripts/generate-report.sh +++ b/scripts/generate-report.sh @@ -2,12 +2,14 @@ # generate-report.sh — Create a plain-text summary report of all pipeline results # Usage: ./scripts/generate-report.sh # -# Reads every step's output once into ${SAMPLE}/summary.json -# (bin/collect_summary.py) and renders the text report from it -# (bin/render_report.py); step 24 renders the HTML report from the same -# summary. A result from an earlier run whose step was skipped or failed in the -# latest run-all.sh run is marked stale with its date. Run after completing all -# (or some) pipeline steps. +# Renders the text report (bin/render_report.py) from ${SAMPLE}/summary.json, +# the summary og summary collect reads from every step's output. When step 24 +# wrote that summary and nothing it reads has changed since (in a run-all.sh +# run, step 24 ran just before), the text report is rendered from it as it is: +# one collect per run, and both reports carry the same values and time. +# Otherwise it collects the summary again first. A result from an earlier run +# whose step was skipped or failed in the latest run-all.sh run is marked +# stale with its date. Run after completing all (or some) pipeline steps. # # Then it writes what changed since the previous run of the sample # (og run delta, docs/delta-report.md): ${SAMPLE}_delta.txt, .html and @@ -34,14 +36,39 @@ if [ ! -f "${SAMPLE_DIR}/run_manifest.tsv" ]; then GENOME_DIR="$GENOME_DIR" bash "${PGP_ROOT}/bin/write_manifest.sh" "$SAMPLE" generate-report.sh fi +# summary_current: 0 when SAMPLE_DIR/summary.json is newer than everything a +# collect reads: every file of the sample folder except what is written from +# the summary or beside it (the reports, the search index, the charts, the +# delta, the logs other than run_status.tsv, and nextflow/, the Nextflow launch +# folder with work/), and bin/ (og and the tables it reads there) and assets/. Anything newer, or a +# summary that is not there, means collect again. +summary_current() { + local s="${SAMPLE_DIR}/summary.json" newer + [ -s "$s" ] && [ ! -L "$s" ] || return 1 + newer=$(cd "$SAMPLE_DIR" && find . \( -path ./nextflow -o -path ./summary.json \) -prune -o ! -type d -newer summary.json \ + ! \( -path './logs/*' ! -path ./logs/run_status.tsv \) \ + ! -name "${SAMPLE}_report.html" ! -name "${SAMPLE}_report.txt" ! -name "${SAMPLE}_search_index.json" \ + ! -name "${SAMPLE}_charts.json" ! -name "${SAMPLE}_delta.txt" ! -name "${SAMPLE}_delta.html" \ + ! -name "${SAMPLE}_delta.json" -print -quit) || return 1 + [ -z "$newer" ] || return 1 + newer=$(find "${PGP_ROOT}/bin" "${PGP_ROOT}/bin/.og" "${PGP_ROOT}/assets" -maxdepth 1 ! -type d -newer "$s" -print -quit 2> /dev/null) \ + || return 1 + [ -z "$newer" ] +} + rm -f "$REPORT" -# assets/: the ACMG SF rules (acmg_sf.tsv) and the carrier list (carrier_genes.tsv). +if summary_current; then + echo "Rendering from summary.json: step 24 wrote it and nothing it reads has changed since" + from=(--summary "/genome/${SAMPLE}/summary.json") +else + # assets/: the ACMG SF rules (acmg_sf.tsv) and the carrier list (carrier_genes.tsv). + from=(--assets /pgp-assets --json "/genome/${SAMPLE}/summary.json") +fi run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets:/pgp-assets:ro" "${PYTHON_IMAGE}" \ python3 /pgp-bin/render_report.py \ - --assets /pgp-assets \ + "${from[@]}" \ --sample "$SAMPLE" \ --sample-dir "/genome/${SAMPLE}" \ - --json "/genome/${SAMPLE}/summary.json" \ -o "/genome/${SAMPLE}/${SAMPLE}_report.txt" if [ ! -f "$REPORT" ]; then @@ -55,7 +82,7 @@ echo "Report saved to: ${REPORT}" # What changed since the previous run, then keep this run as the next baseline. HISTORY="${SAMPLE_DIR}/history" # og picks the previous run (--newest-history): the newest history// -# that holds a summary.json, the rule bin/collect_summary.py's "New since" +# that holds a summary.json, the rule og summary collect's "New since" # follows too. A newer folder without one, from a copy that stopped half # way, is passed over. og is the static binary, so the python image runs it. echo "" From 88d42be7f8b1e8bf9c1e61af2c6de55179f27809 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:32:46 +0100 Subject: [PATCH 071/255] test(summary): check the summary through og summary collect The tests that called the Python section readers now read the section og summary collect writes for the same folder. Checks of a Python internal (the G2 reader, the probable CYP2D6 rule, the poly-C tracts, the imprinted regions, the repeat ranges table) move to og-summary's own cargo tests. The demo maker writes step 26b, 49 and 05 tables without the Python remnants. collect_summary.py imports re for half_call. Bead: pgp-bs9u.45 --- bin/ancestry_fine_report.py | 119 --------- bin/annotsv_postfilter.py | 117 --------- bin/chip_report.py | 62 ----- bin/clip_check.py | 39 --- bin/collect_summary.py | 1 + bin/genotype_rules_indels.py | 210 ---------------- bin/pgx_outside_calls.py | 55 ----- bin/traits_report.py | 70 ------ .../og-summary/src/collect/group_c/chrm.rs | 5 + .../og-summary/src/collect/group_c/roh.rs | 43 ++++ .../og-summary/src/collect/group_c/tests.rs | 21 ++ .../og-summary/src/collect/group_c/traits.rs | 155 ++++++++++++ rust/crates/og-summary/src/collect/pgx.rs | 58 +++++ rust/crates/og-summary/src/collect/scan.rs | 3 +- tests/demo/make_demo_sample.py | 143 +++++++---- tests/e2e/callability-2-genes.sh | 16 +- tests/e2e/fixitfelix-chr21-1.sh | 16 +- tests/fake-docker/hla-kir-toggle.sh | 17 +- tests/test_acmg_sf_rules.py | 38 +-- tests/test_archaic_report.py | 33 +-- tests/test_callability.py | 24 +- tests/test_chip_report.py | 89 +++++-- tests/test_clip_check.py | 56 ----- tests/test_collect_summary.py | 161 ++++++------- tests/test_copy_number.py | 18 +- tests/test_cross_engine_biallelic.py | 29 ++- tests/test_exomiser_report.py | 28 ++- tests/test_genotype_rules_indels.py | 228 ------------------ tests/test_hla_report.py | 12 +- tests/test_locityper_report.py | 9 +- tests/test_lpa_kiv2.py | 27 ++- tests/test_mei_report.py | 37 ++- tests/test_mito_extended.py | 5 +- tests/test_mocha_report.py | 35 ++- tests/test_paralog_loci.py | 19 +- tests/test_pypgx_genes.sh | 2 +- tests/test_render_report.py | 21 +- tests/test_report_charts.py | 36 ++- tests/test_run_state.py | 2 +- tests/test_str_ranges.py | 11 +- tests/test_traits_report.py | 52 +++- tests/test_traits_table.py | 19 +- tests/test_viral_report.py | 26 +- tests/test_vntyper_report.py | 3 +- tests/test_ystr_report.py | 22 +- 45 files changed, 887 insertions(+), 1305 deletions(-) delete mode 100644 bin/ancestry_fine_report.py delete mode 100755 bin/annotsv_postfilter.py delete mode 100755 bin/chip_report.py delete mode 100755 bin/clip_check.py delete mode 100755 bin/genotype_rules_indels.py delete mode 100755 bin/pgx_outside_calls.py delete mode 100755 bin/traits_report.py delete mode 100644 tests/test_clip_check.py delete mode 100644 tests/test_genotype_rules_indels.py diff --git a/bin/ancestry_fine_report.py b/bin/ancestry_fine_report.py deleted file mode 100644 index 12043750f..000000000 --- a/bin/ancestry_fine_report.py +++ /dev/null @@ -1,119 +0,0 @@ -"""ancestry_fine_report.py: what other Python still reads of step 26b. - -Step 26b's commands (panel-map, check-map, order, sample-bed, chroms and -report) run in the Rust binary as og ancestry-fine (rust/crates/og-ancestry). -This module keeps only the names two Python programs import: - - bin/collect_summary.py label, COLOURS and MIN_REFERENCES, for the - report's group names and painting legend - tests/demo/make_demo_sample.py label, CHROM_LEN, painting_shares, - write_rows and painting_png, for the docs - demo, which is built without og - -The group names, the colours and MIN_REFERENCES must stay the same as -rust/crates/og-ancestry/src/lib.rs; tests/test_ancestry_fine_report.py checks -them against what og writes. - -A group name is ours: a set of reference populations we put together and -named, not a nationality. docs/26b-ancestry-fine.md lists them. -""" -import os -import struct -import zlib - -LABELS = { - "continent": { - "AFR": "Africa", - "WEA": "West Eurasia (Europe, Middle East, North Africa)", - "CSA": "Central and South Asia", - "EAS": "East Asia", - "AMR": "Americas (Native American)", - "OCE": "Oceania", - }, - "region": { - "WAF": "West Africa", "EAF": "East Africa", "CAF": "Central African rainforest", "SAF": "Southern Africa", - "NWE": "Northwestern Europe", "NEE": "Northeastern Europe", "IBE": "Iberia", "ITA": "Italy", - "SAR": "Sardinia", "CAU": "Caucasus", "LEV": "Levant", "BDN": "Bedouin", "NAF": "North Africa", - "CSA": "Central and South Asia", "EAS": "East Asia", "SIB": "Siberia", - "AMR": "Americas (Native American)", "OCE": "Oceania", - }, -} - -# Painting colours of the continental groups, and grey where nothing is painted. -COLOURS = {"AFR": (230, 159, 0), "WEA": (0, 114, 178), "CSA": (0, 158, 115), "EAS": (204, 121, 167), - "AMR": (213, 94, 0), "OCE": (86, 180, 233)} -UNPAINTED = (225, 225, 225) - -# GRCh38 lengths of chr1 to chr22, for the painting picture. -CHROM_LEN = [248956422, 242193529, 198295559, 190214555, 181538259, 170805979, 159345973, 145138636, - 138394717, 133797422, 135086622, 133275309, 114364328, 107043718, 101991189, 90338345, - 83257441, 80373285, 58617616, 64444167, 46709983, 50818468] - -# The fewest unrelated reference people a group needs to be modelled -# (rust/crates/og-ancestry/src/lib.rs says why). -MIN_REFERENCES = 10 - - -def label(level, code): - return LABELS.get(level, {}).get(code, code) - - -def write_rows(path, header, rows): - tmp = path + ".tmp" - with open(tmp, "w") as f: - f.write("\t".join(header) + "\n") - for r in rows: - f.write("\t".join(str(x) for x in r) + "\n") - os.replace(tmp, path) - - -def chrom_key(c): - c = c[3:] if c.startswith("chr") else c - return (0, int(c)) if c.isdigit() else (1, c) - - -def painting_shares(rows): - total = sum(r[2] - r[1] for r in rows) - by = {} - for r in rows: - by[r[4]] = by.get(r[4], 0) + r[2] - r[1] - return total, {k: v / total for k, v in by.items()} if total else {} - - -def png_bytes(width, height, pixels): - """A plain RGB PNG of rows of (r, g, b) tuples.""" - raw = b"".join(b"\x00" + bytes(v for px in row for v in px) for row in pixels) - - def chunk(kind, data): - return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xffffffff) - - return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) - + chunk(b"IDAT", zlib.compress(raw, 9)) + chunk(b"IEND", b"")) - - -def painting_png(rows, width=1000, bar=6, gap=6): - """Chromosomes 1 to 22 as two bars each (the two haplotypes), drawn to - scale, each painted segment in its group's colour, the rest grey.""" - scale = (width - 20) / CHROM_LEN[0] - height = gap + 22 * (2 * bar + 1 + gap) - img = [[(255, 255, 255)] * width for _ in range(height)] - for i, n in enumerate(CHROM_LEN): - y0 = gap + i * (2 * bar + 1 + gap) - for hap in (0, 1): - y = y0 + hap * (bar + 1) - for yy in range(y, y + bar): - for x in range(10, 10 + int(n * scale)): - img[yy][x] = UNPAINTED - for chrom, start, end, hap, code in rows: - key = chrom_key(chrom) - if key[0] != 0 or not 1 <= key[1] <= 22: - continue - i = key[1] - 1 - y = gap + i * (2 * bar + 1 + gap) + (hap - 1) * (bar + 1) - x0 = 10 + int(start * scale) - x1 = max(x0 + 1, 10 + int(end * scale)) - colour = COLOURS.get(code, (0, 0, 0)) - for yy in range(y, y + bar): - for x in range(x0, min(x1, width - 10)): - img[yy][x] = colour - return png_bytes(width, height, img) diff --git a/bin/annotsv_postfilter.py b/bin/annotsv_postfilter.py deleted file mode 100755 index 7ef0bee4c..000000000 --- a/bin/annotsv_postfilter.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 -# remnant: kept for collect_summary and the docs demo; deleted in lane R3 -"""annotsv_postfilter.py: what is left of step 05's class 4 and 5 checks once -they moved to `og sv postfilter` (rust/crates/og-steps/src/sv). Only the -names the reports and the docs demo still import are kept: the table's -column names, the FORMAT reader of its sample column, and the join of step -22's consensus support that the reports make when they are written. -`og dev remnants` (tests/parity/remnants.tsv) checks that nothing else -comes back. Standard library only. -""" -import gzip -import os - -CONSENSUS_BP = 1000 -KEEP = ["AnnotSV_ID", "ID", "SV_chrom", "SV_start", "SV_end", "SV_length", "SV_type", "Gene_name", "ACMG_class", - "AnnotSV_ranking_score"] -ADDED = ["Caller", "Exon_edge", "Exon_edge_detail", "Population_AF", "Population_source", "Common", "AnnotSV_class", - "Depth_support", "Rule", "Rule_detail"] - - -def open_text(path): - if path.endswith((".gz", ".bgz")): - return gzip.open(path, "rt", errors="replace") - return open(path, errors="replace") - - -def chrom_key(c): - """AnnotSV writes chromosomes without 'chr'; step 22 writes them with it.""" - return c[3:] if c.lower().startswith("chr") else c - - -def sv_type(row): - t = (row.get("SV_type") or "").upper() - return "DUP" if t.startswith("DUP") else t - - -def info_value(info, key): - for kv in (info or "").split(";"): - k, _, v = kv.partition("=") - if k == key: - return v - return "" - - -def sample_format(row): - """{FORMAT key: value} of the first sample column, the one after FORMAT - in AnnotSV's header; {} when the row has none.""" - cols = sample_columns(row) - if not cols: - return {} - return dict(zip((row.get("FORMAT") or "").split(":"), (row.get(cols[1]) or "").split(":"))) - - -def sample_columns(row): - """["FORMAT", ] of AnnotSV's header, the two columns - sample_format reads; [] when the row has no sample column.""" - keys = list(row) - if "FORMAT" not in keys: - return [] - i = keys.index("FORMAT") + 1 - return [] if i >= len(keys) or keys[i] == "Annotation_mode" else keys[i - 1:i + 1] - - -def read_consensus(path): - """(caller names, records) of step 22's consensus VCF; a record is - (chrom, pos, end, type, supp, supp_vec).""" - names, recs = [], [] - with open_text(path) as f: - for line in f: - if line.startswith("##"): - continue - c = line.rstrip("\n").split("\t") - if line.startswith("#CHROM"): - names = c[9:] - continue - if len(c) < 8: - continue - t = info_value(c[7], "SVTYPE").upper() - try: - pos = int(c[1]) - end = int(info_value(c[7], "END") or pos) - supp = int(info_value(c[7], "SUPP") or 0) - except ValueError: - continue - recs.append((chrom_key(c[0]), pos, end, "DUP" if t.startswith("DUP") else t, supp, - info_value(c[7], "SUPP_VEC"))) - return names, recs - - -def consensus_support(rows, path): - """Support and Support_callers for each checked row from the consensus - at path; None for both when there is no consensus file.""" - if not path or not os.path.isfile(path): - return [{"Support": None, "Support_callers": None} for _ in rows] - names, recs = read_consensus(path) - by = {} - for rec in recs: - by.setdefault((rec[0], rec[3]), []).append(rec) - out = [] - for r in rows: - best = None - try: - start, end = int(r["SV_start"]), int(r["SV_end"]) - except (KeyError, ValueError): - start = end = None - if start is not None: - for rec in by.get((chrom_key(r.get("SV_chrom", "")), sv_type(r)), []): - d = abs(rec[1] - start) + abs(rec[2] - end) - if abs(rec[1] - start) <= CONSENSUS_BP and abs(rec[2] - end) <= CONSENSUS_BP and (best is None or d < best[0]): - best = (d, rec) - if best: - vec = best[1][5] - callers = [names[i] if i < len(names) else f"caller {i + 1}" for i, x in enumerate(vec) if x == "1"] - out.append({"Support": best[1][4], "Support_callers": ",".join(callers)}) - else: - out.append({"Support": 1, "Support_callers": r.get("Caller") or "."}) - return out diff --git a/bin/chip_report.py b/bin/chip_report.py deleted file mode 100755 index 10bccb594..000000000 --- a/bin/chip_report.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""chip_report.py: what the reports still read of step 49 (clonal haematopoiesis) in Python. - -The step itself runs in Rust: `og chip prepare`, `og chip report` and -`og chip check-table` (rust/crates/og-chip, docs/49-chip.md). This module -keeps only the names bin/collect_summary.py and tests/demo/make_demo_sample.py -import: the two tables' columns, the gene table reader and the detection -limit, the smallest VAF a clone needs to be found with probability -DETECT_POWER, taken as at least MIN_ALT_READS reads with the change out of the -mean depth (binomial). og chip computes the same limit the same way. - -Standard library only (it runs in the plain python image). -""" -import math -import sys - -TABLE_COLUMNS = ["gene", "truncating", "missense", "note", "pmids"] -OUT_COLUMNS = ["gene", "chrom", "pos", "ref", "alt", "consequence", "protein_change", "exon", "vaf", - "alt_reads", "depth", "filter", "status", "class", "note"] -DEPTH_COLUMNS = ["gene", "transcript", "chrom", "coding_bases", "mean_depth", "detection_limit_vaf", "status"] -MIN_ALT_READS = 3 # reads with the change a call needs, and the detection limit assumes -DETECT_POWER = 0.8 # the detection limit: the VAF found this often at the mean depth - - -def die(msg): - sys.stderr.write(f"ERROR: {msg}\n") - sys.exit(1) - - -def read_table(path): - """The rows of assets/chip_genes.tsv as dicts (og chip check-table checks the rest of its schema).""" - rows = [] - with open(path) as fh: - lines = [ln.rstrip("\n") for ln in fh if not ln.startswith("#") and ln.strip()] - if not lines or lines[0].split("\t") != TABLE_COLUMNS: - die(f"{path}: the header must be {' '.join(TABLE_COLUMNS)} (tab separated)") - for ln in lines[1:]: - f = ln.split("\t") - if len(f) != len(TABLE_COLUMNS): - die(f"{path}: '{ln[:60]}' has {len(f)} columns, not {len(TABLE_COLUMNS)}") - rows.append(dict(zip(TABLE_COLUMNS, f))) - return rows - - -def p_at_least(k, n, f): - """P(X >= k) for X ~ Binomial(n, f).""" - return 1.0 - sum(math.comb(n, i) * f ** i * (1 - f) ** (n - i) for i in range(k)) - - -def detection_limit(depth, k=MIN_ALT_READS, power=DETECT_POWER): - """The smallest VAF found with probability `power` at `depth` (k reads needed); None below k reads.""" - n = int(depth) - if n < k: - return None - lo, hi = 0.0, 1.0 - for _ in range(60): - mid = (lo + hi) / 2 - if p_at_least(k, n, mid) >= power: - hi = mid - else: - lo = mid - return hi diff --git a/bin/clip_check.py b/bin/clip_check.py deleted file mode 100755 index 91dadaa2a..000000000 --- a/bin/clip_check.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python3 -"""clip_check.py: the names bin/collect_summary.py reads of the cDNA read check. - -The cDNA read check itself is og cdna check (rust/crates/og-bam/src/cdna.rs, -docs/06-clinvar-screen.md): steps 06 and 17 run it, and it writes -_clinvar_cdna_check.tsv and _cpsr_cdna_check.tsv. What is -left here is what the report reads beside those tables: the table's columns, -the note of a flagged hit, CPSR's classes in one spelling and the variant of -a CPSR row. og's own copies of these are in cdna.rs; the parity cases of -og-bam keep the two the same. -""" -import re - -FLAG_NOTE = "possible cDNA/transgene contamination: check the reads" -COLUMNS = ["chrom", "pos", "ref", "alt", "alt_reads", "alt_clipped", "ref_reads", "ref_clipped", - "cdna_check", "cdna_note", "construct_site"] - - -def norm_class(v): - """A CPSR or ClinVar class in one spelling: 'Likely_Pathogenic' and 'LP' - -> 'likely pathogenic'; '' for an empty or NA cell.""" - v = (v or "").replace("_", " ").strip().lower() - return {"p": "pathogenic", "lp": "likely pathogenic", "b": "benign", "lb": "likely benign", - "na": "", ".": ""}.get(v, v) - - -PLP = ("pathogenic", "likely pathogenic") - - -def cpsr_variant(row): - """(chrom, pos, ref, alt) of a CPSR row from VAR_ID or GENOMIC_CHANGE; - None when neither can be read.""" - f = (row.get("VAR_ID") or "").split("_") - if len(f) >= 4 and f[-3].isdigit(): - return "_".join(f[:-3]), int(f[-3]), f[-2], f[-1] - m = re.match(r"^(\S+):g\.(\d+)([A-Za-z]+)>([A-Za-z]+)$", row.get("GENOMIC_CHANGE") or "") - if m: - return m.group(1), int(m.group(2)), m.group(3), m.group(4) - return None diff --git a/bin/collect_summary.py b/bin/collect_summary.py index a492746d8..55bffce51 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -18,6 +18,7 @@ import gzip import json import os +import re import subprocess import tempfile import zlib diff --git a/bin/genotype_rules_indels.py b/bin/genotype_rules_indels.py deleted file mode 100755 index b30aaf65b..000000000 --- a/bin/genotype_rules_indels.py +++ /dev/null @@ -1,210 +0,0 @@ -#!/usr/bin/env python3 -"""genotype_rules_indels.py: the APOL1 G2 deletion, read beside step 42's APOL1 rule. - -Step 42's rule table (assets/genotype_rules.tsv, bin/traits_report.py) reads -single-base sites only, so its APOL1 rule reads G1 (two missense changes) and -says nothing about G2 (rs71785313), a 6-base deletion of N388 and Y389 a few -bases away (Genovese et al. 2010, PMID 20647424). G1 and G2 never share a copy, -and two risk copies (G1/G1, G1/G2 or G2/G2) are the high-risk genotype. This -module reads G2 from the sample's VCF and puts the two together: - - read_g2(records, kind) the G2 copies (0, 1, 2) from the VCF records that - overlap G2, with a status: - called a G2 call, or reference blocks - (gVCF) or reference calls over - the whole G2 span: 0 copies - no_call_record a VCF without reference blocks and - no record over G2: no G2 call, but - a reference base cannot be told - from a position with no reads - not_covered a gVCF with no genotype over part - of the G2 span - not_read a no-call, a filtered G2 call, an - allele other than G2 inside its - span, a REF that is not the - GRCh38 base, or more than one - record calling G2 (one copy - written twice cannot be told - from two copies) - combine(rule_line, g2) the G1 + G2 result and its reading, from step 42's - APOL1 line (G0/G0, G0/G1, G1/G1) and read_g2's answer. - -A deletion in a repeat can be written at several positions. A record is G2 -when applying its allele to GRCh38_CONTEXT gives the same sequence as G2, so a -left-aligned and a right-shifted record both count. The pinned dbSNP record, -DBSNP, is what NCBI's Variation API gave for rs71785313 on 2026-11-06 (the VCF -fields of NC_000022.11:36265995:ATAATTATAA:ATAA); tests/e2e/genotype-rules-2-g2.sh -compares it with dbSNP's VCF of the build versions.env pins, and -tests/test_genotype_rules_indels.py checks the marker against it and against -GRCh38_CONTEXT. Standard library only. -""" -import re - -# rsID, chrom, VCF POS, REF, ALT of rs71785313 in dbSNP's GRCh38 VCF (left-aligned). -DBSNP = ("rs71785313", "chr22", 36265995, "AATAATT", "A") -G2 = {"rule": "APOL1", "name": "G2", "rsid": "rs71785313", "chrom": "chr22", "pos": 36265995, - "ref": "AATAATT", "alt": "A"} -# GRCh38 chr22:36265975-36266025, the plus strand (UCSC hg38, 2026-11-06). dbSNP's -# SPDI for G2 spans 36265996-36266005 (ATAATTATAA becomes ATAA): any 6-base -# deletion inside that span is the same sequence. -CONTEXT_START = 36265975 -GRCh38_CONTEXT = "AGAAGCTAAACATTCTCAACAATAATTATAAGATTCTGCAGGCGGACCAAG" -CONTEXT_END = CONTEXT_START + len(GRCh38_CONTEXT) - 1 -# The G2 span: the anchor base of the left-aligned record to the end of the -# ambiguous span. A genotype over every base of it reads G2. -SPAN = (36265995, 36266005) -G1_COPIES = {"G0/G0": 0, "G0/G1": 1, "G1/G1": 2} -SYMBOLIC = re.compile(r"^(<.*>|\*|\.)$") - - -def apply(pos, ref, alt): - """GRCh38_CONTEXT with ref at pos replaced by alt, or None when ref is not - the GRCh38 bases there or the record does not lie inside the context.""" - i = pos - CONTEXT_START - if i < 0 or pos + len(ref) - 1 > CONTEXT_END or GRCh38_CONTEXT[i:i + len(ref)] != ref: - return None - return GRCh38_CONTEXT[:i] + alt + GRCh38_CONTEXT[i + len(ref):] - - -def marker_errors(m=G2, dbsnp=DBSNP): - """The problems of the G2 marker, as messages: against the pinned dbSNP - record (a swapped pair named as such) and against GRCh38_CONTEXT.""" - errs = [] - rec = (m["rsid"], m["chrom"], m["pos"], m["ref"], m["alt"]) - if rec != tuple(dbsnp): - if (m["ref"], m["alt"]) == (dbsnp[4], dbsnp[3]): - errs.append(f"{m['rsid']}: ref and alt are swapped (marker {m['ref']}>{m['alt']}, " - f"dbSNP REF {dbsnp[3]})") - else: - errs.append(f"{m['rsid']}: the marker is {m['chrom']}:{m['pos']} {m['ref']}>{m['alt']}, " - f"dbSNP {dbsnp[1]}:{dbsnp[2]} {dbsnp[3]}>{dbsnp[4]}") - hap = apply(m["pos"], m["ref"], m["alt"]) - if hap is None: - errs.append(f"{m['rsid']}: REF {m['ref']} is not the GRCh38 bases at {m['chrom']}:{m['pos']}") - elif len(GRCh38_CONTEXT) - len(hap) != 6: - errs.append(f"{m['rsid']}: the marker is not a 6-base deletion") - return errs - - -G2_HAPLOTYPE = apply(G2["pos"], G2["ref"], G2["alt"]) - - -def record_end(r): - """The last base a VCF record covers: INFO END for a reference block, else POS + len(REF) - 1.""" - m = re.search(r"(?:^|;)END=(\d+)", r[7]) if len(r) > 7 else None - return int(m.group(1)) if m else int(r[1]) + len(r[3]) - 1 - - -def overlaps(r): - """Whether a split VCF data line is on chr22 and covers part of the G2 span.""" - return (len(r) > 4 and r[0] == G2["chrom"] and r[1].isdigit() - and int(r[1]) <= SPAN[1] and record_end(r) >= SPAN[0]) - - -def read_g2(records, kind): - """G2 from the VCF records that overlap SPAN (other records are skipped). - kind is 'gvcf' (reference blocks: a base with no record has no reads) or - 'vcf' (variant calls only). Returns {status, copies, gq, text}; copies is - None unless status is called or no_call_record.""" - copies, covered, gqs, problems, seen, g2_records = 0, set(), [], [], False, 0 - for r in records: - if not overlaps(r): - continue - if len(r) < 10: - problems.append("a record over G2 has no sample column") - continue - seen = True - pos, ref = int(r[1]), r[3] - alleles = [ref] + r[4].split(",") - fmt = dict(zip(r[8].split(":"), r[9].split(":"))) - gt = re.split(r"[/|]", fmt.get("GT", ".")) - if any(not g.isdigit() for g in gt): - problems.append(f"a no-call at {r[0]}:{pos}") - continue - idx = [int(g) for g in gt] - if any(i >= len(alleles) for i in idx): - problems.append(f"a genotype at {r[0]}:{pos} names an allele the record lacks") - continue - if all(i == 0 for i in idx): - covered.update(range(pos, record_end(r) + 1)) - gqs.append(fmt.get("GQ", ".")) - continue - if r[6] not in ("PASS", "."): - problems.append(f"a call at {r[0]}:{pos} that failed the caller's filter ({r[6]})") - continue - n = 0 - for i in idx: - if i == 0: - continue - a = alleles[i] - inside = pos >= CONTEXT_START and record_end(r) <= CONTEXT_END - if inside and apply(pos, ref, ref) is None: - problems.append(f"REF {ref} at {r[0]}:{pos} is not the GRCh38 base (is the VCF on GRCh38?)") - break - hap = None if SYMBOLIC.match(a) or not inside else apply(pos, ref, a) - if hap == G2_HAPLOTYPE: - n += 1 - else: - problems.append(f"an allele other than G2 inside its span ({r[0]}:{pos} {ref}>{a})") - break - else: - if n: - g2_records += 1 - if g2_records > 1: - # Two records of the one deletion (say a left-aligned and a shifted copy of - # the same call) cannot be told from two G2 copies. - problems.append("more than one record calls the G2 deletion") - break - copies += n - covered.update(range(pos, record_end(r) + 1)) - gqs.append(fmt.get("GQ", ".")) - gq = ",".join(gqs) if gqs else "." - if problems: - return {"status": "not_read", "copies": None, "gq": gq, - "text": f"G2 (rs71785313) not read: {problems[0]}."} - if copies: - return {"status": "called", "copies": copies, "gq": gq, - "text": f"G2 (rs71785313): {copies} cop{'ies' if copies > 1 else 'y'} of the deletion called."} - if all(p in covered for p in range(SPAN[0], SPAN[1] + 1)): - return {"status": "called", "copies": 0, "gq": gq, - "text": "G2 (rs71785313): the reference over the whole deletion span, no G2 copy."} - if kind == "vcf" and not seen: - return {"status": "no_call_record", "copies": 0, "gq": ".", - "text": "G2 (rs71785313): no G2 deletion among the VCF's calls."} - return {"status": "not_covered", "copies": None, "gq": gq, - "text": "G2 (rs71785313) not read: no genotype over part of the deletion span."} - - -TWO_COPIES = ("Two APOL1 risk copies ({label}): the high-risk genotype of Genovese 2010, which went with " - "focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African " - "Americans. A risk factor, not a diagnosis.") - - -def combine(line, g2): - """{result, reading} of G1 and G2 together, from step 42's APOL1 line and read_g2's answer.""" - g1 = G1_COPIES.get(line.get("result")) if line.get("status") == "called" else None - if g1 is None and g2["status"] == "called" and g2["copies"] == 2: - # Two G2 copies fill both copies of the gene: G2/G2 whatever the G1 sites read. - return {"result": "G2/G2", "reading": TWO_COPIES.format(label="G2/G2") + " The G1 sites were not read, " - "but two G2 copies leave no copy for G1."} - if g1 is None: - return {"result": ".", "reading": "Not read: the G1 sites of the APOL1 rule were not read."} - if g2["copies"] is None: - return {"result": ".", "reading": "The G1 result above stands on its own."} - c = g2["copies"] - if g1 + c > 2: - return {"result": ".", "reading": f"{g1} G1 and {c} G2 copies do not fit two copies of the gene " - "(G1 and G2 never share a copy): not read. Check the calls."} - label = "/".join(["G0"] * (2 - g1 - c) + ["G1"] * g1 + ["G2"] * c) - if g1 + c == 2: - reading = TWO_COPIES.format(label=label) - elif g1 + c == 1: - reading = (f"One APOL1 risk copy ({label}): one copy did not raise the risk in Genovese 2010 " - "(a recessive pattern).") - else: - reading = "No G1 and no G2 copy: no APOL1 kidney risk copy." - if g2["status"] == "no_call_record": - label += " (no G2 call)" - reading += (" No G2 deletion was called, but this VCF has no reference blocks, so a G2 copy at a " - "position with no reads would not show.") - return {"result": label, "reading": reading} diff --git a/bin/pgx_outside_calls.py b/bin/pgx_outside_calls.py deleted file mode 100755 index 9b80128b8..000000000 --- a/bin/pgx_outside_calls.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# remnant: kept for collect_summary (and its test); deleted in lane R3 -"""pgx_outside_calls.py: what is left of step 36's outside calls once they -moved to `og pgx outside-calls`, `og pgx hla-concordance` and `og pgx -pypgx-summary` (rust/crates/og-steps/src/pgx_rules.rs and pypgx_summary.rs). -Only the rules the summary still imports are kept: what counts as a call, -the probable CYP2D6 call of pypgx alone, and the names of step 32's no-call -statuses. `og dev remnants` (tests/parity/remnants.tsv) checks that nothing -else comes back. Standard library only. -""" - -HLA_GENES = ("HLA-A", "HLA-B") -NO_CALL = {"", "none", "n/a", "na", "failed", "indeterminate", "not called", "unknown", "."} -PROBABLE = "probable (one caller: pypgx)" -# Step 32's summary: the status of a gene the reference cannot hold, and the -# quality flag of a call resting on a defining variant below the GQ floor. -NOT_CALLABLE = "not callable on this reference" -LOW_GQ = "low-confidence: check the reads" - - -def is_call(diplotype): - d = (diplotype or "").strip() - return d.lower() not in NO_CALL and "/" in d and ";" not in d - - -def same_diplotype(a, b): - """Equal up to the order of the two alleles.""" - def key(d): - return sorted(x.strip() for x in d.split("/")) - return key(a) == key(b) - - -def probable_cyp2d6(row, cyrius, depth_status): - """{'diplotype', 'phenotype', 'label'} for pypgx's CYP2D6 call when it is - the only usable call, else None. row: pypgx's CYP2D6 summary row (a - dict); cyrius: (genotype, filter), or None when Cyrius did not run; - depth_status: the depth check's status, or None without its file (step 32 - turns the row into Indeterminate when the check fails, so a call in the - row passed it). - - Not probable: no call, a usable Cyrius call (then the two callers decide), - a depth check that did not pass, a defining variant that is missing from - the VCF or not PASS (a summary written before these columns has none), a - star allele pypgx's table lacks, or a defining variant with a GQ below the - floor or no GQ.""" - if not row or not is_call(row.get("Diplotype")): - return None - if cyrius is not None and is_call(cyrius[0]) and cyrius[1] == "PASS": - return None - if depth_status is not None and depth_status != "ok": - return None - if (row.get("Defining_filter") or "") not in ("PASS", "none") or row.get("Quality") not in ("ok", "N/A"): - return None - return {"diplotype": row["Diplotype"].strip(), "phenotype": (row.get("Phenotype") or "").strip(), - "label": PROBABLE} diff --git a/bin/traits_report.py b/bin/traits_report.py deleted file mode 100755 index 0d8e67310..000000000 --- a/bin/traits_report.py +++ /dev/null @@ -1,70 +0,0 @@ -# remnant: kept for collect_summary, demo and catalog tests; deleted in lane R3. -"""Step 42 metadata still read by the Python report collectors. - -The executable implementation is og traits table, phase-sites and check. -Copy-aware marker readers are og cn marker-read. -""" -COLUMNS = ["chrom", "pos", "rsid", "gene", "ref", "alt", "trait", "hom_ref", "het", "hom_alt", - "ancestry", "note", "pmids"] -OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", - "note", "pmids"] -RULE_OUT_COLUMNS = ["rule", "gene", "trait", "rsids", "chrom", "pos", "genotypes", "gq", "phase", "result", - "status", "reading", "ancestry", "note", "pmids"] - -TRAITS_TABLE_RULES = ("APOE",) - -class TableError(Exception): - pass - - -def read_table(path): - """The rows of the traits table as dicts. Lines starting with # are comments; - the first other line is the header, which must be COLUMNS.""" - rows, header = [], None - with open(path, encoding="utf-8") as f: - for n, line in enumerate(f, 1): - line = line.rstrip("\n") - if not line or line.startswith("#"): - continue - cells = line.split("\t") - if header is None: - if cells != COLUMNS: - raise TableError(f"{path}:{n}: header is {cells}, expected {COLUMNS}") - header = cells - continue - if len(cells) != len(COLUMNS): - raise TableError(f"{path}:{n}: {len(cells)} columns, expected {len(COLUMNS)}") - row = dict(zip(COLUMNS, cells)) - row["line"] = n - rows.append(row) - if header is None: - raise TableError(f"{path}: no header line") - return rows - - -MARKERS = { - # KIV-2 4925G>A: the last base of the second KIV-2 exon (its donor splice - # site), G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds - # it at the end of each of the six KIV-2 units (exons 5, 7, ..., 15 of - # ENST00000316300.10; NM_005577.4:c.733G>A at the first, p.Ala245Thr) and - # at the near-identical last exon of KIV-1 (exon 3, c.391): seven copies, - # the block og cn call --locus LPA_KIV2 counts as N + 2 copies (N KIV-2 repeats and - # one KIV-1 per chromosome). dbSNP names the first KIV-2 copy only. - "LPA_KIV2_4925": { - "name": "LPA KIV-2 4925G>A", "chrom": "chr6", "ref": "C", "alt": "T", - "dbsnp": ("rs1484919544", "chr6", 160640667, "C", "T"), - "copies": (160646214, 160640667, 160635123, 160629574, 160624030, 160618484, 160612932), - "context": "GACTCCTTACCTTGTTCGGAA", - "min_depth": 100, "min_alt": 5, "min_fraction": 0.008, "max_absent": 2, - }, - # SMN1 g.27134T>G is NM_000344.4:c.*3+80T>G in intron 7 (plus strand). - # SMN2 holds the same 21 bases 875,420 bases upstream, as at exon 7. - "SMN1_27134": { - "name": "SMN1 g.27134T>G", "chrom": "chr5", "ref": "T", "alt": "G", - "dbsnp": ("rs143838139", "chr5", 70952074, "T", "G"), - "copies": (70952074, 70076654), - "context": "TTTTTGAACATTTAAAAAGTT", - "min_depth": 10, "min_alt": 3, "min_fraction": 0.05, "max_absent": 1, - }, -} -MARKER_STATUSES = ("present", "not seen", "uncertain", "too little depth", "not assessed") diff --git a/rust/crates/og-summary/src/collect/group_c/chrm.rs b/rust/crates/og-summary/src/collect/group_c/chrm.rs index e232bccf6..68739a98a 100644 --- a/rust/crates/og-summary/src/collect/group_c/chrm.rs +++ b/rust/crates/og-summary/src/collect/group_c/chrm.rs @@ -467,6 +467,11 @@ mod tests { assert!(in_polyc(302, "A", "AC")); assert!(!in_polyc(300, "A", "G")); assert!(in_polyc(16189, "T", "C")); + // The tract's last base is in it, the next base is not; an SNV just + // before a tract is not, an insertion anchored there is. + assert!(in_polyc(315, "C", "T")); + assert!(!in_polyc(316, "C", "T")); + assert!(!in_polyc(302, "A", "G")); } #[test] diff --git a/rust/crates/og-summary/src/collect/group_c/roh.rs b/rust/crates/og-summary/src/collect/group_c/roh.rs index 489aa1a7e..f0fb24dc4 100644 --- a/rust/crates/og-summary/src/collect/group_c/roh.rs +++ b/rust/crates/og-summary/src/collect/group_c/roh.rs @@ -443,6 +443,49 @@ mod tests { use super::*; use crate::testutil::Dir; + /// assets/imprinted_regions.tsv holds the rows of [`IMPRINTED_REGIONS`] + /// (tests/test_collect_summary.py checked the Python copy the same way). + #[test] + fn imprinted_regions_are_the_asset_rows() { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../assets/imprinted_regions.tsv" + ); + let text = std::fs::read_to_string(path).unwrap(); + let mut lines = text.lines().filter(|l| !l.starts_with('#')); + let head: Vec<&str> = lines.next().unwrap().split('\t').collect(); + let col = |name: &str| head.iter().position(|h| *h == name).unwrap(); + let num = |x: &str| (!x.is_empty()).then(|| x.parse::().unwrap()); + let rows: Vec<_> = lines + .map(|l| { + let f: Vec<&str> = l.split('\t').collect(); + let span = num(f[col("start")]).zip(num(f[col("end")])); + ( + f[col("chrom")].to_owned(), + span, + f[col("region")].to_owned(), + f[col("maternal_upd")].to_owned(), + f[col("paternal_upd")].to_owned(), + f[col("pmid")].to_owned(), + ) + }) + .collect(); + let want: Vec<_> = IMPRINTED_REGIONS + .iter() + .map(|r| { + ( + r.0.to_owned(), + r.1, + r.2.to_owned(), + r.3.to_owned(), + r.4.to_owned(), + r.5.to_owned(), + ) + }) + .collect(); + assert_eq!(rows, want); + } + #[test] fn awk_reads_the_leading_number() { assert_eq!(awk_num("7000000"), 7_000_000.0); diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index 4a93b5111..82afd2dc8 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -202,3 +202,24 @@ fn a_stale_section_says_from_which_run() { assert!(text(at(roh, &["note"])).contains("step 11 was skipped")); assert_eq!(at(roh, &["data", "largest_mb"]), &Value::Float(7.0)); } + +/// Every locus of the checkout's `bin/str_intermediate_ranges.tsv` is read +/// (tests/test_str_ranges.py checked the Python reader the same way). +#[test] +fn every_row_of_the_repeat_ranges_table_is_read() { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../bin/str_intermediate_ranges.tsv" + ); + let rows = super::util::bin_rows(path).unwrap(); + let text = std::fs::read_to_string(path).unwrap(); + let loci: Vec<&str> = text + .lines() + .filter(|l| !l.starts_with('#') && !l.trim().is_empty()) + .skip(1) + .map(|l| l.split('\t').next().unwrap_or("")) + .collect(); + let read: Vec<&str> = rows.iter().map(|r| r["locus"].as_str()).collect(); + assert_eq!(read, loci); + assert!(loci.len() >= 3, "{loci:?}"); +} diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs index 6a553b36f..4e222e974 100644 --- a/rust/crates/og-summary/src/collect/group_c/traits.rs +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -654,6 +654,161 @@ pub mod apol1 { assert!(g.text.contains("more than one record")); } + /// The GRCh38 bases `a..=b` of the pinned context. + fn ctx(a: i64, b: i64) -> &'static str { + &CONTEXT[(a - CONTEXT_START) as usize..=(b - CONTEXT_START) as usize] + } + + /// The tested record, with its filter and GQ. + fn call(pos: i64, r: &str, a: &str, gt: &str, filter: &str) -> Vec { + let mut x = rec(pos, r, a, gt, "."); + x[6] = filter.to_owned(); + x + } + + fn block(end: i64) -> Vec { + let mut x = rec(36_265_900, "A", "<*>", "0/0", &format!("END={end}")); + x[9] = "0/0:50".into(); + x + } + + // The checks of tests/test_genotype_rules_indels.py, which tested the + // Python G2 reader before this port replaced it. + #[test] + fn left_aligned_and_right_shifted_count_each_copy() { + let right = ( + 36_265_999, + ctx(36_265_999, 36_266_005), + ctx(36_265_999, 36_265_999), + ); + for (p, r, a) in [(POS, REF, ALT), right] { + for (gt, n) in [("0/1", 1), ("1|1", 2)] { + let g = read_g2(&[rec(p, r, a, gt, ".")], false); + assert_eq!((g.status, g.copies), ("called", Some(n)), "{p} {gt}: {g:?}"); + } + } + let two = read_g2( + &[ + rec(POS, REF, ALT, "0/1", "."), + rec(right.0, right.1, right.2, "0/1", "."), + ], + false, + ); + assert_eq!((two.status, two.copies), ("not_read", None)); + } + + #[test] + fn a_reference_block_keeps_its_gq_and_no_record_reads_zero() { + let g = read_g2(&[block(36_266_100)], true); + assert_eq!( + (g.status, g.copies, g.gq.as_str()), + ("called", Some(0), "50") + ); + let g = read_g2(&[], false); + assert_eq!((g.status, g.copies), ("no_call_record", Some(0))); + let g = read_g2(&[], true); + assert_eq!((g.status, g.copies), ("not_covered", None)); + let g = read_g2(&[block(36_266_000)], true); + assert_eq!( + g.status, "not_covered", + "a block that stops inside the span" + ); + } + + #[test] + fn anything_else_inside_the_span_is_not_read() { + let snv = ctx(36_266_001, 36_266_001); + let other = if snv == "G" { "C" } else { "G" }; + for (what, r) in [ + ("a no-call", call(POS, REF, ALT, "./.", "PASS")), + ("a filtered G2 call", call(POS, REF, ALT, "0/1", "RefCall")), + ( + "an SNV inside the span", + call(36_266_001, snv, other, "0/1", "PASS"), + ), + ( + "a 5-base deletion", + call(POS, ctx(POS, 36_266_000), "A", "0/1", "PASS"), + ), + ( + "a REF that is not GRCh38", + call(POS, "CATAATT", "C", "0/1", "PASS"), + ), + ] { + let g = read_g2(&[block(36_266_100), r], true); + assert_eq!((g.status, g.copies), ("not_read", None), "{what}: {g:?}"); + } + } + + #[test] + fn records_outside_the_span_are_skipped() { + let mut chr21 = rec(POS, REF, ALT, "1/1", "."); + chr21[0] = "chr21".into(); + let g = read_g2( + &[ + block(36_266_100), + call(36_266_100, "A", "G", "0/1", "PASS"), + chr21, + ], + true, + ); + assert_eq!((g.status, g.copies), ("called", Some(0))); + // Control: a 6-base deletion anchored right after the span is not G2. + let out = rec( + 36_266_006, + ctx(36_266_006, 36_266_012), + ctx(36_266_006, 36_266_006), + "1/1", + ".", + ); + assert_eq!(read_g2(&[block(36_266_100), out], true).copies, Some(0)); + } + + #[test] + fn combine_reads_every_count_and_refuses_three_copies() { + let called = |c: i64| G2 { + status: "called", + copies: Some(c), + gq: ".".into(), + text: "t".into(), + source: None, + source_first: false, + }; + for (g1, c, label) in [ + ("G0/G0", 0, "G0/G0"), + ("G0/G0", 1, "G0/G2"), + ("G0/G0", 2, "G2/G2"), + ("G0/G1", 0, "G0/G1"), + ("G0/G1", 1, "G1/G2"), + ("G1/G1", 0, "G1/G1"), + ] { + let (got, reading) = combine(g1, "called", &called(c)); + let word = match label.matches("G1").count() + label.matches("G2").count() { + 0 => "No G1 and no G2", + 1 => "One APOL1 risk copy", + _ => "Two APOL1 risk copies", + }; + assert_eq!(got, label); + assert!(reading.contains(word), "{label}: {reading}"); + } + let (got, reading) = combine("G1/G1", "called", &called(1)); + assert_eq!(got, "."); + assert!(reading.contains("never share"), "{reading}"); + assert_eq!(combine(".", "not_covered", &called(1)).0, "."); + let (got, reading) = combine(".", "not_covered", &called(2)); + assert_eq!(got, "G2/G2"); + assert!(reading.contains("Two APOL1 risk copies")); + let unread = G2 { + status: "not_read", + copies: None, + text: "G2 x.".into(), + ..called(0) + }; + let (got, reading) = combine("G0/G1", "called", &unread); + assert_eq!(got, "."); + assert!(reading.contains("stands on its own") && !reading.contains("G2 x.")); + } + #[test] fn combine_counts_both() { let g2 = read_g2(&[rec(POS, REF, ALT, "0/1", ".")], false); diff --git a/rust/crates/og-summary/src/collect/pgx.rs b/rust/crates/og-summary/src/collect/pgx.rs index d59db94c2..c271384a6 100644 --- a/rust/crates/og-summary/src/collect/pgx.rs +++ b/rust/crates/og-summary/src/collect/pgx.rs @@ -661,4 +661,62 @@ mod tests { ); assert!(json_load("[1]").is_ok()); } + + /// The checks of step 32's probable CYP2D6 call, from + /// tests/test_pgx_outside_calls.py, which tested the Python rule before + /// this port replaced it. + #[test] + fn pypgx_alone_is_probable_only_on_clean_evidence() { + let ev = |filter: &str, quality: &str| { + Some([ + "Normal Metabolizer".to_owned(), + filter.to_owned(), + quality.to_owned(), + ]) + }; + let cy = |g: &str, f: &str| Cyrius { + genotype: g.into(), + filter: Some(f.into()), + }; + let p = |e: Option<[String; 3]>, c: Option<&Cyrius>, depth: &str| { + probable("*1/*2", e.as_ref(), c, Some(depth)) + }; + assert_eq!( + p(ev("PASS", "ok"), None, "ok"), + Some(Probable { + diplotype: "*1/*2".into(), + phenotype: "Normal Metabolizer".into() + }) + ); + let none = cy("None", "Not_assigned_to_haplotypes"); + assert!( + p(ev("PASS", "ok"), Some(&none), "ok").is_some(), + "Cyrius ran, no call" + ); + assert!( + p(ev("PASS", "ok"), None, "unreliable").is_none(), + "a failed depth check" + ); + let call = cy("*1/*2", "PASS"); + assert!( + p(ev("PASS", "ok"), Some(&call), "ok").is_none(), + "two callers decide" + ); + for (filter, quality) in [ + ("PASS", "low-confidence: check the reads"), + ("not PASS: chr22:42128945 C>T RefCall", "ok"), + ("not in pypgx's table: *99", "ok"), + ("PASS", "GQ not in the VCF: check the reads"), + ] { + assert!( + p(ev(filter, quality), None, "ok").is_none(), + "{filter} {quality}" + ); + } + assert!( + p(None, None, "ok").is_none(), + "a summary without the columns" + ); + assert!(probable("None", ev("PASS", "ok").as_ref(), None, None).is_none()); + } } diff --git a/rust/crates/og-summary/src/collect/scan.rs b/rust/crates/og-summary/src/collect/scan.rs index 2a0cb9695..b81721f1f 100644 --- a/rust/crates/og-summary/src/collect/scan.rs +++ b/rust/crates/og-summary/src/collect/scan.rs @@ -60,7 +60,8 @@ impl FolderIndex { }; if meta.is_file() { files.insert(rel); - } else if meta.is_dir() && depth + 1 < MAX_DEPTH && !(depth == 0 && rel == SKIP_TOP) { + } else if meta.is_dir() && depth + 1 < MAX_DEPTH && !(depth == 0 && rel == SKIP_TOP) + { stack.push((path, rel, depth + 1)); } } diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index ff98d42fa..e55cb03d3 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -24,14 +24,14 @@ somalier, VerifyBamID2 and pgsc_calc files and the score folder `og prs format` writes (written here in its layout: og runs only inside the pipeline's Linux containers); - - step 33's mosaic rows come from og qc sex-loss, step 26b's painting - from bin/ancestry_fine_report.py, step 35b's table from + - step 33's mosaic rows come from og qc sex-loss, step 26b's table + and painting from og ancestry-fine report, step 35b's table from og paraphase table, step 39's from bin/couple_report.py and step 42's from bin/traits_report.py, each fed made-up inputs; step 46's table and step 35's HBA1/HBA2 table are what og cn call writes for made-up depths (HBA_TABLE and LPA_TABLE; tests/test_hba_copy_number.py and tests/test_lpa_kiv2.py check that og still writes them), step 49's depth - rows from bin/chip_report.py's detection_limit, with made-up counts; the + rows with og chip's detection limit, at made-up depths; the tables of steps 02, 40, 41, 45, 47, 48, 50, 51, 52 and 53 are written here, in the columns their writers use (og for 45 to 53), with the values those writers give for the made-up inputs named in each function. @@ -624,10 +624,11 @@ def write(path, recs, extra=()): f"{min(1, p * 9):.4e}\t{rng.uniform(0, 0.5):.4f}\t{rng.uniform(0, 0.1):.4f}\t{rng.randint(0, 9)}") put(os.path.join(d, "cnvpytor", f"{SAMPLE}_cnvs.txt"), "\n".join(lines) + "\n") - # Step 05's class 4 and 5 SVs, checked (bin/annotsv_postfilter.py's - # columns): none, as in most genomes. - import annotsv_postfilter - cols = annotsv_postfilter.KEEP + annotsv_postfilter.ADDED + # Step 05's class 4 and 5 SVs, checked (the columns og sv postfilter + # writes): none, as in most genomes. + cols = ["AnnotSV_ID", "ID", "SV_chrom", "SV_start", "SV_end", "SV_length", "SV_type", "Gene_name", "ACMG_class", + "AnnotSV_ranking_score", "Caller", "Exon_edge", "Exon_edge_detail", "Population_AF", + "Population_source", "Common", "AnnotSV_class", "Depth_support", "Rule", "Rule_detail"] rows = [] if has("annotsv"): # One class 4 deletion over an invented gene, from Manta, kept by step 05's rules. @@ -802,7 +803,10 @@ def callability(d, rng): written from the repository's catalog, so it always matches it. The ALT-only genes get no row, as in the real step.""" rows = ["#chrom\tstart\tend\tregion\t10X"] - for r in collect_summary.asset_rows("callability_genes.tsv"): + with open(os.path.join(REPO, "assets", "callability_genes.tsv"), encoding="utf-8") as f: + catalog = list(csv.DictReader((x for x in f if not x.startswith("#")), delimiter="\t", + quoting=csv.QUOTE_NONE)) + for r in catalog: if r["start"] == ".": continue start, end = int(r["start"]), int(r["end"]) @@ -852,41 +856,56 @@ def mosaic(d, rng): def ancestry_fine(d, rng): """Step 26b's table (ADMIXTURE fractions with their bootstrap range, then - the painted shares) and its painting, drawn by bin/ancestry_fine_report.py. - Mostly West Eurasian, as step 25's EUR placement above.""" - import ancestry_fine_report as afr + the painted shares) and its painting, written by og ancestry-fine report + from made-up ADMIXTURE and RFMix output. Mostly West Eurasian, as step + 25's EUR placement above.""" continent = {"WEA": 0.942, "CSA": 0.031, "AFR": 0.012, "EAS": 0.008, "AMR": 0.005, "OCE": 0.002} region = {"NWE": 0.463, "IBE": 0.274, "ITA": 0.118, "NEE": 0.052, "SAR": 0.009, "CAU": 0.011, "LEV": 0.008, "BDN": 0.002, "NAF": 0.005, "CSA": 0.031, "WAF": 0.007, "EAF": 0.003, "CAF": 0.001, "SAF": 0.001, "EAS": 0.006, "SIB": 0.002, "AMR": 0.005, "OCE": 0.002} - table = [] - for level, fr in (("continent", continent), ("region", region)): - for code, f in sorted(fr.items(), key=lambda kv: -kv[1]): - se = max(0.002, math.sqrt(f * (1 - f)) * rng.uniform(0.04, 0.08)) - table.append([level, code, afr.label(level, code), f"{f:.4f}", f"{se:.4f}", - f"{max(0.0, f - 1.96 * se):.4f}", f"{min(1.0, f + 1.96 * se):.4f}"]) - # The painting: each haplotype of chr1-22 in segments of 8 to 60 Mb, West - # Eurasian but for a few short ones. - rows = [] - for i, n in enumerate(afr.CHROM_LEN): - for hap in (1, 2): - pos = rng.randint(200_000, 1_500_000) - while pos < n - 2_000_000: - end = min(n - rng.randint(200_000, 1_500_000), pos + rng.randint(8_000_000, 60_000_000)) - code = "WEA" if rng.random() < 0.8 else rng.choice(["CSA", "CSA", "AFR"]) - if code != "WEA": - end = min(end, pos + rng.randint(3_000_000, 12_000_000)) - rows.append([f"chr{i + 1}", pos, end, hap, code]) + # GRCh38 lengths of chr1 to chr22. + chrom_len = [248956422, 242193529, 198295559, 190214555, 181538259, 170805979, 159345973, 145138636, + 138394717, 133797422, 135086622, 133275309, 114364328, 107043718, 101991189, 90338345, + 83257441, 80373285, 58617616, 64444167, 46709983, 50818468] + with tempfile.TemporaryDirectory() as work: + model, adm, paint, out = (os.path.join(work, x) for x in ("model", "admixture", "paint", "out")) + for x in (model, adm, paint): + os.makedirs(x) + levels = [] + for level, fr in (("continent", continent), ("region", region)): + codes = list(fr) + levels.append(f"{level}\t{len(codes)}\t{','.join(codes)}\n") + put(os.path.join(adm, f"{SAMPLE}_{level}.{len(codes)}.Q"), " ".join(f"{fr[c]:.6f}" for c in codes) + "\n") + # 20 bootstrap replicates, each fraction moved by a few percent of its spread. + boot = [] + for _ in range(20): + row = [max(0.0, fr[c] + rng.gauss(0, max(0.002, math.sqrt(fr[c] * (1 - fr[c])) * 0.06))) for c in codes] + boot.append(" ".join(f"{v / sum(row):.6f}" for v in row) + "\n") + put(os.path.join(adm, f"{SAMPLE}_{level}.boot.Q"), "".join(boot)) + put(os.path.join(model, "levels.tsv"), "".join(levels)) + # The painting: each chromosome in windows of 3 to 15 Mb, each haplotype + # West Eurasian but for a few windows. + groups = ["AFR", "CSA", "WEA"] + for i, n in enumerate(chrom_len): + c = f"chr{i + 1}" + rows = ["#Subpopulation order/codes: " + "\t".join(f"{g}={k}" for k, g in enumerate(groups)), + "#chm\tspos\tepos\tsgpos\tegpos\tn snps\t" + f"{SAMPLE}.0\t{SAMPLE}.1"] + pos, end_of = rng.randint(200_000, 1_500_000), n - rng.randint(200_000, 1_500_000) + while pos < end_of - 2_000_000: + end = min(end_of, pos + rng.randint(3_000_000, 15_000_000)) + haps = [2 if rng.random() < 0.85 else rng.choice([1, 1, 0]) for _ in (0, 1)] + rows.append(f"{c}\t{pos}\t{end}\t{pos / 1e6:.3f}\t{end / 1e6:.3f}\t100\t{haps[0]}\t{haps[1]}") pos = end - _, shares = afr.painting_shares(rows) - table += [["painting", code, afr.label("continent", code), f"{sh:.4f}", ".", ".", "."] - for code, sh in sorted(shares.items(), key=lambda kv: -kv[1])] - out = os.path.join(d, "ancestry_fine") - os.makedirs(out, exist_ok=True) - afr.write_rows(os.path.join(out, f"{SAMPLE}_ancestry_fine.tsv"), - ["level", "code", "group", "fraction", "se", "low", "high"], table) - with open(os.path.join(out, f"{SAMPLE}_painting.png"), "wb") as f: - f.write(afr.painting_png(rows)) + put(os.path.join(paint, f"{c}.msp.tsv"), "\n".join(rows) + "\n") + r = subprocess.run([og_binary(), "ancestry-fine", "report", "--sample", SAMPLE, "--model", model, + "--admixture", adm, "--paint", paint, "--panel", "demo", "--out", out], + stdout=subprocess.DEVNULL) + if r.returncode: + raise SystemExit(f"og ancestry-fine report failed (exit {r.returncode})") + dest = os.path.join(d, "ancestry_fine") + os.makedirs(dest, exist_ok=True) + for name in (f"{SAMPLE}_ancestry_fine.tsv", f"{SAMPLE}_painting.png"): + shutil.copy(os.path.join(out, name), os.path.join(dest, name)) def paralogs(d): @@ -1145,29 +1164,48 @@ def lpa(d, rng): put(os.path.join(d, "lpa", f"{SAMPLE}_lpa_kiv2.tsv"), LPA_TABLE) +CHIP_COLUMNS = ["gene", "chrom", "pos", "ref", "alt", "consequence", "protein_change", "exon", "vaf", + "alt_reads", "depth", "filter", "status", "class", "note"] +CHIP_DEPTH_COLUMNS = ["gene", "transcript", "chrom", "coding_bases", "mean_depth", "detection_limit_vaf", "status"] + + +def chip_limit(depth, k=3, power=0.8): + """og chip's detection limit: the smallest VAF that gives K or more reads + with the change with probability POWER at DEPTH (binomial, by bisection).""" + n = int(depth) + + def p_at_least(f): + return 1.0 - sum(math.comb(n, i) * f ** i * (1 - f) ** (n - i) for i in range(k)) + lo, hi = 0.0, 1.0 + for _ in range(60): + mid = (lo + hi) / 2 + lo, hi = (lo, mid) if p_at_least(mid) >= power else (mid, hi) + return hi + + def chip(d, rng): """Step 49: no CHIP call, and one depth row per gene of assets/chip_genes.tsv - at a made-up mean depth near 31, with bin/chip_report.py's own detection limit.""" - import chip_report - table = chip_report.read_table(os.path.join(REPO, "assets", "chip_genes.tsv")) + at a made-up mean depth near 31, with og chip's detection limit.""" + with open(os.path.join(REPO, "assets", "chip_genes.tsv"), encoding="utf-8") as f: + lines = [x for x in f if not x.startswith("#") and x.strip()] rows = [] - for r in table: + for gene in (x.split("\t")[0] for x in lines[1:]): mean = rng.uniform(29.0, 33.0) - rows.append({"gene": r["gene"], "transcript": "", "chrom": "", "coding_bases": str(rng.randint(1500, 7000)), - "mean_depth": f"{mean:.1f}", "detection_limit_vaf": f"{chip_report.detection_limit(mean):.3f}", + rows.append({"gene": gene, "transcript": "", "chrom": "", "coding_bases": str(rng.randint(1500, 7000)), + "mean_depth": f"{mean:.1f}", "detection_limit_vaf": f"{chip_limit(mean):.3f}", "status": "covered"}) calls = [] if has("chip"): - # One small DNMT3A clone, above the detection limit at this depth (bin/chip_report.py's columns). + # One small DNMT3A clone, above the detection limit at this depth (og chip's columns). calls.append({"gene": "DNMT3A", "chrom": "chr2", "pos": "25234373", "ref": "C", "alt": "T", "consequence": "missense_variant", "protein_change": "p.R882H", "exon": "23", "vaf": "0.161", "alt_reads": "8", "depth": "50", "filter": "PASS", "status": "chip", "class": "hotspot", "note": "a published CHIP hotspot"}) - put(os.path.join(d, "chip", f"{SAMPLE}_chip.tsv"), "\t".join(chip_report.OUT_COLUMNS) + "\n" - + "".join("\t".join(c[k] for k in chip_report.OUT_COLUMNS) + "\n" for c in calls)) + put(os.path.join(d, "chip", f"{SAMPLE}_chip.tsv"), "\t".join(CHIP_COLUMNS) + "\n" + + "".join("\t".join(c[k] for k in CHIP_COLUMNS) + "\n" for c in calls)) put(os.path.join(d, "chip", f"{SAMPLE}_chip_depth.tsv"), - "\t".join(chip_report.DEPTH_COLUMNS) + "\n" - + "".join("\t".join(x[c] for c in chip_report.DEPTH_COLUMNS) + "\n" for x in rows)) + "\t".join(CHIP_DEPTH_COLUMNS) + "\n" + + "".join("\t".join(x[c] for c in CHIP_DEPTH_COLUMNS) + "\n" for x in rows)) def archaic(d, rng): @@ -1287,9 +1325,10 @@ def exomiser(d): def traits(d): - """Step 42: TRAIT_ALT_COPIES at the panel's positions, read by bin/traits_report.py.""" - import traits_report - rows = traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv")) + """Step 42: TRAIT_ALT_COPIES at the panel's positions, read by og traits table.""" + with open(os.path.join(REPO, "assets", "traits.tsv"), encoding="utf-8") as f: + rows = list(csv.DictReader((x for x in f if x.strip() and not x.startswith("#")), delimiter="\t", + quoting=csv.QUOTE_NONE)) unknown = sorted(r["rsid"] for r in rows if r["rsid"] not in TRAIT_ALT_COPIES | {"rs4410790": 0, "rs4481887": 0}) if unknown: raise SystemExit(f"assets/traits.tsv has rows TRAIT_ALT_COPIES does not give: {', '.join(unknown)}") diff --git a/tests/e2e/callability-2-genes.sh b/tests/e2e/callability-2-genes.sh index 960555b3e..6fdf87717 100755 --- a/tests/e2e/callability-2-genes.sh +++ b/tests/e2e/callability-2-genes.sh @@ -39,12 +39,18 @@ check_ge "the MAPQ 0 copy has reads of MAPQ 0 in the window" \ # reading SAMPLE_DIR_TABLE: "gene status" lines of a per-gene table read with the test catalog reading() { - python3 - "$CAT" "$1" "${REPO}/bin" <<'PY' -import csv, sys -sys.path.insert(0, sys.argv[3]) -import collect_summary + # og summary collect's callability section over the table, with the test + # catalog as the assets' callability_genes.tsv. + local t + t=$(mktemp -d "${CASE_TMP}/reading.XXXXXX") + mkdir -p "${t}/S/mosdepth" "${t}/assets" + cp "$CAT" "${t}/assets/callability_genes.tsv" + ln -s "$1" "${t}/S/mosdepth/S_callability.bed.gz" + "${REPO}/bin/og" summary collect --sample S --sample-dir "${t}/S" --assets "${t}/assets" --out "${t}/summary.json" > /dev/null + python3 - "$CAT" "${t}/summary.json" <<'PY' +import csv, json, sys rows = list(csv.DictReader(open(sys.argv[1]), delimiter="\t")) -d = collect_summary.callability(rows, sys.argv[2]) +d = json.load(open(sys.argv[2]))["sections"]["callability"]["data"] bad = {g["gene"]: g for g in d["not_read"]} for r in rows: g = bad.get(r["gene"]) diff --git a/tests/e2e/fixitfelix-chr21-1.sh b/tests/e2e/fixitfelix-chr21-1.sh index dc60e7dda..15751896f 100644 --- a/tests/e2e/fixitfelix-chr21-1.sh +++ b/tests/e2e/fixitfelix-chr21-1.sh @@ -181,12 +181,18 @@ grep -v '^#' "${REPO}/assets/callability_genes.tsv" \ | awk -F'\t' 'NR == 1 || $1 == "KCNE1" || $1 == "CBS" || $1 == "U2AF1" || $1 == "CRYAA"' > "$CAT" check_eq "the four genes' rows of assets/callability_genes.tsv" "$(tail -n +2 "$CAT" | cut -f1 | sort | tr '\n' ' ')" "CBS CRYAA KCNE1 U2AF1 " reading() { - python3 - "$CAT" "$1" "${REPO}/bin" <<'PY' -import csv, sys -sys.path.insert(0, sys.argv[3]) -import collect_summary + # og summary collect's callability section over the table, with the test + # catalog as the assets' callability_genes.tsv. + local t + t=$(mktemp -d "${CASE_TMP}/reading.XXXXXX") + mkdir -p "${t}/S/mosdepth" "${t}/assets" + cp "$CAT" "${t}/assets/callability_genes.tsv" + ln -s "$1" "${t}/S/mosdepth/S_callability.bed.gz" + "${REPO}/bin/og" summary collect --sample S --sample-dir "${t}/S" --assets "${t}/assets" --out "${t}/summary.json" > /dev/null + python3 - "$CAT" "${t}/summary.json" <<'PY' +import csv, json, sys rows = list(csv.DictReader(open(sys.argv[1]), delimiter="\t")) -d = collect_summary.callability(rows, sys.argv[2]) +d = json.load(open(sys.argv[2]))["sections"]["callability"]["data"] bad = {g["gene"] for g in d["not_read"]} print(" ".join(f"{r['gene']}={'not_read' if r['gene'] in bad else 'read'}" for r in rows)) PY diff --git a/tests/fake-docker/hla-kir-toggle.sh b/tests/fake-docker/hla-kir-toggle.sh index 3ef604d32..8da798813 100755 --- a/tests/fake-docker/hla-kir-toggle.sh +++ b/tests/fake-docker/hla-kir-toggle.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash # Step 08 run with KIR=true, then again without it, as after a realignment: # the second run must not leave the first run's KIR genotype files, or the -# report (collect_summary.kir_genotypes reads kir_t1k/_kir_genotype.tsv -# or kir/_kir_genotype.tsv, where the Nextflow pipeline publishes it, -# whenever it exists) shows them as this run's. A run that stops early, with +# report (og summary collect reads kir_t1k/_kir_genotype.tsv or +# kir/_kir_genotype.tsv, where the Nextflow pipeline publishes it, +# whenever it exists: rust/crates/og-summary, the hla section's kir) shows them +# as this run's. A run that stops early, with # the HLA database removed or on a missing BAM index, must not leave them either. # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" @@ -52,10 +53,14 @@ mkdir -p "${G}/kir/IPD-KIR_${KIR_DB_RELEASE}" printf 'CC IPD-KIR Release Version %s\n' "$KIR_DB_RELEASE" > "${G}/kir/IPD-KIR_${KIR_DB_RELEASE}/kir.dat" K="${G}/sample1/kir_t1k" -# shown: what the report's summary reads for KIR (None = no KIR block). +# shown: True when a file the report's summary reads for KIR is there (the +# two paths of og summary collect's KIR reader, first one first). shown() { - python3 -c 'import sys; sys.path.insert(0, sys.argv[1]); import collect_summary as c; print(c.kir_genotypes(sys.argv[2], "sample1") is not None)' \ - "${REPO_ROOT}/bin" "${G}/sample1" + if [ -e "${G}/sample1/kir_t1k/sample1_kir_genotype.tsv" ] || [ -e "${G}/sample1/kir/sample1_kir_genotype.tsv" ]; then + echo True + else + echo False + fi } KIR=true run_expect 0 kir-on "${SCRIPTS}/08-hla-typing.sh" sample1 diff --git a/tests/test_acmg_sf_rules.py b/tests/test_acmg_sf_rules.py index 358e065a1..b3b07f6bd 100644 --- a/tests/test_acmg_sf_rules.py +++ b/tests/test_acmg_sf_rules.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""The ACMG SF v3.3 reporting rules (assets/acmg_sf.tsv, bin/collect_summary.py -acmg_sf_tier) and the one-person carrier list (assets/carrier_genes.tsv), on +"""The ACMG SF v3.3 reporting rules (assets/acmg_sf.tsv, og summary collect's +ACMG SF tier, rust/crates/og-summary) and the one-person carrier list (assets/carrier_genes.tsv), on synthetic sample folders. Checks: @@ -33,6 +33,7 @@ class 5 deletion in MUTYH (no SV rule) is not; 9. without the assets folder the ClinVar hits and the clinical counts stay, and both lists say why they were not made. """ +import csv import gzip import os import shutil @@ -102,10 +103,23 @@ def sf_of(d): def sf_of_clinvar(d): - """The SF tier of a folder with ClinVar hits only (no clinical VCF): the - Nextflow layout gives sec_clinical the clinical VCF, so the tier is read - straight from the hits here.""" - return collect_summary.acmg_sf_tier(d, "S", None) + """The SF tier of a folder with ClinVar hits and a clinical VCF with no + record (the Nextflow layout: the clinical section is the VCF's count, and + the tier comes from the hits alone).""" + v = f"{d}/clinical/S_clinical.vcf.gz" + if not os.path.exists(v): + put(v, VCF_HEAD, gz=True) + return sf_of(d) + + +def catalog(folder): + """The rows of FOLDER/acmg_sf.tsv, and {gene: rule} with its rule cells + read (the variants as collect_summary.parse_alleles reads them).""" + with open(os.path.join(folder, "acmg_sf.tsv"), encoding="utf-8") as f: + rows = list(csv.DictReader((x for x in f if not x.startswith("#")), delimiter="\t", quoting=csv.QUOTE_NONE)) + return rows, {r["gene"]: {"inheritance": r["inheritance"], "report_if": r["report_if"], + "table_text": r["table_text"], "sv_cn": r["sv_cn"] == "yes", + "variants": collect_summary.parse_alleles(r["variants"])} for r in rows} def reported(sf, gene): @@ -122,8 +136,7 @@ def main(): # 1. the catalog p = os.path.join(REPO, "assets", "acmg_sf.tsv") try: - rows = collect_summary.asset_rows("acmg_sf.tsv") - rules = collect_summary.acmg_sf_rules() + rows, rules = catalog(os.path.join(REPO, "assets")) except Exception as e: # the pre-fix state: no catalog, no loader rows, rules = [], {} check("catalog: assets/acmg_sf.tsv loads", False, f"{type(e).__name__}: {e}") @@ -153,12 +166,7 @@ def main(): if os.path.isfile(p): with open(p) as f, open(os.path.join(dropped, "acmg_sf.tsv"), "w") as g: g.write("".join(x for x in f if not x.startswith("TTN\t"))) - keep = collect_summary.ASSETS_DIR - collect_summary.ASSETS_DIR = dropped - try: - n = len(collect_summary.acmg_sf_rules()) - finally: - collect_summary.ASSETS_DIR = keep + n = len(catalog(dropped)[1]) check("catalog control: a copy without TTN has 83 genes, not 84", n == 83, n) # 2. MUTYH: chr1 MUTYH p.Gly396Asp and p.Tyr179Cys (GRCh38) @@ -287,12 +295,10 @@ def main(): # 9. without the assets folder keep = collect_summary.ASSETS_DIR collect_summary.ASSETS_DIR = os.path.join(work, "no-assets") - collect_summary._ASSETS.clear() try: summ = collect_summary.collect("S", d) finally: collect_summary.ASSETS_DIR = keep - collect_summary._ASSETS.clear() cv, cl = summ["sections"]["clinvar"], summ["sections"]["clinical"] check("no assets: the ClinVar hits stay", cv["state"] == "ok" and cv["data"]["count"] == 4, cv["state"]) check("no assets: both lists say why they were not made", diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index cffe87e77..9fbeb2ea9 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -23,7 +23,7 @@ 3. report: Altai and Vindija segments merge into Neanderthal ones, a Denisova segment that overlaps one is not counted as Denisovan, segments under 50 kb or LOD 4 are dropped, the share is of the analysed span, and - the genes of the longest segments are named; bin/collect_summary.py + the genes of the longest segments are named; og summary collect reads the summary and stops on a damaged one, and bin/render_report.py prints both shares; 4. with IBDmix itself (IBDMIX_IMAGE, docker): a planted 600 kb stretch where @@ -387,29 +387,30 @@ def table_info(out, extra=()): import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 + +def archaic(d): + """The archaic section og summary collect writes for folder D.""" + return cs.collect("S1", d)["sections"]["archaic"] + + sdir = f"{TMP}/sample" os.makedirs(f"{sdir}/archaic") shutil.copy(f"{TMP}/out/S1_archaic.tsv", f"{sdir}/archaic/S1_archaic.tsv") -path, data = cs.sec_archaic(sdir, "S1") -check("collect_summary reads the summary", path and data.get("neanderthal_percent") == "5.00" - and data.get("denisovan_segments") == 1 and len(data.get("longest", [])) == 2, data) -check("the archaic section is in SECTIONS as step 43", any(x[0] == "archaic" and x[2] == "43" for x in cs.SECTIONS)) +sec = archaic(sdir) +data = sec["data"] +check("og summary collect reads the summary", sec["state"] == "ok" and data.get("neanderthal_percent") == "5.00" + and data.get("denisovan_segments") == 1 and len(data.get("longest", [])) == 2, sec) +check("the archaic section is in SECTIONS as step 43", any(x[0] == "archaic" and x[2] == "43" for x in cs.SECTIONS) + and sec["step"] == "43") lines = rr.archaic_lines(data) check("the report lines give both shares", lines[0].startswith("Neanderthal: 5.00% (0.5 Mb in 1 segments)") and lines[1].startswith("Denisovan: 2.00%"), lines) write(f"{sdir}/archaic/S1_archaic.tsv", "key\tvalue\nneanderthal_percent\tlots\n") -try: - cs.sec_archaic(sdir, "S1") - check("a damaged summary is unreadable, not a share", False) -except cs.Unreadable: - check("a damaged summary is unreadable, not a share", True) +check("a damaged summary is unreadable, not a share", archaic(sdir)["state"] == "unreadable", archaic(sdir)) write(f"{sdir}/archaic/S1_archaic.tsv", "key\tvalue\nneanderthal_segments\n") -try: - cs.sec_archaic(sdir, "S1") - check("a row cut after its key is unreadable, not a crash", False) -except cs.Unreadable: - check("a row cut after its key is unreadable, not a crash", True) -check("no file: the section is missing", cs.sec_archaic(f"{TMP}/nowhere", "S1") == (None, {})) +check("a row cut after its key is unreadable, not a crash", archaic(sdir)["state"] == "unreadable", archaic(sdir)) +nowhere = archaic(f"{TMP}/nowhere") +check("no file: the section is missing", (nowhere["state"], nowhere["data"]) == ("missing", {}), nowhere) # --- 4. the real IBDmix ------------------------------------------------------------- NEA = (2000000, 2600000) diff --git a/tests/test_callability.py b/tests/test_callability.py index c0696d9b7..2fd1aace9 100644 --- a/tests/test_callability.py +++ b/tests/test_callability.py @@ -6,7 +6,7 @@ tests/e2e/callability-1-catalog.sh rebuilds it and compares). Step 16b counts, per gene, the bases with 10 or more reads of MAPQ 20 or more (_callability.bed.gz, mosdepth's thresholds table), and -bin/collect_summary.py turns that into the report's section. +og summary collect (rust/crates/og-summary) turns that into the report's section. 1. the catalog: its header, one row per gene, known reasons, an HGNC id per row, a position on chr1-22, X or Y unless ALT-only, and owners as the build script's rule gives them (assets/paralog_loci.tsv, @@ -33,6 +33,7 @@ import csv import gzip import os +import shutil import subprocess import sys import tempfile @@ -117,9 +118,22 @@ def write_thresholds(path, catalog, zero=(), drop=()): # --- 2 to 4. the reader ---------------------------------------------------------------- +def og_callability(catalog, table): + """The callability data og summary collect reads from TABLE as step 16b's + table, in a sample folder of its own (og reads the repository's catalog, + which is CATALOG).""" + assert catalog is cat + sd = tempfile.mkdtemp(dir=os.path.dirname(table)) + os.makedirs(os.path.join(sd, "mosdepth")) + shutil.copy(table, os.path.join(sd, "mosdepth", "S_callability.bed.gz")) + sec = collect_summary.collect("S", sd)["sections"]["callability"] + return sec["data"] if sec["state"] == "ok" else {"not_read": [], "not_read_owned": [], "read": 0, + "not_on_reference": [], "state": sec} + + def alt_read_bug(catalog, table): """The planted bug: an ALT-only gene taken as read.""" - d = collect_summary.callability(catalog, table) + d = og_callability(catalog, table) d["not_read"] = [g for g in d["not_read"] if g["status"] != "not on this reference"] d["read"] += len(d["not_on_reference"]) return d @@ -160,12 +174,12 @@ def count_matches(reader, tsv): write_thresholds(tsv, cat, zero={MAPQ0, OWNED}) check("2. control: a reader that takes an ALT-only gene as read fails check 2", bool(mapq0_and_alt(alt_read_bug, tsv)), "the planted bug passed") - problems = mapq0_and_alt(collect_summary.callability, tsv) + problems = mapq0_and_alt(og_callability, tsv) check("2. a gene of MAPQ 0 reads is not read, a unique one is read, an ALT-only one is never read", not problems, problems) check("3. control: the planted bug's list differs from the catalog query", not count_matches(alt_read_bug, tsv)[0]) - ok, d = count_matches(collect_summary.callability, tsv) + ok, d = count_matches(og_callability, tsv) check("3. the genes not read with no owner are the catalog query's", ok, d["not_read"][:5]) check("3. owned genes not read are listed apart (HBA1 by the HBA count), GSTT1 is not one of them", any(g["gene"] == OWNED and g["owner"] == "cn_loci:HBA" for g in d["not_read_owned"]) @@ -209,7 +223,7 @@ def count_matches(reader, tsv): # og cn finds bin/cn_loci.tsv beside its own binary. Guard's unit-tests job # builds og in a temporary folder, so the tests run a copy that sits next # to the catalog, as bin/.og/og-* does in a checkout. - import atexit, shutil # noqa: E401 + import atexit # noqa: E401 _og_dir = tempfile.mkdtemp(prefix="og-cn-") atexit.register(shutil.rmtree, _og_dir, True) shutil.copy2(OG, os.path.join(_og_dir, "og")) diff --git a/tests/test_chip_report.py b/tests/test_chip_report.py index da506988c..42340069d 100644 --- a/tests/test_chip_report.py +++ b/tests/test_chip_report.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 """Step 49 (clonal haematopoiesis): og chip on synthetic bcftools output and on -assets/chip_genes.tsv, and what bin/chip_report.py still holds for the reports. +assets/chip_genes.tsv, and what the reports read of it. Without og (OG unset) only part A runs: - A. bin/chip_report.py's detection limit: about 14% at 30x (binomial, 3 - reads, 8 in 10), lower at 60x, none under 3x; the summary reader and - the report lines show the call and the limit (bin/collect_summary.py - sec_chip, bin/render_report.py chip_lines), the median limit included. + A. the detection limit, computed here as an independent binomial (3 + reads, 8 in 10): about 14% at 30x, lower at 60x, none under 3x; the + summary (og summary collect, which needs OG; without it part A stops + there) and the report lines show the call and the limit + (bin/render_report.py chip_lines), the median limit included. With OG naming the og binary, also: 1. og chip check-table passes assets/chip_genes.tsv; a malformed class token, a gene listed twice, a gene that counts no class and a bad @@ -23,7 +24,7 @@ 5. the depth table reads a base with no pileup line as depth 0, and a gene off the reference as not_on_reference; the summary sentence says the limit, that CHIP starts at 2%, that a negative result does not rule a - clone out, and that it is not a diagnosis; og's limit is the Python's; + clone out, and that it is not a diagnosis; og's limit is the binomial's; 6. prepare picks the MANE Select transcript of a synthetic Ensembl GFF3 (not the other one), renames the contigs (2 -> chr2), pads the coding exons by 10 bases, leaves out a contig the reference lacks, and stops on a gene @@ -32,6 +33,7 @@ Run: OG=rust/target/release/og python3 tests/test_chip_report.py """ +import math import os import subprocess import sys @@ -39,11 +41,14 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import chip_report as cr # noqa: E402 import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 TABLE = os.path.join(REPO, "assets", "chip_genes.tsv") +# The two tables og chip report writes, in column order. +OUT_COLUMNS = ["gene", "chrom", "pos", "ref", "alt", "consequence", "protein_change", "exon", "vaf", + "alt_reads", "depth", "filter", "status", "class", "note"] +DEPTH_COLUMNS = ["gene", "transcript", "chrom", "coding_bases", "mean_depth", "detection_limit_vaf", "status"] OG = os.environ.get("OG", "") FAILS = 0 @@ -66,29 +71,65 @@ def og(*args): return subprocess.run([OG, "chip", *args], capture_output=True, text=True) +def p_at_least(k, n, f): + """P(X >= k) for X ~ Binomial(n, f).""" + return 1.0 - sum(math.comb(n, i) * f ** i * (1 - f) ** (n - i) for i in range(k)) + + +def detection_limit(depth, k=3, power=0.8): + """The smallest VAF found with probability POWER at DEPTH (K reads needed), by + bisection; None below K reads. The test's own oracle for og chip's limit.""" + n = int(depth) + if n < k: + return None + lo, hi = 0.0, 1.0 + for _ in range(60): + mid = (lo + hi) / 2 + if p_at_least(k, n, mid) >= power: + hi = mid + else: + lo = mid + return hi + + +def table_genes(path): + """The genes of assets/chip_genes.tsv.""" + with open(path) as fh: + lines = [ln for ln in fh if not ln.startswith("#") and ln.strip()] + return [ln.split("\t")[0] for ln in lines[1:]] + + +def chip_section(d, sample): + """The chip section's data og summary collect reads from folder D.""" + if not OG: + print("skip: OG is not set (OG=rust/target/release/og): the summary needs og summary collect") + sys.exit(1 if FAILS or os.environ.get("OG_REQUIRED") else 0) + return cs.collect(sample, d)["sections"]["chip"]["data"] + + def chip_tables(d, sample, calls, depth): """A sample folder with the two tables, as og chip report writes them.""" os.makedirs(os.path.join(d, sample, "chip")) write(os.path.join(d, sample, "chip"), f"{sample}_chip.tsv", - "\t".join(cr.OUT_COLUMNS) + "\n" + "".join("\t".join(c) + "\n" for c in calls)) + "\t".join(OUT_COLUMNS) + "\n" + "".join("\t".join(c) + "\n" for c in calls)) write(os.path.join(d, sample, "chip"), f"{sample}_chip_depth.tsv", - "\t".join(cr.DEPTH_COLUMNS) + "\n" + "".join("\t".join(r) + "\n" for r in depth)) + "\t".join(DEPTH_COLUMNS) + "\n" + "".join("\t".join(r) + "\n" for r in depth)) return os.path.join(d, sample) -# A. what Python keeps -lim30, lim60 = cr.detection_limit(30), cr.detection_limit(60) +# A. the detection limit and what the reports read +lim30, lim60 = detection_limit(30), detection_limit(60) check("the 30x detection limit is between 13% and 16%", 0.13 < lim30 < 0.16, lim30) -check("P(>=3 reads) at that VAF is 0.8", abs(cr.p_at_least(3, 30, lim30) - 0.8) < 1e-6, cr.p_at_least(3, 30, lim30)) +check("P(>=3 reads) at that VAF is 0.8", abs(p_at_least(3, 30, lim30) - 0.8) < 1e-6, p_at_least(3, 30, lim30)) check("the 60x limit is lower", lim60 < lim30, (lim60, lim30)) -check("no limit below 3x", cr.detection_limit(2.9) is None) +check("no limit below 3x", detection_limit(2.9) is None) with tempfile.TemporaryDirectory() as d: call = ["DNMT3A", "chr2", "1150", "C", "T", "missense", "R882H", "2", "0.130", "6", "46", "PASS", "chip", "missense", "hotspot R882"] s = chip_tables(d, "S", [call], [["DNMT3A", "T2", "chr2", "392", "30.0", f"{lim30:.3f}", "covered"], ["JAK2", "T3", "chr9", "0", "", "", "not_on_reference"]]) - _, sec = cs.sec_chip(s, "S") - check("sec_chip reads the chip call and the median limit", + sec = chip_section(s, "S") + check("the summary reads the chip call and the median limit", [c["protein_change"] for c in sec.get("calls", [])] == ["R882H"] and abs(sec["median_limit"] - 0.137) < 0.001 and sec["not_covered"] == ["JAK2"], sec) text = "\n".join(rr.chip_lines(sec, "S")) @@ -96,13 +137,13 @@ def chip_tables(d, sample, calls, depth): "DNMT3A R882H: VAF 0.130 (6 of 46 reads)" in text and "about 14% or more" in text and "JAK2" in text, text) # With an even count of covered genes the median depth is the mean of the two # middle depths, and the reported limit must be the one at that depth. - e = chip_tables(d, "E", [], [[f"G{x}", "T", "chr1", "10", f"{x}.0", f"{cr.detection_limit(x):.4f}", "covered"] + e = chip_tables(d, "E", [], [[f"G{x}", "T", "chr1", "10", f"{x}.0", f"{detection_limit(x):.4f}", "covered"] for x in (20, 30, 60, 80)]) - _, sec4 = cs.sec_chip(e, "E") + sec4 = chip_section(e, "E") check("with 4 covered genes (20, 30, 60, 80x) the limit is the one at the median 45x", - sec4.get("median_depth") == 45 and abs(sec4["median_limit"] - cr.detection_limit(45)) < 1e-9, sec4) + sec4.get("median_depth") == 45 and abs(sec4["median_limit"] - detection_limit(45)) < 1e-9, sec4) check("no call reads 'No CHIP call.'", "No CHIP call." in rr.chip_lines({"calls": [], "median_limit": None}, "S")) - check("the table reader reads assets/chip_genes.tsv", "DNMT3A" in [r["gene"] for r in cr.read_table(TABLE)]) + check("assets/chip_genes.tsv holds DNMT3A", "DNMT3A" in table_genes(TABLE)) if not OG: print("skip: OG is not set (OG=rust/target/release/og): only part A ran") @@ -213,15 +254,15 @@ def status_of(c): p = og("report", "--genes", TABLE, "--gff", GFF_PATH, "--bed", bed, "--calls", EMPTY, "--depth", dep, "--sample", "S", "--out", out, "--depth-out", dout) lines = open(dout).read().splitlines() if p.returncode == 0 else [] - rows = {ln.split("\t")[0]: dict(zip(cr.DEPTH_COLUMNS, ln.split("\t"))) for ln in lines[1:]} + rows = {ln.split("\t")[0]: dict(zip(DEPTH_COLUMNS, ln.split("\t"))) for ln in lines[1:]} check("a base with no pileup line counts as depth 0 (mean 15)", rows.get("DNMT3A", {}).get("mean_depth") == "15.0", rows.get("DNMT3A")) check("a gene with no reads is too_shallow", rows.get("TET2", {}).get("status") == "too_shallow" and rows["TET2"]["mean_depth"] == "0.0", rows.get("TET2")) check("a gene off the reference is not_on_reference", rows.get("JAK2", {}).get("status") == "not_on_reference", rows.get("JAK2")) - check("every table gene has a depth row", sorted(rows) == sorted(r["gene"] for r in cr.read_table(TABLE))) - check("og's 15x limit is the Python's", rows.get("DNMT3A", {}).get("detection_limit_vaf") == f"{cr.detection_limit(15):.3f}", + check("every table gene has a depth row", sorted(rows) == sorted(table_genes(TABLE))) + check("og's 15x limit is the binomial's", rows.get("DNMT3A", {}).get("detection_limit_vaf") == f"{detection_limit(15):.3f}", rows.get("DNMT3A")) s = p.stdout.splitlines()[-1] if p.stdout else "" for want in ("about 26%", "15x", "2%", "does not rule", "Not a diagnosis"): @@ -274,11 +315,11 @@ def status_of(c): and "about 14%" in p3.stdout, p3.stdout + p3.stderr) lines = open(out).read().splitlines() if p3.returncode == 0 else [] check("the calls table has the columns in order and one chip row", - lines[:1] == ["\t".join(cr.OUT_COLUMNS)] and len(lines) == 2 and "\tchip\t" in lines[1], lines) + lines[:1] == ["\t".join(OUT_COLUMNS)] and len(lines) == 2 and "\tchip\t" in lines[1], lines) dl = open(dout).read().splitlines() if p3.returncode == 0 else [] check("the depth table has DNMT3A at 30x and JAK2 not on the reference", len(dl) == 3 and "\t30.0\t" in dl[1] and dl[2].endswith("not_on_reference"), dl) - _, sec = cs.sec_chip(os.path.join(d, "S"), "S") + sec = chip_section(os.path.join(d, "S"), "S") check("the reports read og's tables", [c["protein_change"] for c in sec.get("calls", [])] == ["R882H"] and sec["not_covered"] == ["JAK2"], sec) p4 = og("report", "--genes", genes, "--gff", out_gff, "--bed", out_bed, "--calls", os.path.join(d, "none.tsv"), diff --git a/tests/test_clip_check.py b/tests/test_clip_check.py deleted file mode 100644 index a6ba6b233..000000000 --- a/tests/test_clip_check.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -"""bin/clip_check.py: what is left of it after og cdna check took the read -check (rust/crates/og-bam, its unit tests and parity cases): the names -bin/collect_summary.py reads beside step 06's and step 17's tables. - - - norm_class spells every CPSR and ClinVar class one way; - - cpsr_variant reads VAR_ID, else GENOMIC_CHANGE, else nothing; - - COLUMNS and FLAG_NOTE are what og cdna check writes (the header and the - note of a flagged row of an og-bam parity golden). - -Run: python3 tests/test_clip_check.py -""" -import os -import sys - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import clip_check as cc # noqa: E402 - -FAILS = [] - - -def expect(desc, got, want): - if got == want: - print(f"PASS {desc}") - else: - print(f"FAIL {desc}: got {got!r}, want {want!r}") - FAILS.append(desc) - - -def main(): - expect("norm_class: underscores, case and spaces", cc.norm_class(" Likely_Pathogenic "), "likely pathogenic") - expect("norm_class: LP", cc.norm_class("LP"), "likely pathogenic") - expect("norm_class: NA and None are empty", (cc.norm_class("NA"), cc.norm_class(None)), ("", "")) - expect("norm_class: a class it does not know stays", cc.norm_class("VUS"), "vus") - expect("PLP", cc.PLP, ("pathogenic", "likely pathogenic")) - expect("cpsr_variant: VAR_ID", cc.cpsr_variant({"VAR_ID": "13_32340300_G_A"}), ("13", 32340300, "G", "A")) - expect("cpsr_variant: a contig with an underscore", cc.cpsr_variant({"VAR_ID": "HLA_A_5_C_T"}), - ("HLA_A", 5, "C", "T")) - expect("cpsr_variant: GENOMIC_CHANGE when VAR_ID does not read", - cc.cpsr_variant({"VAR_ID": ".", "GENOMIC_CHANGE": "2:g.47800000A>G"}), ("2", 47800000, "A", "G")) - expect("cpsr_variant: neither", cc.cpsr_variant({"GENOMIC_CHANGE": "2:g.A>G"}), None) - golden = os.path.join(REPO, "rust/crates/og-bam/tests/data/reads/want/check-spliced/files/out.tsv") - with open(golden) as fh: - lines = [line.rstrip("\n").split("\t") for line in fh] - expect("COLUMNS: the header og cdna check writes", lines[0], cc.COLUMNS) - expect("FLAG_NOTE: the note of its flagged rows", {r[9] for r in lines[1:] if r[8] == "flagged"}, {cc.FLAG_NOTE}) - if FAILS: - print(f"{len(FAILS)} check(s) failed") - return 1 - print("all checks passed") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index c5ffcc667..99be691f1 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -212,13 +212,33 @@ def og_postfilter(args): sys.path.insert(0, os.path.join(REPO, "bin")) sys.path.insert(0, os.path.join(REPO, "tests")) sys.path.insert(0, os.path.join(REPO, "tests", "schema")) -import clip_check # noqa: E402 import collect_summary # noqa: E402 import render_report # noqa: E402 import validate # noqa: E402 from og_binary import og_binary # noqa: E402 +def cdna_table(): + """The header og cdna check writes and the note of its flagged rows, from + an og-bam golden (rust/crates/og-bam/tests/data/reads).""" + with open(os.path.join(REPO, "rust/crates/og-bam/tests/data/reads/want/check-spliced/files/out.tsv")) as f: + rows = [line.rstrip("\n").split("\t") for line in f] + return rows[0], {r[9] for r in rows[1:] if r[8] == "flagged"}.pop() + + +CDNA_COLUMNS, CDNA_FLAG_NOTE = cdna_table() +# The columns of step 42's traits table (og traits panel). +TRAITS_OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", + "note", "pmids"] + + +def traits_rows(): + """The rows of assets/traits.tsv, step 42's panel, as dicts.""" + with open(os.path.join(REPO, "assets", "traits.tsv"), encoding="utf-8") as f: + return list(csv.DictReader((x for x in f if x.strip() and not x.startswith("#")), delimiter="\t", + quoting=csv.QUOTE_NONE)) + + def og_tool(*args): """og's exit code for a step tool that once lived in collect_summary.py (og qc sample, og prs table, og mito annotate): they write the tables @@ -256,6 +276,16 @@ def put(path, text, gz=False, age_days=0): os.utime(path, (t, t)) +# gzip's magic and header flags, then no deflate stream: not readable as gzip. +BROKEN_GZ = b"\x1f\x8b\x08\x00not gzip\n" + + +def put_bytes(path, data): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(data) + + VCF_HEAD = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" HITS = VCF_HEAD + ( "chr1\t10\t1\tC\tT\t50\tPASS\tGENEINFO=GENEA:11;CLNSIG=Pathogenic;CLNREVSTAT=criteria_provided,_single_submitter\tGT\t0/1\n" @@ -437,6 +467,15 @@ def render(d, s="S"): return summ, render_report.text_report(summ), render_report.html_report(summ) +def section(d, key, s="S", sex=None): + """One section of the summary og summary collect writes for folder D.""" + return collect_summary.collect(s, d, sex)["sections"][key] + + +# The five repeat loci the expansions section always lists, in this order. +EH_LOCI = ["HTT", "FMR1", "C9ORF72", "ATXN1", "DMPK"] + + def bgzf_block(data): """One BGZF block (a gzip member with the BC extra field) holding data.""" c = zlib.compressobj(6, zlib.DEFLATED, -15) @@ -449,7 +488,7 @@ def write_tabixed_vcf(path, header, blocks): """A bgzipped VCF of one BGZF block per item of blocks (lists of record lines, the header in the first block) and its .tbi: bin 0 of each sequence spans its lines (og mito annotate reads the bins), and the - linear index (bin/collect_summary.py reads that).""" + linear index (og summary collect reads that).""" raw, lin, span, coff = b"", {}, {}, 0 for i, lines in enumerate(blocks): data = (header if i == 0 else "").encode() @@ -505,7 +544,7 @@ def construct_scan_section(work, schema): put(f"{d}/clinvar/S_clinvar_hits.vcf", VCF_HEAD + ( f"chr7\t117548628\t1\tGTTT\tG\t50\tPASS\t{cftr}\tGT:GQ:DP\t1/.:45:30\n" f"chr21\t25897626\t2\tTC\tGA\t50\tPASS\t{app}\tGT:GQ:DP\t0/1:50:35\n")) - put(f"{d}/clinvar/S_clinvar_cdna_check.tsv", "\t".join(clip_check.COLUMNS) + "\n" + put(f"{d}/clinvar/S_clinvar_cdna_check.tsv", "\t".join(CDNA_COLUMNS) + "\n" "chr7\t117548628\tGTTT\tG\t\t\t\t\tnot_checked\tnot a SNV, MNV or simple indel\t\n" "chr21\t25897626\tTC\tGA\t12\t0\t10\t0\tclear\t\tAPP Swedish K670N/M671L\n") put(f"{d}/clinvar/S_clinvar_cdna_scan.tsv", "# cdna_scan: read 3 exon regions on 1 contigs; 1 loci listed\n" @@ -526,7 +565,6 @@ def construct_scan_section(work, schema): check("split poly-T: both reports print that wording, neither prints it as het", f"{half} Pathogenic" in txt and f"" in html and "chr7:117548628 GTTT>G het" not in txt, [l for l in txt.splitlines() if "117548628" in l]) - check("split poly-T: one copy of two (not hemizygous)", collect_summary.allele_copies("1/.") == (1, 2)) carriers = {x["gene"]: x for x in summ["sections"]["carrier"]["data"].get("carriers", [])} check("split poly-T: the carrier list words it the same way", carriers.get("CFTR", {}).get("genotype") == half, summ["sections"]["carrier"]["data"]) @@ -654,15 +692,12 @@ def clinvar_quality(work, schema): og_z = og_clinvar("hits", f"{z}/clinvar/S_clinvar_hits.vcf") check("call floor: a floor of GQ 0 and DP 0 marks nothing, so the mark above depends on the floor", cz["low_confidence"] == 0 and cz["count"] == 4 and "\tyes" not in (og_z or ""), (cz["low_confidence"], og_z)) - check("call floor: a hits file without the floor line uses GQ 30 and DP 10", - collect_summary.call_floor(f"{work}/cdna/S/clinvar/S_clinvar_hits.vcf") == {"GQ": 30, "DP": 10}) af = {g: (by[g]["af"], by[g]["common"]) for g in by} check("gnomAD AF: read through the tabix index from both BGZF blocks; above 1% is common; no VEP record is unknown", af == {"GENEA": (0.03, True), "GENEB": (0.0002, False), "GENEC": (0.0001, False), "GENED": (None, False)} and (by["GENEA"]["gnomad_exome_af"], by["GENEA"]["gnomad_genome_af"]) == (0.03, 0.02), af) - got_lin = collect_summary.tabix_linear_index(f"{d}/vep/S_vep.vcf.gz") - check("tabix: the linear index is read as written (chr2 starts in the second BGZF block)", - got_lin == lin and got_lin["chr2"][0] >> 16 > 0 and len(got_lin["chr1"]) == 3, (got_lin, lin)) + check("tabix: the test's index puts chr2 in the second BGZF block, so the AFs above came through it", + lin["chr2"][0] >> 16 > 0 and len(lin["chr1"]) == 3, lin) check("gnomAD AF: the summary names the VEP file", cv["frequency"]["source"] == "vep/S_vep.vcf.gz" and cv["frequency"]["note"] is None, cv["frequency"]) check("both reports: the AF and the common mark on the hit, unknown on the hit VEP lacks", @@ -741,9 +776,9 @@ def cpsr_beside_clinvar(work, schema): put(f"{d}/cpsr/S.cpsr.grch38.classification.tsv.gz", CPSR_COLS + CPSR_ROWS, gz=True) put(f"{d}/cpsr/S.cpsr.grch38.html", "\n") put(f"{d}/clinvar/S_clinvar_hits.vcf", FLOOR_HITS) - put(f"{d}/cpsr/S_cpsr_cdna_check.tsv", "\t".join(clip_check.COLUMNS) + "\n" + put(f"{d}/cpsr/S_cpsr_cdna_check.tsv", "\t".join(CDNA_COLUMNS) + "\n" "13\t32340300\tG\tA\t12\t0\t10\t0\tclear\t\n" - f"2\t47800000\tA\tG\t9\t8\t11\t0\tflagged\t{clip_check.FLAG_NOTE}\n") + f"2\t47800000\tA\tG\t9\t8\t11\t0\tflagged\t{CDNA_FLAG_NOTE}\n") put(f"{d}/run_manifest.tsv", "data\tclinvar_file_date\t20261001\ndata\tcpsr_clinvar_release\t20250309\n") summ, txt, html = render(d) c = summ["sections"]["cpsr"]["data"] @@ -861,11 +896,6 @@ def ranked(keep): rc == 0 and sorted(tags_nc) == [100, 200, 300, 400, 500, 700], sorted(tags_nc)) else: print("[NOTE] OG is not set, so og vcf csq-rank does not run here; its own tests pin these tags") - agree = collect_summary.csq_high_impact_genes(f"{d}/clinical/S_clinical.vcf.gz") - check("the summary agrees with og vcf csq-rank on which records are HIGH and which calls are weak", - {p_ for p_, t in want.items() if t[0] == "HIGH"} == {k[1] for k in agree} - and {p_ for p_, t in want.items() if t[3] == "weak"} == {k[1] for k, x in agree.items() if x["weak"]}, - (want, agree)) summ, txt, html = render(d) sf = summ["sections"]["clinical"]["data"]["acmg_sf"] by = {x["gene"]: x for x in sf["high_impact"]} @@ -1135,7 +1165,7 @@ def annotsv_section(work, schema): check("annotsv: without step 22 the summary validates", not validate.validate(schema, json.loads(json.dumps(summ))), validate.validate(schema, json.loads(json.dumps(summ)))) # a corrupt consensus: the rows stay, the note names the file - put(f"{d}/sv_merged/S_sv_consensus.vcf.gz", "not gzip") + put_bytes(f"{d}/sv_merged/S_sv_consensus.vcf.gz", BROKEN_GZ) sec = render(d)[0]["sections"]["annotsv"] check("annotsv: an unreadable consensus keeps the rows and says so", sec["state"] == "ok" and sec["data"]["count"] == 3 and "could not be read" in (sec["data"]["consensus_note"] or ""), @@ -1173,7 +1203,7 @@ def run(name, gt): "CLNREVSTAT=criteria_provided,_multiple_submitters,_no_conflicts\tGT:GQ:DP:PS\t1|0:50:30:45330000\n") with open(f"{d}/annotsv/S_sv_class4_5_checked.tsv") as f: head = f.readline().rstrip("\n").split("\t") - two = (collect_summary.sec_carrier(d, "S")[1].get("two_hits") or {}).get("genes", []) + two = (section(d, "carrier")["data"].get("two_hits") or {}).get("genes", []) return head, [x for x in two if x["gene"] == "MUTYH"] head, t = run("trans", "0|1") @@ -1186,14 +1216,10 @@ def run(name, gt): check("sv phase: a deletion genotyped 0/0 pairs with nothing", t == [], t) # A phased GT with one copy not called says nothing about cis or trans. - check("sv phase: '.|1' gives no copy, '0|1' gives copy 1", - (collect_summary.phase_haps(".|1"), collect_summary.phase_haps("0|1")) == (None, frozenset({1})), - (collect_summary.phase_haps(".|1"), collect_summary.phase_haps("0|1"))) - dl = {"inside": False, "chrom": "chr1", "start": 100, "end": 200, "gt": ".|1", "ps": "90"} - v = {"chrom": "chr1", "pos": 300, "gt": "0|1", "ps": "90", "copies": 1, "ploidy": 2} - verdict, why = collect_summary.pair_phase(dl, v) - check("sv phase: a deletion '.|1' and a hit '0|1' in one phase set are not called in cis", - verdict == "possibly biallelic" and "in cis" not in why, (verdict, why)) + _, t = run("halfcall", ".|1") + check("sv phase: a deletion '.|1' and a hit '1|0' in one phase set: phase unknown, not called in cis", + len(t) == 1 and t[0]["verdict"] == "possibly biallelic" and t[0]["why"].startswith("phase unknown") + and "in cis" not in t[0]["why"], t) def report_followups_sections(work, schema): @@ -1201,11 +1227,10 @@ def report_followups_sections(work, schema): flagged motif loci and the loci the repeat catalog leaves without thresholds reach both reports.""" import html as htmlmod - import traits_report # The genotype rules: TAS2R38 homozygous for the taster haplotype PAV # (G, G, C on the plus strand; rs1726866's G is the reference base). t = os.path.join(work, "rules", "S") - rows = traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv")) + rows = traits_rows() g = os.path.join(work, "rules_g.tsv") put(g, "chr7\t141972804\tT\tC\tC/C\t40\nchr7\t141972905\tG\t.\tG/G\t40\nchr7\t141973545\tC\tG\tG/G\t40\n" "chr19\t44908684\tT\t.\tT/T\t40\nchr19\t44908822\tC\t.\tC/C\t40\n") @@ -1315,7 +1340,7 @@ def made_up(name, tg, t, ugt, r117h=None, ugt_pc=None, stranger=False, traits_te {"LocusId": "CFTR", "NoThresholds": "the CFTR tract"}, {"LocusId": "UGT1A1", "NoThresholds": "the UGT1A1 repeat"}])) if r117h or traits_text: - put(f"{x}/traits/{name}_traits.tsv", traits_text or "\t".join(collect_summary.traits_report.OUT_COLUMNS) + put(f"{x}/traits/{name}_traits.tsv", traits_text or "\t".join(TRAITS_OUT_COLUMNS) + "\n" + f"rs78655421\tCFTR\tchr7\t117530975\tCFTR R117H\t{r117h}\tcalled\tr\tno\tn\t32404922\n", age_days=side_age) if ugt_pc: @@ -1763,7 +1788,7 @@ def parascopy_section(work, schema): and (pd.get("hba") or {}).get("alpha_globin_genes") == "4", pd.get("loci")) bound = os.path.join(g, "paralogs", "S_notch2_copy_number.tsv") put(bound, SMN_COLS + "chr1\t120000000\t120100000\tNOTCH2\tPASS\t<5\t30\t.\t.\t.\t*\n") - lb = collect_summary.parascopy_locus("notch2", bound, None) + lb = {x["table"]: x for x in section(g, "parascopy")["data"].get("loci", [])}.get("notch2", {}) check("loci: a bound such as <5 is not resolved, and the note names the whole-number rule", lb["status"] == "not resolved" and lb["cn"] is None and lb["text"].endswith(": no region with a whole copy number, filter PASS and quality 20 or more"), lb) @@ -1905,16 +1930,8 @@ def hla_optitype_kir_section(work, schema): def report_sections_mito_roh_telomere_cnv(work, schema): """ROH on autosomes outside the bands, the telomere read floor, CNVpytor's filtered count, the D-loop poly-C tracts and the chrM annotation.""" - import re as _re - # The masked bands: og's step 11 code holds the one copy that step 11 - # and the ROH module run (og roh summary); the summary's line agrees. - rs = open(os.path.join(REPO, "rust/crates/og-summary/src/collect/group_c/roh.rs")).read() - body = rs[rs.index("pub const MASKED_BANDS"):] - body = body[:body.index("];")] - rust_bands = " ".join(f"{c}:{int(a.replace('_', ''))}-{int(b.replace('_', ''))}" - for c, a, b in _re.findall(r'\("(chr\w+)", ([0-9_]+), ([0-9_]+)\)', body)) - check("bands: og's MASKED_BANDS is the line of bin/collect_summary.py", - rust_bands == collect_summary.MASKED_BANDS, rust_bands[:80]) + # The masked bands: og's step 11 code holds the one copy that step 11, + # the ROH module (og roh summary) and the summary run. for f in ("scripts/11-roh-analysis.sh", "modules/local/roh/main.nf"): text = open(os.path.join(REPO, f)).read() check(f"bands: {f} holds no copy of the bands and runs og roh summary", @@ -1943,15 +1960,6 @@ def report_sections_mito_roh_telomere_cnv(work, schema): "Left out in centromere and heterochromatin bands: 46.5 MB" in txt and "No population allele frequencies were used" in txt and "Segments on chrX/chrY, not counted: 1" in txt and "No population allele" in html, txt) - # Red control: masked_bands() with no bands, so nothing is masked. - orig = collect_summary.masked_bands - collect_summary.masked_bands = lambda text="": orig("") - try: - r0 = collect_summary.sec_roh(d, "S")[1] - finally: - collect_summary.masked_bands = orig - check("ROH red control: without the band mask the gvar segment is the largest again", - r0["largest_mb"] == 25.0 and len(r0["autosomal_over_5mb"]) == 4, r0) # Step 11's summary (og roh summary) on the same file lists the same segments. if OG: out = subprocess.run([OG, "roh", "summary", "--sample", "S", "--roh", f"{d}/vcf/S_roh.txt", @@ -1976,16 +1984,8 @@ def report_sections_mito_roh_telomere_cnv(work, schema): t["low_reads"] is low and t["depth"] == 30.0 and t["read_floor"] == 2000 and warned is low and ("too few telomeric reads" in html) is low and f"Intratelomeric reads: {n:,}" in txt, (t, txt)) put(f"{d}/telomere/S/S/S_summary.tsv", head + "S\ttumor\t620000000\t150\tn=7\t10\t40\t48-52\t150000000\t1.5\n") - saved = collect_summary.TELOMERE_FLOOR_PER_30X - collect_summary.TELOMERE_FLOOR_PER_30X = 0 - try: - t0 = collect_summary.sec_telomere(d, "S")[1] - finally: - collect_summary.TELOMERE_FLOOR_PER_30X = saved - check("telomere red control: with the floor at 0 the 40-read sample is not flagged, so the warning comes from it", - t0["low_reads"] is False, t0) put(f"{d}/telomere/S/S/S_summary.tsv", "PID\tsample\tintratelomeric_reads\ttel_content\nS\ttumor\t61873\t334\n") - t1 = collect_summary.sec_telomere(d, "S")[1] + t1 = section(d, "telomere")["data"] check("telomere: no total_reads or read length gives the count and says the check did not run", t1.get("intratelomeric_reads") == 61873 and "low_reads" not in t1 and "not checked" in t1["read_check"], t1) @@ -2040,18 +2040,6 @@ def report_sections_mito_roh_telomere_cnv(work, schema): in txt and "Length heteroplasmy in the D-loop poly-C tracts" in html, txt) check("mito: no annotation table says so, and why", "ClinVar and gnomAD: not annotated (no annotation table" in txt and "MITO_VARIANTS" in txt, txt) - saved = collect_summary.POLYC_TRACTS - collect_summary.POLYC_TRACTS = () - try: - m0 = collect_summary.sec_mito(d, "S")[1] - finally: - collect_summary.POLYC_TRACTS = saved - check("mito red control: without the tracts the count is 3 again", m0["heteroplasmic"] == 3, m0) - check("mito red control: without the tracts the poly-C calls are listed sites too, so the check above can fail", - len(m0["heteroplasmic_sites"]) == 3 and m0["heteroplasmic_sites"] != mi["heteroplasmic_sites"], m0) - check("mito: an SNV just before a tract is not in it, an insertion anchored there is", - not collect_summary.in_polyc(302, "A", "G") and collect_summary.in_polyc(302, "A", "AC") - and collect_summary.in_polyc(315, "C", "T") and not collect_summary.in_polyc(316, "C", "T")) # chrM annotation (step 20): ClinVar with chrM, tabixed; gnomAD's chrM sites. cv = "##fileformat=VCFv4.1\n##fileDate=2026-10-05\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n" @@ -2113,8 +2101,6 @@ def roh_upd_karyotype(work, schema): rows = list(csv.DictReader((l for l in f if not l.startswith("#")), delimiter="\t")) asset = [(r["chrom"], int(r["start"]) if r["start"] else None, int(r["end"]) if r["end"] else None, r["region"], r["maternal_upd"], r["paternal_upd"], r["pmid"]) for r in rows] - check("ROH catalog: assets/imprinted_regions.tsv holds the rows of collect_summary.IMPRINTED_REGIONS", - asset and asset == [tuple(x) for x in getattr(collect_summary, "IMPRINTED_REGIONS", [])], asset[:2]) check("ROH catalog: the six imprinted chromosomes, each row with a PMID", [r[0] for r in asset] == ["chr6", "chr7", "chr11", "chr14", "chr15", "chr20"] and all(x.isdigit() for r in asset for x in r[6].split(";")), asset) @@ -2145,7 +2131,7 @@ def roh_upd_karyotype(work, schema): validate.validate(schema, json.loads(json.dumps(summ)))) # Red control: the run on chr15 cut to 15q21 onwards misses 15q11-q13, so no row joins. put(f"{d}/vcf/S_roh.txt", upd.replace("chr15\t1\t101991189\t101991189", "chr15\t40000001\t101991189\t61991189")) - u0 = collect_summary.sec_roh(d, "S")[1].get("upd_flag") or {} + u0 = section(d, "roh")["data"].get("upd_flag") or {} check("ROH UPD red control: a chr15 run outside 15q11-q13 is flagged but joins no row", u0.get("chrom") == "chr15" and u0.get("imprinted") == [], u0) _, txt0, html0 = render(d) @@ -2164,7 +2150,7 @@ def roh_upd_karyotype(work, schema): for t in (txt5, html5)), txt5) # chr7 joins as a whole chromosome. put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr7\t70000001\t110000000\t40000000\t900\t50\n") - u7 = collect_summary.sec_roh(d, "S")[1].get("upd_flag") or {} + u7 = section(d, "roh")["data"].get("upd_flag") or {} check("ROH UPD: 40 Mb on chr7 joins the whole-chromosome Silver-Russell row", [x["maternal_upd"] for x in u7.get("imprinted", [])] == ["Silver-Russell syndrome"], u7) @@ -2185,12 +2171,12 @@ def roh_upd_karyotype(work, schema): check("ROH spread: with a band the F_ROH note says what the band is", all("The band is the row" in t for t in (txt, html)), txt) # Short runs only: F_ROH 0, no sign of related parents. put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t13000000\t3000000\t900\t50\n") - r = collect_summary.sec_roh(d, "S")[1] + r = section(d, "roh")["data"] check("ROH: runs under 5 Mb only give F_ROH 0 and no sign of related parents", r.get("f_roh") == 0 and r.get("relationship") == "no sign of related parents", r) # A run of exactly 5 Mb counts, as in step 11's summary (5 Mb or more). put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t15000000\t5000000\t900\t50\n") - r = collect_summary.sec_roh(d, "S")[1] + r = section(d, "roh")["data"] check("ROH: a run of exactly 5 Mb counts in F_ROH and the long-run list, as in step 11", r.get("long_mb") == 5.0 and len(r.get("autosomal_over_5mb") or []) == 1, r) @@ -2216,16 +2202,13 @@ def roh_upd_karyotype(work, schema): # somalier's ratios from 10 chrX sites or fewer name nothing: its own sex call needs more. for n in (10, "", "inf"): put(f"{k}/qc/S_sample_qc.tsv", qc.format(n=n, x="1.02", y="0.49")) - got = collect_summary.sec_sample_qc(k, "S")[1].get("karyotype") + got = section(k, "sample_qc")["data"].get("karyotype") check(f"karyotype: somalier's chrX 2.04, chrY 0.98 from x_sites '{n}' names nothing", got is None, got) # A run of infinite length makes the ROH file unreadable, not a crash in the band lookup. put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t13000000\tinf\t900\t50\n") - try: - r = collect_summary.sec_roh(d, "S") - except collect_summary.Unreadable as e: - r = e - check("ROH: an RG row of length inf is unreadable", isinstance(r, collect_summary.Unreadable), r) + r = section(d, "roh") + check("ROH: an RG row of length inf is unreadable", r["state"] == "unreadable", r) def main(): @@ -2390,9 +2373,10 @@ def main(): check("combined allele: the summary validates", not validate.validate(schema, json.loads(json.dumps(summ_cv))), validate.validate(schema, json.loads(json.dumps(summ_cv)))) - # 7. a corrupt file + # 7. a corrupt file: gzip's magic, then no gzip stream (a file named + # .gz that holds plain text is read as text: gzip goes by the magic) c = os.path.join(work, "corrupt", "S") - put(f"{c}/vcf/S.vcf.gz", "not gzip\n") + put_bytes(f"{c}/vcf/S.vcf.gz", BROKEN_GZ) put(f"{c}/clinvar/S_clinvar_hits.vcf", HITS) summ_c, txt_c, _ = render(c) check("corrupt VCF: section unreadable, the report still renders", @@ -2625,7 +2609,7 @@ def qc(sex, ped, x_n, x_het, x_hom_alt, declared, y_depth="0.0", mid="0.010"): ed.get("flagged") == [{"locus": "LOCUS_B", "repeat_count": "20/75", "status": "full_mutation", "sizing": "unknown"}], ed) check("Stranger: a record without STR_STATUS is counted apart, not flagged", ed.get("no_status") == 1, ed) - check("Stranger: the five key loci stay listed", [k["locus"] for k in ed["key_loci"]] == collect_summary.EH_LOCI, ed) + check("Stranger: the five key loci stay listed", [k["locus"] for k in ed["key_loci"]] == EH_LOCI, ed) check("Stranger: both reports list LOCUS_B full_mutation and the short-read caveat", "LOCUS_B 20/75 full_mutation" in txt_x and "" in html_x and "can be wrong at some loci" in txt_x and "can be wrong at some loci" in html_x, txt_x) @@ -2681,7 +2665,7 @@ def strec(rep, cn, so, ci, adsp, st, nmax="20", pmin="40"): zd = summ_z["sections"]["expansions"]["data"] zl = {l["locus"]: l for l in zd.get("loci", [])} check("every locus: ten records give ten rows, the five key loci first", - [l["locus"] for l in zd.get("loci", [])][:5] == collect_summary.EH_LOCI and len(zl) == 10, zd.get("loci")) + [l["locus"] for l in zd.get("loci", [])][:5] == EH_LOCI and len(zl) == 10, zd.get("loci")) check("every locus: the catalog's normal maximum from STR_NORMAL_MAX", zl.get("HTT", {}).get("normal_max") == "26", zl.get("HTT")) check("sizing: a FLANKING-only call is flanking_only, not sized", zl.get("LOCUS_FL", {}).get("sizing") == "flanking_only", @@ -2731,7 +2715,7 @@ def strec(rep, cn, so, ci, adsp, st, nmax="20", pmin="40"): put(f"{e}/expansion_hunter/S_eh.vcf", VCF_HEAD) txt_e = render(e)[1] check("text: a VCF without records still lists the five key loci as not in output", - all(f"{k:<8} not in output" in txt_e for k in collect_summary.EH_LOCI), txt_e) + all(f"{k:<8} not in output" in txt_e for k in EH_LOCI), txt_e) put(f"{z}/expansion_hunter/S_rfc1_motif.tsv", open(f"{z}/expansion_hunter/S_rfc1_motif.tsv").read(), age_days=3) zd2 = render(z)[0]["sections"]["expansions"]["data"] check("RFC1: a motif table older than Stranger's output is not read", @@ -2864,9 +2848,8 @@ def strec(rep, cn, so, ci, adsp, st, nmax="20", pmin="40"): check("mosaic: an unknown mosaic_call is unreadable", sec_m["state"] == "unreadable" and "mosaic_call" in (sec_m["note"] or ""), sec_m) # 18. the traits panel (step 42) - import traits_report t = os.path.join(work, "traits", "S") - rows = traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv")) + rows = traits_rows() by_id = {r["rsid"]: r for r in rows} lct, ear = by_id["rs4988235"], by_id["rs17822931"] g = os.path.join(work, "traits_g.tsv") diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py index 5e1d250dc..04c53623c 100644 --- a/tests/test_copy_number.py +++ b/tests/test_copy_number.py @@ -905,7 +905,7 @@ def intervals(cat, read): check("step 35 interval block with a failed bedcov: exit 0, every row check_failed", out.returncode == 0 and len(lines) == len(iv) and all(x.split("\t")[10] == "check_failed" for x in lines), (out.returncode, out.stderr[-400:])) - # collect_summary: step 35's section carries the interval rows with a change + # og summary collect: step 35's section carries the interval rows with a change import collect_summary as cs sd = os.path.join(t, "sample") os.makedirs(os.path.join(sd, "paralogs")) @@ -914,14 +914,14 @@ def intervals(cat, read): "homologous_regions\n") if os.path.exists(good): os.replace(good, os.path.join(sd, "paralogs", "S_cn_intervals.tsv")) - _, data = cs.sec_parascopy(sd, "S") + data = cs.collect("S", sd)["sections"]["parascopy"]["data"] ci = data.get("cn_intervals") or {} - check("collect_summary: step 35's section lists the interval rows with a change (AZFc) and counts the rest", + check("summary: step 35's section lists the interval rows with a change (AZFc) and counts the rest", [c["locus"] for c in ci.get("changes", [])] == ["AZFc_DAZ"] and ci.get("rows") == len(iv), ci) os.remove(os.path.join(sd, "paralogs", "S_smn_copy_number.tsv")) - _, data = cs.sec_parascopy(sd, "S") + data = cs.collect("S", sd)["sections"]["parascopy"]["data"] ci = (data or {}).get("cn_intervals") or {} - check("collect_summary: the interval rows stay when step 35 wrote no other table", + check("summary: the interval rows stay when step 35 wrote no other table", [c["locus"] for c in ci.get("changes", [])] == ["AZFc_DAZ"] and ci.get("rows") == len(iv), ci) @@ -1242,7 +1242,7 @@ def psv(cat, read): "to the couple screen (--cyp21a2).") in md} check(f"couple screen, {desc}: a gap line for each partner without a CYP21A2 table", got == want, (got, md[-600:])) - # collect_summary: step 35's section carries the CYP21A2 table + # og summary collect: step 35's section carries the CYP21A2 table import collect_summary as cs sd = os.path.join(t, "sample") os.makedirs(os.path.join(sd, "paralogs")) @@ -1251,15 +1251,15 @@ def psv(cat, read): "homologous_regions\n") if os.path.exists(tabs["deletion"]): os.replace(tabs["deletion"], os.path.join(sd, "paralogs", "S_cyp21a2_psv.tsv")) - _, data = cs.sec_parascopy(sd, "S") + data = cs.collect("S", sd)["sections"]["parascopy"]["data"] c = data.get("cyp21a2") or {} - check("collect_summary: step 35's section carries the CYP21A2 table (deletion, 1 copy)", + check("summary: step 35's section carries the CYP21A2 table (deletion, 1 copy)", (c.get("finding"), c.get("cyp21a2_copies")) == ("deletion", "1"), c) # and the reports print why a table without MAPQ 0 reads is not assessed import render_report if os.path.exists(tabs["no_mapq0"]): os.replace(tabs["no_mapq0"], os.path.join(sd, "paralogs", "S_cyp21a2_psv.tsv")) - _, data = cs.sec_parascopy(sd, "S") + data = cs.collect("S", sd)["sections"]["parascopy"]["data"] got = render_report.cyp21a2_line(data.get("cyp21a2")) check("report: the CYP21A2 line without MAPQ 0 reads is not assessed and says the BAM has few or no multi-mapping reads", ": not assessed (the BAM has few or no multi-mapping reads here" in got and "deletion," not in got, got) diff --git a/tests/test_cross_engine_biallelic.py b/tests/test_cross_engine_biallelic.py index 777232b48..2b89415be 100644 --- a/tests/test_cross_engine_biallelic.py +++ b/tests/test_cross_engine_biallelic.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Two hits in one gene across callers (bin/collect_summary.py two_hits): a +"""Two hits in one gene across callers (og summary collect's two-hit rule): a called deletion plus a small variant, on synthetic sample folders. Nothing here describes a person: the calls are made up at real gene positions. @@ -111,7 +111,7 @@ def sample(work, name, hits, annotsv=None): def carrier(d): - return collect_summary.sec_carrier(d, "S")[1] + return collect_summary.collect("S", d)["sections"]["carrier"]["data"] def two(d, gene): @@ -119,7 +119,12 @@ def two(d, gene): def sf(d): - return collect_summary.acmg_sf_tier(d, "S", None) + """The SF tier of the folder: from the hits and the clinical VCF, which is + a VCF with no record when the case gives none (the Nextflow layout).""" + v = f"{d}/clinical/S_clinical.vcf.gz" + if not os.path.exists(v): + put(v, VCF_HEAD, gz=True) + return collect_summary.collect("S", d)["sections"]["clinical"]["data"].get("acmg_sf") or {} def main(): @@ -294,8 +299,8 @@ def main(): f"chr1\t{SNV[1]}\t.\tC\tT\t50\tPASS\tCSQ=T|stop_gained|HIGH|MUTYH|0.0001\tGT:GQ:DP:AD:PS\t" "0|1:50:30:15,15:45330000\n") d = sample(work, "high_trans", [], sv(DEL, "MUTYH", "1|0", ps=45330000)) - clin = put(f"{d}/clinical/S_clinical.vcf", csq) - rep = [x for x in collect_summary.acmg_sf_tier(d, "S", clin)["high_impact"] if x["gene"] == "MUTYH"] + put(f"{d}/clinical/S_clinical.vcf.gz", csq, gz=True) + rep = [x for x in sf(d)["high_impact"] if x["gene"] == "MUTYH"] check("a HIGH-impact hit phased in trans with the deletion: biallelic in the SF tier", len(rep) == 1 and (rep[0].get("two_hit") or "").startswith("biallelic with DEL"), rep) # one hit in cis, one in trans: only the trans one carries the note @@ -308,15 +313,17 @@ def main(): and "chr1:45330500 A>G" in rows and rows["chr1:45330500 A>G"] is None, rows) # an unreadable carrier list: the SF card says what was not read d = sample(work, "no_list", [hit(SNV, "MUTYH", "0/1")], sv(DEL, "MUTYH")) - real = collect_summary.carrier_genes - - def broken(): - raise collect_summary.Unreadable("carrier_genes.tsv row 1: broken") - collect_summary.carrier_genes = broken + broken = os.path.join(work, "assets-broken") + shutil.copytree(collect_summary.ASSETS_DIR, broken) + with open(os.path.join(broken, "carrier_genes.tsv")) as f: + text = f.read() + with open(os.path.join(broken, "carrier_genes.tsv"), "w") as f: + f.write(text.replace("gene\tchrom\tstart", "name\tchrom\tstart", 1)) + real, collect_summary.ASSETS_DIR = collect_summary.ASSETS_DIR, broken try: s = sf(d) finally: - collect_summary.carrier_genes = real + collect_summary.ASSETS_DIR = real check("an unreadable carrier list: a note on the SF card", "carrier_genes.tsv" in (s.get("two_hit_note") or "") and any("carrier_genes.tsv" in x for x in render_report.sf_apart_lines(s)), s.get("two_hit_note")) diff --git a/tests/test_exomiser_report.py b/tests/test_exomiser_report.py index b516b6a5d..c13e52e96 100644 --- a/tests/test_exomiser_report.py +++ b/tests/test_exomiser_report.py @@ -38,6 +38,21 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) OG = os.environ.get("OG", "") sys.path.insert(0, os.path.join(REPO, "bin")) +import collect_summary as cs # noqa: E402 + + +class Unreadable(Exception): + """A section og summary collect could not read (its state is unreadable).""" + + +def sec(d, s, key): + """(source path, data) of section KEY of the summary og summary collect writes + for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" + x = cs.collect(s, d)["sections"][key] + if x["state"] == "unreadable": + raise Unreadable(x["note"]) + return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] + FAILS = [] @@ -318,7 +333,6 @@ def case_report(): def case_reports(): - import collect_summary as cs import render_report as rr with tempfile.TemporaryDirectory() as d: sd = os.path.join(d, "S1", "exomiser") @@ -326,14 +340,14 @@ def case_reports(): r = report(sd, ROWS) shutil.copy(os.path.join(sd, "top.tsv"), os.path.join(sd, "S1_exomiser_top.tsv")) os.rename(os.path.join(sd, "terms.tsv"), os.path.join(sd, "S1_exomiser_terms.tsv")) - p, data = cs.sec_exomiser(os.path.join(d, "S1"), "S1") + p, data = sec(os.path.join(d, "S1"), "S1", "exomiser") check("summary: the section reads the table", p is not None and r.returncode == 0) check("summary: ranks with their variants", [(g["rank"], g["gene"], len(g["variants"])) for g in data["genes"]] == [("1", "GENEA", 1), ("2", "GENEB", 2), ("3", "GENEC", 1)], str(data["genes"])) check("summary: the terms", data["terms"] == [{"id": "HP:0001156", "label": "Brachydactyly"}], str(data["terms"])) check("summary: the section is listed under step 48", - ("exomiser", "Symptom-driven ranking (Exomiser)", "48", cs.sec_exomiser) in cs.SECTIONS + ("exomiser", "Symptom-driven ranking (Exomiser)", "48") in cs.SECTIONS and "48" in cs.RUN_ALL_STEPS) text = "\n".join(rr.exomiser_text(data, "S1")) check("text report: rank, gene and variant", @@ -345,9 +359,9 @@ def case_reports(): bad = os.path.join(sd, "S1_exomiser_top.tsv") write(bad, "rank\tgene\n1\tGENEA\n") try: - cs.sec_exomiser(os.path.join(d, "S1"), "S1") + sec(os.path.join(d, "S1"), "S1", "exomiser") check("summary: a table without its columns is unreadable", False, "no error") - except cs.Unreadable: + except Unreadable: check("summary: a table without its columns is unreadable", True) # The -stub table of EXOMISER_REPORT reads as "no gene passed", not unreadable. import re @@ -359,9 +373,9 @@ def case_reports(): check("stub: the top table carries the report's columns", head == out_header, repr(head)) write(bad, head) try: - _, data = cs.sec_exomiser(os.path.join(d, "S1"), "S1") + _, data = sec(os.path.join(d, "S1"), "S1", "exomiser") check("stub: the summary reads it as no gene", data.get("genes") == [], str(data)) - except cs.Unreadable as e: + except Unreadable as e: check("stub: the summary reads it as no gene", False, str(e)) diff --git a/tests/test_genotype_rules_indels.py b/tests/test_genotype_rules_indels.py deleted file mode 100644 index 35fccfd72..000000000 --- a/tests/test_genotype_rules_indels.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python3 -"""bin/genotype_rules_indels.py (APOL1 G2 beside step 42's APOL1 rule), and its path -through bin/collect_summary.py and bin/render_report.py. - -Checks: - 1. the G2 marker matches the pinned dbSNP record and the GRCh38 context; controls: - a marker with ref and alt swapped, one at another position and one whose REF is - not the GRCh38 bases each fail, naming the mistake; - 2. read_g2 on synthetic VCF records: the left-aligned dbSNP record and a - right-shifted record of the same deletion each count (het 1, hom 2); a gVCF - reference block over the span gives 0 copies, called, with its GQ; a - variant-only VCF with no record gives no_call_record; a gVCF with no record - gives not_covered; a no-call, a filtered G2 call, an SNV inside the span, a - 5-base deletion, a REF that is not GRCh38 and two het records of the one - deletion give not_read; a record outside - the span is skipped; control: a 6-base deletion just outside the span is not G2; - 3. combine: every G1 x G2 count gives its label and risk-copy reading; three risk - copies are not read; a G1 rule not called is not read, unless G2 has two copies - (G2/G2); G2 not read keeps the G1 result alone; a '.' result renders 'not read'; - no_call_record says the count rests on the absence of a call; - 4. collect_summary reads G2 from the sample's gVCF (bgzipped with a .tbi, and - plain gzip without one) or its VCF, and puts it on the APOL1 rule line only; - render_report shows "With G2" in the HTML and the text report; control: the - same folder with a reference gVCF gives another result. - -Run: python3 tests/test_genotype_rules_indels.py -""" -import gzip -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import genotype_rules_indels as m # noqa: E402 -import render_report as rr # noqa: E402 - -FAILS = 0 - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") - if not ok: - FAILS += 1 - - -def ctx(a, b): - """GRCh38 bases a..b (1-based, inclusive) from the pinned context.""" - return m.GRCh38_CONTEXT[a - m.CONTEXT_START:b - m.CONTEXT_START + 1] - - -def rec(pos, ref, alt, gt, filt="PASS", info=".", gq="30"): - return ["chr22", str(pos), ".", ref, alt, "30", filt, info, "GT:GQ", f"{gt}:{gq}"] - - -LEFT = (36265995, "AATAATT", "A") # dbSNP's record -RIGHT = (36265999, ctx(36265999, 36266005), ctx(36265999, 36265999)) # the same deletion, shifted right -BLOCK = rec(36265900, "A", "<*>", "0/0", ".", "END=36266100", "50") - - -def marker_checks(): - check("1. the G2 marker matches dbSNP and GRCh38", m.marker_errors() == [], m.marker_errors()) - check("the G2 haplotype is GRCh38 with 6 bases deleted", len(m.GRCh38_CONTEXT) - len(m.G2_HAPLOTYPE) == 6) - swapped = dict(m.G2, ref="A", alt="AATAATT") - e = m.marker_errors(swapped) - check("control: ref and alt swapped fails, named as swapped", any("ref and alt are swapped" in x for x in e), e) - moved = dict(m.G2, pos=36265996) - e = m.marker_errors(moved) - check("control: another position fails against dbSNP and GRCh38", - any("dbSNP chr22:36265995" in x for x in e) and any("not the GRCh38 bases" in x for x in e), e) - e = m.marker_errors(dict(m.G2, ref="AATAATA"), dbsnp=("rs71785313", "chr22", 36265995, "AATAATA", "A")) - check("control: a REF that is not the GRCh38 bases fails even when dbSNP agrees", - any("not the GRCh38 bases" in x for x in e), e) - - -def read_checks(): - for name, (p, r, a) in (("left-aligned", LEFT), ("right-shifted", RIGHT)): - for gt, n in (("0/1", 1), ("1|1", 2)): - g = m.read_g2([rec(p, r, a, gt)], "vcf") - check(f"2. {name} G2 {gt}: {n} copies", (g["status"], g["copies"]) == ("called", n), g) - g = m.read_g2([rec(*LEFT, "0/1"), rec(*RIGHT, "0/1")], "vcf") - check("two het records of the one deletion (left-aligned and shifted): not read, never two copies", - (g["status"], g["copies"]) == ("not_read", None) and "more than one record" in g["text"], g) - g = m.read_g2([BLOCK], "gvcf") - check("a gVCF reference block over the span: 0 copies, called, its GQ", - (g["status"], g["copies"], g["gq"]) == ("called", 0, "50"), g) - g = m.read_g2([], "vcf") - check("a VCF with no record: no_call_record, 0 copies", (g["status"], g["copies"]) == ("no_call_record", 0), g) - g = m.read_g2([], "gvcf") - check("a gVCF with no record: not_covered", (g["status"], g["copies"]) == ("not_covered", None), g) - half = rec(36265900, "A", "<*>", "0/0", ".", "END=36266000", "50") - g = m.read_g2([half], "gvcf") - check("a gVCF block that stops inside the span: not_covered", g["status"] == "not_covered", g) - snv = ctx(36266001, 36266001) - other = "G" if snv != "G" else "C" - bad = {"a no-call": rec(*LEFT, "./."), "a filtered G2 call": rec(*LEFT, "0/1", "RefCall"), - "an SNV inside the span": rec(36266001, snv, other, "0/1"), - "a 5-base deletion": rec(36265995, ctx(36265995, 36266000), "A", "0/1"), - "a REF that is not GRCh38": rec(36265995, "CATAATT", "C", "0/1")} - for desc, r in bad.items(): - g = m.read_g2([BLOCK, r], "gvcf") - check(f"{desc}: not_read", (g["status"], g["copies"]) == ("not_read", None), g) - g = m.read_g2([BLOCK, rec(36266100, "A", "G", "0/1"), ["chr21", "36265995", ".", "AATAATT", "A", ".", - "PASS", ".", "GT", "1/1"]], "gvcf") - check("records outside the span or on another contig are skipped", (g["status"], g["copies"]) == ("called", 0), g) - # control: a 6-base deletion right after the span (anchor 36266006) is not G2 - out = rec(36266006, ctx(36266006, 36266012), ctx(36266006, 36266006), "1/1") - g = m.read_g2([BLOCK, out], "gvcf") - check("control: a 6-base deletion outside the span does not count as G2", g["copies"] == 0, g) - - -def combine_checks(): - want = {("G0/G0", 0): "G0/G0", ("G0/G0", 1): "G0/G2", ("G0/G0", 2): "G2/G2", ("G0/G1", 0): "G0/G1", - ("G0/G1", 1): "G1/G2", ("G1/G1", 0): "G1/G1"} - for (g1, c), label in want.items(): - res = m.combine({"status": "called", "result": g1}, {"status": "called", "copies": c, "text": "t"}) - risk = label.count("G1") + label.count("G2") - word = {0: "No G1 and no G2", 1: "One APOL1 risk copy", 2: "Two APOL1 risk copies"}[risk] - check(f"3. {g1} + {c} G2: {label}", res["result"] == label and word in res["reading"], res) - res = m.combine({"status": "called", "result": "G1/G1"}, {"status": "called", "copies": 1, "text": "t"}) - check("G1/G1 with a G2 copy (three risk copies): not read", res["result"] == "." and "never share" in res["reading"], - res) - res = m.combine({"status": "not_covered", "result": "."}, {"status": "called", "copies": 1, "text": "t"}) - check("the G1 rule not called: not read", res["result"] == ".", res) - res = m.combine({"status": "not_covered", "result": "."}, {"status": "called", "copies": 2, "text": "t"}) - check("the G1 rule not called with two G2 copies: G2/G2, two risk copies", - res["result"] == "G2/G2" and "Two APOL1 risk copies" in res["reading"], res) - res = m.combine({"status": "called", "result": "G0/G1"}, {"status": "not_read", "copies": None, "text": "G2 x."}) - check("G2 not read: the G1 result stands alone, without repeating G2's text", - res["result"] == "." and "stands on its own" in res["reading"] and "G2 x." not in res["reading"], res) - for res in (m.combine({"status": "called", "result": "G1/G1"}, {"status": "called", "copies": 1, "text": "t"}), - m.combine({"status": "not_covered", "result": "."}, {"status": "called", "copies": 1, "text": "t"})): - shown = rr.g2_result({"g2": dict(res, status="called", text="t")}) - check(f"render_report: a '.' result shows 'not read' ({res['reading'][:30]})", shown == "not read", shown) - res = m.combine({"status": "called", "result": "G0/G1"}, m.read_g2([], "vcf")) - check("no_call_record: the label says no G2 call and the reading why", - res["result"] == "G0/G1 (no G2 call)" and "no reference blocks" in res["reading"], res) - - -def bgzip_with_tbi(path, lines): - """A bgzipped VCF and its .tbi from bgzip and tabix when they exist, else None - (tests/e2e/genotype-rules-2-g2.sh covers the .tbi path with the pinned image).""" - if not (shutil.which("bgzip") and shutil.which("tabix")): - return None - with open(path[:-3], "w") as f: - f.writelines(lines) - subprocess.run(["bgzip", "-f", path[:-3]], check=True) - subprocess.run(["tabix", "-p", "vcf", path], check=True) - return path - - -def report_checks(): - import collect_summary as cs - import render_report as rr - from test_traits_report import traits_tables - head = "##fileformat=VCFv4.2\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - g1 = [("chr22", "36265860", "A", "G", "A/G", "40"), ("chr22", "36265988", "T", "G", "T/G", "40")] - work = tempfile.mkdtemp() - try: - def folder(name, gvcf_lines, kind): - d = os.path.join(work, name) - os.makedirs(os.path.join(d, "traits")) - os.makedirs(os.path.join(d, "vcf")) - p = os.path.join(d, "traits", "S_genotype_rules.tsv") - g = os.path.join(work, name + "_g.tsv") - with open(g, "w") as f: - f.writelines("\t".join(x) + "\n" for x in g1) - traits_tables(g, os.path.join(d, "traits", "S_traits.tsv"), p) - v = os.path.join(d, "vcf", "S.g.vcf.gz" if kind == "gvcf" else "S.vcf.gz") - with gzip.open(v, "wt") as f: - f.write(head + "".join("\t".join(r) + "\n" for r in gvcf_lines)) - return d - - het = rec(*LEFT, "0/1") - d = folder("het", [het], "gvcf") - rows = cs.genotype_rules(d, "S") - apol1 = next(r for r in rows if r["rule"] == "APOL1") - others = [r["rule"] for r in rows if "g2" in r and r["rule"] != "APOL1"] - check("4. collect_summary: G0/G1 plus a G2 copy in the gVCF reads G1/G2", - apol1["g2"]["result"] == "G1/G2" and apol1["g2"]["source"] == "vcf/S.g.vcf.gz", apol1.get("g2")) - check("only the APOL1 line carries g2", others == [], others) - page = "\n".join(rr.rules_html({"rules": rows}, "S")) - check("render_report HTML: With G2: G1/G2 and the high-risk reading", - "With G2: G1/G2" in page and "Two APOL1 risk copies" in page, page[:400]) - text = "\n".join(rr.traits_text({"called": 0, "not_covered": 0, "other_allele": 0, "rows": [], - "rules": rows}, "S")) - check("render_report text: With G2 (rs71785313): G1/G2", "With G2 (rs71785313): G1/G2" in text, text) - d = folder("ref", [BLOCK], "gvcf") - apol1 = next(r for r in cs.genotype_rules(d, "S") if r["rule"] == "APOL1") - check("control: a reference gVCF gives G0/G1", apol1["g2"]["result"] == "G0/G1", apol1.get("g2")) - d = folder("vcf", [], "vcf") - apol1 = next(r for r in cs.genotype_rules(d, "S") if r["rule"] == "APOL1") - check("a VCF with no G2 record: G0/G1 (no G2 call), from vcf/S.vcf.gz", - apol1["g2"]["result"] == "G0/G1 (no G2 call)" and apol1["g2"]["source"] == "vcf/S.vcf.gz", - apol1.get("g2")) - d = os.path.join(work, "none") - os.makedirs(os.path.join(d, "traits")) - shutil.copy(os.path.join(work, "het", "traits", "S_genotype_rules.tsv"), os.path.join(d, "traits")) - apol1 = next(r for r in cs.genotype_rules(d, "S") if r["rule"] == "APOL1") - check("no VCF at all: G2 not read", apol1["g2"]["status"] == "not_read" and apol1["g2"]["result"] == ".", - apol1.get("g2")) - tb = os.path.join(work, "tbi", "vcf", "S.g.vcf.gz") - os.makedirs(os.path.dirname(tb)) - pad = [rec(36000000 + i * 1000, "A", "<*>", "0/0", ".", f"END={36000000 + i * 1000 + 999}") for i in range(300)] - lines = [head] + ["\t".join(r) + "\n" for r in sorted(pad + [rec(*LEFT, "1/1")], key=lambda r: int(r[1]))] - if bgzip_with_tbi(tb, lines): - got = m.read_g2(cs.vcf_window(tb, "chr22", m.CONTEXT_START, m.SPAN[1]), "gvcf") - check("vcf_window through the .tbi finds the G2 record (2 copies)", got["copies"] == 2, got) - else: - print("[SKIP] no bgzip and tabix here: the .tbi path runs where they are installed") - finally: - shutil.rmtree(work) - - -def main(): - marker_checks() - read_checks() - combine_checks() - report_checks() - print(f"\n{'FAILED' if FAILS else 'ALL PASSED'}: {FAILS} failure(s)") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_hla_report.py b/tests/test_hla_report.py index aef51e80b..94e1f3812 100644 --- a/tests/test_hla_report.py +++ b/tests/test_hla_report.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """The HLA section of the reports (step 08): bin/hla_alleles.tsv, what -bin/collect_summary.py derives from T1K's genotype TSV, and what +og summary collect derives from T1K's genotype TSV, and what bin/render_report.py prints. Synthetic T1K rows only. Checks: @@ -103,6 +103,11 @@ def t1k(rows): return "".join(out) +# The label og summary collect gives T1K's HFE row. +HLA_HFE_LABEL = ("T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the " + "variant calls (ClinVar section)") + + def summary(work, name, rows): d = os.path.join(work, name, "S") os.makedirs(os.path.join(d, "hla_t1k"), exist_ok=True) @@ -118,7 +123,8 @@ def main(): # 1. the table errs = table_errors(TABLE) check("bin/hla_alleles.tsv passes its schema and lists every required allele", not errs, errs) - check("collect_summary reads the same rows", [r["allele"] for r in cs.hla_read_table()] + derived = summary(work, "table", [("HLA-B", "B*57:01:01", 40, "B*08:01:01", 40)])["sections"]["hla"]["data"] + check("og summary collect reads the same rows", [r["allele"] for r in derived["derived"]["alleles"]] == [l.split("\t")[0] for l in open(TABLE) if not l.startswith("#") and l.strip()][1:]) good = open(TABLE).read() for desc, bad in (("a wrong kind", good.replace("\tdrug\t", "\tdrugs\t", 1)), @@ -205,7 +211,7 @@ def state(summ, allele): check("a quality-0 allele with 12 candidates is stored with three and the count", loci["HLA-DRB5"]["alleles"] == ["DRB5*01:01,DRB5*01:02,DRB5*01:03"] and loci["HLA-DRB5"]["candidates"] == [12], loci["HLA-DRB5"]) - check("the HFE row has its label", loci["HLA-HFE"].get("label") == cs.HLA_HFE_LABEL, loci["HLA-HFE"]) + check("the HFE row has its label", loci["HLA-HFE"].get("label") == HLA_HFE_LABEL, loci["HLA-HFE"]) check("cap summary validates", not validate.validate(schema, json.loads(json.dumps(cap))), validate.validate(schema, json.loads(json.dumps(cap)))) txt, html = render_report.text_report(cap), render_report.html_report(cap) diff --git a/tests/test_locityper_report.py b/tests/test_locityper_report.py index 5bbae64a9..122feb26a 100755 --- a/tests/test_locityper_report.py +++ b/tests/test_locityper_report.py @@ -429,13 +429,8 @@ def og_cases(t): cyp = [x for x in s["sections"]["locityper"]["data"]["loci"] if x["locus"] == "CYP2D6"][0] check("5. a primary call older than the last run says it is stale", "(stale: from an earlier run)" in cyp["primary"], (s["sections"]["cyrius"]["state"], cyp)) - hla = {"loci": [{"gene": "DRB1", "alleles": ["*15:01", "*03:01"], "low_confidence": True}, - {"gene": "DQB1", "alleles": ["*06:02", "*02:01"], "low_confidence": False}]} - prim = collect_summary.locityper_primary({"hla": {"state": "ok", "data": hla}, "pypgx": {"state": "missing"}, - "cyrius": {"state": "missing"}}) - check("5. a low-confidence T1K call says so in the primary text, a confident one does not", - prim["HLA-DRB1"] == "T1K (step 08): *15:01/*03:01 (low confidence)" - and prim["HLA-DQB1"] == "T1K (step 08): *06:02/*02:01", prim) + # A low-confidence T1K call in the primary text: og-summary's own test + # (collect::group_c::tests, the Locityper primary calls). os.makedirs(os.path.join(t, "none", "S")) s = collect_summary.collect("S", os.path.join(t, "none", "S")) check("5. without step 53 there is no card and no text section", diff --git a/tests/test_lpa_kiv2.py b/tests/test_lpa_kiv2.py index 03ab6f15f..ae634adb0 100644 --- a/tests/test_lpa_kiv2.py +++ b/tests/test_lpa_kiv2.py @@ -52,6 +52,7 @@ Run: OG=rust/target/release/og python3 tests/test_lpa_kiv2.py """ +import csv import math import os import subprocess @@ -75,6 +76,11 @@ from test_copy_number import cn # noqa: E402 pinned differential oracle FAILS = 0 +# The seven copies of KIV-2 4925G>A on GRCh38 (rust/crates/og-cn/src/markers.rs LPA_KIV2_4925). +KIV2_4925_COPIES = (160646214, 160640667, 160635123, 160629574, 160624030, 160618484, 160612932) +# The columns of step 42's traits table (og traits table --out). +TRAITS_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", + "note", "pmids"] # The two LPA rows; named once here so no line of this file reads as an rsID with a genotype. R1, R2 = "rs10455872", "rs3798220" ROW = cn.load_catalog(CATALOG)["LPA_KIV2"] @@ -153,11 +159,10 @@ def cv(vals): def pileup(path, per_site, extra=""): - """mpileup lines at the seven copies of 4925G>A (traits_report.MARKERS); + """mpileup lines at the seven copies of 4925G>A (KIV2_4925_COPIES); per_site is (C reads, T reads) at each.""" - import traits_report with open(path, "w") as f: - for p in traits_report.MARKERS["LPA_KIV2_4925"]["copies"]: + for p in KIV2_4925_COPIES: c, t_ = per_site bases = "^]" + "C" * c + "t" * t_ + "$" f.write(f"chr6\t{p}\tN\t{c + t_}\t{bases}\t{'I' * (c + t_)}\n") @@ -166,10 +171,11 @@ def pileup(path, per_site, extra=""): def traits(): """6. assets/traits.tsv holds the two LPA risk variants the report counts.""" - import collect_summary - import traits_report - table = {r["rsid"]: r for r in traits_report.read_table(os.path.join(REPO, "assets", "traits.tsv"))} - for rs, alt in collect_summary.LPA_RISK_ALLELES.items(): + with open(os.path.join(REPO, "assets", "traits.tsv"), encoding="utf-8") as f: + table = {r["rsid"]: r for r in csv.DictReader((line for line in f if line.strip() and not line.startswith("#")), + delimiter="\t", quoting=csv.QUOTE_NONE)} + # The two risk alleles og summary collect counts for the Lp(a) flag. + for rs, alt in {"rs10455872": "G", "rs3798220": "C"}.items(): r = table.get(rs, {}) check(f"traits.tsv: {rs} is an LPA row whose alt ({alt}) is the risk allele the report counts", r.get("gene") == "LPA" and r.get("alt") == alt, r) @@ -277,7 +283,6 @@ def main(): traits() import collect_summary import render_report - import traits_report # 7. reports def sample(t, name, n_copies, flank=30.0, genotypes=None): @@ -291,11 +296,11 @@ def sample(t, name, n_copies, flank=30.0, genotypes=None): if genotypes: os.makedirs(os.path.join(d, "traits")) with open(os.path.join(d, "traits", "S_traits.tsv"), "w") as f: - f.write("\t".join(traits_report.OUT_COLUMNS) + "\n") + f.write("\t".join(TRAITS_COLUMNS) + "\n") for rs, gt in genotypes.items(): - row = {k: "." for k in traits_report.OUT_COLUMNS} + row = {k: "." for k in TRAITS_COLUMNS} row.update(rsid=rs, gene="LPA", genotype=gt, status="called" if gt != "." else "not_covered") - f.write("\t".join(row[k] for k in traits_report.OUT_COLUMNS) + "\n") + f.write("\t".join(row[k] for k in TRAITS_COLUMNS) + "\n") summ = collect_summary.collect("S", d) return summ, render_report.text_report(summ), render_report.html_report(summ) diff --git a/tests/test_mei_report.py b/tests/test_mei_report.py index d87f1c5cb..34873cdc9 100644 --- a/tests/test_mei_report.py +++ b/tests/test_mei_report.py @@ -179,20 +179,37 @@ def status(result, gene): # 5. the summary and the reports import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 + +class Unreadable(Exception): + """A section og summary collect could not read (its state is unreadable).""" + + +def sec(d, s, key): + """(source path, data) of section KEY of the summary og summary collect writes + for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" + x = cs.collect(s, d)["sections"][key] + if x["state"] == "unreadable": + raise Unreadable(x["note"]) + return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] + + +MEI_NOT_ASSESSED = ("Mobile element insertions (Alu, L1 and SVA, the FKTN, TAF1 and BBS1 SVA insertions among them): " + "not looked for (step 50, MEGAnE, is opt-in)") + with tempfile.TemporaryDirectory() as d: os.makedirs(f"{d}/S") summ = cs.collect("S", f"{d}/S") check("5 without step 50 the not-assessed list names mobile element insertions", - cs.MEI_NOT_ASSESSED in summ["not_assessed"] and cs.MEI_NOT_ASSESSED.startswith("Mobile element insertions"), + MEI_NOT_ASSESSED in summ["not_assessed"] and MEI_NOT_ASSESSED.startswith("Mobile element insertions"), True) - check("5 no table: no section", cs.sec_mei(f"{d}/S", "S"), (None, {})) + check("5 no table: no section", sec(f"{d}/S", "S", "mei"), (None, {})) # A step 50 that ran and failed was looked for: it is listed as failed, not as opt-in. os.makedirs(f"{d}/F/logs") with open(f"{d}/F/logs/run_status.tsv", "w") as f: f.write(f"meta\tstarted_epoch\t{time.time() - 60}\nstep\t50\tfailed (exit 1)\n") summ = cs.collect("F", f"{d}/F") check("5 a failed step 50 is listed under not run, not as not looked for", - (summ["sections"]["mei"]["state"], cs.MEI_NOT_ASSESSED in summ["not_assessed"], + (summ["sections"]["mei"]["state"], MEI_NOT_ASSESSED in summ["not_assessed"], any("step 50 failed" in x for x in summ["not_run"])), ("failed", False, True)) os.makedirs(f"{d}/S/mei") for name, text in (("m.vcf", HEAD + FKTN_SVA + rec("chr20", 5040478, "SINE/Alu")), ("sites.tsv", SITES), @@ -202,14 +219,14 @@ def status(result, gene): subprocess.run([OG, "mei", "report", "--sample", "S", "--vcf", f"{d}/m.vcf", "--sites", f"{d}/sites.tsv", "--depth", f"{d}/depth.tsv", "--out-calls", f"{d}/S/mei/S_mei.tsv", "--out-sites", f"{d}/S/mei/S_mei_sites.tsv"], check=True) - path, data = cs.sec_mei(f"{d}/S", "S") + path, data = sec(f"{d}/S", "S", "mei") check("5 the summary finds the table", path.endswith("mei/S_mei_sites.tsv"), True) check("5 the summary counts the calls per element", (data["calls"], data["by_element"]), (2, {"Alu": 1, "SVA": 1})) check("5 the summary has each catalog row", [(x["gene"], x["status"]) for x in data["sites"]], [("FKTN", "seen"), ("TAF1", "not seen")]) summ = cs.collect("S", f"{d}/S") check("5 with step 50's tables, mobile element insertions are not listed as not assessed", - cs.MEI_NOT_ASSESSED in summ["not_assessed"], False) + MEI_NOT_ASSESSED in summ["not_assessed"], False) try: page = rr.html_report(summ) except Exception as e: # the whole step 24 page is lost if the card raises @@ -226,23 +243,23 @@ def status(result, gene): good_calls = open(calls_path).read() with open(calls_path, "w") as f: f.write(good_calls.splitlines()[0] + "\n") - check("5 a calls table with its header and no row is zero calls", cs.sec_mei(f"{d}/S", "S")[1]["calls"], 0) + check("5 a calls table with its header and no row is zero calls", sec(f"{d}/S", "S", "mei")[1]["calls"], 0) for name, text in (("an empty calls table", ""), ("a calls table without its columns", "chrom\tpos\n")): with open(calls_path, "w") as f: f.write(text) try: - cs.sec_mei(f"{d}/S", "S") + sec(f"{d}/S", "S", "mei") check(f"5 {name} is unreadable", "read", "Unreadable") - except cs.Unreadable: + except Unreadable: check(f"5 {name} is unreadable", "Unreadable", "Unreadable") with open(calls_path, "w") as f: f.write(good_calls) with open(f"{d}/S/mei/S_mei_sites.tsv", "w") as f: f.write("gene\tstatus\nFKTN\tseen\n") try: - cs.sec_mei(f"{d}/S", "S") + sec(f"{d}/S", "S", "mei") check("5 a sites table without its columns is unreadable", "read", "Unreadable") - except cs.Unreadable: + except Unreadable: check("5 a sites table without its columns is unreadable", "Unreadable", "Unreadable") # 6. the real catalog diff --git a/tests/test_mito_extended.py b/tests/test_mito_extended.py index a409b153e..47304a04a 100644 --- a/tests/test_mito_extended.py +++ b/tests/test_mito_extended.py @@ -215,8 +215,7 @@ def sample_dir(tmp, name, records, deletions=None, idxstats=None): def mito_section(d, name): import collect_summary - _, data = collect_summary.sec_mito(d, name) - return data + return collect_summary.collect(name, d)["sections"]["mito"]["data"] def reports(d, name): @@ -247,13 +246,11 @@ def test_confirmed_snv(tmp): "Confirmed disease variants (assets/mito_confirmed.tsv): 0" in text) import collect_summary saved, collect_summary.ASSETS_DIR = collect_summary.ASSETS_DIR, os.path.join(tmp, "no-assets") - collect_summary._ASSETS.clear() try: data = mito_section(d, "Snone") text, html = reports(d, "Snone") finally: collect_summary.ASSETS_DIR = saved - collect_summary._ASSETS.clear() check("without the list the summary has no confirmed count, only why", "confirmed" not in data and "mito_confirmed.tsv is not in" in (data.get("confirmed_note") or ""), str(data)) check("without the list neither report shows a count of confirmed variants", diff --git a/tests/test_mocha_report.py b/tests/test_mocha_report.py index 55a94a56a..80aa58c86 100644 --- a/tests/test_mocha_report.py +++ b/tests/test_mocha_report.py @@ -48,6 +48,23 @@ import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 +class Unreadable(Exception): + """A section og summary collect could not read (its state is unreadable).""" + + +def sec(d, s, key): + """(source path, data) of section KEY of the summary og summary collect writes + for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" + x = cs.collect(s, d)["sections"][key] + if x["state"] == "unreadable": + raise Unreadable(x["note"]) + return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] + + +MCA_NOT_ASSESSED = ("Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share " + "of the blood cells): step 51 (MoChA) did not assess this sample") + + FAILS = [] STATS_HEAD = ("sample_id\tcomputed_gender\tcall_rate\tcov_median\tcov_sd\tcov_auto\tbaf_corr\tbaf_conc\tbaf_auto" "\tn_sites\tn_hets\n") @@ -148,7 +165,7 @@ def cases_1_to_6(): # 6. both reports _, _, _, d = run([call(), call(chrom="chrX")]) - path, data = cs.sec_mocha(f"{d}/S", "S") + path, data = sec(f"{d}/S", "S", "mosaic") check("6 the section lists the reported call only", [(x["type"], x["chrom"], x["cell_fraction"]) for x in data["calls"]], [("CN-LOH", "chr20", "0.150")]) check("6 and counts the other", data["not_reported"], 1) @@ -157,24 +174,24 @@ def cases_1_to_6(): "about 0.150 of the cells") check("6 the first line gives the floor", lines[0].startswith("Reported at a cell fraction of 0.050 or more at 30.0x"), True) summ = cs.collect("S", f"{d}/S") - check("6 assessed: no not-assessed line", cs.MCA_NOT_ASSESSED in summ["not_assessed"], False) + check("6 assessed: no not-assessed line", MCA_NOT_ASSESSED in summ["not_assessed"], False) os.remove(f"{d}/S/mosaic/S_mocha_summary.tsv") summ = cs.collect("S", f"{d}/S") - check("6 without step 51: the not-assessed line", cs.MCA_NOT_ASSESSED in summ["not_assessed"], True) + check("6 without step 51: the not-assessed line", MCA_NOT_ASSESSED in summ["not_assessed"], True) _, _, _, d = run([call()], baf_auto=0.05) - data = cs.sec_mocha(f"{d}/S", "S")[1] + data = sec(f"{d}/S", "S", "mosaic")[1] check("6 not assessed: the text says why", rr.mosaic_mca_lines(data)[0].startswith("Not assessed: phased BAF"), True) - check("6 and the not-assessed line stays", cs.MCA_NOT_ASSESSED in cs.collect("S", f"{d}/S")["not_assessed"], True) + check("6 and the not-assessed line stays", MCA_NOT_ASSESSED in cs.collect("S", f"{d}/S")["not_assessed"], True) check("6 no reported call: the text says none is reported, not that none exists", - rr.mosaic_mca_lines(cs.sec_mocha(f"{run([call(cf=0.04)])[3]}/S", "S")[1])[1].startswith("No mosaic alteration reported"), True) + rr.mosaic_mca_lines(sec(f"{run([call(cf=0.04)])[3]}/S", "S", "mosaic")[1])[1].startswith("No mosaic alteration reported"), True) check("6 the pipeline-wide not-assessed list keeps large mosaic alterations out of its chrM-only line", - [x for x in cs.NOT_ASSESSED if x.startswith("Mosaic")], ["Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)"]) + [x for x in cs.collect("S", f"{d}/S")["not_assessed"] if x.startswith("Mosaic")], ["Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)"]) def unreadable(d): try: - cs.sec_mocha(f"{d}/S", "S") - except cs.Unreadable as e: + sec(f"{d}/S", "S", "mosaic") + except Unreadable as e: return str(e) return None diff --git a/tests/test_paralog_loci.py b/tests/test_paralog_loci.py index c6fc5a529..58c9c2ac0 100755 --- a/tests/test_paralog_loci.py +++ b/tests/test_paralog_loci.py @@ -26,7 +26,7 @@ genes it does not list are the 17 of NOT_IN_CALLABILITY; 4. the report readers: the not-assessed row step 35 writes (its own lines, cut from scripts/35-paralogs.sh and run) gives a reading that names the - reason in bin/collect_summary.py, and in bin/paraphase_report.py and + reason in the summary (og summary collect), and in bin/paraphase_report.py and bin/couple_report.py (the Paraphase side-by-side and the couple screen), and a called table still reads as before. Control: the old row with "." for its place is not found over SMN1 exon 7 there. A catalog table @@ -262,6 +262,15 @@ def unlisted(cal_rows): # 4. the report reader on step 35's tables sys.path.insert(0, os.path.join(REPO, "bin")) import collect_summary # noqa: E402 + + def parascopy(d): + """Step 35's section of the summary og summary collect writes for folder D.""" + return collect_summary.collect("S", d)["sections"]["parascopy"]["data"] + + def gba1(d): + """The gba1 catalog row of that section.""" + return next(x for x in parascopy(d)["loci"] if x["table"] == "gba1") + head11 = ("chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\t" "homologous_regions\n") with tempfile.TemporaryDirectory() as d: @@ -279,7 +288,7 @@ def unlisted(cal_rows): os.environ, region="chr5:70951946-70951946", model="SMN1.gz", reason=reason, out=t)) import paraphase_report # noqa: E402 import couple_report # noqa: E402 - _, data = collect_summary.sec_parascopy(d, "S") + data = parascopy(d) check("report: a not-assessed SMN table reads as not assessed, with the reason", data["reading"] == f"SMN1 and SMN2 not assessed: {reason}: no carrier reading" and data["smn1"]["cn"] is None, data.get("reading")) @@ -296,7 +305,7 @@ def unlisted(cal_rows): with open(t, "w") as f: f.write(head11 + "chr5\t70925000\t70953000\tSMN1\tPASS\t4\t40\tPASS\t1,3\t35,30\t" "chr5:70049000-70077000:-\n") - _, data = collect_summary.sec_parascopy(d, "S") + data = parascopy(d) check("report: a called SMN table still reads one SMN1 copy as a carrier", data["smn1"]["cn"] == 1 and "carrier" in data["reading"] and "not assessed" not in data["reading"], data.get("reading")) @@ -310,12 +319,12 @@ def unlisted(cal_rows): "homologous_regions": "*"} with open(g, "w") as f: f.write("\t".join(cols) + "\n" + "\t".join(vals[c] for c in cols) + "\n") - loc = collect_summary.parascopy_locus("gba1", g, None) + loc = gba1(d) check(f"report: a catalog table without {drop} is unreadable, not 'not resolved'", loc["status"] == "unreadable" and loc["cn"] is None, loc) with open(g, "w") as f: f.write(head11 + "chr1\t155231478\t155244699\tGBA\tPASS\t4\t40\tPASS\t2,2\t30,30\t*\n") - loc = collect_summary.parascopy_locus("gba1", g, None) + loc = gba1(d) check("control: the same table with every column is called", loc["status"] == "called" and loc["cn"] == 4, loc) if FAILS: diff --git a/tests/test_pypgx_genes.sh b/tests/test_pypgx_genes.sh index 18810d0dc..a1273b452 100755 --- a/tests/test_pypgx_genes.sh +++ b/tests/test_pypgx_genes.sh @@ -241,7 +241,7 @@ def problems(lines): os.makedirs(os.path.join(d, "pypgx")) with open(os.path.join(d, "pypgx", "s1_pypgx_summary.tsv"), "w") as f: f.write("\n".join("\t".join(r) for r in [hdr] + lines) + "\n") - _, data = collect_summary.sec_pypgx(d, "s1") + data = collect_summary.collect("s1", d)["sections"]["pypgx"]["data"] # The rows of the block (gene, text), with CNVpytor's section beside pypgx's: # pypgx owns every bam gene, so each has one row, and it is pypgx's. block = render_report.pgx_copy_number({"pypgx": {"state": "ok", "data": data}, diff --git a/tests/test_render_report.py b/tests/test_render_report.py index beb0497c0..1d85a1758 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -133,6 +133,21 @@ def check(desc, ok, detail=""): FAILS += 1 +PRS_NOT_ADJUSTED = "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums" + + +def haplogroup_explore(kind, hg): + """The explore links og summary collect writes for one Y ('y') or mitochondrial ('mt') haplogroup call.""" + with tempfile.TemporaryDirectory() as d: + if kind == "y": + put(f"{d}/y_haplogroup/S_y_haplogroup.txt", f"Sample_name\tHg\tValid_markers\tQC-score\nS\t{hg}\t9\t1\n") + key = "y_haplogroup" + else: + put(f"{d}/mito/S_haplogroup.txt", f"SampleID\tHaplogroup\nS\t{hg}\n") + key = "haplogroup" + return collect_summary.collect("S", d)["sections"][key]["data"]["explore"] + + def put(path, text, gz=False): os.makedirs(os.path.dirname(path), exist_ok=True) with (gzip.open(path, "wt") if gz else open(path, "w")) as f: @@ -330,7 +345,7 @@ def chapter_checks(work): ("missing", "missing"), ("ok", "failed"))]) for kind, hg in (("y", "R-M269*(xR-L23)"), ("mt", "H1a+16189")): - rows = render_report.explore_html({"haplogroup": hg, "explore": collect_summary.haplogroup_explore(kind, hg)}) + rows = render_report.explore_html({"haplogroup": hg, "explore": haplogroup_explore(kind, hg)}) marks = re.findall(r'\(([^)]*)\)', "".join(rows)) want = [render_report.OFFSITE_HAPLOGROUP] * (len(marks) - 1) + [render_report.OFFSITE] check(f"chapters: {hg}: the FamilyTreeDNA and YFull links say the address carries the haplogroup, the docs link does not", @@ -740,7 +755,7 @@ def chrx_of(res): and "PGS000002 (3)" not in txt, txt) check("no panel: the HTML row says raw score only", "" in html) check("no panel: not-assessed names the missing percentiles", - collect_summary.PRS_NOT_ADJUSTED in summ["not_assessed"]) + PRS_NOT_ADJUSTED in summ["not_assessed"]) d = f"{work}/sample_adj/S" put(f"{d}/prs/S_prs_summary.tsv", "\t".join(PRS_COLUMNS) + "\n" "Coronary artery disease\tPGS000001\t1.25\t2\t3\t66.7\t94.4\tEUR\t0\tgvcf\n") @@ -752,7 +767,7 @@ def chrx_of(res): check("panel: the note names the group and the panel, no raw-score line, no chrX line", "among the EUR samples of the pgsc_1000G_v1 reference panel" in txt and "Raw score only" not in txt and "Raw score only" not in html and "Rows on chrX" not in txt, txt) - check("panel: no not-assessed line about percentiles", collect_summary.PRS_NOT_ADJUSTED not in summ["not_assessed"]) + check("panel: no not-assessed line about percentiles", PRS_NOT_ADJUSTED not in summ["not_assessed"]) caveat = ("Most PRS scores were developed in European-ancestry populations and predict less well for " "other ancestries, even with an ancestry-adjusted percentile.") check("panel: the ancestry-transfer caveat in both reports", caveat in txt and caveat in html, txt) diff --git a/tests/test_report_charts.py b/tests/test_report_charts.py index 075e36827..23dc05b2d 100755 --- a/tests/test_report_charts.py +++ b/tests/test_report_charts.py @@ -30,7 +30,7 @@ scripts/24-html-report.sh) pass it with _charts.json beside the report, and two renders of the demo give the same file apart from generated_utc; - 9. run_delta.py sees no change between a summary without + 9. og run delta sees no change between a summary without heteroplasmic_sites and one with it; 10. the live page (bin/owngenome_live.py with bin/owngenome_live_charts.py and assets/live/plugins/charts.js): the plugin serves _charts.json and @@ -48,15 +48,16 @@ import os import re import shutil +import subprocess import sys import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import collect_summary # noqa: E402 import render_report # noqa: E402 import report_charts # noqa: E402 -import run_delta # noqa: E402 + +OG = os.environ.get("OG") or os.path.join(REPO, "bin", "og") FAILS = 0 GREENS = ("#d5f5e3", "#1d3b2a", "#8ee0ae", "#196f3d", "green") @@ -278,8 +279,8 @@ def unit_checks(): nd.get("ClinVar hits on chrM") == 1 and nd.get("ClinVar hits on other contigs (alt, unplaced)") == 1 and "Not drawn:" in g["svg"] and not any(m["chrom"] in ("chrM", "chr22_KI270879v1_alt") for m in g["marks"]), nd) - check("4. genome: the 5 MB floor is collect_summary.ROH_MIN_BP", - report_charts.ROH_MIN_MB * 1e6 == collect_summary.ROH_MIN_BP) + # The floor of the roh section (og-summary, collect/group_c/roh.rs ROH_MIN_BP). + check("4. genome: the 5 MB floor is the roh section's", report_charts.ROH_MIN_MB * 1e6 == 5e6) for region in ("chr3:10000000_22300000", "3:1-2", "chr3:20-10"): bad = copy.deepcopy(s) bad["sections"]["roh"]["data"]["autosomal_over_5mb"][0]["region"] = region @@ -426,14 +427,31 @@ def demo_checks(work): check("7. the hits table carries the id the charts link to, and its rows keep their shape", page.count('') == 1 and len(re.findall(r"" in html, True) check("7 the card lists the low call too", "" in html, True) @@ -328,9 +342,9 @@ def good(gb="0", dp="20"): with open(os.path.join(sd, "y_str", "S1_y_str.tsv"), "w") as f: f.write(t) try: - cs.sec_y_str(sd, "S1") + sec(sd, "S1", "y_str") check("7 an unknown confidence is unreadable", "read", "Unreadable") -except cs.Unreadable: +except Unreadable: check("7 an unknown confidence is unreadable", "Unreadable", "Unreadable") try: page = rr.html_report(cs.collect("S1", sd)) From 2545b38bf3ba63ef2b1027f9f88f11a0f40f62a9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:32:53 +0100 Subject: [PATCH 072/255] refactor(summary): delete the Python the summary switch left unused With summary.json written by og summary collect, nothing imports pgx_outside_calls.py, ancestry_fine_report.py, clip_check.py, annotsv_postfilter.py, chip_report.py, traits_report.py or genotype_rules_indels.py any more. They go, with test_clip_check.py and test_genotype_rules_indels.py (their checks are og's cargo tests now). The e2e cases that read a pinned record from them read og-summary's and og-cn's constants. collect_summary.py is listed as a remnant for the report, couple, index and live code until their own ports. Bead: pgp-bs9u.45 --- .github/workflows/guard.yml | 2 +- assets/genotype_rules.tsv | 2 +- assets/imprinted_regions.tsv | 4 +- bin/render_report.py | 2 +- scripts/ci/e2e-case-paths.tsv | 186 +++++++++++----------- tests/e2e/genotype-rules-2-g2.sh | 35 ++-- tests/e2e/genotype-rules-2-lpa-null.sh | 14 +- tests/e2e/genotype-rules-2-smn1-marker.sh | 14 +- tests/e2e/str-extra-repeats-1-readings.sh | 6 +- tests/parity/remnants.tsv | 6 +- tests/test_ancestry_fine_report.py | 68 +------- tests/test_e2e_scope.sh | 18 +-- tests/test_pgx_outside_calls.py | 32 +--- 13 files changed, 160 insertions(+), 229 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 4dfdc9daf..7f6ae2a0b 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -624,7 +624,7 @@ jobs: tests/test_results_index.py \ tests/test_pgx_outside_calls.py tests/test_hla_second_caller.py tests/test_paraphase_report.py \ tests/test_copy_number.py tests/test_hba_copy_number.py \ - tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_traits_table.py tests/test_genotype_rules_indels.py tests/test_callability.py; do + tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_traits_table.py tests/test_callability.py; do rc=0 echo "::group::${t}" case "$t" in diff --git a/assets/genotype_rules.tsv b/assets/genotype_rules.tsv index fb8eb55cf..bd99e77bf 100644 --- a/assets/genotype_rules.tsv +++ b/assets/genotype_rules.tsv @@ -41,7 +41,7 @@ haplotype TAS2R38 AVI C,A,T combination TAS2R38 PAV/PAV PAV/PAV Two copies of PAV, the taster form: in Bufe 2005 this group rated PTC and PROP the most bitter. combination TAS2R38 PAV/AVI PAV/AVI One PAV and one AVI copy: a taster. In Bufe 2005 this group recognised PTC and PROP as bitter at concentrations close to those of PAV/PAV; how bitter they rated them lay between the two homozygous groups and varied widely. combination TAS2R38 AVI/AVI AVI/AVI Two copies of AVI, the non-taster form: in Bufe 2005 the AVI receptor did not respond to PTC, and this group recognised PTC as bitter only at far higher concentrations. PROP responses varied more. -rule APOL1 APOL1 APOL1 kidney risk variants (G1 read, G2 not read) no yes G1 is two missense changes that sit on one copy, S342G (rs73885319) and I384M (rs60910145), in perfect linkage disequilibrium in Genovese 2010, so they are read together without phase. G2 (rs71785313) is a 6-base deletion of N388 and Y389; this rule table reads single-base sites only, so this rule does not read G2 and a G2 copy reads as G0 here; the HTML report reads G2 from the sample's VCF beside this result (bin/genotype_rules_indels.py). G1 and G2 never share a copy. In African Americans, two risk copies (G1 or G2) went with focal segmental glomerulosclerosis (odds ratio 10.5) and hypertension-attributed end-stage kidney disease (odds ratio 7.3), and one copy did not raise the risk (a recessive pattern). Both variants are common on African chromosomes and were absent from the European chromosomes studied. 20647424 +rule APOL1 APOL1 APOL1 kidney risk variants (G1 read, G2 not read) no yes G1 is two missense changes that sit on one copy, S342G (rs73885319) and I384M (rs60910145), in perfect linkage disequilibrium in Genovese 2010, so they are read together without phase. G2 (rs71785313) is a 6-base deletion of N388 and Y389; this rule table reads single-base sites only, so this rule does not read G2 and a G2 copy reads as G0 here; the HTML report reads G2 from the sample's VCF beside this result (og summary collect). G1 and G2 never share a copy. In African Americans, two risk copies (G1 or G2) went with focal segmental glomerulosclerosis (odds ratio 10.5) and hypertension-attributed end-stage kidney disease (odds ratio 7.3), and one copy did not raise the risk (a recessive pattern). Both variants are common on African chromosomes and were absent from the European chromosomes studied. 20647424 site APOL1 rs73885319 chr22 36265860 A G site APOL1 rs60910145 chr22 36265988 T G haplotype APOL1 G0 A,T diff --git a/assets/imprinted_regions.tsv b/assets/imprinted_regions.tsv index b83ec1c89..c71bb24dd 100644 --- a/assets/imprinted_regions.tsv +++ b/assets/imprinted_regions.tsv @@ -1,6 +1,6 @@ # Imprinting disorders caused by uniparental disomy (UPD), one row per chromosome or region. -# Read by bin/collect_summary.py (IMPRINTED_REGIONS holds the same rows; tests/test_collect_summary.py -# checks that the two agree) to name what a run of homozygosity over most of one chromosome could mean. +# Read by og summary collect (IMPRINTED_REGIONS in rust/crates/og-summary holds the same rows; its +# tests check that the two agree) to name what a run of homozygosity over most of one chromosome could mean. # start and end: GRCh38, 0-based start, exclusive end, the UCSC cytoBand.hg38.txt bands of the region; # empty for a row that applies to the whole chromosome. A row with a region applies when a run on the # flagged chromosome overlaps it. "-" is no known imprinting disorder for UPD from that parent. diff --git a/bin/render_report.py b/bin/render_report.py index eac22b4f4..e949a759b 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -404,7 +404,7 @@ def cyp2d6_verdict(c): def probable_text(p): - """The probable CYP2D6 call (pgx_outside_calls.probable_cyp2d6) as one line.""" + """The probable CYP2D6 call (og summary collect's cyp2d6 block) as one line.""" return (f"{p.get('diplotype')}" + (f", {p['phenotype']}" if p.get("phenotype") else "") + "; not passed to PharmCAT, so no drug guidance rests on it") diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 7f5b093d3..50dd13368 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -40,143 +40,143 @@ 51-text-report - 60-nextflow - 90-report-assets - -acmg-sf-rules-1-reports assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh -align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/render_report.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +acmg-sf-rules-1-reports assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh +align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh alignment-markdup-gvcf-sex-1-align scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-2-ploidy scripts/ci/fake-docker/lib.sh -alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh +alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-4-bwamem2 scripts/02a-alignment-bwamem2.sh scripts/ci/fake-docker/lib.sh -ancestry-fine-grained-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgs_scores.tsv assets/stub/sample.vcf.gz bin/ancestry_fine_report.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancestry_fine/* modules/local/html_report/* modules/local/prs/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/26b-ancestry-fine.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/og bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -annotsv-delly-depth-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh tests/fixtures/annotsv/class45.sh -annotsv-report-section-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh tests/fixtures/annotsv/class45.sh -annotsv-sv-inputs-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -archaic-ancestry-1 assets/pgs_scores.tsv bin/og bin/collect_summary.py bin/traits_report.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh +ancestry-fine-grained-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgs_scores.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/ancestry_fine/* modules/local/html_report/* modules/local/prs/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/26b-ancestry-fine.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/og bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +annotsv-delly-depth-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh tests/fixtures/annotsv/class45.sh +annotsv-report-section-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh tests/fixtures/annotsv/class45.sh +annotsv-sv-inputs-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh +archaic-ancestry-1 assets/pgs_scores.tsv bin/og bin/collect_summary.py modules/local/archaic/* modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/43-archaic.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh bash-step-08-hla - bash-step-09-str assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh bash-step-10-telomere scripts/10-telomere-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh bash-step-11-roh scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh bash-step-14-imputation scripts/14-imputation-prep.sh scripts/ci/fake-docker/lib.sh bash-step-19-delly scripts/19-delly.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -bash-step-20-numt assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh -bash-step-bash42 assets/construct_variants.tsv bin/clip_check.py bin/cpsr_sample_id bin/og scripts/03d-octopus.sh scripts/04a-tiddit.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh +bash-step-20-numt assets/callability_genes.tsv bin/collect_summary.py scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh +bash-step-bash42 assets/construct_variants.tsv bin/cpsr_sample_id bin/og scripts/03d-octopus.sh scripts/04a-tiddit.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh bloodtype-1 assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og modules/local/bloodtype/* modules/local/vcf_precheck/* scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh bumps-drop-in-baseline scripts/ci/fake-docker/lib.sh callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh -callability-2-genes assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -chip-clonal-1 assets/chip_genes.tsv bin/chip_report.py bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +callability-2-genes assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +chip-clonal-1 assets/chip_genes.tsv bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh chip-mt-1-roundtrip assets/par_grch38.bed bin/og scripts/40-export-consumer.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz ci-e2e-2-locityper assets/locityper_loci.bed bin/og scripts/53-locityper.sh scripts/setup.sh -clinical-filter-mane-and-quality-1-tiers assets/construct_variants.tsv bin/clip_check.py bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh +clinical-filter-mane-and-quality-1-tiers assets/construct_variants.tsv bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh clinical-filter-mane-and-quality-2-vep-norm scripts/13-vep-annotation.sh scripts/ci/fake-docker/lib.sh -clinical-filter-mane-and-quality-3-modules assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/clinical_filter/* modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -clinvar-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/clip_check.py bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh -clinvar-combined-allele-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/vcf/combined_allele_clinvar.vcf tests/fixtures/vcf/combined_allele_sample.vcf -clinvar-cpsr-quality-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/clip_check.py bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh -cn-engine-1-parity assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og bin/traits_report.py scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cn-guard-1-mapq0 bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh +clinical-filter-mane-and-quality-3-modules assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/clinical_filter/* modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +clinvar-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh +clinvar-combined-allele-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/vcf/combined_allele_clinvar.vcf tests/fixtures/vcf/combined_allele_sample.vcf +clinvar-cpsr-quality-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh +cn-engine-1-parity assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cn-guard-1-mapq0 bin/cn_loci.tsv bin/og modules/local/parascopy/* scripts/35-paralogs.sh cn-intervals-1-catalog assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh -cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs -construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/chip_report.py bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +cn-intervals-2-hg002 assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/og modules/local/deepvariant/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cn-report-1-tables assets/carrier_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py scripts/24-html-report.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh +comphet-phasing-1-reads assets/construct_variants.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/slivar/* modules/local/vcf_precheck/* modules/local/vep/* scripts/13-vep-annotation.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs +construct-scan-1-both assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +consumer-export-1 assets/chip_genes.tsv assets/par_grch38.bed bin/og modules/local/chip/* modules/local/export_consumer/* modules/local/vcf_precheck/* scripts/40-export-consumer.sh scripts/49-chip.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz containers-software-versions scripts/ci/fake-docker/lib.sh scripts/ci/gen-containers-config.sh -couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/parascopy/* scripts/35-paralogs.sh -exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +couple-carrier-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/paralog_loci.tsv assets/pedigree.example.tsv assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/expansion_hunter/* modules/local/hla_typing/* modules/local/html_report/* modules/local/parascopy/* modules/local/somalier/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/09-expansion-hunter.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/35-paralogs.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/couple/* modules/local/delly/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/og modules/local/parascopy/* scripts/35-paralogs.sh +exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh -families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv -family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv +family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv bin/og scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh -followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py bin/traits_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh -followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh -followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py bin/traits_report.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz bin/og +followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh +followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh +followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz bin/og genome-lookup-1 bin/og scripts/ci/fake-docker/lib.sh scripts/owngenome-lookup.sh genotype-rule-rows-1-str-extra assets/str_extra_loci.json assets/stub/db/catalog.json scripts/09-expansion-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -genotype-rules-1-phase assets/genotype_rules.tsv assets/traits.tsv bin/traits_report.py modules/local/traits/* modules/local/vcf_precheck/* scripts/42-traits.sh scripts/ci/check-rule-alleles.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -genotype-rules-2-g2 assets/genotype_rules.tsv bin/collect_summary.py bin/genotype_rules_indels.py bin/traits_report.py -genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/traits_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh -genotype-rules-2-smn1-marker bin/collect_summary.py bin/og bin/render_report.py bin/traits_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh -haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/traits_report.py bin/yleaf_run.py modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/vcf_precheck/* modules/local/y_haplogroup/* scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh -hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +genotype-rules-1-phase assets/genotype_rules.tsv assets/traits.tsv modules/local/traits/* modules/local/vcf_precheck/* scripts/42-traits.sh scripts/ci/check-rule-alleles.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +genotype-rules-2-g2 assets/genotype_rules.tsv bin/collect_summary.py +genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh +genotype-rules-2-smn1-marker bin/collect_summary.py bin/og bin/render_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh +haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/yleaf_run.py modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/vcf_precheck/* modules/local/y_haplogroup/* scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh +hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/og scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh -mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/og bin/traits_report.py scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh -nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh -nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/traits_report.py modules/local/indexcov/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/traits_report.py scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh -nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/annotsv_postfilter.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh -nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/annotsv_postfilter.py bin/clip_check.py bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -nf-wiring-1-report-tables assets/acmg_sf.tsv assets/test_samplesheet.csv assets/stub/* bin/clip_check.py bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py bin/traits_report.py conf/test_all.config conf/test.config modules/local/align_minimap2/* modules/local/clinical_filter/* modules/local/clinvar_screen/* modules/local/cpsr/* modules/local/html_report/* modules/local/slivar/* modules/local/vep/* scripts/24-html-report.sh tests/fixtures/consumer_export/sites.tsv.gz +lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh +mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/og scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh +nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.html assets/viewer/index.html bin/collect_summary.py modules/local/indexcov/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh +nextflow-hardening-2-nf assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/viewer/index.html bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/yleaf_run.py modules/local/clinvar_screen/* modules/local/delly/* modules/local/html_report/* modules/local/indexcov/* modules/local/manta/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/survivor_merge/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/y_haplogroup/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/10-telomere-hunter.sh scripts/12-mito-haplogroup.sh scripts/15-duphold.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh +nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv bin/cn_loci.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/annotsv/* modules/local/clinvar_screen/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/mito_variants/* modules/local/pypgx/* modules/local/slivar/* modules/local/stranger/* modules/local/vcf_precheck/* modules/local/vep/* scripts/05-annotsv.sh scripts/06-clinvar-screen.sh scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/13-vep-annotation.sh scripts/15-duphold.sh scripts/16b-mosdepth.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +nf-wiring-1-report-tables assets/acmg_sf.tsv assets/test_samplesheet.csv assets/stub/* bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py conf/test_all.config conf/test.config modules/local/align_minimap2/* modules/local/clinical_filter/* modules/local/clinvar_screen/* modules/local/cpsr/* modules/local/html_report/* modules/local/slivar/* modules/local/vep/* scripts/24-html-report.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-2-sv-str-tools bin/cnvpytor_to_vcf.py bin/family_report.py bin/og modules/local/cnvpytor/* modules/local/duphold/* modules/local/expansion_hunter/* modules/local/indexcov/* modules/local/manta/* modules/local/vcf_precheck/* output-contract-1-hg002 assets/* bin/* modules/* scripts/* -paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og bin/traits_report.py modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralogs-paraphase-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/html_report/* modules/local/paraphase/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/24-html-report.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +paralogs-paraphase-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/html_report/* modules/local/paraphase/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/24-html-report.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz pgx-1-depth-check bin/og scripts/21-cyrius.sh scripts/32-pypgx.sh scripts/ci/alt-depth-ab.sh scripts/ci/fake-docker/lib.sh -pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/og bin/traits_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/pgx_outside_calls.py bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -pgx-6-gvcf-column assets/live/index.html assets/par_grch38.bed assets/pgs_scores.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/indexcov/* modules/local/pharmcat/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/16-indexcov.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/og scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +pgx-6-gvcf-column assets/live/index.html assets/par_grch38.bed assets/pgs_scores.tsv assets/viewer/index.html bin/collect_summary.py bin/og modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/indexcov/* modules/local/pharmcat/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/16-indexcov.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cn_loci.tsv bin/collect_summary.py bin/og modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh pharmcat-indel-refblocks-1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh tests/fixtures/pharmcat/refblocks.g.vcf pharmcat-mito-input-and-log-1-mtrnr1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh platelet-antigens-1 assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -prs-1-agreement assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh -prs-2-ancestry assets/pgs_scores.tsv bin/collect_summary.py bin/render_report.py bin/traits_report.py scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -prs-3-panel-measure assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -prs-runopts-and-trace-1 assets/par_grch38.bed assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py bin/traits_report.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/og bin/pgx_outside_calls.py scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -qc-1-sample-qc bin/collect_summary.py bin/og bin/traits_report.py scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -qc-3-cram assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py modules/local/cram_archive/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/34-cram-archive.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +prs-1-agreement assets/pgs_scores.tsv bin/collect_summary.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh +prs-2-ancestry assets/pgs_scores.tsv bin/collect_summary.py bin/render_report.py scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +prs-3-panel-measure assets/pgs_scores.tsv bin/collect_summary.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +prs-runopts-and-trace-1 assets/par_grch38.bed assets/pgs_scores.tsv bin/collect_summary.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summary.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +qc-1-sample-qc bin/collect_summary.py bin/og scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +qc-3-cram assets/callability_genes.tsv bin/collect_summary.py modules/local/cram_archive/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/34-cram-archive.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh reference-noalt scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -report-followups-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/traits.tsv bin/annotsv_postfilter.py bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/annotsv/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/manta/* modules/local/traits/* modules/local/vcf_precheck/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/09-expansion-hunter.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz -report-sections-mito-roh-telomere-cnv-1-mito assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/10-telomere-hunter.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh -reports-and-pgx-correctness-clinical assets/construct_variants.tsv bin/clip_check.py bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh -reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/traits_report.py scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -reports-and-pgx-correctness-reports assets/stub/db/clinvar.vcf.gz assets/stub/db/hla.dat bin/collect_summary.py bin/render_report.py bin/traits_report.py bin/write_manifest.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh +report-followups-1-sections assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.vcf.gz assets/sv_rule_regions.tsv assets/traits.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/annotsv/* modules/local/expansion_hunter/* modules/local/export_consumer/* modules/local/html_report/* modules/local/manta/* modules/local/traits/* modules/local/vcf_precheck/* scripts/04-manta.sh scripts/05-annotsv.sh scripts/09-expansion-hunter.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +report-sections-mito-roh-telomere-cnv-1-mito assets/callability_genes.tsv bin/collect_summary.py scripts/10-telomere-hunter.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh +reports-and-pgx-correctness-clinical assets/construct_variants.tsv bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh +reports-and-pgx-correctness-pgx assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +reports-and-pgx-correctness-reports assets/stub/db/clinvar.vcf.gz assets/stub/db/hla.dat bin/collect_summary.py bin/render_report.py bin/write_manifest.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh reports-and-pgx-correctness-revel scripts/30-vcfanno.sh scripts/ci/fake-docker/lib.sh -run-all-launcher assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/vep/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/13-vep-annotation.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +run-all-launcher assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* modules/local/vep/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/13-vep-annotation.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh run-all-sample-qc-and-stale-index-1-index-times assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -setup-gencode-exons-and-nextflow-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/cpsr_sample_id bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +setup-gencode-exons-and-nextflow-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh single-source-lib-ownership scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/expansion_hunter/* modules/local/stranger/* modules/local/vcf_precheck/* scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/traits_report.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh +str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh sv-mito-telomere-steps-1-sv-merge scripts/22-survivor-merge.sh scripts/ci/fake-docker/lib.sh tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in -sv-mito-telomere-steps-2-mito assets/callability_genes.tsv bin/collect_summary.py bin/traits_report.py scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh +sv-mito-telomere-steps-2-mito assets/callability_genes.tsv bin/collect_summary.py scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh sv-mito-telomere-steps-3-y-haplogroup assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh sv-mito-telomere-steps-4-clinvar-tier bin/og scripts/23-clinical-filter.sh scripts/ci/fake-docker/lib.sh sv-mito-telomere-steps-5-giab scripts/benchmark-variants.sh scripts/ci/fake-docker/lib.sh sv-mito-telomere-steps-6-scatter scripts/03a-gatk-haplotypecaller.sh scripts/03b-freebayes.sh scripts/ci/fake-docker/lib.sh -traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vcf-precheck-ref-spotcheck-1 assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/vcf/grch37_no_contigs.vcf vendor-vcf-intake-1-bash bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/clip_check.py bin/collect_summary.py bin/og bin/traits_report.py modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/clip_check.py bin/cn_loci.tsv bin/collect_summary.py bin/og bin/pgx_outside_calls.py bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/og modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh viral-reads-1 bin/og modules/local/vcf_precheck/* modules/local/viral/* scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh -vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/traits_report.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/traits_report.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh rust-z2e-readers bin/og tests/e2e/lib.sh scripts/ci/image-smoke.sh scripts/ci/nf-trace-requests.tsv rust-c2b-traits assets/traits.tsv assets/genotype_rules.tsv bin/og modules/local/traits/* scripts/42-traits.sh diff --git a/tests/e2e/genotype-rules-2-g2.sh b/tests/e2e/genotype-rules-2-g2.sh index 5932e63c3..ee560b54b 100644 --- a/tests/e2e/genotype-rules-2-g2.sh +++ b/tests/e2e/genotype-rules-2-g2.sh @@ -1,20 +1,23 @@ #!/usr/bin/env bash -# APOL1 G2 (rs71785313) beside step 42's APOL1 rule (bin/genotype_rules_indels.py), +# APOL1 G2 (rs71785313) beside step 42's APOL1 rule (og summary collect's G2 +# reader, the apol1 module of rust/crates/og-summary/src/collect/group_c/traits.rs), # with the real bcftools images: -# - dbSNP orientation, online: the record pinned in the module (DBSNP) is what -# dbSNP's GRCh38 VCF of the build versions.env pins holds for rs71785313 (same -# position and REF, the module's ALT among dbSNP's); control: the same compare on -# a copy with REF and ALT swapped fails; +# - dbSNP orientation, online: the record pinned in that module (POS, REF, ALT) is +# what dbSNP's GRCh38 VCF of the build versions.env pins holds for rs71785313 +# (same position and REF, the module's ALT among dbSNP's); control: the same +# compare on a copy with REF and ALT swapped fails; # - the report's reader through a real .tbi: a bgzipped, indexed gVCF (bcftools -# view -Oz, bcftools index -t) with a G2 deletion, read by collect_summary's -# genotype_rules, gives G1/G2 beside a G0/G1 rule line; control: the same -# folder with a reference block over G2 gives G0/G1. +# view -Oz, bcftools index -t) with a G2 deletion, read by og summary collect, +# gives G1/G2 beside a G0/G1 rule line; control: the same folder with a +# reference block over G2 gives G0/G1. # Writes one folder under GENOME_DIR, removed at the end; no other case reads it. . "$(dirname "$0")/lib.sh" URL="https://ftp.ncbi.nih.gov/snp/archive/${CONSUMER_DBSNP_BUILD:?}/VCF/GCF_000001405.40.gz" -PINNED=$(python3 -c "import sys; sys.path.insert(0, '${REPO}/bin'); import genotype_rules_indels as m; \ -print(m.DBSNP[0], m.DBSNP[2], m.DBSNP[3], m.DBSNP[4], sep='\t')") +# The pinned record: the POS, REF and ALT constants of og-summary's apol1 module. +PINNED=$(awk '/^pub mod apol1/ { on = 1 } on && /^ const (POS|REF|ALT):/ { v = $NF; gsub(/[";_]/, "", v); x[$2] = v } + END { printf "rs71785313\t%s\t%s\t%s\n", x["POS:"], x["REF:"], x["ALT:"] }' \ + "${REPO}/rust/crates/og-summary/src/collect/group_c/traits.rs") echo "pinned: ${PINNED}" # same_record PINNED LIVE: the pinned rsid, POS and REF equal dbSNP's, and the pinned ALT is one of dbSNP's. @@ -73,12 +76,12 @@ PY done read_result() { - python3 - "$REPO" "${D}/$1" "$1" <<'PY' -import os, subprocess, sys -repo, d, s = sys.argv[1:] -sys.path.insert(0, os.path.join(repo, "bin")) -import collect_summary as cs -r = next(x for x in cs.genotype_rules(d, s) if x["rule"] == "APOL1") + "${REPO}/bin/og" summary collect --sample "$1" --sample-dir "${D}/$1" --assets "${REPO}/assets" \ + --out "${CASE_TMP}/$1.summary.json" > /dev/null + python3 - "${CASE_TMP}/$1.summary.json" <<'PY' +import json, sys +rules = json.load(open(sys.argv[1]))["sections"]["traits"]["data"]["rules"] +r = next(x for x in rules if x["rule"] == "APOL1") print(r["result"], r["g2"]["result"], r["g2"]["source"], r["g2"]["status"], sep="\t") PY } diff --git a/tests/e2e/genotype-rules-2-lpa-null.sh b/tests/e2e/genotype-rules-2-lpa-null.sh index f92c2bc11..891a1c72b 100755 --- a/tests/e2e/genotype-rules-2-lpa-null.sh +++ b/tests/e2e/genotype-rules-2-lpa-null.sh @@ -39,14 +39,20 @@ sample() { rm -rf "${G:?}/$1"; mkdir -p "${G}/$1/aligned" cp "$2" "${G}/$1/aligned/$1_sorted.bam"; cp "$2.bai" "${G}/$1/aligned/$1_sorted.bam.bai" } -SITES=$(python3 -c "import sys; sys.path.insert(0, '${REPO}/bin'); import traits_report as t; \ -print(','.join(map(str, t.MARKERS['LPA_KIV2_4925']['copies'])))") +# marker KEY copies|dbsnp: the copies (comma-separated) or the dbSNP record +# (rsID, POS, REF, ALT) of a copy-aware marker, from og-cn's table of them. +marker() { + awk -v k="$1" -v what="$2" '$0 ~ "key: \"" k "\"" { on = 1 } on && /^};/ { exit } on { b = b $0 "\n" } END { + if (what == "copies") { c = b; sub(/.*copies: &\[/, "", c); sub(/\].*/, "", c); gsub(/[ \n_]/, "", c); sub(/,$/, "", c); print c } + else { d = b; sub(/.*dbsnp: \(/, "", d); sub(/\).*/, "", d); gsub(/[ "_]/, "", d); split(d, x, ","); print x[1] "\t" x[3] "\t" x[4] "\t" x[5] } }' \ + "${REPO}/rust/crates/og-cn/src/markers.rs" +} +SITES=$(marker LPA_KIV2_4925 copies) check_eq "seven copies of the 4925G>A site" "$(tr ',' '\n' <<< "$SITES" | wc -l | tr -d ' ')" 7 # --- dbSNP orientation, online -------------------------------------------------------- URL="https://ftp.ncbi.nih.gov/snp/archive/${CONSUMER_DBSNP_BUILD:?}/VCF/GCF_000001405.40.gz" -PINNED=$(python3 -c "import sys; sys.path.insert(0, '${REPO}/bin'); import traits_report as t; \ -d = t.MARKERS['LPA_KIV2_4925']['dbsnp']; print(d[0], d[2], d[3], d[4], sep='\t')") +PINNED=$(marker LPA_KIV2_4925 dbsnp) same_record() { awk -F'\t' 'NR == 1 { id = $1; pos = $2; ref = $3; alt = $4; next } $1 == id && $2 == pos && $3 == ref && index("," $4 ",", "," alt ",") { ok = 1 } diff --git a/tests/e2e/genotype-rules-2-smn1-marker.sh b/tests/e2e/genotype-rules-2-smn1-marker.sh index 38cc4b746..e0e9134b5 100755 --- a/tests/e2e/genotype-rules-2-smn1-marker.sh +++ b/tests/e2e/genotype-rules-2-smn1-marker.sh @@ -30,14 +30,20 @@ W="${GENOME_DIR}/${P}" rm -rf "$W"; mkdir -p "${W}/aligned" "${W}/paralogs" retry() { local i; for i in 1 2 3 4; do "$@" && return 0; [ "$i" -lt 4 ] && { echo " try ${i} of 4 failed; again in $((i * 15)) s"; sleep $((i * 15)); }; done; return 1; } col() { awk -F'\t' -v k="$2" '$1 == k {print $2}' "$1" 2>/dev/null; } -SITES=$(python3 -c "import sys; sys.path.insert(0, '${REPO}/bin'); import traits_report as t; \ -print(','.join(map(str, t.MARKERS['SMN1_27134']['copies'])))") +# marker KEY copies|dbsnp: the copies (comma-separated) or the dbSNP record +# (rsID, POS, REF, ALT) of a copy-aware marker, from og-cn's table of them. +marker() { + awk -v k="$1" -v what="$2" '$0 ~ "key: \"" k "\"" { on = 1 } on && /^};/ { exit } on { b = b $0 "\n" } END { + if (what == "copies") { c = b; sub(/.*copies: &\[/, "", c); sub(/\].*/, "", c); gsub(/[ \n_]/, "", c); sub(/,$/, "", c); print c } + else { d = b; sub(/.*dbsnp: \(/, "", d); sub(/\).*/, "", d); gsub(/[ "_]/, "", d); split(d, x, ","); print x[1] "\t" x[3] "\t" x[4] "\t" x[5] } }' \ + "${REPO}/rust/crates/og-cn/src/markers.rs" +} +SITES=$(marker SMN1_27134 copies) check_eq "two copies of the marker site (SMN1, SMN2)" "$SITES" "70952074,70076654" # --- dbSNP orientation, online -------------------------------------------------------- URL="https://ftp.ncbi.nih.gov/snp/archive/${CONSUMER_DBSNP_BUILD:?}/VCF/GCF_000001405.40.gz" -PINNED=$(python3 -c "import sys; sys.path.insert(0, '${REPO}/bin'); import traits_report as t; \ -d = t.MARKERS['SMN1_27134']['dbsnp']; print(d[0], d[2], d[3], d[4], sep='\t')") +PINNED=$(marker SMN1_27134 dbsnp) same_record() { awk -F'\t' 'NR == 1 { id = $1; pos = $2; ref = $3; alt = $4; next } $1 == id && $2 == pos && $3 == ref && index("," $4 ",", "," alt ",") { ok = 1 } diff --git a/tests/e2e/str-extra-repeats-1-readings.sh b/tests/e2e/str-extra-repeats-1-readings.sh index 14c9806da..65981889f 100644 --- a/tests/e2e/str-extra-repeats-1-readings.sh +++ b/tests/e2e/str-extra-repeats-1-readings.sh @@ -120,12 +120,12 @@ unset REF_FASTA python3 - "$G" "$REPO" <<'PY' import os, sys g, repo = sys.argv[1], sys.argv[2] -sys.path.insert(0, os.path.join(repo, "bin")) -import traits_report +# The columns of step 42's traits table (og traits table). +OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", "note", "pmids"] os.makedirs(f"{g}/SXA/traits", exist_ok=True) os.makedirs(f"{g}/SXA/cpic", exist_ok=True) with open(f"{g}/SXA/traits/SXA_traits.tsv", "w") as f: - f.write("\t".join(traits_report.OUT_COLUMNS) + "\n") + f.write("\t".join(OUT_COLUMNS) + "\n") f.write("rs78655421\tCFTR\tchr7\t117530975\tCFTR R117H\tG/G\tcalled\tNo R117H copy.\tno\t-\t32404922\n") with open(f"{g}/SXA/cpic/SXA_phenotypes.tsv", "w") as f: f.write("Gene\tDiplotype\tPhenotype\tStatus\nUGT1A1\t*1/*28\tIntermediate Metabolizer\tnon-normal\n") diff --git a/tests/parity/remnants.tsv b/tests/parity/remnants.tsv index 268a9ce99..721483261 100644 --- a/tests/parity/remnants.tsv +++ b/tests/parity/remnants.tsv @@ -5,8 +5,6 @@ # filenamesimporterslane bin/family_report.py COLUMNS,MISSING,SEX,NAME,BINS,die,missing,read_pedigree,build_members,bad_name,kinship_table,declared_relationship,degree,read_par,in_par bin/couple_report.py,bin/pedigree_svg.py,tests/test_family_report.py R4 bin/genome_lookup.py GENE_KEYS,RSID_RE,norm_chrom,parse_query,names_of,walk_mentions,flat bin/results_index.py,tests/test_results_index.py R5c -bin/pgx_outside_calls.py HLA_GENES,NO_CALL,PROBABLE,NOT_CALLABLE,LOW_GQ,is_call,same_diplotype,probable_cyp2d6 bin/collect_summary.py,tests/test_pgx_outside_calls.py R3 -bin/paraphase_report.py SMN1_EXON7,parascopy_smn1,compare_smn1 bin/collect_summary.py,bin/couple_report.py,tests/test_paralog_loci.py,tests/test_paraphase_report.py R4 +bin/paraphase_report.py SMN1_EXON7,parascopy_smn1,compare_smn1 bin/couple_report.py,tests/test_paralog_loci.py,tests/test_paraphase_report.py R4 bin/cnvpytor_to_vcf.py DEPTH_MIN_SIZE,delly_depth_contradicts bin/couple_report.py R4 -bin/annotsv_postfilter.py CONSENSUS_BP,KEEP,ADDED,open_text,chrom_key,sv_type,info_value,sample_format,sample_columns,read_consensus,consensus_support bin/collect_summary.py,tests/demo/make_demo_sample.py R3 -bin/traits_report.py COLUMNS,OUT_COLUMNS,RULE_OUT_COLUMNS,TRAITS_TABLE_RULES,TableError,read_table,MARKERS,MARKER_STATUSES bin/collect_summary.py,tests/demo/make_demo_sample.py,tests/test_traits_report.py,tests/test_traits_table.py,tests/test_collect_summary.py,tests/test_lpa_kiv2.py R3 +bin/collect_summary.py HERE,ASSETS_DIR,SECTIONS,og,collect,STARS,CALL_FLOOR,COMMON_AF,RUN_ALL_STEPS,SEVERITY_KINDS,Unreadable,open_text,vcf_records,info_map,read_tsv,clinvar_variants,clinvar_combined_note,low_confidence,half_call,clinvar_zygosity,clinvar_row,hla_quality,HLA_MAX_CANDIDATES,EXPLORE_DOCS,csq_rank,allele_key,parse_alleles bin/couple_report.py,bin/owngenome_live.py,bin/render_report.py,bin/results_index.py,tests/demo/make_demo_sample.py,tests/test_acmg_sf_rules.py,tests/test_ancestry_fine_report.py,tests/test_archaic_report.py,tests/test_callability.py,tests/test_chip_report.py,tests/test_collect_summary.py,tests/test_copy_number.py,tests/test_cross_engine_biallelic.py,tests/test_exomiser_report.py,tests/test_haplogroup_explore.py,tests/test_hla_report.py,tests/test_live_journal.py,tests/test_live_results.py,tests/test_locityper_report.py,tests/test_lpa_kiv2.py,tests/test_mei_report.py,tests/test_mito_extended.py,tests/test_mocha_report.py,tests/test_paralog_loci.py,tests/test_render_report.py,tests/test_results_index.py,tests/test_run_state.py,tests/test_traits_report.py,tests/test_viewer.py,tests/test_viral_report.py,tests/test_vntyper_report.py,tests/test_ystr_report.py R6b diff --git a/tests/test_ancestry_fine_report.py b/tests/test_ancestry_fine_report.py index 5e0831e28..1f00da38c 100755 --- a/tests/test_ancestry_fine_report.py +++ b/tests/test_ancestry_fine_report.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 -"""og ancestry-fine, step 26b's commands, on made-up inputs, and what is left -of bin/ancestry_fine_report.py: +"""og ancestry-fine, step 26b's commands, on made-up inputs: panel-map a group of fewer than 10 unrelated people (one Oceanian, three San) is no reference and no painting reference, and the @@ -16,10 +15,7 @@ report fractions with the bootstrap range, merged painting rows, the painted shares, a PNG of the right size, the left-out group and the chromosome RFMix failed on, in the table, the text and - both pipeline reports (collect_summary.py, render_report.py) - the module the group names, colours and MIN_REFERENCES collect_summary.py - reads from it are the ones og writes, and its painting_png - draws the same pixels as og's S_painting.png + both pipeline reports (og summary collect, render_report.py) The Rust unit tests (rust/crates/og-ancestry) and tests/parity check each command in detail; this test runs them as the pipeline does, through bin/og. @@ -35,7 +31,6 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import ancestry_fine_report as af # noqa: E402 OG = os.environ.get("OG", "") FAILS = [] @@ -58,48 +53,6 @@ def write(path, text): f.write(text) -def png_pixels(data): - """The RGB rows of an 8-bit RGB PNG, every filter undone.""" - w, h = struct.unpack(">II", data[16:24]) - idat, pos = b"", 8 - while pos < len(data): - n, kind = struct.unpack(">I4s", data[pos:pos + 8]) - if kind == b"IDAT": - idat += data[pos + 8:pos + 8 + n] - pos += 12 + n - raw, stride, rows, prev = zlib.decompress(idat), 3 * w, [], bytearray(3 * w) - for y in range(h): - f, line = raw[y * (stride + 1)], bytearray(raw[y * (stride + 1) + 1:(y + 1) * (stride + 1)]) - for i in range(stride): - a = line[i - 3] if i >= 3 else 0 - b, c = prev[i], prev[i - 3] if i >= 3 else 0 - if f == 1: - line[i] = (line[i] + a) & 255 - elif f == 2: - line[i] = (line[i] + b) & 255 - elif f == 3: - line[i] = (line[i] + (a + b) // 2) & 255 - elif f == 4: - p = a + b - c - pa, pb, pc = abs(p - a), abs(p - b), abs(p - c) - line[i] = (line[i] + (a if pa <= pb and pa <= pc else b if pb <= pc else c)) & 255 - rows.append(bytes(line)) - prev = line - return (w, h, rows) - - -def first_difference(a, b): - """None if two PNGs hold the same pixels, else where they first differ.""" - (wa, ha, ra), (wb, hb, rb) = png_pixels(a), png_pixels(b) - if (wa, ha) != (wb, hb): - return f"size {wa}x{ha} vs {wb}x{hb}" - for y, (la, lb) in enumerate(zip(ra, rb)): - for x in range(wa): - if la[3 * x:3 * x + 3] != lb[3 * x:3 * x + 3]: - return f"pixel ({x}, {y}): {tuple(la[3 * x:3 * x + 3])} vs {tuple(lb[3 * x:3 * x + 3])}" - return None - - def run_all(tmp): # --- panel-map and check-map -------------------------------------------------- # 10 unrelated Yoruba, 3 unrelated San, one unrelated Papuan and two @@ -114,7 +67,7 @@ def run_all(tmp): "--paint-per-group", "50", "--out", samples) expect("panel-map: exits 0", (r.returncode, r.stderr), (0, "")) expect("panel-map: the install log names the groups left out", - f"Left out of the model, fewer than {af.MIN_REFERENCES} unrelated reference people: " + "Left out of the model, fewer than 10 unrelated reference people: " "Oceania (continental and regional group), Southern Africa (regional group)" in r.stdout, True) rows = [line.split("\t") for line in open(samples).read().splitlines()] by = {row[0]: row for row in rows[1:]} @@ -244,20 +197,7 @@ def run_all(tmp): expect("report: the text says chr19 was not painted, and chr20 still was", ("Not painted: RFMix failed on chr19" in text, "of chr20:" in text), (True, True)) - # --- the module and og agree on the names collect_summary.py reads --------------- - expect("module: each group name in og's table is the module's label", - [row[2] for row in table[1:] if row[0] in ("continent", "region", "painting")], - [af.label("region" if row[0] == "region" else "continent", row[1]) - for row in table[1:] if row[0] in ("continent", "region", "painting")]) - expect("module: MIN_REFERENCES is 10, as og's", af.MIN_REFERENCES, 10) - expect("module: a colour for each continental group", sorted(af.COLOURS), sorted(af.LABELS["continent"])) - # The legend collect_summary.py draws from af.COLOURS, and the docs demo's - # painting_png, must match og's picture: the same rows give the same - # pixels (the compressed bytes may differ). - demo = af.painting_png([(c, int(s), int(e), int(hp), code) for c, s, e, hp, code in bed]) - expect("module: painting_png draws og's pixels, colour for colour", first_difference(demo, png), None) - - # Both pipeline reports (collect_summary.py, then render_report.py) carry them. + # Both pipeline reports (og summary collect, then render_report.py) carry them. import collect_summary # noqa: E402 import render_report # noqa: E402 sd = os.path.join(tmp, "S") diff --git a/tests/test_e2e_scope.sh b/tests/test_e2e_scope.sh index b7717c684..9ea0189b6 100755 --- a/tests/test_e2e_scope.sh +++ b/tests/test_e2e_scope.sh @@ -34,7 +34,7 @@ paths() { printf '# a comment\n\n' printf '%s\n' "01-base${TAB}-" "02-base${TAB}-" "lpa-1${TAB}modules/local/lpa/* scripts/46-lpa.sh" \ "nextflow-from-fastq-1-bash${TAB}scripts/01b-fastp-qc.sh" "nextflow-from-fastq-2-nextflow${TAB}-" \ - "traits-1${TAB}-" "traits-2${TAB}bin/traits_report.py" + "traits-1${TAB}-" "traits-2${TAB}bin/og" } paths > "$E2E_PATHS" @@ -53,14 +53,14 @@ broken "a second row for a case" "traits-2 has more than one row" "^NONE" "trait broken "a row for a case that is not a file" "row for gone" "^NONE" "gone${TAB}main.nf" broken "a base row with globs" "base case 01-base" "^01-base${TAB}" "01-base${TAB}main.nf" broken "a glob that matches no file" "bin/no_such_helper.py matches no tracked file" "^traits-2${TAB}" \ - "traits-2${TAB}bin/traits_report.py bin/no_such_helper.py" + "traits-2${TAB}bin/og bin/no_such_helper.py" broken "a module glob that matches no file" "modules/local/nope/\\* matches no tracked file" "^lpa-1${TAB}" \ "lpa-1${TAB}modules/local/nope/*" broken "a row of one column" "has 1 columns" "^traits-2${TAB}" "traits-2" broken "a glob that matches a file outside the e2e scope" "docs/testing.md matches docs/testing.md, which is outside the e2e scope" \ - "^traits-2${TAB}" "traits-2${TAB}bin/traits_report.py docs/testing.md" + "^traits-2${TAB}" "traits-2${TAB}bin/og docs/testing.md" broken "a wildcard glob that reaches outside the e2e scope" "docs/\\* matches docs/" "^traits-2${TAB}" \ - "traits-2${TAB}bin/traits_report.py docs/*" + "traits-2${TAB}bin/og docs/*" mkdir -p "${WORK}/nosmoke" cp "${E2E_CASE_DIR}"/*.sh "${WORK}/nosmoke/" rm "${WORK}/nosmoke/nextflow-from-fastq-2-nextflow.sh" @@ -86,7 +86,7 @@ expect "a change outside the scope runs nothing" "" tests/test_cpic_parser.py .g expect "main.nf runs every shard case" "lpa-1 ${SMOKE} traits-1 traits-2" main.nf docs/testing.md expect "a workflow file runs every shard case" "lpa-1 ${SMOKE} traits-1 traits-2" workflows/pgx.nf expect "the e2e runner runs every shard case" "lpa-1 ${SMOKE} traits-1 traits-2" tests/e2e/lib.sh -expect "one helper runs the smoke cases, its case and the case that needs it" "${SMOKE} traits-1 traits-2" bin/traits_report.py +expect "one helper runs the smoke cases, its case and the case that needs it" "${SMOKE} traits-1 traits-2" bin/og expect "a file in one module runs that module's case" "lpa-1 ${SMOKE}" modules/local/lpa/main.nf expect "an in-scope path no case runs runs the smoke cases" "${SMOKE}" scripts/ci/check-name.sh expect "a committed fixture a case reads is in the scope" "${SMOKE}" tests/fixtures/consumer_export/sites.tsv.gz @@ -96,11 +96,11 @@ expect "a base step runs the smoke cases only (the base leg runs every base case printf '%s\n' traits-2 > "${WORK}/rows.txt" ROWS="${WORK}/rows.txt" expect "a changed table row runs its case and what it needs" "${SMOKE} traits-1 traits-2" docs/x.md expect "a deleted case file runs the smoke cases" "${SMOKE}" tests/e2e/gone-1.sh -if grep -q 'traits-1: needed by traits-2' "${WORK}/why.txt" 2> /dev/null || { pick bin/traits_report.py > /dev/null; grep -q 'traits-2: runs bin/traits_report.py' "${WORK}/why.txt"; }; then +if grep -q 'traits-1: needed by traits-2' "${WORK}/why.txt" 2> /dev/null || { pick bin/og > /dev/null; grep -q 'traits-2: runs bin/og' "${WORK}/why.txt"; }; then ok "select says why it picked each case" else bad "select gave no reasons: $(cat "${WORK}/why.txt")"; fi paths | grep -v "^traits-1${TAB}" > "${WORK}/norow.tsv" -printf '%s\n' bin/traits_report.py > "${WORK}/changed.txt" +printf '%s\n' bin/og > "${WORK}/changed.txt" if E2E_PATHS="${WORK}/norow.tsv" "$S" select "${WORK}/changed.txt" > /dev/null 2>&1; then bad "select ran with a case that has no row" else ok "select refuses a table with a case that has no row"; fi @@ -111,7 +111,7 @@ diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -3 +3,2 @@ -# an old comment -+traits-2${TAB}bin/traits_report.py ++traits-2${TAB}bin/og +lpa-1${TAB}- diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv --- a/scripts/ci/e2e-case-times.tsv @@ -131,7 +131,7 @@ have=$(awk '/^ push:/ {p = 1; next} p && /^ [a-z_]+:/ {exit} p && /^ - / {sub(/^ - /, ""); gsub(/\047/, ""); print}' "${REPO}/.github/workflows/e2e.yml" | sort) if [ "$("$S" scope)" = "$have" ]; then ok "the push paths of e2e.yml are the scope" else bad "push paths and scope differ: $(diff <("$S" scope) <(echo "$have") || true)"; fi -printf '%s\n' bin/traits_report.py > "${WORK}/changed.txt" +printf '%s\n' bin/og > "${WORK}/changed.txt" "$S" select "${WORK}/changed.txt" > "${WORK}/real.txt" 2> /dev/null if [ -s "${WORK}/real.txt" ] && E2E_SELECT="${WORK}/real.txt" "${REPO}/scripts/ci/e2e-shards.sh" plan 2 > /dev/null; then ok "a real pick splits: $(paste -sd' ' "${WORK}/real.txt")" diff --git a/tests/test_pgx_outside_calls.py b/tests/test_pgx_outside_calls.py index 9d4fe22a7..33664a9b1 100644 --- a/tests/test_pgx_outside_calls.py +++ b/tests/test_pgx_outside_calls.py @@ -31,11 +31,9 @@ 11. (the star alleles of a diplotype and the defining-variant evidence of one call are og's own unit tests: cargo test -p og-steps pypgx_summary) 13. pypgx alone with the depth check ok and PASS defining variants: the - reports' probable call (bin/pgx_outside_calls.py, kept for the - summary), while the outside-call file stays empty and step 36's row - stays indeterminate; no probable call with a failed depth check, a - usable Cyrius call, a low-GQ or non-PASS defining variant, or a - summary without the columns + outside-call file stays empty and step 36's row stays indeterminate + (the reports' probable call itself is og summary collect's rule, + tested in rust/crates/og-summary/src/collect/pgx.rs) 14. the whole subcommand on pypgx results archives, pypgx's own tables (rows of pypgx 0.26.0's gene and allele tables, tests/fixtures/pgx/ summary/) and an indexed VCF: GSTT1 not callable on this reference @@ -228,8 +226,8 @@ def medications(text): report_has("10 with the table missing, the report says to run step 36 again", gone, "Run step 36 again, then steps 07 and 27.") # --- step 32's summary and the probable CYP2D6 call ------------------------------- -sys.path.insert(0, os.path.join(REPO, "bin")) -import pgx_outside_calls as poc # noqa: E402 +# The rule for pypgx's call alone (probable_cyp2d6) is og summary collect's, +# tested in rust/crates/og-summary/src/collect/pgx.rs. SUMMARY_HEADER = ["Gene", "Diplotype", "Phenotype", "CNV_call", "Source", "Status", "Defining_variants", "Min_GQ", "Weakest_variant", "Defining_filter", "Quality"] @@ -244,26 +242,6 @@ def same(desc, got, want): # 13. -ROW = {"Gene": "CYP2D6", "Diplotype": "*1/*2", "Phenotype": "Normal Metabolizer", "Defining_filter": "PASS", - "Quality": "ok"} -same("13 pypgx alone, depth ok, PASS defining variants: probable", - poc.probable_cyp2d6(ROW, None, "ok"), - {"diplotype": "*1/*2", "phenotype": "Normal Metabolizer", "label": "probable (one caller: pypgx)"}) -same("13 Cyrius ran and made no call: still one caller, probable", - (poc.probable_cyp2d6(ROW, ("None", "Not_assigned_to_haplotypes"), "ok") or {}).get("label"), poc.PROBABLE) -same("13 red control: a failed depth check gives no probable call", poc.probable_cyp2d6(ROW, None, "unreliable"), None) -same("13 a usable Cyrius call: the two callers decide, no probable call", - poc.probable_cyp2d6(ROW, ("*1/*2", "PASS"), "ok"), None) -same("13 a low-GQ defining variant: no probable call", - poc.probable_cyp2d6(dict(ROW, Quality=poc.LOW_GQ), None, "ok"), None) -same("13 a defining variant not PASS: no probable call", - poc.probable_cyp2d6(dict(ROW, Defining_filter="not PASS: chr22:42128945 C>T RefCall"), None, "ok"), None) -same("13 an allele pypgx's table lacks: no probable call", - poc.probable_cyp2d6(dict(ROW, Defining_filter="not in pypgx's table: *99"), None, "ok"), None) -same("13 a defining variant with no GQ: no probable call", - poc.probable_cyp2d6(dict(ROW, Quality="GQ not in the VCF: check the reads"), None, "ok"), None) -same("13 a summary without the columns: no probable call", - poc.probable_cyp2d6({"Diplotype": "*1/*2", "Phenotype": "NM"}, None, "ok"), None) with tempfile.TemporaryDirectory() as tmp: summ = os.path.join(tmp, "pypgx.tsv") with open(summ, "w") as f: From d2bb659b9fca14b3af643de622843cd2938a94d5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:37:43 +0100 Subject: [PATCH 073/255] docs: name og summary collect where the docs named the Python collector Bead: pgp-bs9u.45 --- docs/05-annotsv.md | 2 +- docs/08-hla-typing.md | 2 +- docs/09-str-expansions.md | 2 +- docs/11-roh-analysis.md | 4 ++-- docs/17-cpsr.md | 2 +- docs/23-clinical-filter.md | 2 +- docs/24-html-report.md | 2 +- docs/33-sample-qc.md | 2 +- docs/36-pgx-consensus.md | 2 +- docs/42-traits.md | 4 ++-- docs/interpreting-results.md | 2 +- docs/nextflow.md | 2 +- docs/run-state.md | 2 +- docs/rust.md | 7 +++---- rust/README.md | 2 +- tests/demo/make_demo_sample.py | 2 +- 16 files changed, 20 insertions(+), 21 deletions(-) diff --git a/docs/05-annotsv.md b/docs/05-annotsv.md index 2d4798eb0..eb61026c3 100644 --- a/docs/05-annotsv.md +++ b/docs/05-annotsv.md @@ -126,7 +126,7 @@ Some class 4 and 5 calls are common or unreliable for a reason AnnotSV does not The regions are in `assets/sv_rule_regions.tsv`, one row each (`chrom`, `start` 0-based, `end`, `rule`, `action`, `name`, `source`): 2,409 coding tandem repeats, the six IG and TR loci, the MHC and 10 pseudogenes. `scripts/ci/build-sv-rule-regions.sh` writes it from four pinned downloads, each checked against its sha256: TRExplorer v2.1's GRCh38 tandem repeat catalog (the file upstream rebuilt in place on 2026-10-30, sha256 `df9c23c4…`; every repeat of 50 bp or more on a MANE Select coding exon of GENCODE 50), GENCODE 50's IG and TR genes, the MHC row of the GRCh38 assembly regions report, and HGNC's complete set of 2026-10-06 (each pseudogene whose alias or previous symbol is the approved symbol of a gene with a GenCC curation). The file is about 250 KB and ships with the repository, so the step downloads nothing. The e2e case `followups-ci-3-sv-rule-regions` rebuilds it with `scripts/ci/build-sv-rule-regions.sh --check`, which fails when one byte differs, so a hand edit or a drifted pin cannot ship. Step 05 mounts it into the AnnotSV image and the Nextflow `ANNOTSV` gets it as an input. -How many callers saw each SV comes from step 22's consensus (`sv_merged/_sv_consensus.vcf.gz`), which runs after this step, and beside `ANNOTSV` in Nextflow. So the reports (`bin/collect_summary.py`) join it when they are written: an SV matches the consensus record of the same type whose two breakpoints lie within 1,000 bp of its own, step 22's distance, and its support is that record's `SUPP` with the callers of `SUPP_VEC`. An SV in no consensus record was made by one caller and shows as "single caller". Without the consensus file the support is "not known", never 1. `og sv postfilter --consensus` adds the same `Support` and `Support_callers` columns to the table. +How many callers saw each SV comes from step 22's consensus (`sv_merged/_sv_consensus.vcf.gz`), which runs after this step, and beside `ANNOTSV` in Nextflow. So the reports (`og summary collect`) join it when they are written: an SV matches the consensus record of the same type whose two breakpoints lie within 1,000 bp of its own, step 22's distance, and its support is that record's `SUPP` with the callers of `SUPP_VEC`. An SV in no consensus record was made by one caller and shows as "single caller". Without the consensus file the support is "not known", never 1. `og sv postfilter --consensus` adds the same `Support` and `Support_callers` columns to the table. Both reports list every class 4 and 5 SV in a section of their own, with its support, the exon-edge flag and the frequency. The bash report (`scripts/24-html-report.sh`, which `run-all.sh` runs) reads the table from the sample folder. The Nextflow `HTML_REPORT` gets the table from `ANNOTSV` and, with `survivor_merge` in `--tools`, the consensus, so a report written by `nextflow run` alone has the same section. A report of a run without AnnotSV lists it under the steps not run. diff --git a/docs/08-hla-typing.md b/docs/08-hla-typing.md index 3c766f372..5926f8849 100644 --- a/docs/08-hla-typing.md +++ b/docs/08-hla-typing.md @@ -120,7 +120,7 @@ docker run --rm \ ## What the Report Derives -The report (step 24 and the text report) lists each T1K row and then derives three things from it. The code is `sec_hla` and the helpers below it in `bin/collect_summary.py`; `tests/test_hla_report.py` checks every rule on synthetic T1K rows. The genotypes are unphased: T1K gives two alleles per gene, not which chromosome each sits on. +The report (step 24 and the text report) lists each T1K row and then derives three things from it. The code is the HLA section of `og summary collect` (`rust/crates/og-summary/src/collect/group_c/hla.rs`); `tests/test_hla_report.py` checks every rule on synthetic T1K rows. The genotypes are unphased: T1K gives two alleles per gene, not which chromosome each sits on. **Coeliac DQ.** From the DQA1 and DQB1 rows: diff --git a/docs/09-str-expansions.md b/docs/09-str-expansions.md index d0bcd4f2c..c46b8a99e 100644 --- a/docs/09-str-expansions.md +++ b/docs/09-str-expansions.md @@ -47,7 +47,7 @@ Two short repeats that change how other findings read are not disease expansions Each entry has a `NoThresholds` reason and no Stranger thresholds: step 9b gives them no status, and the step 09 log prints them as "Not graded" with that reason. The counts are in the VCF (`REPCN`) and the JSON, one record per repeat: the report names the two CFTR records by their `VARID`, `CFTR_TG` and `CFTR_T`. -The reports read the counts (`bin/collect_summary.py`, under the repeat expansions section, `readings` in `summary.json`): +The reports read the counts (`og summary collect`, under the repeat expansions section, `readings` in `summary.json`): - **CFTR**: the TG and poly-T counts of both copies, such as `TG11-7T/TG11-7T`. ExpansionHunter writes the two tracts as two records, so when both the TG and the poly-T counts differ between the copies, which TG sits with which poly-T is not known: the line says `TG11/TG12 5T/7T` and names both pairings. A 5T copy gets the 5T line: 5T is a variable-penetrance allele, more often disease-causing with TG12 or TG13 on the same copy, and it matters when the other copy carries a CF-causing variant. The line then reads R117H (rs78655421), a row of step 42's traits table: no R117H copy; R117H and 5T in cis when one of them is on both copies; otherwise phase unknown, because R117H lies 17.6 kb from the tract, farther than a read pair spans. No 5T copy gives no 5T line, but the poly-T still matters beside R117H (milder on a 7T or 9T copy, CF-causing with 5T in cis), so the line still reads R117H and says which poly-T copy it sits on. A traits table from an earlier run is read and said to be so; one of a failed or unreadable step 42 is not read. - **UGT1A1**: the TA count of each copy as its allele (`*36`, `*1`, `*28`, `*37`), and the number of decreased-function copies (`*28`, `*37`) compared with PharmCAT's UGT1A1 call of step 27, which reads `*28` from the small-variant calls with the reference-block fill of `og pgx refblock-fill`. The line says whether the two agree; when they differ, read the reads before using either. A TA length outside TA5 to TA8 has no named allele, so the line makes no function claim and does not compare it. A step 27 table from an earlier run is compared and said to be so; one of a failed or unreadable step is not compared. PharmCAT's call is still the one step 27's drug guidance uses. diff --git a/docs/11-roh-analysis.md b/docs/11-roh-analysis.md index 0435e38b2..20a8146bb 100644 --- a/docs/11-roh-analysis.md +++ b/docs/11-roh-analysis.md @@ -42,7 +42,7 @@ For chip data (no `FORMAT/PL`), the script adds `-G30`. ### Centromere and heterochromatin bands -Few markers map in GRCh38's centromeres (cytoband stain `acen`), its large heterochromatin blocks (`gvar`, such as 1q12, 9q12 and 16q11.2) and the acrocentric stalks (`stalk`), so `bcftools roh` reads long homozygous runs there that are artefacts. The script, the Nextflow module and the reports take the part of each segment that lies in one of these bands out of its length; a segment wholly inside one counts as nothing. The bands are the `acen`, `gvar` and `stalk` rows of UCSC's `cytoBand.hg38.txt` (the file `setup.sh` installs for step 10), neighbouring rows merged into one span around each centromere (two on chrY), written into `scripts/11-roh-analysis.sh`, `modules/local/roh/main.nf` and `bin/collect_summary.py` as the same line (a unit test checks they agree). On chr1 the span is 121.7-143.2 Mb, on chr9 42.2-61.5 Mb, on chr18 15.4-21.5 Mb. +Few markers map in GRCh38's centromeres (cytoband stain `acen`), its large heterochromatin blocks (`gvar`, such as 1q12, 9q12 and 16q11.2) and the acrocentric stalks (`stalk`), so `bcftools roh` reads long homozygous runs there that are artefacts. The script, the Nextflow module and the reports take the part of each segment that lies in one of these bands out of its length; a segment wholly inside one counts as nothing. The bands are the `acen`, `gvar` and `stalk` rows of UCSC's `cytoBand.hg38.txt` (the file `setup.sh` installs for step 10), neighbouring rows merged into one span around each centromere (two on chrY), written into `scripts/11-roh-analysis.sh`, `modules/local/roh/main.nf` and `og summary collect` as the same line (a unit test checks they agree). On chr1 the span is 121.7-143.2 Mb, on chr9 42.2-61.5 Mb, on chr18 15.4-21.5 Mb. The reports count autosomes only: Segments, Total and Largest keep chr1 to chr22, so they leave out chrX and chrY (a man's single X reads as one long run) and any other contig (chrM, chrEBV, chrUn_*), and the part of each segment in a band. They say how many chrX and chrY segments and how many megabases in bands were left out. @@ -89,7 +89,7 @@ A flagged chromosome is looked up in `assets/imprinted_regions.tsv`, the chromos | 15q11-q13, chr15:20,500,000-33,400,000 (SNRPN, UBE3A) | Prader-Willi syndrome | Angelman syndrome | 22237428, 27615419 | | chr20 (20q13.32, GNAS) | Mulchandani-Bhoj-Conlin syndrome (growth failure) | Pseudohypoparathyroidism type 1B | 26248010, 29959430 | -`bin/collect_summary.py` holds the same rows (a unit test checks that the two agree), because the Nextflow report task reads `bin/` but not `assets/`. A flagged chromosome with no row still matters: a recessive variant inside the run, if there is one, would be homozygous. +`og summary collect` holds the same rows (`IMPRINTED_REGIONS` in `rust/crates/og-summary`; a unit test checks that the two agree). A flagged chromosome with no row still matters: a recessive variant inside the run, if there is one, would be homozygous. What this cannot tell, and the reports say so: diff --git a/docs/17-cpsr.md b/docs/17-cpsr.md index e542e1feb..abf28e100 100644 --- a/docs/17-cpsr.md +++ b/docs/17-cpsr.md @@ -120,7 +120,7 @@ Run by hand like this, `--sample_id` must be 3 to 40 characters long. The script - `${SAMPLE}.cpsr.grch38.html` — Interactive HTML report with classified variants - `${SAMPLE}.cpsr.grch38.classification.tsv.gz` — Tab-separated variant classifications (gzipped; read it with `zcat`) - Every variant in the panel genes gets one of the five ACMG/AMP classes: Pathogenic, Likely pathogenic, VUS, Likely benign, Benign -- In the TSV, `CLASSIFICATION` is the final class and `ASSERTION_AUTHORITY` says whose it is: ClinVar's, unless ClinVar has no record or a conflicted one, then CPSR's own (`CPSR_CLASSIFICATION`, which every variant has). PCGR 2.2.5 named the final column `FINAL_CLASSIFICATION`; the reports (`bin/collect_summary.py`) read either. The file names are the same in both versions. +- In the TSV, `CLASSIFICATION` is the final class and `ASSERTION_AUTHORITY` says whose it is: ClinVar's, unless ClinVar has no record or a conflicted one, then CPSR's own (`CPSR_CLASSIFICATION`, which every variant has). PCGR 2.2.5 named the final column `FINAL_CLASSIFICATION`; the reports (`og summary collect`) read either. The file names are the same in both versions. - `${SAMPLE}_cpsr_cdna_check.tsv` — step 6's cDNA read check on CPSR's P/LP rows (final class or CPSR's own), in the columns of step 6's `clinvar/${SAMPLE}_clinvar_cdna_check.tsv`. The rows are read from `VAR_ID` (or `GENOMIC_CHANGE`) and keep CPSR's contig names (`13`, not `chr13`); the BAM is read under either name. It needs the sample's BAM, `aligned/${SAMPLE}_sorted.bam` with its `.bai` (`ALIGN_DIR` picks another folder), and the GENCODE exons `setup.sh` installs; without either, every row is "not checked" with the reason. In Nextflow step 6's `CLINVAR_CDNA_CHECK` process runs it after `CPSR` (source `cpsr`) under the name `CPSR_CDNA_CHECK`, so the run's process table and the live view count it under step 17, with the samplesheet's BAM and `--gencode_exons`, and publishes the table to `cpsr/`; the HTML report puts it beside each P/LP row there too. Its image comes from a `withName: 'CPSR_CDNA_CHECK'` selector (a setting for step 6's `CLINVAR_CDNA_CHECK` does not reach it). ### What the reports show diff --git a/docs/23-clinical-filter.md b/docs/23-clinical-filter.md index 66f03986c..4f4eec752 100644 --- a/docs/23-clinical-filter.md +++ b/docs/23-clinical-filter.md @@ -86,7 +86,7 @@ The Nextflow `CLINICAL_FILTER` module applies the same tiers, ranking, frequency ### Secondary-findings genes (ACMG SF v3.3) -Both reports list, beside the clinical filter's counts, the secondary findings in the 84 genes of the ACMG SF v3.3 list (Lee et al., Genet Med 2025;27(8):101454; the genes in which the ACMG recommends reporting pathogenic variants found by chance, such as BRCA1, LDLR, MYBPC3 or TTN): the step 6 ClinVar hits in those genes, the clinical filter's records with a HIGH-impact consequence on the ranked transcript in them, and step 5's AnnotSV class 4 and 5 structural variants in the genes whose rule allows them. Each gene's rule from Table 1 of the list is a row of `assets/acmg_sf.tsv`: one variant in a dominant gene, two in a recessive one (MUTYH, BTD, ATP7B and others; a homozygote, or two variants of unknown phase), HFE only as homozygous p.C282Y, GLA and OTC also in a heterozygous female, ABCD1 hemizygous, homozygous or with two heterozygous variants. A variant the rule does not report is listed apart with the reason; a pseudodeficiency allele is not counted, and a BTD finding with the p.Asp444His hypomorph says partial, not profound. Of the reported HIGH-impact records the reports count as "rare HIGH impact" only the rare ones with a sound call; a weak call or an unknown frequency is listed with its mark and counted apart, and a common one is left out. The list version is `ACMG_SF_VERSION` in `bin/collect_summary.py`. It is a list to review with a clinician: a rare HIGH-impact variant is not a ClinVar classification. [Interpreting results](interpreting-results.md#carrier-status-is-not-disease) gives every rule. Step 17 (CPSR) reports the ACMG secondary findings with its own classification. +Both reports list, beside the clinical filter's counts, the secondary findings in the 84 genes of the ACMG SF v3.3 list (Lee et al., Genet Med 2025;27(8):101454; the genes in which the ACMG recommends reporting pathogenic variants found by chance, such as BRCA1, LDLR, MYBPC3 or TTN): the step 6 ClinVar hits in those genes, the clinical filter's records with a HIGH-impact consequence on the ranked transcript in them, and step 5's AnnotSV class 4 and 5 structural variants in the genes whose rule allows them. Each gene's rule from Table 1 of the list is a row of `assets/acmg_sf.tsv`: one variant in a dominant gene, two in a recessive one (MUTYH, BTD, ATP7B and others; a homozygote, or two variants of unknown phase), HFE only as homozygous p.C282Y, GLA and OTC also in a heterozygous female, ABCD1 hemizygous, homozygous or with two heterozygous variants. A variant the rule does not report is listed apart with the reason; a pseudodeficiency allele is not counted, and a BTD finding with the p.Asp444His hypomorph says partial, not profound. Of the reported HIGH-impact records the reports count as "rare HIGH impact" only the rare ones with a sound call; a weak call or an unknown frequency is listed with its mark and counted apart, and a common one is left out. The list version is `ACMG_SF_VERSION` in `og summary collect` (`rust/crates/og-summary`). It is a list to review with a clinician: a rare HIGH-impact variant is not a ClinVar classification. [Interpreting results](interpreting-results.md#carrier-status-is-not-disease) gives every rule. Step 17 (CPSR) reports the ACMG secondary findings with its own classification. ## Runtime diff --git a/docs/24-html-report.md b/docs/24-html-report.md index 889f675f2..d5551fecb 100644 --- a/docs/24-html-report.md +++ b/docs/24-html-report.md @@ -10,7 +10,7 @@ The pipeline produces output across many directories in different formats (VCF, ## Tool -`bin/collect_summary.py` reads every step's output once into `${SAMPLE}/summary.json`, and `bin/render_report.py` renders the report from it. `scripts/generate-report.sh` renders the text report from the same summary, so the two reports cannot disagree about a count. Both are Python standard library only. +`og summary collect` (`rust/crates/og-summary`, run by `bin/render_report.py`) reads every step's output once into `${SAMPLE}/summary.json`, and `bin/render_report.py` renders the report from it. `scripts/generate-report.sh` renders the text report from the same summary, so the two reports cannot disagree about a count. Both are Python standard library only. ## Docker Image diff --git a/docs/33-sample-qc.md b/docs/33-sample-qc.md index 94e085561..2aed8fd77 100644 --- a/docs/33-sample-qc.md +++ b/docs/33-sample-qc.md @@ -19,7 +19,7 @@ Contamination only warns. A contaminated sample is still your sample; its calls ## Docker Image - `SOMALIER_IMAGE` - `VERIFYBAMID2_IMAGE` -- `PYTHON_IMAGE` (the verdict, from `bin/collect_summary.py`, and the mosaic estimate, from `og qc sex-loss`, the Rust binary `scripts/setup.sh --og` builds) +- `PYTHON_IMAGE` (the verdict, from `og summary collect`, and the mosaic estimate, from `og qc sex-loss`, the Rust binary `scripts/setup.sh --og` builds) Pinned in `versions.env`; [Image versions](versions.md) lists the current tags. diff --git a/docs/36-pgx-consensus.md b/docs/36-pgx-consensus.md index 64779cb30..59427ba6a 100644 --- a/docs/36-pgx-consensus.md +++ b/docs/36-pgx-consensus.md @@ -63,7 +63,7 @@ For CYP2D6 the `Reason` is one of: the depth check's message (multi-mapped reads ## Notes - Without Cyrius, CYP2D6 is always `indeterminate` here: that is the point, not a fault. Install Cyrius (`setup.sh --cyrius`, non-commercial licence) to get an agreed call; `run-all.sh` then runs step 21 in a default run. -- **A probable call, never an outside call.** When pypgx alone called CYP2D6 (Cyrius did not run, or made no usable call), the depth check passed, and every variant that defines its star alleles is in the VCF as a `PASS` call at or above the GQ floor (step 32's `Defining_filter` and `Quality` columns), both reports show pypgx's diplotype and phenotype labelled `probable (one caller: pypgx)`. This table still says `indeterminate` for it and nothing reaches PharmCAT, so no drug guidance rests on it. The rule is `probable_cyp2d6` in `bin/pgx_outside_calls.py`, which the reports import; a failed depth check, a usable Cyrius call (then the two callers decide), a defining variant that is missing, not `PASS`, below the GQ floor or without a GQ, a star allele pypgx's table lacks, or a summary from before these columns gives no probable call. +- **A probable call, never an outside call.** When pypgx alone called CYP2D6 (Cyrius did not run, or made no usable call), the depth check passed, and every variant that defines its star alleles is in the VCF as a `PASS` call at or above the GQ floor (step 32's `Defining_filter` and `Quality` columns), both reports show pypgx's diplotype and phenotype labelled `probable (one caller: pypgx)`. This table still says `indeterminate` for it and nothing reaches PharmCAT, so no drug guidance rests on it. The rule is the probable call of `og summary collect` (`rust/crates/og-summary/src/collect/pgx.rs`); a failed depth check, a usable Cyrius call (then the two callers decide), a defining variant that is missing, not `PASS`, below the GQ floor or without a GQ, a star allele pypgx's table lacks, or a summary from before these columns gives no probable call. - A Cyrius call counts only with Filter `PASS`; `None`, an ambiguous call (two diplotypes) or `CYP2D6_depth_unreliable` is no call. - The two diplotypes must be the same up to the order of the alleles. The one passed to PharmCAT is pypgx's. - PharmCAT turns the HLA alleles into its own phenotypes (`*57:01 positive`, `*58:01 negative` and so on) and gives the drug guidance for them; step 27 marks those genes `[outside call]`. diff --git a/docs/42-traits.md b/docs/42-traits.md index ad3aa25cf..b43df3ed9 100644 --- a/docs/42-traits.md +++ b/docs/42-traits.md @@ -108,12 +108,12 @@ The rules today: The six rules after TAS2R38 need no phase. Their sites lie close together on one copy (the two APOL1 G1 changes were in perfect linkage disequilibrium in Genovese et al. 2010; the 9p21 and 17q21.31 tags each mark one haplotype), or, for HFE, a double heterozygote reads as one change on each copy. A copy the rule does not describe, such as a 9p21 copy with only one of the two risk alleles, is `other_haplotype`, not read. What each rule leaves out, said in its note: -- APOL1 G2 (rs71785313) is a 6-base deletion. The rule table reads single-base sites only, so the rule itself reads G1, and a G2 copy reads as G0 there. The HTML and text reports read G2 beside the rule (`bin/genotype_rules_indels.py`, through `bin/collect_summary.py`) from the sample's gVCF, or its VCF when there is no gVCF, and show the G1 plus G2 result under "With G2": G1/G2, G2/G2 and G1/G1 are the high-risk genotype. Two G2 copies read G2/G2 even when the G1 sites were not read. A VCF without reference blocks cannot tell a reference base from a position with no reads, so there a result with no G2 copy is marked `(no G2 call)`. The Nextflow report gets the sample's gVCF too (DeepVariant's, or the samplesheet's `gvcf` column), so both runs read the gVCF when there is one. +- APOL1 G2 (rs71785313) is a 6-base deletion. The rule table reads single-base sites only, so the rule itself reads G1, and a G2 copy reads as G0 there. The HTML and text reports read G2 beside the rule (`og summary collect`, the `apol1` module of `rust/crates/og-summary/src/collect/group_c/traits.rs`) from the sample's gVCF, or its VCF when there is no gVCF, and show the G1 plus G2 result under "With G2": G1/G2, G2/G2 and G1/G1 are the high-risk genotype. Two G2 copies read G2/G2 even when the G1 sites were not read. A VCF without reference blocks cannot tell a reference base from a position with no reads, so there a result with no G2 copy is marked `(no G2 call)`. The Nextflow report gets the sample's gVCF too (DeepVariant's, or the samplesheet's `gvcf` column), so both runs read the gVCF when there is one. - SERPINA1: M is any copy with neither S nor Z; rarer deficiency and null alleles are not read. A serum level confirms a deficiency. - HFE: the C282Y homozygote is also a secondary finding (`assets/acmg_sf.tsv`, [step 23](23-clinical-filter.md)); the rule gives the reading of every genotype. - GC: the protein levels are those of Powe et al. (2013), whose monoclonal assay was questioned in later letters. A protein type, not a vitamin D deficiency. -Two more markers of the planned list are not rules here, because each sits in sequence the genome holds more than once and a gVCF genotype at one position cannot read it. They are read from the BAM by the step that counts the copies, as a fraction of copies rather than a genotype (`MARKERS` in `bin/traits_report.py`, which pins each one's dbSNP b157 record and the GRCh38 sequence around every copy): +Two more markers of the planned list are not rules here, because each sits in sequence the genome holds more than once and a gVCF genotype at one position cannot read it. They are read from the BAM by the step that counts the copies, as a fraction of copies rather than a genotype (`MARKERS` in `rust/crates/og-cn/src/markers.rs`, which pins each one's dbSNP b157 record and the GRCh38 sequence around every copy): - **LPA KIV-2 4925G>A** (Coassin et al. 2017, PMID 28444229), a splice variant that lowers the Lp(a) made from the chromosome that carries it. GRCh38 holds the site in each of its six KIV-2 units and in KIV-1. [Step 46](46-lpa.md) reads it beside the KIV-2 count, and the reports print it under the count. - **SMN1 g.27134T>G** (rs143838139; Luo et al. 2014, PMID 23788250), a marker of SMN1 duplication alleles that SMN2 holds too. [Step 35](35-paralogs.md) reads it beside the SMN1 copy number, and the reports print it under the SMN1 copies. It refines the chance of a silent 2+0 carrier for a two-copy result, and its line always says that it does not prove 2+0. diff --git a/docs/interpreting-results.md b/docs/interpreting-results.md index 9aa0a7bd7..576e963c2 100644 --- a/docs/interpreting-results.md +++ b/docs/interpreting-results.md @@ -76,7 +76,7 @@ The most common "pathogenic" finding in any genome is **heterozygous carrier sta - BTD with p.Asp444His: reported as partial, not profound, biotinidase deficiency; - AnnotSV class 4 and 5 structural variants (step 5) in BRCA1, BRCA2, MLH1, MSH2 and LDLR reach the same card, and so does a deletion of the 3' end of EPCAM (exons 8 and 9), which silences MSH2 (Kovacs et al., Hum Mutat 2009;30(2):197-203). -**Two hits across callers: a deletion plus a variant on the other copy.** Small-variant callers see one hit when the other copy of the gene is deleted. So the carrier list also joins the deletions the other engines call with the ClinVar hits, gene by gene (`bin/collect_summary.py`, `two_hits`). The deletions come from AnnotSV's class 4 and 5 deletions (step 5), step 35's interval table (the recurrent regions of `bin/cn_loci.tsv`, such as 1q21.1), its HBA1/HBA2 count and Parascopy's SMN1 count. A recessive gene that a deletion and a hit both reach gets one line: +**Two hits across callers: a deletion plus a variant on the other copy.** Small-variant callers see one hit when the other copy of the gene is deleted. So the carrier list also joins the deletions the other engines call with the ClinVar hits, gene by gene (`og summary collect`, `two_hits`). The deletions come from AnnotSV's class 4 and 5 deletions (step 5), step 35's interval table (the recurrent regions of `bin/cn_loci.tsv`, such as 1q21.1), its HBA1/HBA2 count and Parascopy's SMN1 count. A recessive gene that a deletion and a hit both reach gets one line: - **biallelic** when the reads show the two hits on different copies. Either the hit is called homozygous inside the deletion, so it sits on the one copy left (the line and the carrier list call it hemizygous), or both calls carry read-backed phase (a `|` genotype with the same phase set, PS) on different copies; - **possibly biallelic** otherwise: the phase is unknown. Most short-read calls carry no phase, so this is the usual line. Step 05's checked table keeps no genotype column yet, and step 35's counts have no phase, so in a pipeline run a deletion pairs as possibly biallelic unless the hit is called homozygous inside it. A parent's genome (step 38, [family analysis](multi-sample.md#family-analysis-step-38-opt-in)) or long reads can settle it; diff --git a/docs/nextflow.md b/docs/nextflow.md index 0f15b62c5..07224e7ed 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -359,7 +359,7 @@ Where a module and its script differ on purpose: | Mito variants (20) | extracts the chrM reads with `samtools view`; annotates the calls with `clinvar/clinvar_chr.vcf.gz` and the gnomAD chrM sites under `gnomad_mito/` when they are installed | with GATK `PrintReads`. Mutect2 applies its own read filters to either, so the calls are expected to match; CI does not compare them. Both mark possible NuMTs at the median autosomal depth from mosdepth (step 16b, or `MOSDEPTH` when `mosdepth` is in `--tools`). `MITO_VARIANTS_ANNOTATE` runs the same annotation (`og mito annotate`) with the files of `--mito_clinvar` and `--gnomad_mito`; without them the table names each file as not given and leaves its columns `.`. `run-all.sh` passes `--mito_clinvar` when `clinvar/clinvar_chr.vcf.gz` and its `.tbi` are installed, and `--gnomad_mito` when the gnomAD chrM sites are | | AnnotSV (05) | annotates step 15's duphold-filtered calls; without them, or when Manta's calls are newer, Manta's calls, and says so | `ANNOTSV` always reads `DUPHOLD_FILTER`'s output: `annotsv` in `--tools` needs `duphold` and `manta` | | Y haplogroup (37) | reads the sex step 16 (indexcov) infers | runs on the rows whose samplesheet sex is male, which `INDEXCOV` has checked against the reads | -| HTML report (24) | renders every section from `bin/collect_summary.py`'s summary | `HTML_REPORT` runs the same code on the outputs of this run's steps, among them the cDNA read check of the ClinVar hits, the CYP2D6 depth check, the phased compound-het pairs, the RFC1 motif count, AnnotSV's class 4 and 5 table, the chrM annotation and step 35's HBA1/HBA2 count, CYP21A2 and gene-dosage tables. It also gets the sample's gVCF (DEEPVARIANT's or the `gvcf` column's, linked as `vcf/.g.vcf.gz`, where step 24 finds step 03's) and the release in nextflow.config's manifest, as a `run pipeline_version` row of the report's run manifest (`bin/write_manifest.sh` writes the same row for `run-all.sh`). It also gets the VEP VCF (each ClinVar hit's gnomAD AF and rsIDs), CPSR's classification table and the cDNA read check of its P/LP rows, `ALIGN_MINIMAP2`'s read count, and the clinical filter's and slivar's summary tables. The slivar card has no compound-het variant and gene counts: the Nextflow slivar module writes no `_compound_hets.tsv`. For every section, run `GENOME_DIR= scripts/24-html-report.sh ` on the Nextflow output | +| HTML report (24) | renders every section from the summary `og summary collect` writes | `HTML_REPORT` runs the same code on the outputs of this run's steps, among them the cDNA read check of the ClinVar hits, the CYP2D6 depth check, the phased compound-het pairs, the RFC1 motif count, AnnotSV's class 4 and 5 table, the chrM annotation and step 35's HBA1/HBA2 count, CYP21A2 and gene-dosage tables. It also gets the sample's gVCF (DEEPVARIANT's or the `gvcf` column's, linked as `vcf/.g.vcf.gz`, where step 24 finds step 03's) and the release in nextflow.config's manifest, as a `run pipeline_version` row of the report's run manifest (`bin/write_manifest.sh` writes the same row for `run-all.sh`). It also gets the VEP VCF (each ClinVar hit's gnomAD AF and rsIDs), CPSR's classification table and the cDNA read check of its P/LP rows, `ALIGN_MINIMAP2`'s read count, and the clinical filter's and slivar's summary tables. The slivar card has no compound-het variant and gene counts: the Nextflow slivar module writes no `_compound_hets.tsv`. For every section, run `GENOME_DIR= scripts/24-html-report.sh ` on the Nextflow output | | CNVpytor (18) | mounts each resource file over the image's data folder | copies the files into the image's `site-packages`, so it needs a writable container | | Cyrius (21) | runs Cyrius from the install `setup.sh --cyrius` made | the same install, given as `--cyrius_install` (see [Cyrius, opt-in](#cyrius-opt-in)) | diff --git a/docs/run-state.md b/docs/run-state.md index 1f481ff45..8474435c5 100644 --- a/docs/run-state.md +++ b/docs/run-state.md @@ -26,7 +26,7 @@ When `run-all.sh` ends, however it ends, a `planned` row gives way to the step's `assets/steps.tsv` has one row per step: its number, its label as `run-all.sh` prints it, its `--tools` name, whether `run-all.sh` records it in `run_status.tsv`, its Nextflow processes, the report sections it feeds, the steps whose output it reads, and its usual hours. The hours are the upper end of the step's time in the [runtime table](hardware-requirements.md#runtime-per-step); `hours_from` says whether that figure was measured or is the page's estimate, and is empty when the table has no time for the step. On a measured row, `hours_low` is the lower end of the same measured time, always below `hours`; a part that runs once per reference, such as building an index, is left out of it. -`tests/test_run_state.py` keeps the catalogue true: every step `run-all.sh` plans has its row with the same label and tool, the recorded steps are the ones `bin/collect_summary.py` checks for stale results, every report section belongs to the row of its step, every process in `modules/local` belongs to a step, and the tools are the ones `main.nf` accepts. +`tests/test_run_state.py` keeps the catalogue true: every step `run-all.sh` plans has its row with the same label and tool, the recorded steps are the ones `og summary collect` checks for stale results, every report section belongs to the row of its step, every process in `modules/local` belongs to a step, and the tools are the ones `main.nf` accepts. ## What a step's state means diff --git a/docs/rust.md b/docs/rust.md index 5f0793205..0e0de2374 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -21,7 +21,7 @@ rust/ crates/og-qc/ og qc sex-loss: mosaic loss of Y or X for step 33 crates/og-setup/ what setup.sh runs once: the STRchive catalog, zips, the genetic map crates/og-steps/ the step helpers: constraint-join, prs format, step 40's consumer export; og family (step 38); og pgx outside-calls|hla-concordance|pypgx-summary (steps 08, 32, 36); og paraphase (step 35b) - crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect, the whole summary.json with its post pass, or one group with --only a|b|c (no caller yet); og roh summary (step 11) + crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect, the whole summary.json with its post pass (render_report.py runs it for steps 24 and HTML_REPORT and generate-report.sh), or one group with --only a|b|c; og roh summary (step 11) crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read @@ -36,7 +36,7 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`; `clip_check.py` keeps only the names `collect_summary.py` imports), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports, and `annotsv_postfilter.py` only the column names and the consensus join the reports import. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (the commands of `chip_report.py`, which keeps only the detection limit and the column names the reports import). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py` (which keeps `MARKERS` for the summary). Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`; `bin/ancestry_fine_report.py` keeps only the names `collect_summary.py` and the docs demo import. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`, which keeps the CYP2D6 and HLA rules the reports import), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (were the commands of `chip_report.py`). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py`. Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. @@ -247,5 +247,4 @@ The shared `og-io` reader consumes LF, CRLF and lone CR records with storage bou Step 42 uses `og traits table`, `og traits phase-sites` and `og traits check`. The readers retain only catalog sites. The trait and genotype-rule output -columns and readings stay the same. `bin/traits_report.py` keeps report -metadata until the Python summary and report collectors move. +columns and readings stay the same. diff --git a/rust/README.md b/rust/README.md index c69f31c2d..ce5fae19c 100644 --- a/rust/README.md +++ b/rust/README.md @@ -10,7 +10,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically, and the tools that read BAMs: the reads at a variant (`reads`: one CIGAR walk per read, SA pieces, filters per tool), `og cdna check`, `og cdna scan`, `og comphet phase`, `og bam contigs`, the CYP2D6 depth check and the RFC1 motif count | | `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-qc` | `og qc sex-loss`: step 33's mosaic loss of Y or X, with CPython's medians, `fmean` and exact `variance` | -| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), and without `--only` the whole summary.json with its post pass (the readings that join sections, the CYP2D6 block, `not_run`, `not_assessed`, `recheck`, `new_since`), beside the Python collector, which still writes summary.json; `og roh summary` (step 11) | +| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), and without `--only` the whole summary.json with its post pass (the readings that join sections, the CYP2D6 block, `not_run`, `not_assessed`, `recheck`, `new_since`), which steps 24 and HTML_REPORT and `generate-report.sh` write through `bin/render_report.py`; `og roh summary` (step 11) | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | | `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-bloodtype` | step 41, blood groups: `og bloodtype report`, `og bloodtype check-markers`, `og bloodtype systems` | diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index e55cb03d3..c6efbb918 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -27,7 +27,7 @@ - step 33's mosaic rows come from og qc sex-loss, step 26b's table and painting from og ancestry-fine report, step 35b's table from og paraphase table, step 39's from bin/couple_report.py and step 42's - from bin/traits_report.py, each fed made-up inputs; step 46's table and + from og traits table, each fed made-up inputs; step 46's table and step 35's HBA1/HBA2 table are what og cn call writes for made-up depths (HBA_TABLE and LPA_TABLE; tests/test_hba_copy_number.py and tests/test_lpa_kiv2.py check that og still writes them), step 49's depth From 17728ec4c929bc29ba1d92f7512301507efedd19 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:48:41 +0100 Subject: [PATCH 074/255] test(report): check that a run collects the summary once The fake-docker run-all case now checks that generate-report.sh renders from the summary step 24 wrote, that both reports carry one generated time, and that a step output newer than summary.json makes it collect again. Bead: pgp-bs9u.45 --- tests/fake-docker/run-all-partial-failure.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index 098370891..5deda814f 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -150,6 +150,22 @@ for r in "$TXT" "$HTML"; do grep -q 'STALE: from an earlier run.*step 19 was not finished' "$r" \ || miss "${r##*/} does not mark step 19's old result stale, but Delly did not finish in this run" done +# One collect per run: generate-report.sh renders the text report from the +# summary step 24 wrote just before it, so both reports carry one time. +grep -q '^Rendering from summary.json' "${S}/logs/generate-report.log" \ + || miss "generate-report.sh collected the summary again after step 24 wrote it: $(head -3 "${S}/logs/generate-report.log")" +t_txt=$(sed -n 's/^ Generated: //p' "$TXT" | head -1) +t_html=$(grep -o 'Generated: [^<]*' "$HTML" | head -1 | sed 's/^Generated: //') +[ -n "$t_txt" ] && [ "$t_txt" = "$t_html" ] || miss "the text report says generated '${t_txt}', the HTML report '${t_html}'" +# The reverse: a step output newer than summary.json makes it collect again. +roh=$(find "$S" -name 'sample1_roh.txt' -type f | head -1) +[ -n "$roh" ] || fail "no ROH output to touch" +touch "$roh" +run_expect 0 recollect "${SCRIPTS}/generate-report.sh" sample1 +if grep -q '^Rendering from summary.json' "${CASE_WORK}/recollect.out" 2> /dev/null; then + miss "generate-report.sh rendered from summary.json, but a step output changed after it was written" +fi +touch -d '30 days ago' "$roh" # back to its age, for the cases below # --- 3. the same failure on a first run: nothing cached ------------------------------------ awk -F'\t' -v OFS='\t' '{$2 += $3; $3 = 0; print}' "${CASE_WORK}/resume.tsv" > "${CASE_WORK}/first.tsv" From 2d09b4f983e1e18a9bd5d1692898b927061e05f5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:29:24 +0100 Subject: [PATCH 075/255] perf(summary): read the clinical and other sections one at a time og summary collect read groups b and c at once, and group c on a pool of up to eight threads, so a run's peak memory was the sum of the sections in flight. The sample VCF, the one input that takes time, stays on its own thread; every other section is read one at a time meanwhile. On a synthetic folder with every side table 50 times its demo size the peak footprint fell from 119-130 MB to 105-112 MB, CPU time unchanged. Bead: pgp-bs9u.45 --- rust/crates/og-summary/src/collect/group_b.rs | 39 ++++++++----------- rust/crates/og-summary/src/collect/group_c.rs | 39 +++++-------------- rust/crates/og-summary/src/collect/post.rs | 12 +++--- 3 files changed, 33 insertions(+), 57 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_b.rs b/rust/crates/og-summary/src/collect/group_b.rs index 394178cf5..81426f5bb 100644 --- a/rust/crates/og-summary/src/collect/group_b.rs +++ b/rust/crates/og-summary/src/collect/group_b.rs @@ -6,8 +6,8 @@ //! section, the ACMG tier, the carrier list and CPSR's call floor; the //! clinical VCF feeds the record count and the ACMG tier; AnnotSV's table //! and step 35's copy-number tables feed both two-hit readings. The -//! ClinVar section (with its VEP lookups) and the clinical section (with -//! the clinical VCF) are read on their own threads. +//! sections are read one at a time, so the peak memory is the biggest +//! section's, not the sum. use super::acmg::{self, ClinicalVcf}; use super::carrier::Carrier; @@ -128,15 +128,15 @@ pub fn read(input: &Input, folder: &Folder) -> Vec
{ }; let side = |rel: &str| one(rel.to_owned()); - let (clinvar, clinical, rest) = std::thread::scope(|scope| { - let clinvar = scope.spawn(|| -> Read { - let Some(h) = &hits else { return Ok(None) }; - let data = ClinVar::read(h, d, &side, &s)?; - with_mtime(&h.path, h.rel.clone(), Data::ClinVar(data)) - }); - let clinical = scope.spawn(|| -> Read { - let at = clinical_table.clone().or_else(|| clinical_vcf_at.clone()); - let Some((p, rel)) = at else { return Ok(None) }; + // One section at a time, so the peak memory is the biggest section's. + let clinvar: Read = match &hits { + None => Ok(None), + Some(h) => ClinVar::read(h, d, &side, &s) + .and_then(|data| with_mtime(&h.path, h.rel.clone(), Data::ClinVar(data))), + }; + let clinical: Read = match clinical_table.clone().or_else(|| clinical_vcf_at.clone()) { + None => Ok(None), + Some((p, rel)) => { let vcf = clinical_vcf_at.as_ref().map(|(v, _)| ClinicalVcf::read(v)); let tier = acmg::Inputs { assets: &input.assets, @@ -144,20 +144,15 @@ pub fn read(input: &Input, folder: &Folder) -> Vec
{ clinical: vcf.as_ref(), deletions: &deletions, }; - let data = Clinical::read( + Clinical::read( clinical_table.as_ref().map(|(p, _)| p.as_str()), vcf.as_ref(), &tier, - )?; - with_mtime(&p, rel, Data::Clinical(data)) - }); - let rest = read_rest(input, &s, index, &find, &hits, &deletions); - let join = |h: std::thread::ScopedJoinHandle<'_, Read>| { - h.join() - .unwrap_or_else(|_| Err("internal error: the section reader stopped".into())) - }; - (join(clinvar), join(clinical), rest) - }); + ) + .and_then(|data| with_mtime(&p, rel, Data::Clinical(data))) + } + }; + let rest = read_rest(input, &s, index, &find, &hits, &deletions); let [carrier, pharmcat, cpic, pypgx, cyrius, cpsr, slivar] = rest; let reads = [ clinvar, carrier, pharmcat, cpic, pypgx, cyrius, cpsr, clinical, slivar, diff --git a/rust/crates/og-summary/src/collect/group_c.rs b/rust/crates/og-summary/src/collect/group_c.rs index d9c8c9506..edf94ecc7 100644 --- a/rust/crates/og-summary/src/collect/group_c.rs +++ b/rust/crates/og-summary/src/collect/group_c.rs @@ -200,36 +200,15 @@ pub fn read_all(c: &Ctx<'_>) -> Vec { tables::mei, tables::mosaic, ]; - // The readers are independent: one thread each, over a small pool, so a - // big input (a VCF, a table of thousands of rows) never holds up the - // others. A panic stays in its section. - let jobs = std::thread::available_parallelism().map_or(4, |n| n.get().min(8)); - let next = std::sync::atomic::AtomicUsize::new(0); - let slots: Vec>> = (0..readers.len()) - .map(|_| std::sync::Mutex::new(None)) - .collect(); - std::thread::scope(|scope| { - for _ in 0..jobs { - scope.spawn(|| { - loop { - let i = next.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let Some(r) = readers.get(i) else { break }; - let out = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| r(c))) - .unwrap_or_else(|_| Err("internal error: the reader stopped".to_owned())); - if let Ok(mut slot) = slots[i].lock() { - *slot = Some(out); - } - } - }); - } - }); - slots - .into_iter() - .map(|m| { - m.into_inner() - .ok() - .flatten() - .unwrap_or_else(|| Err("internal error: the reader stopped".to_owned())) + // One reader at a time: a run's peak memory is then the biggest single + // section, not the sum of all of them. The sample VCF, the one input + // that takes time, is read on its own thread meanwhile (`read_a`). A + // panic stays in its section. + readers + .iter() + .map(|r| { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| r(c))) + .unwrap_or_else(|_| Err("internal error: the reader stopped".to_owned())) }) .collect() } diff --git a/rust/crates/og-summary/src/collect/post.rs b/rust/crates/og-summary/src/collect/post.rs index 6065d5402..305bc87ac 100644 --- a/rust/crates/og-summary/src/collect/post.rs +++ b/rust/crates/og-summary/src/collect/post.rs @@ -120,14 +120,17 @@ no standard caller sees them"; const MCA_NOT_ASSESSED: &str = "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share \ of the blood cells): step 51 (MoChA) did not assess this sample"; -/// Reads the whole summary of a sample folder. The three groups are read -/// at once over one walk of the folder. An error is a run status or a -/// manifest that is there and cannot be read. +/// Reads the whole summary of a sample folder, over one walk of the +/// folder. An error is a run status or a manifest that is there and cannot +/// be read. pub fn collect(input: &Input) -> Result { let folder = Folder::open(input)?; + // Group a reads the sample VCF, the input that takes time, on its own + // thread; groups b and c read their sections meanwhile, one at a time, + // so the peak memory stays near the biggest section's. let (a, b, mut c) = std::thread::scope(|scope| { let a = scope.spawn(|| read_a(input, &folder)); - let b = scope.spawn(|| group_b::read(input, &folder)); + let b = group_b::read(input, &folder); let c = read_c(input, &folder); let stopped = |specs: &[super::model::Spec]| -> Vec
{ specs @@ -142,7 +145,6 @@ pub fn collect(input: &Input) -> Result { .collect() }; let a = a.join().unwrap_or_else(|_| stopped(&super::model::GROUP_A)); - let b = b.join().unwrap_or_else(|_| stopped(&group_b::GROUP_B)); (a, b, c) }); let declared_sex = declared(input, &folder, qc_declared(&a)); From 7c06808cada67696c24232b5675bcf59aa5468bd Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:29:24 +0100 Subject: [PATCH 076/255] refactor(report): delete run_delta.py, which only the old collector imported og run delta makes the comparison and og summary collect writes New since. The two run-delta checks of test_collect_summary.py 21b now run og run delta; test_run_eta.py no longer checks the old command line. Bead: pgp-bs9u.45 --- bin/owngenome_live_recheck.py | 3 +- bin/run_delta.py | 288 ---------------------------------- docs/hardware-requirements.md | 2 + scripts/ci/e2e-case-paths.tsv | 4 +- tests/test_collect_summary.py | 27 +++- tests/test_render_report.py | 2 +- tests/test_run_eta.py | 4 +- 7 files changed, 31 insertions(+), 299 deletions(-) delete mode 100755 bin/run_delta.py diff --git a/bin/owngenome_live_recheck.py b/bin/owngenome_live_recheck.py index fed062fdb..5b235d76a 100644 --- a/bin/owngenome_live_recheck.py +++ b/bin/owngenome_live_recheck.py @@ -52,7 +52,8 @@ def recheck_state(run_dir, state, events): except (OSError, ValueError, AttributeError): return {} # A check older than the run's run_manifest.tsv describes an earlier run: what it calls newer may - # be what this run used. Not shown, as in the reports (bin/run_delta.py read_recheck). + # be what this run used. Not shown, as in the reports (og run delta, read_recheck + # in rust/crates/og-runlog/src/delta.rs). if utc14(manifest_written(run_dir)) > utc14(st.get("checked_utc")): return {} steps = {} diff --git a/bin/run_delta.py b/bin/run_delta.py deleted file mode 100755 index 8b7d51ca0..000000000 --- a/bin/run_delta.py +++ /dev/null @@ -1,288 +0,0 @@ -#!/usr/bin/env python3 -"""run_delta.py: what changed since the previous run of the same sample. - -The command is `og run delta` (rust/crates/og-runlog, docs/delta-report.md): -scripts/generate-report.sh runs it after the text report, before it keeps a -copy of this run's summary.json and run_manifest.tsv under -/history//, and it writes .txt, .html and -.json. This module is the same comparison for bin/collect_summary.py, -whose "New since" badges need it. Standard library only. - -What it compares: - inputs the run manifest (bin/write_manifest.sh): versions.env values, the - image each *_IMAGE names and the digest it resolved to, the data - rows (ClinVar file date, VEP cache, CPSR bundle, pypgx bundle, HLA - database, ancestry panel, PGS headers), the pipeline commit and the - declared sex. When and by which script it was written are left out. - sections a report section that ran in one run and not in the other. - ClinVar hits added or dropped (by chrom:pos ref>alt), a hit whose - significance changed (reclassified), whose review status or stars - changed (re-starred), or whose genotype changed. - CPSR the count of each classification. - PGx each CPIC gene's diplotype and phenotype, pypgx's CYP2D6 call and - the CYP2D6 consensus. - PRS scores added or dropped, and a percentile that moved by - PRS_BAND points or more, or appeared or went away. -For each result change it lists the changed inputs that feed that section, or -says that the inputs it tracks are the same and the call itself differs. -changed_sections() names the summary sections the changes touch, and -read_recheck() the last database re-check (recheck/state.json beside the -summary, bin/owngenome_recheck.py). -""" -import json -import os - -PRS_BAND = 5.0 # percentile points - -# Changed inputs that can explain a change in a section: an input whose name -# contains one of these (case-insensitive). The pipeline commit explains any. -WHY_TOKENS = { - "clinvar": ["clinvar", "DEEPVARIANT", "BCFTOOLS", "MINIMAP2", "BWA", "GATK", "FASTP"], - "cpsr": ["PCGR", "VEP", "DEEPVARIANT"], - "cpic": ["PHARMCAT", "PYPGX", "CYRIUS", "DEEPVARIANT", "BCFTOOLS"], - "pypgx": ["PYPGX", "pypgx_bundle"], - "cyp2d6": ["PYPGX", "pypgx_bundle", "CYRIUS", "PHARMCAT"], - "prs": ["pgs:", "PGSC", "PLINK2", "ancestry_panel", "DEEPVARIANT"], -} -IGNORED_RUN_KEYS = ("written_utc", "written_by") -# The summary section of each result area (diff_states names its own). -SECTION_OF_AREA = {"ClinVar": "clinvar", "CPSR": "cpsr", "PGx (CPIC)": "cpic", "PGx (pypgx)": "pypgx", - "PGx (CYP2D6 consensus)": "pypgx", "PRS": "prs"} - - -def read_manifest(path): - """{(section, key): value} of a run_manifest.tsv; {} when there is none.""" - rows = {} - if not path or not os.path.isfile(path): - return rows - with open(path, errors="replace") as f: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) >= 3 and not c[0].startswith("#"): - rows[(c[0], c[1])] = "\t".join(c[2:]) - return rows - - -def manifest_items(rows): - """The manifest as {item: value}, one entry per thing that can change. - An image is keyed by its *_IMAGE name, its value is 'ref (digest)', so a - new tag and a new digest of the same tag are each one change.""" - out = {} - for (sec, key), val in rows.items(): - if sec == "run": - if key not in IGNORED_RUN_KEYS: - out[key] = val - elif sec == "version": - if key.endswith("_IMAGE"): - out[key] = f"{val} ({rows.get(('image', val), 'not in the manifest')})" - else: - out[key] = val - elif sec == "data": - out[key] = val - return out - - -def change(area, kind, item, before, after, why=None, section=None): - c = {"area": area, "kind": kind, "item": item, "before": before, "after": after} - if why is not None: - c["why"] = why - if section is not None: - c["section"] = section # the summary.json section it changes: the report marks it "new since" - return c - - -def diff_inputs(prev, cur): - a, b = manifest_items(prev), manifest_items(cur) - out = [] - for k in sorted(set(a) | set(b)): - if a.get(k) != b.get(k): - out.append(change("inputs", "input", k, a.get(k, "absent"), b.get(k, "absent"))) - return out - - -def why_for(area, inputs): - tokens = [t.lower() for t in WHY_TOKENS.get(area, [])] + ["git_commit"] - hit = [c for c in inputs if any(t in c["item"].lower() for t in tokens)] - if not hit: - return "the inputs this delta tracks for it are the same; the call itself differs" - return "changed inputs: " + "; ".join(f"{c['item']} {c['before']} -> {c['after']}" for c in hit) - - -def section(s, key): - return ((s.get("sections") or {}).get(key)) or {} - - -def has_data(sec): - return sec.get("state") in ("ok", "stale") - - -def diff_states(p, c): - out = [] - keys = sorted(set(p.get("sections") or {}) | set(c.get("sections") or {})) - for k in keys: - a, b = section(p, k), section(c, k) - # A section only one summary knows (a newer version added it) and that did not run reads as - # missing on both sides, so a pipeline upgrade lists no noise rows. - sa, sb = a.get("state", "missing"), b.get("state", "missing") - if sa != sb: - out.append(change(b.get("title") or a.get("title") or k, "section", k, sa, sb, section=k)) - return out - - -def hit_key(h): - return f"{h.get('chrom')}:{h.get('pos')} {h.get('ref')}>{h.get('alt')}" - - -def diff_clinvar(p, c): - a, b = section(p, "clinvar"), section(c, "clinvar") - if not (has_data(a) and has_data(b)): - return [] - ha = {hit_key(h): h for h in a.get("data", {}).get("hits", [])} - hb = {hit_key(h): h for h in b.get("data", {}).get("hits", [])} - out = [] - for k in sorted(set(ha) | set(hb)): - x, y = ha.get(k), hb.get(k) - gene = (y or x).get("gene", ".") - item = f"{k} ({gene})" - if x is None: - out.append(change("ClinVar", "added", item, "absent", f"{y.get('significance')}, {y.get('stars')} stars")) - elif y is None: - out.append(change("ClinVar", "dropped", item, f"{x.get('significance')}, {x.get('stars')} stars", "absent")) - else: - if x.get("significance") != y.get("significance"): - out.append(change("ClinVar", "reclassified", item, x.get("significance"), y.get("significance"))) - if x.get("stars") != y.get("stars") or x.get("review_status") != y.get("review_status"): - out.append(change("ClinVar", "re-starred", item, - f"{x.get('stars')} stars ({x.get('review_status')})", - f"{y.get('stars')} stars ({y.get('review_status')})")) - if x.get("genotype") != y.get("genotype"): - out.append(change("ClinVar", "genotype", item, x.get("genotype"), y.get("genotype"))) - return out - - -def diff_cpsr(p, c): - a, b = section(p, "cpsr"), section(c, "cpsr") - if not (has_data(a) and has_data(b)): - return [] - ca, cb = a.get("data", {}).get("classification") or {}, b.get("data", {}).get("classification") or {} - return [change("CPSR", "classification count", k, ca.get(k, 0), cb.get(k, 0)) - for k in sorted(set(ca) | set(cb)) if ca.get(k, 0) != cb.get(k, 0)] - - -def diff_cpic(p, c): - a, b = section(p, "cpic"), section(c, "cpic") - if not (has_data(a) and has_data(b)): - return [] - ga = {g.get("gene"): g for g in a.get("data", {}).get("genes", [])} - gb = {g.get("gene"): g for g in b.get("data", {}).get("genes", [])} - out = [] - for k in sorted(set(ga) | set(gb), key=str): - x, y = ga.get(k), gb.get(k) - fx = f"{x.get('diplotype')} ({x.get('phenotype')})" if x else "absent" - fy = f"{y.get('diplotype')} ({y.get('phenotype')})" if y else "absent" - if fx != fy: - out.append(change("PGx (CPIC)", "call", k, fx, fy)) - return out - - -def diff_cyp2d6(p, c): - out = [] - a, b = section(p, "pypgx"), section(c, "pypgx") - if has_data(a) and has_data(b): - x, y = a.get("data", {}).get("cyp2d6"), b.get("data", {}).get("cyp2d6") - if x != y: - out.append(change("PGx (pypgx)", "call", "CYP2D6", x, y)) - x = ((p.get("cyp2d6") or {}).get("consensus") or {}).get("result") - y = ((c.get("cyp2d6") or {}).get("consensus") or {}).get("result") - if x != y: - out.append(change("PGx (CYP2D6 consensus)", "call", "CYP2D6", x or "none", y or "none")) - return out - - -def pct(v): - try: - return float(v) - except (TypeError, ValueError): - return None - - -def shown(v): - return "none" if v is None else v - - -def diff_prs(p, c): - a, b = section(p, "prs"), section(c, "prs") - if not (has_data(a) and has_data(b)): - return [] - sa = {r.get("pgs_id"): r for r in a.get("data", {}).get("scores", [])} - sb = {r.get("pgs_id"): r for r in b.get("data", {}).get("scores", [])} - out = [] - for k in sorted(set(sa) | set(sb), key=str): - x, y = sa.get(k), sb.get(k) - item = f"{k} ({(y or x).get('condition', '')})" - if x is None or y is None: - out.append(change("PRS", "added" if x is None else "dropped", item, - "absent" if x is None else "scored", "scored" if x is None else "absent")) - continue - px, py = pct(x.get("percentile")), pct(y.get("percentile")) - if (px is None) != (py is None) or (px is not None and abs(px - py) >= PRS_BAND): - # A percentile of 0 is a value: only a missing one reads "none". - out.append(change("PRS", "percentile", item, shown(x.get("percentile")), shown(y.get("percentile")))) - return out - - -def compare(prev_summary, prev_manifest, summary, manifest): - inputs = diff_inputs(prev_manifest, manifest) - results = [] - for area, fn in (("clinvar", diff_clinvar), ("cpsr", diff_cpsr), ("cpic", diff_cpic), - ("cyp2d6", diff_cyp2d6), ("prs", diff_prs)): - for ch in fn(prev_summary, summary): - ch["why"] = why_for(area, inputs) - ch["section"] = SECTION_OF_AREA.get(ch["area"], area) - results.append(ch) - return inputs + diff_states(prev_summary, summary) + results - - -def changed_sections(changes): - """The summary sections a delta changes, in order: what the report marks 'New since'.""" - return list(dict.fromkeys(c["section"] for c in changes if c.get("section"))) - - -def newest_history(sample_dir): - """The newest history// folder with a summary.json, or None.""" - h = os.path.join(sample_dir, "history") - try: - names = sorted(n for n in os.listdir(h) if os.path.isfile(os.path.join(h, n, "summary.json"))) - except OSError: - return None - return os.path.join(h, names[-1]) if names else None - - -def read_recheck(path): - """/recheck/state.json (bin/owngenome_recheck.py) as a short block, or None.""" - try: - with open(path) as f: - st = json.load(f) - except (OSError, ValueError): - return None - if not isinstance(st, dict) or not isinstance(st.get("items"), list): - return None - # A check of an earlier run: /run_manifest.tsv was written after it, so its "newer than - # the run used" may already be what this run used. Not shown; the next check replaces it. - man = read_manifest(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(path))), "run_manifest.tsv")) - if utc14(man.get(("run", "written_utc"))) > utc14(st.get("checked_utc")): - return None - return {"checked_utc": st.get("checked_utc"), - "items": [{k: i.get(k) for k in ("item", "label", "class", "ran_with", "upstream", "action")} - for i in st["items"] if isinstance(i, dict)], - "affected_steps": st.get("affected_steps") or [], - "rerun": st.get("rerun") if isinstance(st.get("rerun"), dict) else None} - - -def utc14(text): - """'2026-09-02 12:00:00 UTC' or '2026-09-02T12:00:00Z' as '20260902120000'; '' without one.""" - return "".join(c for c in str(text or "") if c.isdigit())[:14] - - -if __name__ == "__main__": - raise SystemExit("bin/run_delta.py has no command line any more: run bin/og run delta (docs/delta-report.md)") diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index ba4ef1c61..67a7b09ee 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -256,6 +256,8 @@ Both ends of each measured figure are also in `assets/steps.tsv` (`hours_low` an So the reports cost about 25 s per sample, not 1-3 minutes, and almost all of it is collecting the summary, which each of the two scripts does once. Under `cProfile` (one HTML run), 82% of the collection was `sec_variants` reading every record of the sample's small-variant VCF in Python, and 9% the ROH section. Rendering is under half a second. The HTML report was 2.0-2.7 MiB and the search index 8.3-12.2 MiB. +Since then the Rust `og summary collect` writes `summary.json`, and `generate-report.sh` renders from the `summary.json` step 24 wrote unless a step output is newer than it, so a run collects each sample once. `run_delta.py` is gone: `og run delta` makes the comparison. + The command, per genome (`` holds the sample folder, `` is empty and writable, run from the repo root): ```sh diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 50dd13368..f4dd8e3cd 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -159,7 +159,7 @@ setup-gencode-exons-and-nextflow-clip-check-1 assets/construct_variants.tsv asse short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh single-source-lib-ownership scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/expansion_hunter/* modules/local/stranger/* modules/local/vcf_precheck/* scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh +str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh sv-mito-telomere-steps-1-sv-merge scripts/22-survivor-merge.sh scripts/ci/fake-docker/lib.sh tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in sv-mito-telomere-steps-2-mito assets/callability_genes.tsv bin/collect_summary.py scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh sv-mito-telomere-steps-3-y-haplogroup assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh @@ -175,7 +175,7 @@ vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_gene vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh viral-reads-1 bin/og modules/local/vcf_precheck/* modules/local/viral/* scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/run_delta.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh rust-z2e-readers bin/og tests/e2e/lib.sh scripts/ci/image-smoke.sh scripts/ci/nf-trace-requests.tsv diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 99be691f1..20eb59617 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -82,7 +82,7 @@ frequency caveat on every hit; 21b. each hit carries the rsIDs of VEP's Existing_variation ('rs123&COSV1' gives ['rs123']; another allele's ids are not taken), [] with no VEP record or no VEP file, and the run delta - (bin/run_delta.py) reads the same with or without them; + (og run delta) reads the same with or without them; 22. CPSR beside ClinVar (step 17): the P/LP rows by final or own class, the rows where CLINVAR_CLASSIFICATION and CPSR_CLASSIFICATION disagree on P/LP with CPSR's ACMG codes, the final P/LP rows CPSR's own rules class @@ -729,8 +729,24 @@ def clinvar_quality(work, schema): def clinvar_rsids(work, schema): """21b. the rsIDs of each ClinVar hit, from VEP's Existing_variation.""" - import run_delta d = os.path.join(work, "clinvar-rsids", "S") + + def changes(prev, cur): + """og run delta's changes from summary PREV to summary CUR.""" + w = os.path.join(work, "clinvar-rsids-delta") + shutil.rmtree(w, ignore_errors=True) + for name, summary in (("prev", prev), ("cur", cur)): + os.makedirs(f"{w}/{name}") + with open(f"{w}/{name}/summary.json", "w") as f: + json.dump(summary, f) + p = subprocess.run([collect_summary.og(), "run", "delta", "--sample", "S", "--summary", + f"{w}/cur/summary.json", "--out", f"{w}/delta", "--previous", f"{w}/prev"], + capture_output=True, text=True) + if p.returncode != 0: + return p.stderr + with open(f"{w}/delta.json") as f: + return json.load(f)["changes"] + put(f"{d}/clinvar/S_clinvar_hits.vcf", FLOOR_HITS) vep_head = "##fileformat=VCFv4.2\n" + CSQ_EV_HEAD + VCF_HEAD.split("\n", 1)[1] write_tabixed_vcf(f"{d}/vep/S_vep.vcf.gz", vep_head, [ @@ -748,12 +764,13 @@ def clinvar_rsids(work, schema): bare = json.loads(json.dumps(summ)) for h in bare["sections"]["clinvar"]["data"]["hits"]: h.pop("rsids") - check("ClinVar rsIDs: the run delta reads the same with or without them", - run_delta.compare(bare, {}, summ, {}) == run_delta.compare(summ, {}, summ, {}), "") + got = changes(bare, summ) + check("ClinVar rsIDs: the run delta reads the same with or without them", got == [], got) moved = json.loads(json.dumps(summ)) moved["sections"]["clinvar"]["data"]["hits"][0]["significance"] = "Benign" + got = changes(bare, moved) check("ClinVar rsIDs: red control, the run delta does see a reclassified hit", - run_delta.compare(bare, {}, moved, {}) != run_delta.compare(summ, {}, summ, {}), "") + isinstance(got, list) and any(c["kind"] == "reclassified" for c in got), got) shutil.rmtree(f"{d}/vep") none = collect_summary.collect("S", d)["sections"]["clinvar"]["data"]["hits"] check("ClinVar rsIDs: without step 13 every hit has [] and the section still reads", diff --git a/tests/test_render_report.py b/tests/test_render_report.py index 1d85a1758..36add70fe 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -405,7 +405,7 @@ def chapter_checks(work): check("chapters: red control: a dropped card is caught", sorted(re.findall(r"

([^<]*)

", html.replace("

Traits

", ""))) != GOLDEN_TITLES) - # "New since" and the database re-check (bin/owngenome_recheck.py, bin/run_delta.py through collect_summary) + # "New since" and the database re-check (bin/owngenome_recheck.py, og summary collect) since = json.loads(json.dumps(summ)) since["new_since"] = {"previous_utc": "2026-01-01T00:00:00Z", "sections": ["clinvar"]} since["recheck"] = {"checked_utc": "2026-02-02T00:00:00Z", "affected_steps": ["06", "24"], "rerun": None, diff --git a/tests/test_run_eta.py b/tests/test_run_eta.py index ded2e7e0b..f3cc0a6ca 100644 --- a/tests/test_run_eta.py +++ b/tests/test_run_eta.py @@ -26,7 +26,7 @@ 15. without --sample, the trace of a run that run-all.sh has not finished is left out of the step times, and counts once run_status.tsv says it finished; a run_status.tsv that cannot be read leaves out the newest trace, never a crash. - 16. bin/run_eta.py, run_state.py, run_headroom.py and run_delta.py, run as commands, + 16. bin/run_eta.py, run_state.py and run_headroom.py, run as commands, exit 1 and name their og run command, never a silent exit 0. Each check is then run against a planted mistake (a mutant of run_eta.py or a broken config) and must fail there: a check that cannot fail is no check. @@ -293,7 +293,7 @@ def unreadable(path): return out -OLD_CLIS = {"run_eta.py": "eta", "run_state.py": "state", "run_headroom.py": "headroom", "run_delta.py": "delta"} +OLD_CLIS = {"run_eta.py": "eta", "run_state.py": "state", "run_headroom.py": "headroom"} def old_cli_refuses(path, cmd): From bf525e2d2be2cc2118e5dd0551467b1dbc2e4ae0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:29:39 +0100 Subject: [PATCH 077/255] docs(changelog): og writes the summary the reports read, once per run Bead: pgp-bs9u.45 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bf731b53..2a462b156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again only when a file it reads is newer than that summary.json. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: + - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. + - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. + - The Rust `og` binary now writes the whole summary.json: `og summary collect` without `--only` reads all 48 sections over one walk of the sample folder, then runs the post pass. The post pass joins sections: the repeat readings with the traits and CPIC states, Locityper's primary calls with HLA, pypgx and Cyrius, and CNVpytor's chrY filter with the declared sex. It also builds the CYP2D6 block and writes `not_run`, `not_assessed`, `recheck` and `new_since`. Without `--out` it writes `summary.json` in the sample folder, as the Python did; `--only` still needs `--out`. Nothing calls it yet: the reports still read the summary that `bin/collect_summary.py` writes. The whole file is the same as the Python's on 8 of 9 new parity cases, compared as JSON with key order and only `generated_utc` masked; the ninth holds the deliberate fix below. On a synthetic folder with every section it takes 0.01 to 0.03 s and 15 MB, where the Python took 0.15 to 0.21 s and 42 to 44 MB. With a 1,000,000-record VCF it takes 0.44 to 0.62 s and 17 to 20 MB, where the Python took 1.5 s and 42 to 44 MB. What differs from the Python: - The post pass keeps the summary. The Python ran it outside its per-section error handling, so one row of an unexpected shape there lost summary.json and both reports. The Rust parts read with defaults and have no such error to raise; a part that returns an error makes only its own section unreadable, with a note that names the file. - A section reader that stops on an error the Python did not catch keeps the summary too. A telomere table with an infinite read length stopped the Python with an OverflowError and no summary.json; og marks only the telomere section unreadable. From 84027ab0e4f670e867c6b60c9bba31cdee4725b5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:41:38 +0100 Subject: [PATCH 078/255] test(report): read the HTML report's generated time up to its separator The HTML header puts the latest run after the generated time on the same line, so the comparison with the text report took both. Bead: pgp-bs9u.45 --- tests/fake-docker/run-all-partial-failure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index 5deda814f..556872618 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -155,7 +155,7 @@ done grep -q '^Rendering from summary.json' "${S}/logs/generate-report.log" \ || miss "generate-report.sh collected the summary again after step 24 wrote it: $(head -3 "${S}/logs/generate-report.log")" t_txt=$(sed -n 's/^ Generated: //p' "$TXT" | head -1) -t_html=$(grep -o 'Generated: [^<]*' "$HTML" | head -1 | sed 's/^Generated: //') +t_html=$(grep -o 'Generated: [^<&]*' "$HTML" | head -1 | sed 's/^Generated: //; s/ *$//') [ -n "$t_txt" ] && [ "$t_txt" = "$t_html" ] || miss "the text report says generated '${t_txt}', the HTML report '${t_html}'" # The reverse: a step output newer than summary.json makes it collect again. roh=$(find "$S" -name 'sample1_roh.txt' -type f | head -1) From 46d78bacd0425d28bf17e7ef9af21f3627174e33 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:37:25 +0100 Subject: [PATCH 079/255] fix(live): keep a closed child's flagged-card count out of family.json and count its acts as its page does A parent's page listed a gated child's flagged-card count, which changes with the cards the unopened results flag. family.json now gives attention null for a child whose own page keeps the report closed (the family plugin and the demo builder alike). /runs.json counts a child's results to act on from the summary its results.json reads, also while summary.json is older than a finished step. The server, the home page and the person page decide who is a child by one rule (child_of). Adults and no-act children are unchanged. --- bin/owngenome_live.py | 49 +++++++++++++++++++++++++++++++++++ bin/owngenome_live_family.py | 10 +++++-- bin/owngenome_live_results.py | 14 +++------- tests/demo/build_live_demo.py | 4 +++ 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/bin/owngenome_live.py b/bin/owngenome_live.py index 1b3edbcf0..9af6f86ad 100755 --- a/bin/owngenome_live.py +++ b/bin/owngenome_live.py @@ -105,6 +105,19 @@ def valid_run(name): return bool(name) and name not in (".", "..") and bool(RUN_RE.match(name)) +def child_of(families, name): + """Whether NAME's pages treat it as a child, the one rule of assets/live/core.js (childIn, preparePerson): + the first family, by name, whose members list NAME decides, by a father or a mother there; a run no + readable family lists is a child while some family's members cannot be read (an empty member list). + families: [{"name", "members": [{"sample", "father", "mother"}]}], as /runs.json lists them.""" + fams = sorted(families or [], key=lambda f: f.get("name") or "") + for f in fams: + for m in f.get("members") or []: + if m.get("sample") == name: + return bool(m.get("father") or m.get("mother")) + return any(not (f.get("members") or []) for f in fams) + + def utc_now(): return datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -529,6 +542,7 @@ def __init__(self, app, cfg, password_hash): self.headline_cache = {} self.attention_cache = {} self.live_overview = {} # run -> (the headline cache key, live_results.overview of that summary) + self.summary_overviews = {} # run -> (the summary summary() gave, live_results.overview of it) self.allowed = set(LOOPBACK_HOSTS) | {h.lower() for h in cfg.allowed_host or []} if cfg.bind not in ("0.0.0.0", "::"): self.allowed.add(cfg.bind.strip("[]").lower()) @@ -817,6 +831,35 @@ def overview(self, d): self.attention_cache[p] = (stamp, n) return n, mtime + def child(self, name, runs=None): + """child_of() over the families the server lists now.""" + return child_of([r for r in (runs or self.runs()).values() if r.get("kind") == "family"], name) + + def summary_overview(self, name, d, st): + """live_results.overview of the summary results.json reads now (summary()), cached with that summary; + None while there is none.""" + s = self.summary(name, d, st) + if not s: + return None + hit = self.summary_overviews.get(name) + if not hit or hit[0] is not s: + try: + import live_results + hit = (s, live_results.overview(s)) + except Exception as e: # one bad summary must not take a page down + print(f"owngenome_live: overview of {name} not built: {type(e).__name__}", file=sys.stderr) + hit = (s, None) + self.summary_overviews[name] = hit + return hit[1] + + def gated(self, name, d, st, runs=None): + """Whether NAME's own page keeps its report closed (assets/live/core.js): a child whose results hold a + result to act on, counted from the summary its results.json reads (summary.json while that has none).""" + if not self.child(name, runs): + return False + o = self.summary_overview(name, d, st) or self.overview(d)[0] + return bool(((o or {}).get("levels") or {}).get("act")) + def runs_index(self): """/runs.json: each run's progress, and the families with their members, for the home page. No result beyond the attention count, the counts by level, the titles of the first four @@ -827,6 +870,8 @@ def runs_index(self): self.headline_cache.pop(k, None) for k in [k for k in list(self.live_overview) if k not in runs]: self.live_overview.pop(k, None) + for k in [k for k in list(self.summary_overviews) if k not in runs]: + self.summary_overviews.pop(k, None) keep = {os.path.join(r["dir"] or os.path.join(self.root, n), "summary.json") for n, r in runs.items()} for k in [k for k in list(self.attention_cache) if k not in keep]: self.attention_cache.pop(k, None) @@ -864,6 +909,10 @@ def runs_index(self): lo = (hit[0], None) self.live_overview[name] = lo o = lo[1] or o + # A child's results to act on decide whether its own page is closed: count them from the summary + # its results.json reads, so the home page closes the child whenever the child's page does. + if self.child(name, runs): + o = self.summary_overview(name, r["dir"] or os.path.join(self.root, name), st) or o item["attention"] = o["attention"] if o else None item["results"] = mtime is not None if o: # the Journal home: counts by level, the first findings, the lead diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py index 5c21a38e5..277a64f35 100644 --- a/bin/owngenome_live_family.py +++ b/bin/owngenome_live_family.py @@ -526,13 +526,19 @@ def page_data(req, m): out = family_page(name, d) live = getattr(req, "live", None) runs = {} + listed = live.runs() if live else {} for p in out["state"].get("members") or []: - r = live.run(p["sample"]) if live else None + r = listed.get(p["sample"]) if live else None if not r or r.get("kind") == "family": continue st = live.base_state(r) planned = [s for s in st["steps"] if s["state"] != "not planned"] - n, mtime = live.attention(r["dir"] or os.path.join(live.root, r["name"])) + rd = r["dir"] or os.path.join(live.root, r["name"]) + n, mtime = live.attention(rd) + # A child whose own page keeps the report closed has no count of flagged cards here: the count changes + # with which cards the unopened results flag, and a parent's page would show it. + if live.gated(p["sample"], rd, st, listed): + n = None runs[p["sample"]] = {"url": "/runs/%s/" % p["sample"], "state": st["run"]["state"], "planned": len(planned), "done": sum(1 for s in planned if s["state"] == "done"), "attention": n, "results": mtime is not None} diff --git a/bin/owngenome_live_results.py b/bin/owngenome_live_results.py index 4e6cf8610..f8f8db57a 100644 --- a/bin/owngenome_live_results.py +++ b/bin/owngenome_live_results.py @@ -95,17 +95,9 @@ def finding(req, m): def child_run(live, name): - """Whether the person page treats NAME as a child's run: a family lists it with a father or a mother, or a - family's members cannot be read and no readable family lists NAME.""" - fams = [r for r in live.runs().values() if r.get("kind") == "family"] - listed = False - for f in fams: - for m in f.get("members") or []: - if m.get("sample") == name: - listed = True - if m.get("father") or m.get("mother"): - return True - return not listed and any(not (f.get("members") or []) for f in fams) + """Whether the person page treats NAME as a child's run: the one rule of the page and the home page + (bin/owngenome_live.py child_of).""" + return live.child(name) def live_results_mod(): diff --git a/tests/demo/build_live_demo.py b/tests/demo/build_live_demo.py index 709a418ea..5eac6c200 100644 --- a/tests/demo/build_live_demo.py +++ b/tests/demo/build_live_demo.py @@ -348,6 +348,10 @@ def family_json(root, index): by = {r["name"]: r for r in index["runs"]} out["runs"] = {p["sample"]: {k: by[p["sample"]][k] for k in ("url", "state", "planned", "done", "attention", "results")} for p in out["state"].get("members") or [] if p["sample"] in by} + # As the family plugin serves it: no count of flagged cards for a child whose own page keeps the report closed. + for name, r in out["runs"].items(): + if owngenome_live.child_of(index.get("families"), name) and (by[name].get("levels") or {}).get("act"): + r["attention"] = None seq = [p["sample"] for p in out["state"].get("members") or [] if p.get("sequenced")] sums = {} for name in seq: From bd89ccf043359ad8f1f4719264e8fe3e3966b240 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:37:25 +0100 Subject: [PATCH 080/255] fix(live): give adult and no-act pages the requests, order, options and search they have without the gate The event stream opens when live.js asks for it and its messages wait for the gate's decision. A file the gate reads first keeps its answer whatever the status, read with the options of the view that reads it (a replay's frames.json with the default cache). person.js answers the charts and the search as before the gate, so a child's open page says what search leaves out. A guessed finding address releases to the run's page. The unused veil CSS goes. --- assets/live/core.js | 109 +++++++++++++++++++++------------- assets/live/live.css | 4 -- assets/live/person.js | 20 ++++++- assets/live/plugins/charts.js | 9 +-- assets/live/plugins/search.js | 9 +-- 5 files changed, 97 insertions(+), 54 deletions(-) diff --git a/assets/live/core.js b/assets/live/core.js index 6bdae7386..5c5e0fd7f 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -155,7 +155,8 @@ // unopened. The page is then a function of the opened results alone and of how many are closed: no view (person.js, // finding.js, map.js, family-page.js, home.js, live.js, the plugins) receives the child's report, because every // view reads through fetch and EventSource and this gate holds both. Adults and children with no result to act on - // pass through untouched: each file is fetched as often, and in the same order, as without the gate. + // pass through untouched: each file is fetched as often, in the same order and with the same options as without + // the gate, and the event stream opens when the view asks for it. var originalFetch = window.fetch.bind(window); var records = {}, current = null, ownsPage = false, eventSources = [], shared = {}, cleaned = false; var CLOSED_TEXT = "This child's report stays closed until every result to act on in adult life is opened on purpose."; @@ -178,18 +179,24 @@ if (r.status === 401) { window.location.href = '/login?next=' + encodeURIComponent(location.pathname); } return r.ok ? r.json() : null; } + // The options the views read their JSON with (PGPUI.getJSON); live.js reads a replay's frames.json with none. + var NO_STORE = { credentials: 'same-origin', cache: 'no-store' }; function jsonAt(url) { - return originalFetch(url, { credentials: 'same-origin', cache: 'no-store' }).then(read).catch(function () { return null; }); + return originalFetch(url, NO_STORE).then(read).catch(function () { return null; }); } - // A file the gate reads before the views: kept once, so the view's own first read of it costs no request. - function sharedJSON(url) { + // A file the gate reads before the views, with the options the view that reads it uses: its answer is kept once, + // whatever its status (a 404 or a failed request too), so the view's own first read of it costs no request. The + // view gets the response itself; the gate reads a copy of an answer that is ok and leaves any other unread, as + // the views do. + function sharedJSON(url, options) { url = absolute(url); - return originalFetch(url, { credentials: 'same-origin', cache: 'no-store' }).then(function (r) { - return r.text().then(function (body) { - if (r.ok) { shared[url] = { body: body, status: r.status, type: r.headers.get('Content-Type') || 'application/json' }; } - return read({ status: r.status, ok: r.ok, json: function () { return Promise.resolve(JSON.parse(body)); } }); - }); - }).catch(function () { return null; }); + return originalFetch(url, options).then(function (r) { + shared[url] = { response: r.ok ? r.clone() : r }; + return r.ok ? read(r) : read({ status: r.status, ok: false }); + }, function () { shared[url] = { failed: true }; return null; }).catch(function () { return null; }); + } + function keptResponse(kept) { + return kept.failed ? Promise.reject(new TypeError('Failed to fetch')) : Promise.resolve(kept.response); } function random() { var b = new Uint32Array(1); crypto.getRandomValues(b); return b[0] / 4294967296; } // The results to act on, one record each: the findings of level "act" (bin/live_results.py). null when unreadable. @@ -212,7 +219,8 @@ return out; } function makeRecord(url, child, r, old) { - var acts = actsOf(r), rec = { url: url, child: child, acts: acts, opened: {}, order: [], perm: [] }; + var acts = actsOf(r), rec = { url: url, child: child, acts: acts, opened: {}, order: [], perm: [], + ids: r && Array.isArray(r.findings) ? r.findings.map(function (f) { return f && f.id; }) : [] }; if (!acts) { return rec; } // An act whose record changed is closed again; an unchanged one keeps its opening for this page. rec.order = old ? old.order.filter(function (id) { return same(byId(old, id), byId(rec, id)); }) : []; @@ -223,7 +231,7 @@ } function record(url, child, cached) { url = absolute(url); - return (cached ? sharedJSON : jsonAt)(url + 'results.json').then(function (r) { + return (cached ? sharedJSON(url + 'results.json', NO_STORE) : jsonAt(url + 'results.json')).then(function (r) { var old = records[url]; var rec = makeRecord(url, child !== undefined ? child : old ? old.child : true, r, old); records[url] = rec; @@ -293,10 +301,11 @@ }); document.title = 'Results closed: OwnGenome'; } + function runPath() { return new URL('../../', absolute(findingPath)).pathname; } function renderGate() { cleanPage(); // A finding's address names its finding: a closed page shows the run's address instead, the same for any id. - if (findingPath) { history.replaceState(null, '', new URL('../../', absolute(findingPath)).pathname + location.hash); } + if (findingPath) { history.replaceState(null, '', runPath() + location.hash); } normalizeHash(current); var wrap = el('main', { 'class': 'wrap', id: 'report-gate' }); add(wrap, el('h1', {}, 'Results about a child')); @@ -316,7 +325,11 @@ var ids = current.acts.map(function (f) { return f.id; }), b = new Uint8Array(16); crypto.getRandomValues(b); var nonce = Array.from(b).map(function (v) { return v.toString(16).padStart(2, '0'); }).join(''); - var target = findingPath || location.pathname; + // A finding address releases to the finding's page only when the results have that finding; any other id + // goes to the run's page, as the server answers 404 for it once the report is open. + var id = null; + try { id = findingPath ? decodeURIComponent(findingPath.split('/finding/')[1].replace(/\/$/, '')) : null; } catch (e) { id = null; } + var target = !findingPath ? location.pathname : (current.ids || []).indexOf(id) >= 0 ? findingPath : runPath(); sessionStorage.setItem(TRANSFER, JSON.stringify({ path: target, stamp: stamp, nonce: nonce, name: window.name.indexOf(NONCE) === 0 ? '' : window.name, order: current.order.map(function (id) { return ids.indexOf(id); }) })); window.name = NONCE + nonce; @@ -337,18 +350,25 @@ } // The person and finding pages: the run's state, its family's roles and, for a child, its results, read in the // order the views read them (the finding page asks for its results at once, the person page once the state has - // answered). An adult's results are not read here: the views read them as they would without the gate. + // answered) and with the views' options. The family is read once the page's scripts have run, so live.js has + // opened its event stream first, as it does without the gate. An adult's results are not read here: the views read + // them as they would without the gate. + var parsed = new Promise(function (done) { + if (document.readyState === 'complete') { done(); } else { document.addEventListener('DOMContentLoaded', done); window.addEventListener('load', done); } + }); function preparePerson() { var url = absolute(findingPath ? '../../' : './'); var replaying = root.getAttribute('data-mode') === 'replay'; - var stateRead = sharedJSON(url + (replaying ? 'frames.json' : 'state.json')); + // live.js reads a replay's frames.json with no options; finding.js reads it, and state.json, as JSON (NO_STORE). + var stateRead = sharedJSON(url + (replaying ? 'frames.json' : 'state.json'), replaying && !findingPath ? undefined : NO_STORE); var results = findingPath ? record(url, undefined, true) : null; - return stateRead.then(function (raw) { + return Promise.all([stateRead, parsed]).then(function (x) { + var raw = x[0]; var state = replaying ? raw && raw.frames && raw.frames.slice(-1)[0] : raw; state = replaying ? state && state.state : state; var fam = state && state.plugins && state.plugins.family; var entry = fam && fam.kind === 'member' && (fam.families || [])[0]; - return entry ? sharedJSON(absolute(entry.url, url) + 'family.json').then(function (f) { + return entry ? sharedJSON(absolute(entry.url, url) + 'family.json', NO_STORE).then(function (f) { var member = f && f.state && (f.state.members || []).filter(function (m) { return m.sample === root.getAttribute('data-run'); })[0]; // A family that cannot be read, or does not list this person, cannot say they are not a child. return !member || !!(member.father || member.mother); @@ -432,21 +452,22 @@ } return f; } - // The home page decides from runs.json alone, as it always has: a run is a child's when a readable family lists - // one of its parents (or, while a family's members cannot be read, when no readable family lists it), and its - // summary is closed when runs.json counts or names a result to act on. + // Who is a child, one rule for the person page (preparePerson: the first of state.json's families, then its + // family.json), this home page and the server (bin/owngenome_live.py child_of): the first family, by name, that + // lists the person decides, by a father or a mother there; a person no readable family lists is a child while + // some family's members cannot be read. + function childIn(fams, name) { + function me(f) { return (f.members || []).filter(function (m) { return m.sample === name; })[0]; } + var f = fams.filter(me)[0]; + return f ? !!(me(f).father || me(f).mother) : fams.some(function (x) { return !(x.members || []).length; }); + } + // The home page decides from runs.json alone: a child's summary is closed when runs.json counts or names a result + // to act on (the server counts a child's from the summary its results.json reads). function projectRuns(data) { - var idx = copy(data), fams = idx.families || [], child = {}, listed = {}; - fams.forEach(function (f) { - (f.members || []).forEach(function (m) { - listed[m.sample] = true; - if ((f.members || []).some(function (x) { return x.sample === m.father || x.sample === m.mother; })) { child[m.sample] = true; } - }); - }); - var unknown = fams.some(function (f) { return !(f.members || []).length; }); + var idx = copy(data), fams = idx.families || []; idx.runs = (idx.runs || []).map(function (r) { var n = (r.levels && r.levels.act) || (r.top || []).filter(function (x) { return x && x.level === 'act'; }).length; - if (!(child[r.name] || (unknown && !listed[r.name])) || !n) { return r; } + if (!childIn(fams, r.name) || !n) { return r; } return Object.assign(progressOf(r), { gated: true, results: true, screened: true, attention: null, levels: { act: n }, top: [{ id: '', level: 'act', title: CLOSED_TEXT }], lead: CLOSED_TEXT, lead_closed: CLOSED_TEXT }); }); @@ -481,8 +502,7 @@ if (ownsPage || closedRun(requestURL)) { return new Promise(function () {}); } var kept = shared[requestURL]; delete shared[requestURL]; - var got = kept ? Promise.resolve(new Response(kept.body, { status: kept.status, headers: { 'Content-Type': kept.type } })) - : originalFetch(url, options); + var got = kept ? keptResponse(kept) : originalFetch(url, options); return got.then(function (response) { if (ownsPage) { return new Promise(function () {}); } var path = new URL(requestURL).pathname; @@ -514,16 +534,25 @@ }); window.addEventListener('pageshow', function (e) { if (e.persisted && cleaned) { location.reload(); } }); var NativeEvents = window.EventSource; + // A view's event stream opens the moment the view asks for it, as without the gate (live.js asks right after + // state.json). What it says waits until the gate knows whether it owns the page: then it reaches the view, in + // order, or the stream closes unread. After that it reaches the view only while the gate does not own the page. window.EventSource = function (url, options) { - var proxy = this, source = null; - proxy.readyState = 0; - proxy.close = function () { if (source) { source.close(); } proxy.readyState = 2; }; + var proxy = this, held = [], closed = false; + var source = ownsPage || closedRun(absolute(url)) ? null : new NativeEvents(url, options); + Object.defineProperty(proxy, 'readyState', { get: function () { return closed || !source ? 2 : source.readyState; } }); + proxy.close = function () { closed = true; if (source) { source.close(); } }; + if (!source) { return; } + eventSources.push(source); + function deliver(kind, e) { if (!ownsPage && !closed && proxy['on' + kind]) { proxy['on' + kind](e); } } + ['message', 'error', 'open'].forEach(function (kind) { + source['on' + kind] = function (e) { if (held) { held.push([kind, e]); } else { deliver(kind, e); } }; + }); ready.then(function () { - if (ownsPage || closedRun(absolute(url)) || proxy.readyState === 2) { return; } - source = new NativeEvents(url, options); eventSources.push(source); - source.onmessage = function (e) { if (!ownsPage && proxy.onmessage) { proxy.onmessage(e); } }; - source.onerror = function (e) { proxy.readyState = source.readyState; if (proxy.onerror) { proxy.onerror(e); } }; - source.onopen = function (e) { proxy.readyState = source.readyState; if (proxy.onopen) { proxy.onopen(e); } }; + var q = held; + held = null; + if (ownsPage || closedRun(absolute(url))) { source.close(); return; } + q.forEach(function (x) { deliver(x[0], x[1]); }); }); }; window.EventSource.CLOSED = NativeEvents.CLOSED; diff --git a/assets/live/live.css b/assets/live/live.css index 6c0959048..f810abbde 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -202,10 +202,6 @@ section.sec { padding: 56px 0 8px; border-top: 1px solid var(--hair); margin-top .waitnote { border-left: 2px solid var(--brass-dim); padding: 4px 0 4px 14px; font-size: 13.5px; color: var(--ink-2); } .notassessed { border: 1px solid rgba(250, 178, 25, .35); border-radius: 8px; padding: 12px 14px; background: rgba(250, 178, 25, .05); font-size: 14px; color: var(--ink-2); margin: 0 0 12px; } .notassessed b { color: var(--rev); font-weight: 600; } -.veil { border: 1px dashed var(--hair-2); border-radius: 8px; padding: 14px 16px; display: flex; gap: 14px; align-items: center; margin: 14px 0; - background: repeating-linear-gradient(135deg, rgba(255, 255, 255, .015) 0 6px, transparent 6px 12px); } -.veil .vt { font-size: 13.5px; color: var(--ink-2); flex: 1; } -.veil .vt b { color: var(--ink); font-weight: 600; } /* section 8 and 9: assessed, no finding; what it cannot see */ .nf { list-style: none; padding: 0; margin: 0; } diff --git a/assets/live/person.js b/assets/live/person.js index 2eefb1dd8..1a05b90fc 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -319,8 +319,24 @@ add(b, text); return b; } - // This renderer receives a child's report only once the shared data gate (core.js) has released it, so it - // closes nothing itself: no card row, chart mark or search hit. + // This renderer receives a child's report only once the shared data gate (core.js) has released it, with every + // result to act on opened, so it closes no card row itself. The charts (plugins/charts.js) and the search box + // (plugins/search.js) still ask it, as before the gate, and on a page that keeps results closed until opened (a + // child's, or one whose family roles are not known) get the same answers: before results.json has answered, + // every mark and hit is left out; then only a tag (results.veil) that names no finding of these results, since every + // result to act on here is open. Asked with no key, the search hears that the page keeps results closed, and says so. + function tagClosed(id) { return closedPage() && !findingById(id); } + window.PGP.markClosed = function (chart, mark) { + if (!closedPage()) { return false; } + if (!results) { return true; } + var id = ((results.veil && results.veil.marks && results.veil.marks[chart]) || {})[mark]; + return !!(id && tagClosed(id)); + }; + window.PGP.searchHidden = function (key, anchor, tag) { + if (!closedPage()) { return false; } + if (!results || key === null) { return true; } + return !!(tag && tagClosed(tag)); + }; function card(c) { var g = gate(c); diff --git a/assets/live/plugins/charts.js b/assets/live/plugins/charts.js index 6808a282d..28a896f3f 100644 --- a/assets/live/plugins/charts.js +++ b/assets/live/plugins/charts.js @@ -4,10 +4,11 @@ // a mark opens a panel with the mark's detail, the chart's caveat, a link to the reads in the // genome viewer (viewer/#) and a link to the section of the report. On a person's page each // chart goes into its card (a [data-chart-slot=] person.js draws), else into this plugin's -// panel. A mark the page keeps closed (a result to act on about a child, person.js's PGP.markClosed) is -// left out of the drawing, its title and its detail with it, wherever the figure sits; the figure is drawn -// again with it once the result is opened. No inline script, no style attribute, no eval: the SVG is -// parsed as XML and holds no script. +// panel. A mark the page keeps closed (person.js's PGP.markClosed: on a child's page, every mark until the +// results have answered; a child's report with a result to act on still closed never reaches the page, see +// the report gate in core.js) is left out of the drawing, its title and its detail with it, wherever the +// figure sits; the figure is drawn again with it once the page can show it. No inline script, no style +// attribute, no eval: the SVG is parsed as XML and holds no script. (function () { 'use strict'; if (!window.PGP) { return; } diff --git a/assets/live/plugins/search.js b/assets/live/plugins/search.js index 997907fff..5c7719d34 100644 --- a/assets/live/plugins/search.js +++ b/assets/live/plugins/search.js @@ -85,10 +85,11 @@ return li; } - // A hit from a section the page keeps closed (a result to act on about a child, person.js's - // PGP.searchHidden) is left out, and so is a suggested term that only such sections name: the search - // must not name what the page veils. The page says so whether or not anything was left out. - // A hit tagged with a result to act on (it.veil) is left out while that result is closed, whatever its section. + // A hit the page keeps closed (person.js's PGP.searchHidden: on a child's page, every hit until the results + // have answered, and a hit tagged (it.veil) with a result to act on that is not open) is left out, and so is a + // suggested term that only such hits name. A child's report with a result to act on still closed never reaches + // the page (the report gate in core.js). A page that keeps results closed says so whether or not anything was + // left out. function hidden(it) { return !!(root.PGP.searchHidden && root.PGP.searchHidden(it.section, it.anchor, it.veil || null)); } function closedNote() { return root.PGP.searchHidden && root.PGP.searchHidden(null, null) From 6e55602cd4f65510f45714838957bf1a0691ecbd Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:37:25 +0100 Subject: [PATCH 081/255] test(live): check the gate in live mode, on a family that cannot be read, on search and on the ticket Live-mode request order on the real server, the parent's page whichever card holds a child's results, the stale-summary home count, a 404 family.json, the fetch cache mode, a search on a no-act child, a guessed finding address and a mutant that keeps the opening ticket. --- tests/test_live_demo.py | 143 ++++++++++++++++++++----- tests/test_owngenome_live_family.py | 160 ++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+), 29 deletions(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index c187af9ca..fead519be 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -1416,7 +1416,10 @@ def snap(): assert b.wait("!document.documentElement.dataset.pgpGated && !!document.querySelector('#results .finding')"), 'last opening never released' # Search something while open; closing must replace this whole document and its caches. b.wait("document.querySelector('input[type=search]') && !document.querySelector('input[type=search]').disabled") - b.eval("var s=document.querySelector('input[type=search]'); s.value='KCNH2'; s.dispatchEvent(new Event('input')); true") + said = type_search(b, 'KCNH2') + if SEARCH_CLOSED not in said: + # A child's opened report searches as before the gate: it says what it leaves out. + transitions.append((count, order, stage, 'search lost its closed note')) b.eval("PGPUI.closeAct(%s); true" % json.dumps(acts[i]['id'])) assert b.wait("document.documentElement.dataset.pgpGated === 'true'"), 'close did not clean up' if b.eval("!!document.querySelector('input[type=search],.search-results,.search-status,canvas,svg,img')"): @@ -1480,6 +1483,15 @@ def gate_addresses(b, base, out, acts, visit, snap): visit('/runs/'+son+'/finding/'+fid+'/') s = snap() seen.append({k: s[k] for k in ('dom', 'text', 'title', 'address', 'accessibility')}) + # Opened on a guessed address, the report opens on the run's page, as main answers no such finding. + landed = {} + for fid in (acts[0]['id'], 'zz-not-a-finding'): + visit('/runs/'+son+'/finding/'+fid+'/') + open_gate(b) + b.wait("document.readyState === 'complete' && document.documentElement.dataset.pgpStatus === 'ready' && !document.documentElement.dataset.pgpGated") + landed[fid] = b.eval('location.pathname') + check('gate addresses: opened on a guessed finding address, the report opens on the run page', + landed == {acts[0]['id']: '/runs/'+son+'/finding/'+acts[0]['id']+'/', 'zz-not-a-finding': '/runs/'+son+'/'}, landed) finally: for d in made: shutil.rmtree(d) @@ -1587,7 +1599,8 @@ def gate_controls(b, base, out): seen = {} def visit(route=None): - b.go('about:blank'); assert b.wait("location.href === 'about:blank'") + # A release's own location.replace can land after about:blank; go there again until it stays. + assert any(b.go('about:blank') or b.wait("location.href === 'about:blank'") for _ in range(3)), 'about:blank never stayed' b.go(base+(route or '/runs/'+son+'/')) if not route: assert b.wait("document.documentElement.dataset.pgpGated === 'true'"), 'fixture never gated' @@ -1688,6 +1701,18 @@ def refresh_cleanup(): with open(rj, 'w') as f: json.dump(data, f) + def ticket(): + # The opening ticket crosses one load only: after closing an opened report again, the closed page keeps no + # ticket (it names the opened results' places in the report and a digest of every result's record). + visit(); open_gate(b) + assert b.wait("!!document.querySelector('#results .finding')"), 'full report never loaded' + fid = next(f['id'] for f in data['findings'] if f['level'] == 'act') + b.eval('PGPUI.closeAct('+json.dumps(fid)+'); true') + assert b.wait("document.documentElement.dataset.pgpGated === 'true' && document.readyState === 'complete'"), 'never closed' + left = b.eval("[sessionStorage.getItem('og-report-opening'), window.name]") + seen['ticket'] = left[0] + return left[0] is None and not left[1].startswith('og-report-opening:') + gate_open = "if (ownsPage || closedRun(requestURL)) { return new Promise(function () {}); }" gate_late = "return got.then(function (response) {\n if (ownsPage) { return new Promise(function () {}); }" mutations = [ @@ -1706,7 +1731,9 @@ def refresh_cleanup(): ('the family view receives a gated child\'s trio rows', ['copy((raw[key] || []).filter(function (r) { return !gated(r.child); }))'], ['copy(raw[key] || [])'], family_rows, lambda: seen.get('rows')), ('skip cleanup after closing', ['ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return true;'], - ['return true;'], cleanup, lambda: seen.get('query') == 'KCNH2')] + ['return true;'], cleanup, lambda: seen.get('query') == 'KCNH2'), + ('keep the opening ticket after reading it', [' sessionStorage.removeItem(TRANSFER);\n'], [''], ticket, + lambda: seen.get('ticket') is not None)] try: for desc, befores, afters, probe, broke in mutations: for before in befores: @@ -1743,19 +1770,27 @@ def refresh_cleanup(): "coverage": {"state": "ok", "data": {"mean_depth": 31.2}}}} -def gate_transparency(b, base, out): - """The gate leaves adult and no-act pages equal to the renderer without it: DOM, attributes, address, canvas and - the requests, in order and counted.""" - son = OTHERS[1] - js = os.path.join(out, 'assets', 'core.js') - files = [os.path.join(out, 'runs', S, 'results.json'), os.path.join(out, 'runs', son, 'results.json'), - os.path.join(out, 'runs.json')] - source = open(js).read() - saved = {p: open(p).read() for p in files} - # The same current renderer with the loader absent is the transparent reference. +# Each network read a page script starts, in the order it starts it, below every wrapper of the page's own: the +# path, and for fetch its cache mode and credentials. Installed before any script of the page runs. +REQUEST_LOG = r"""(function () { + var log = window.ogRequests = [], f = window.fetch, E = window.EventSource; + window.fetch = function (u, o) { + log.push(['fetch', new URL(u && u.url || u, location.href).pathname, (o && o.cache) || 'default', (o && o.credentials) || 'same-origin']); + return f.apply(this, arguments); + }; + if (E) { + var W = function (u, o) { log.push(['events', new URL(u, location.href).pathname]); return new E(u, o); }; + W.prototype = E.prototype; W.CONNECTING = E.CONNECTING; W.OPEN = E.OPEN; W.CLOSED = E.CLOSED; + window.EventSource = W; + } +}());""" + + +def bare_core(source): + """core.js with the report gate cut out: the transparent reference a page without the gate is compared to.""" marker = ' // --- the report gate' assert source.count(marker) == 1, 'gate marker moved' - bare = source[:source.index(marker)] + ''' + return source[:source.index(marker)] + ''' PGPUI.ready = Promise.resolve(); PGPUI.gateOwnsPage = function () { return false; }; PGPUI.openedActs = function () { return {}; }; @@ -1763,7 +1798,31 @@ def gate_transparency(b, base, out): }()); ''' - def capture(route, apoe=False, timeout=20): + +SEARCH_CLOSED = 'Results this page keeps closed are left out of the search until they are opened.' + + +def type_search(b, term): + """Type TERM in the header search once its index has loaded; the text of what it shows.""" + assert b.wait("document.querySelector('input[type=search]') && !document.querySelector('input[type=search]').disabled"), 'search never loaded' + b.eval("var s=document.querySelector('input[type=search]'); s.value=%s; s.dispatchEvent(new Event('input')); true" % json.dumps(term)) + assert b.wait("document.documentElement.dataset.pgpSearch !== undefined"), 'search never answered' + return b.eval("document.querySelector('.search-results').textContent") + + +def gate_transparency(b, base, out): + """The gate leaves adult and no-act pages equal to the renderer without it: DOM, attributes, address, canvas and + the requests, counted, each with its fetch options, and the files the gate reads first in their order.""" + son = OTHERS[1] + js = os.path.join(out, 'assets', 'core.js') + fam = os.path.join(out, 'runs', FAM, 'family.json') + files = [os.path.join(out, 'runs', S, 'results.json'), os.path.join(out, 'runs', son, 'results.json'), + os.path.join(out, 'runs.json'), fam] + source = open(js).read() + saved = {p: open(p).read() for p in files} + bare = bare_core(source) + + def capture(route, apoe=False, timeout=20, search=None, family=True): b.go('about:blank'); assert b.wait("location.href === 'about:blank'") b.go(base + route) if route == '/': @@ -1772,10 +1831,13 @@ def capture(route, apoe=False, timeout=20): assert b.wait("document.documentElement.dataset.pgpStatus === 'ready' && !!document.querySelector('h1')", timeout=timeout), 'finding never drawn' else: assert b.wait("document.documentElement.dataset.pgpResults && !!document.querySelector('[data-pgp-map=drawn]')", timeout=timeout), 'open report not drawn' - assert b.wait("!!document.querySelector('#rail a[href*=DEMO-FAMILY]') && !!document.documentElement.dataset.pgpCharts"), 'family or charts never settled' + assert b.wait("(%s || !!document.querySelector('#rail a[href*=DEMO-FAMILY]')) && !!document.documentElement.dataset.pgpCharts" + % ('false' if family else 'true')), 'family or charts never settled' if apoe: # A redraw while the address names a place: the contents rail must come back as it did. b.eval("document.querySelector('details[data-gate=apoe] summary').click(); true") + if search: + type_search(b, search) time.sleep(0.5) got = b.eval("""(function () { var clone=document.documentElement.cloneNode(true); @@ -1783,13 +1845,14 @@ def capture(route, apoe=False, timeout=20): n.removeAttribute('data-pgp-map-paint-ms'); n.removeAttribute('data-pgp-map-parse-ms'); }); return [clone.outerHTML, location.hash, Array.from(document.querySelectorAll('canvas')).map(function(c){return c.toDataURL();}), - // The browser lists a request when it ends: sort by when each one started. - performance.getEntriesByType('resource').slice().sort(function (a, z) { return a.startTime - z.startTime; }) - .map(function (r) { return new URL(r.name).pathname; })]; + performance.getEntriesByType('resource').map(function (r) { return new URL(r.name).pathname; }), window.ogRequests || []]; }())""") # Scripts and the views' own files load in parallel, in any order, with or without the gate: count every - # request, and keep the order of the files the gate reads first. - got[3] = [sorted(got[3]), [p for p in got[3] if p.endswith(('/state.json', '/frames.json', '/family.json', '/results.json'))]] + # request with its options, and keep the order of the files the gate reads first. + reads = got.pop() + first = [r for r in reads if r[1].endswith(('/state.json', '/frames.json', '/family.json', '/results.json', '/events'))] + # Chrome asks for a fallback favicon after load whenever it likes; no script of the page asks for it. + got[3] = [sorted(p for p in got[3] if p != '/favicon.ico'), sorted(json.dumps(r) for r in reads), first] return got def write(p, d): @@ -1819,13 +1882,27 @@ def synthetic(): lead_closed=o['lead_closed'], screened=o['screened']) write(files[2], idx) + def no_family(): + # A family that cannot be read (its family.json answers 404): the page asks for it once, as without the gate. + os.remove(fam) + + def no_act_no_family(): + no_act() + no_family() + + term = located_term(out) first = next(f['id'] for f in json.loads(saved[files[0]])['findings'] if f['level'] != 'act') - cases = [('adult', None, '/runs/'+S+'/', False), ('adult, a redraw at a section address', None, '/runs/'+S+'/#s3', True), - ('adult finding page', None, '/runs/'+S+'/finding/'+first+'/', False), - ('child with no act', no_act, '/runs/'+son+'/', False), ('home with no gated child', no_act, '/', False), - ('fresh adult with an act and a same-gene ranking', synthetic, '/runs/'+S+'/', False)] + cases = [('adult', None, '/runs/'+S+'/', {}), ('adult, a redraw at a section address', None, '/runs/'+S+'/#s3', {'apoe': True}), + ('adult finding page', None, '/runs/'+S+'/finding/'+first+'/', {}), + ('child with no act, a search typed', no_act, '/runs/'+son+'/', {'search': term}), + ('home with no gated child', no_act, '/', {}), + ('fresh adult with an act and a same-gene ranking', synthetic, '/runs/'+S+'/', {}), + ('adult whose family cannot be read', no_family, '/runs/'+S+'/', {'family': False}), + ('adult finding page whose family cannot be read', no_family, '/runs/'+S+'/finding/'+first+'/', {}), + ('child with no act whose family cannot be read', no_act_no_family, '/runs/'+son+'/', {'family': False})] + log = b.call('Page.addScriptToEvaluateOnNewDocument', {'source': REQUEST_LOG}, b.session)['identifier'] try: - for label, prep, route, apoe in cases: + for label, prep, route, how in cases: for p in files: with open(p, 'w') as f: f.write(saved[p]) @@ -1835,14 +1912,21 @@ def synthetic(): for text in (bare, source): with open(js, 'w') as f: f.write(text) - pair.append(capture(route, apoe)) + pair.append(capture(route, **how)) detail = [('dom', 'hash', 'canvas', 'requests')[i] for i in range(4) if pair[0][i] != pair[1][i]] - if pair[0][3] != pair[1][3]: - detail += [pair[0][3], pair[1][3]] + for k in range(3): + if pair[0][3][k] != pair[1][3][k]: + x, y = pair[0][3][k], pair[1][3][k] + detail += [('without the gate only', [r for r in x if r not in y]), ('with it only', [r for r in y if r not in x]), x, y] if pair[0][0] != pair[1][0]: at = next((i for i, (a, z) in enumerate(zip(pair[0][0], pair[1][0])) if a != z), 0) detail += [pair[0][0][max(0, at-80):at+120], pair[1][0][max(0, at-80):at+120]] check('gate transparency: '+label+' has identical DOM, attributes, address, canvas and requests', pair[0] == pair[1], detail) + if how.get('search'): + # A child's page keeps results closed until opened, with or without results to act on, and says so + # under the hits, as before the gate. + check('gate transparency: a search on a child with no act says that closed results are left out', + SEARCH_CLOSED in pair[1][0], term) for p in files: with open(p, 'w') as f: f.write(saved[p]) @@ -1859,6 +1943,7 @@ def synthetic(): still_equal = False control('gate transparency: '+desc, still_equal) finally: + b.call('Page.removeScriptToEvaluateOnNewDocument', {'identifier': log}, b.session) with open(js, 'w') as f: f.write(source) for p in files: diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index f7f419ba4..dd63bb54a 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -31,6 +31,11 @@ gate_checks on the family page), here so that browser time is not added to test_live_demo.py's; 8. the real server answers every finding id of a child with a result to act on with the same page, and keeps the 404 for an adult's id that is not a finding. + 9. the report gate on the real server, in live mode: one rule of who is a child; the home page counts a + child's results to act on from the summary its results.json reads (a stale summary.json too); family.json + has no flagged-card count for a child whose report is closed, so a parent's page is the same whichever card + holds the child's results; and an adult's and a no-act child's pages make the requests of the page without + the gate, in its order, the event stream included. Checks 2 and 4 are run again on a copy of the fixture with two parents swapped and must fail there: a check that cannot fail is no check. @@ -353,6 +358,7 @@ def get(path): check("the family page loads family.js", code == 200 and b"/assets/plugins/family.js" in body, code) finding_route_checks(live, get, root) browser_checks(port, os.path.dirname(root)) + gate_server_checks(live, get, root, port, os.path.dirname(root)) finally: web.shutdown() ingest.server_close() @@ -439,6 +445,160 @@ def browser_checks(port, work): b.close() +def plp(gene, variant): + return {"gene": gene, "variant": variant, "final": "Pathogenic", "clinvar": "Pathogenic"} + + +# Two reports with the same two results to act on, held by different cards: two CPSR rows, or one CPSR row and a +# class 5 deletion (which flags the structural-variant card). A parent's page must not tell them apart. +SV = {"state": "ok", "data": {"count": 1, "consensus": 1, "single_caller": 0, "exon_edge_checked": False, "exon_edge": 0, + "common": 0, "rows": [{"sv": "chr17:43044295-43125483", "type": "DEL", "gt": "0/1", + "length": -81188, "acmg_class": 5, "genes": ["BRCA1"]}]}} +TWO_IN_ONE_CARD = {"sections": {"cpsr": {"state": "ok", "data": {"plp": [plp("BRCA1", "c.68_69del"), plp("TP53", "c.743G>A")]}}}} +TWO_CARDS = {"sections": {"cpsr": {"state": "ok", "data": {"plp": [plp("BRCA1", "c.68_69del")]}}, "annotsv": SV}} +NO_ACT = {"sections": {"clinvar": {"state": "ok", "data": {"hits": []}}, "annotsv": dict(SV, data=dict(SV["data"], count=0, rows=[]))}} + + +def gate_server_checks(live, get, root, port, work): + """The report gate's server side and its live-mode requests (docstring, check 9).""" + import owngenome_live + import live_results + sys.path.insert(0, os.path.join(REPO, "tests")) + fams = [{"name": "b", "members": [{"sample": "x", "father": None, "mother": None}]}, + {"name": "a", "members": [{"sample": "x", "father": "p", "mother": None}]}, + {"name": "c", "members": []}] + child_of = getattr(owngenome_live, "child_of", lambda *a: None) + got = [child_of(fams, "x"), child_of(fams[:1], "x"), child_of(fams, "y"), child_of(fams[:2], "y")] + check("one rule of who is a child: the first family by name decides; a run no readable family lists is a child " + "while a family cannot be read", got == [True, False, True, False], got) + check("the finding route asks the same rule", __import__("owngenome_live_results").child_run(live, "father_a") + and not __import__("owngenome_live_results").child_run(live, "gf_a")) + made = [] + + def run(name, summary=None): + d = os.path.join(root, name) + os.makedirs(os.path.join(d, "logs"), exist_ok=True) + if not os.path.exists(os.path.join(d, "logs", "run_status.tsv")): + with open(os.path.join(d, "logs", "run_status.tsv"), "w") as f: + f.write("meta\tstarted_epoch\t1767347940\nstep\t06\tplanned\n") + made.append(d) + p = os.path.join(d, "summary.json") + if summary is None: + if os.path.exists(p): + os.remove(p) + else: + with open(p, "w") as f: + json.dump(summary, f) + + def family(): + return json.loads(get("/runs/family_a/family.json")[2]) + + def index(): + return {r["name"]: r for r in json.loads(get("/runs.json")[2])["runs"]} + + real = live.summary + try: + # father_a is a child of gf_a and gm_a; gf_a an adult with a run here. + run("gf_a") + # A stale summary.json with no result to act on, while the summary results.json reads (summary(): the step + # files read again) has one. The home page must count it, so it closes the child as the child's page does. + run("child_a", NO_ACT) + live.summary = lambda name, d, st: TWO_IN_ONE_CARD if name == "child_a" else real(name, d, st) + acts = (index().get("child_a") or {}).get("levels", {}).get("act") + check("home: a child's results to act on are counted from the summary its results.json reads, not a stale " + "summary.json", acts == 2, acts) + check("family.json: no flagged-card count for that child", family()["runs"]["child_a"]["attention"] is None, + family()["runs"]["child_a"]) + live.summary = real + run("child_a") + # The same two results to act on in one card or in two: the flagged-card count differs (0 and 1). + counts = [live_results.overview(x)["attention"] for x in (TWO_IN_ONE_CARD, TWO_CARDS)] + assert counts[0] != counts[1], counts + seen = [] + for summary in (TWO_IN_ONE_CARD, TWO_CARDS): + run("father_a", summary) + seen.append(family()["runs"].get("father_a")) + check("family.json: a child's entry is the same whichever card holds its results to act on, with no count", + seen[0] == seen[1] and seen[0] and seen[0]["attention"] is None, seen) + run("father_a", NO_ACT) + n = family()["runs"]["father_a"]["attention"] + check("family.json: a child with no result to act on keeps its count", n == live_results.overview(NO_ACT)["attention"], n) + live_browser_checks(live, port, work, run, real) + finally: + live.summary = real + for d in made: + shutil.rmtree(d) + run("child_a") + + +def live_browser_checks(live, port, work, run, real): + """In live mode (state.json, then the event stream): an adult's and a no-act child's page make the requests of + the page without the gate, in its order; a parent's page is the same whichever card holds a child's results.""" + import test_live_demo as demo + chrome = demo.find_chrome() + if not chrome: + if os.environ.get("CI") == "true": + check("live gate browser checks have Chrome in CI", False) + else: + print("[SKIP] no Chrome or Chromium for the live gate checks") + return + assets = os.path.join(work, "gate-assets") + shutil.copytree(os.path.join(REPO, "assets", "live"), assets) + js = os.path.join(assets, "core.js") + source = open(js).read() + keep = live.cfg.assets + live.cfg.assets = assets + # gf_a's own report, so its page draws the contents and the family beside them. + adult = live.summary = lambda name, d, st: NO_ACT if name == "gf_a" else real(name, d, st) + b = demo.Chrome(chrome, os.path.join(work, "gate-chrome")) + base = "http://127.0.0.1:%d" % port + t0 = demo.time.time() + try: + b.call("Page.enable", session=b.session) # scripts for each new document need the Page domain + log = b.call("Page.addScriptToEvaluateOnNewDocument", {"source": demo.REQUEST_LOG}, b.session)["identifier"] + + def capture(route, text): + with open(js, "w") as f: + f.write(text) + b.go("about:blank") + assert b.wait("location.href === 'about:blank'") + b.go(base + route) + assert b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " + "!!document.querySelector('#rail a[href*=family_a]')"), route + " never drawn" + demo.time.sleep(1.0) + return b.eval("[document.documentElement.outerHTML, location.href, (window.ogRequests || []).map(function (r) { " + "return r.join(' '); })]") + pages = {} + for label, route, summary in (("an adult", "/runs/gf_a/", None), ("a no-act child", "/runs/father_a/", NO_ACT)): + # The child's results come from its summary as results.json reads it (a step file read now). + live.summary = lambda name, d, st: summary if summary and name == "father_a" else adult(name, d, st) + try: + pair = [capture(route, demo.bare_core(source)), capture(route, source)] + finally: + live.summary = adult + pages[label] = pair + check("live mode: the page of %s makes the requests of the page without the gate, in its order, and is the " + "same page" % label, pair[0] == pair[1], [p[2] for p in pair] if pair[0][2] != pair[1][2] else "the pages differ") + first = [r.split(" ")[1].rsplit("/", 1)[-1] for r in pages["an adult"][0][2]][:3] + print(" live mode, an adult's reads without the gate: " + ", ".join(pages["an adult"][0][2])) + check("live mode: without the gate the page asks for state.json, then the event stream, then family.json", + first == ["state.json", "events", "family.json"], first) + # A parent's page, whichever card holds the child's two results to act on. + seen = [] + for summary in (TWO_IN_ONE_CARD, TWO_CARDS): + run("father_a", summary) + seen.append(capture("/runs/gf_a/", source)[0]) + check("live mode: a parent's page is the same whichever card holds a child's results to act on", + seen[0] == seen[1], "the pages differ") + b.call("Page.removeScriptToEvaluateOnNewDocument", {"identifier": log}, b.session) + finally: + live.summary = real + live.cfg.assets = keep + b.close() + shutil.rmtree(assets, ignore_errors=True) + print(" live gate browser checks: %.2f s" % (demo.time.time() - t0)) + + # A stale blood-group section with two systems past ABO and RhD, for child_a's summary.json. STALE_SUMMARY = {"sections": {"bloodtype": {"state": "stale", "note": "older than its input", "data": { "abo": "A1", "rhd": "D+", "systems": [{"system": "ABO", "phenotype": "A1"}, {"system": "RhD", "phenotype": "D+"}, From d657833ebf8fc06b7661a6ba67f7525db18401a5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:37:25 +0100 Subject: [PATCH 082/255] docs(live): describe the gate's one exception, the parent's count, and what the page keeps as before --- CHANGELOG.md | 2 +- docs/live-site.md | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73b131764..295268acc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -125,7 +125,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Each output is written whole or not at all: a run that stops leaves no part-written table behind. A read-only `family_plan.tsv` is replaced; a folder that cannot be written still stops the plan. - Step 38, step 39 with a families file, `run-all.sh` with `FAMILIES` and the family workflow now need `og` (`scripts/setup.sh --og`). -- A child's live report now has one gate where the page reads its data, instead of a veil on each place that could name a result. While any of a child's results to act on is unopened, the page shows a fixed explanation, the number of such results, one "Show the result" button each, and the own record of each result opened on purpose; nothing else of the child's report reaches the page, and the page is the same whichever result is closed. Which result a button opens is drawn at random on each load, and the address names buttons (`#f/closed-1`), never a result. The server answers every finding address of such a child with the same page. Opening the last result shows the full report with its own addresses; closing one again, a reload or Back closes them all. The cost: such a child sees none of the rest of their report until every result to act on is opened. The home page shows a gated child's tile with fixed text, the count and the run's progress; the family page shows the child's buttons and leaves out the child's rows, stretches, couple screen and cells (the side by side reads "closed"), while the parents' values stay. Adults and children with no result to act on see the same pages as before, with the same requests. APOE keeps its own warning. No output file, field or option changes; `results.json` keeps its `veil` field, which the pages no longer read. The gate guards the page in the browser: the static files still reach a browser that asks for them. +- A child's live report now has one gate where the page reads its data, instead of a veil on each place that could name a result. While any of a child's results to act on is unopened, the page shows a fixed explanation, the number of such results, one "Show the result" button each, and the own record of each result opened on purpose; nothing else of the child's report reaches the page, and the page is the same whichever result is closed. Which result a button opens is drawn at random on each load, and the address names buttons (`#f/closed-1`), never a result. The server answers every finding address of such a child with the same page. Opening the last result shows the full report with its own addresses. Closing one again closes the report and that result, and the others stay opened; a reload or Back closes them all. The cost: such a child sees none of the rest of their report until every result to act on is opened. The home page shows a gated child's tile with fixed text, the count and the run's progress; the family page shows the child's buttons and leaves out the child's rows, stretches, couple screen and cells (the side by side reads "closed"), while the parents' values stay. Adults and children with no result to act on see the same pages as before, with the same requests in the same order. One exception: another member's page, a parent's for example, no longer shows the flagged-card count of a child whose report is closed, because that count changes with which cards the unopened results flag. APOE keeps its own warning. Two outputs change, and no field is renamed or removed: `family.json` gives `attention: null` for such a child, and `/runs.json` counts a child's results from the summary its `results.json` reads, also while `summary.json` is older than a finished step. `results.json` keeps its `veil` field, which the gate does not read. The gate guards the page in the browser: the static files still reach a browser that asks for them. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. diff --git a/docs/live-site.md b/docs/live-site.md index feb14b64e..33d0e7822 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -62,7 +62,7 @@ A stopped run's page says so under the name and in the summary line, with the da While the run goes, the page still opens on the report being written. A short band under the name says how far the run is ("23 of 27 are done, about 8–13 hours remain"), with the steps done as a number and a bar, the time elapsed, the time to go from the usual step times and the retries, and links to the step timeline. Each section and each line of the contents says whether it is written, being written or waiting. The run itself, below, closes the page after section 10, folded until opened (or reached through `#progress`, the band or the contents); a page with no results yet keeps it open. -**A child's report gate.** A child's report stays closed while any of the child's results to act on in adult life (the findings of level `act`) is unopened. A child is a person their family lists with a father or a mother. A person whose family roles cannot be read is treated as a child, and so is a child whose results cannot be read. While the gate is shut, the page shows only: +**A child's report gate.** A child's report stays closed while any of the child's results to act on in adult life (the findings of level `act`) is unopened. A child is a person their family lists with a father or a mother. When several families list the person, the first by name decides. A person whose family roles cannot be read is treated as a child, and so is a person no readable family lists while some family cannot be read. A child whose results cannot be read stays closed too. The person page, the home page and the server decide this by the same rule. While the gate is shut, the page shows only: - a fixed explanation and the number of results to act on ("1 result to act on", "2 results to act on"); - one "Show the result" button per closed result, all in one block; @@ -72,11 +72,13 @@ Nothing else of the child's report reaches the page: no cards, tables, lead, cou Which result a button opens is drawn at random on each page load, so neither a button's place nor what it opens follows the report's order. The buttons have opaque names in the address, `#f/closed-1` and `#f/opened-1`. Any other `#f/` and any query land on `#f/closed-1`. On a finding address of a gated child the server answers the same page for every well-formed id, and the page then shows the run's own address and the same closed page, so a guessed address cannot tell whether the child has that result. -Opening the last result loads the page again, and the full report shows, as an adult's does, with its own addresses (`#f/`). A map or section address returns to the default view. Only which results were opened crosses that load: a one-use ticket in session storage with a digest of the results' records, bound to that one load by a random value in the window's name. The page reads and removes it as it starts, before any network read, so a slow server does not lose the opening. Closing a result again (the genome map's "Close it again") clears the page at once and loads it again closed, without its search, caches or views. A result to act on that appears or changes while the report is open closes it the same way. A reload, or a page shown again with Back, closes every result again. +Opening the last result loads the page again, and the full report shows, as an adult's does, with its own addresses (`#f/`). A map or section address returns to the default view. On a finding address, the finding's page opens when the results have that finding; any other id opens the run's page, as the server answers 404 for it once the report is open. The open report keeps results closed until opened, as before the gate, so its search says "Results this page keeps closed are left out of the search until they are opened." under the hits. Only which results were opened crosses that load: a one-use ticket in session storage with a digest of the results' records, bound to that one load by a random value in the window's name. The page reads and removes it as it starts, before any network read, so a slow server does not lose the opening. Closing a result again (the genome map's "Close it again") clears the page at once and loads it again closed, without its search, caches or views; that result is closed, and the others stay opened. A result to act on that appears or changes while the report is open closes it the same way. A reload, or a page shown again with Back, closes every result again. -The cost: a child with an unopened result to act on sees none of the rest of their report until every one is opened. Adults and children with no result to act on never meet the gate. The gate reads the files their page reads anyway and hands each view its first read from that copy, so each file is fetched as often, and in the same order, as without it. +The cost: a child with an unopened result to act on sees none of the rest of their report until every one is opened. Adults and children with no result to act on never meet the gate. The gate reads the files their page reads anyway, with the same options, and hands each view its first answer for that file, whatever its status. So each file is fetched as often, in the same order and with the same options as without the gate. The event stream opens when `live.js` asks for it, right after `state.json`; its messages wait until the gate knows whether the page is a child's, then reach the page in order. The views get their answers once the gate has decided, after the family has answered. Without the gate, an answer can reach the page earlier or later, and a later redraw can then replace a mark the address set (the contents' `aria-current`). The requests and the page's data are the same; only the moment a view gets an answer differs. -The home page decides from `/runs.json` alone. A gated child is a child of a readable family, or, while a family's members cannot be read, a run no readable family lists, whose entry counts or names a result to act on. Its tile and attention rows show the fixed text and the count of results to act on, and link to the run's page. The run's progress (its state, the steps done, planned, running and failed, and its times) stays, because it says how far the run is, never what it found. +One count is left out for a child whose report is closed. Other members' pages, a parent's for example, list the family's members with how many cards each report flags. That count changes with which cards the child's unopened results flag, so `family.json` gives `attention: null` for that child, and the page shows the name without a count. For adults and children with no result to act on, `family.json` is as before. + +The home page decides from `/runs.json` alone. A gated child is a child (the rule above, read from the families of `/runs.json`) whose entry counts or names a result to act on. The server counts a child's results from the summary its `results.json` reads. While `summary.json` is older than a finished step, that is the step files read again, so the home page closes a child whenever the child's page does. Its tile and attention rows show the fixed text and the count of results to act on, and link to the run's page. The run's progress (its state, the steps done, planned, running and failed, and its times) stays, because it says how far the run is, never what it found. On the family page, chapter IV shows a gated child's buttons only, and each child opens on its own. The family view gets the family without that child's report data: no de novo or compound-het rows, stretches, trio status or trio files of the child, no relationship or shared-segment row with the child, and no couple screen of a couple the child is part of. The side by side keeps a row only when an open member has a value in it, in a fixed order, and the child's cell reads "closed". Plan rows that name the child are left out. The parents' other plan rows and the couple reports of couples with no gated partner stay. Other reports of the family folder are left out while a child is gated, because they can hold the child's data. A member with no run here has no report, so the gate does not close anything of theirs. The pedigree and the members' progress stay. @@ -94,7 +96,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. For a child with a result to act on, the server answers every well-formed id with the page, which stays closed while the report gate is shut. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags, beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. While a child's report gate is shut, chapter IV shows that child's buttons only (the report gate above). V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. While a child's report gate is shut, the child's cell reads "closed" in every row, and a row stays only when an open member has a value in it. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags (never for a child whose report is closed), beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. While a child's report gate is shut, chapter IV shows that child's buttons only (the report gate above). V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. While a child's report gate is shut, the child's cell reads "closed" in every row, and a row stays only when an open member has a value in it. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). The report gate is the same at every width. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. @@ -108,9 +110,9 @@ Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with it The reads cross the network to the browser that logged in: only the byte ranges igv.js asks for, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS); without it the password and the reads cross the network in clear. Opening the viewer writes one access-log row; the byte ranges igv.js reads write none. -`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. For a gated child the page replaces `levels`, `top` and `lead` with fixed text before the home page draws them (the report gate above). +`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. For a child these fields come from the summary the child's `results.json` reads, even when `summary.json` is older. For a gated child the page replaces `levels`, `top` and `lead` with fixed text before the home page draws them (the report gate above). -The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (the rows outside a result's own card that name a result to act on, and `spans`, the results to act on with their gene and position; the file keeps it, and the pages no longer read it: the report gate reads the findings of level `act`). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. +The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (the rows outside a result's own card that name a result to act on, and `spans`, the results to act on with their gene and position; the report gate reads the findings of level `act` instead; on a page that keeps results closed until opened, a child's or one whose family roles are not known, the charts and the search box still leave out a mark or hit whose tag names no finding of these results). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. ## Privacy and what is stored From 0aab619e656787283ae8ef042520bcc53c248635 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:45:00 +0100 Subject: [PATCH 083/255] fix(report): collect again after a removed file, and keep the temporary summary private generate-report.sh rendered from step 24's summary.json when no file was newer, so a step output removed after the collect went unnoticed. A subfolder newer than the summary (other than logs/) now means collect again, and the fake-docker run-all case checks it. collect_summary.collect() without OUT gave og a file in the shared temporary folder, and og's rename over it dropped mkstemp's owner-only mode. It now uses a private temporary folder. Bead: pgp-bs9u.45 --- CHANGELOG.md | 2 +- bin/collect_summary.py | 12 +++++------- scripts/generate-report.sh | 7 +++++-- tests/fake-docker/run-all-partial-failure.sh | 9 +++++++++ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a462b156..f60e0c1db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again only when a file it reads is newer than that summary.json. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/bin/collect_summary.py b/bin/collect_summary.py index 55bffce51..e0f524e66 100755 --- a/bin/collect_summary.py +++ b/bin/collect_summary.py @@ -19,6 +19,7 @@ import json import os import re +import shutil import subprocess import tempfile import zlib @@ -97,9 +98,9 @@ def collect(sample, sample_dir, declared_sex=None, out=None): written).""" tmp = None if out is None: - fd, tmp = tempfile.mkstemp(prefix="summary.", suffix=".json") - os.close(fd) - out = tmp + # A private folder: og writes a sibling file and renames it over OUT. + tmp = tempfile.mkdtemp(prefix="summary.") + out = os.path.join(tmp, "summary.json") cmd = [og(), "summary", "collect", "--sample", sample, "--sample-dir", sample_dir, "--assets", ASSETS_DIR, "--out", out] if declared_sex: @@ -118,10 +119,7 @@ def collect(sample, sample_dir, declared_sex=None, out=None): return json.load(f) finally: if tmp: - try: - os.unlink(tmp) - except OSError: - pass + shutil.rmtree(tmp, ignore_errors=True) # ClinVar review status -> stars. og clinvar hits (rust/crates/og-clinvar) holds the same table diff --git a/scripts/generate-report.sh b/scripts/generate-report.sh index 5e5d3161d..4c4415538 100755 --- a/scripts/generate-report.sh +++ b/scripts/generate-report.sh @@ -41,11 +41,14 @@ fi # the summary or beside it (the reports, the search index, the charts, the # delta, the logs other than run_status.tsv, and nextflow/, the Nextflow launch # folder with work/), and bin/ (og and the tables it reads there) and assets/. Anything newer, or a -# summary that is not there, means collect again. +# summary that is not there, means collect again. A subfolder newer than the +# summary (other than logs/) had a file added or removed since: collect again. summary_current() { local s="${SAMPLE_DIR}/summary.json" newer [ -s "$s" ] && [ ! -L "$s" ] || return 1 - newer=$(cd "$SAMPLE_DIR" && find . \( -path ./nextflow -o -path ./summary.json \) -prune -o ! -type d -newer summary.json \ + newer=$(cd "$SAMPLE_DIR" && find . \( -path ./nextflow -o -path ./summary.json \) -prune \ + -o -type d ! -path . ! -path ./logs -newer summary.json -print -quit \ + -o ! -type d -newer summary.json \ ! \( -path './logs/*' ! -path ./logs/run_status.tsv \) \ ! -name "${SAMPLE}_report.html" ! -name "${SAMPLE}_report.txt" ! -name "${SAMPLE}_search_index.json" \ ! -name "${SAMPLE}_charts.json" ! -name "${SAMPLE}_delta.txt" ! -name "${SAMPLE}_delta.html" \ diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index 556872618..c60a3df9e 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -166,6 +166,15 @@ if grep -q '^Rendering from summary.json' "${CASE_WORK}/recollect.out" 2> /dev/n miss "generate-report.sh rendered from summary.json, but a step output changed after it was written" fi touch -d '30 days ago' "$roh" # back to its age, for the cases below +# A file removed after the collect makes it collect again too. +extra="${roh%/*}/sample1_extra.txt" +: > "$extra" +run_expect 0 recollect-added "${SCRIPTS}/generate-report.sh" sample1 +rm -f "$extra" +run_expect 0 recollect-removed "${SCRIPTS}/generate-report.sh" sample1 +if grep -q '^Rendering from summary.json' "${CASE_WORK}/recollect-removed.out" 2> /dev/null; then + miss "generate-report.sh rendered from summary.json, but a file was removed after it was written" +fi # --- 3. the same failure on a first run: nothing cached ------------------------------------ awk -F'\t' -v OFS='\t' '{$2 += $3; $3 = 0; print}' "${CASE_WORK}/resume.tsv" > "${CASE_WORK}/first.tsv" From 686a0c644ce9e46541d9d527caf50f09ca5fd935 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 19:15:05 +0100 Subject: [PATCH 084/255] fix(report): collect again when a section's whole folder is gone Removing a step folder after the collect changes only the sample folder's own time, which the reports change too, so generate-report.sh rendered the removed results from summary.json. It now also collects again when a file a section names as its source is gone. The fake-docker run-all case checks an unchanged folder, a newer output, a removed file and a removed folder, each from a summary.json that is the newest file. Bead: pgp-bs9u.45 --- CHANGELOG.md | 2 +- scripts/generate-report.sh | 11 +++++-- tests/fake-docker/run-all-partial-failure.sh | 33 ++++++++++++++------ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f60e0c1db..1eea6ada4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source is gone. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/scripts/generate-report.sh b/scripts/generate-report.sh index 4c4415538..f035ece08 100755 --- a/scripts/generate-report.sh +++ b/scripts/generate-report.sh @@ -42,10 +42,17 @@ fi # delta, the logs other than run_status.tsv, and nextflow/, the Nextflow launch # folder with work/), and bin/ (og and the tables it reads there) and assets/. Anything newer, or a # summary that is not there, means collect again. A subfolder newer than the -# summary (other than logs/) had a file added or removed since: collect again. +# summary (other than logs/) had a file added or removed since, and a section's +# source that is gone (its file or its whole folder removed) means the same: +# collect again. summary_current() { - local s="${SAMPLE_DIR}/summary.json" newer + local s="${SAMPLE_DIR}/summary.json" newer src [ -s "$s" ] && [ ! -L "$s" ] || return 1 + # Each section's "source", at the sections' depth of og's one-space indent. + while IFS= read -r src; do + case $src in *\\*) return 1 ;; esac + [ -e "${SAMPLE_DIR}/${src}" ] || return 1 + done < <(sed -n 's/^ "source": "\([^"]*\)",\{0,1\}$/\1/p' "$s") newer=$(cd "$SAMPLE_DIR" && find . \( -path ./nextflow -o -path ./summary.json \) -prune \ -o -type d ! -path . ! -path ./logs -newer summary.json -print -quit \ -o ! -type d -newer summary.json \ diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index c60a3df9e..ea8d7b30c 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -157,24 +157,39 @@ grep -q '^Rendering from summary.json' "${S}/logs/generate-report.log" \ t_txt=$(sed -n 's/^ Generated: //p' "$TXT" | head -1) t_html=$(grep -o 'Generated: [^<&]*' "$HTML" | head -1 | sed 's/^Generated: //; s/ *$//') [ -n "$t_txt" ] && [ "$t_txt" = "$t_html" ] || miss "the text report says generated '${t_txt}', the HTML report '${t_html}'" -# The reverse: a step output newer than summary.json makes it collect again. +# The reverse: a change after summary.json was written makes it collect +# again. Each check first makes summary.json the newest file, as step 24 +# leaves it (the run's history/ copy came after it), and waits a second, so +# only the change it makes is newer. +fresh() { touch "${S}/summary.json" && /bin/sleep 1; } +rendered_from() { grep -q '^Rendering from summary.json' "${CASE_WORK}/$1.out" 2> /dev/null; } +fresh +run_expect 0 current "${SCRIPTS}/generate-report.sh" sample1 +rendered_from current || miss "generate-report.sh collected again, but nothing changed after summary.json was written" +# A step output newer than summary.json. roh=$(find "$S" -name 'sample1_roh.txt' -type f | head -1) [ -n "$roh" ] || fail "no ROH output to touch" +fresh touch "$roh" run_expect 0 recollect "${SCRIPTS}/generate-report.sh" sample1 -if grep -q '^Rendering from summary.json' "${CASE_WORK}/recollect.out" 2> /dev/null; then - miss "generate-report.sh rendered from summary.json, but a step output changed after it was written" -fi +! rendered_from recollect || miss "generate-report.sh rendered from summary.json, but a step output changed after it was written" touch -d '30 days ago' "$roh" # back to its age, for the cases below -# A file removed after the collect makes it collect again too. +# A file removed: its folder is newer than summary.json. extra="${roh%/*}/sample1_extra.txt" : > "$extra" -run_expect 0 recollect-added "${SCRIPTS}/generate-report.sh" sample1 +fresh rm -f "$extra" run_expect 0 recollect-removed "${SCRIPTS}/generate-report.sh" sample1 -if grep -q '^Rendering from summary.json' "${CASE_WORK}/recollect-removed.out" 2> /dev/null; then - miss "generate-report.sh rendered from summary.json, but a file was removed after it was written" -fi +! rendered_from recollect-removed || miss "generate-report.sh rendered from summary.json, but a file was removed after it was written" +# A section's whole folder removed: only the sample folder itself is newer, +# so the section's missing source must tell. +src=$(sed -n 's|^ "source": "\([^"/]*\)/.*|\1|p' "${S}/summary.json" | head -1) +[ -n "$src" ] && [ -d "${S}/${src}" ] || fail "summary.json names no section folder: $(grep -m3 '"source"' "${S}/summary.json")" +fresh +mv "${S}/${src}" "${CASE_WORK}/moved-away" +run_expect 0 recollect-folder "${SCRIPTS}/generate-report.sh" sample1 +mv "${CASE_WORK}/moved-away" "${S}/${src}" +! rendered_from recollect-folder || miss "generate-report.sh rendered from summary.json, but ${src}/, a section's folder, was removed after it was written" # --- 3. the same failure on a first run: nothing cached ------------------------------------ awk -F'\t' -v OFS='\t' '{$2 += $3; $3 = 0; print}' "${CASE_WORK}/resume.tsv" > "${CASE_WORK}/first.tsv" From eaf6bfc6b5e6874b47310eb5d5592c562ca5dc23 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 21:16:18 +0100 Subject: [PATCH 085/255] ci(guard): read the SMN1 marker from the test's own table The traits control reached the marker through traits_report, which this branch deletes; test_traits_report.py holds MARKERS itself now. Bead: pgp-bs9u.45 --- .github/workflows/guard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 878d9be5e..a2aee4fac 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -663,7 +663,7 @@ jobs: assert rc == 1 and test["FAILS"] == 1 assert f"[FAIL] {rid}, reference blocks only: {label}" in log.getvalue() print(f"Control: {rid} WRONG_REFERENCE failed its independent expectation") - marker = test["tr"].MARKERS["SMN1_27134"] + marker = test["MARKERS"]["SMN1_27134"] rs, chrom, pos, ref, alt = marker["dbsnp"] marker["dbsnp"] = rs, chrom, pos + 1, ref, alt test["FAILS"] = 0 From 7aaf3a940170d18f3a94a46d7672e50caa480af1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sat, 14 Nov 2026 22:34:31 +0100 Subject: [PATCH 086/255] fix(report): collect again when the run status or a bin or assets table is gone The summary records that it read logs/run_status.tsv, which the subfolder test leaves out with logs/, and the tables in bin/ and assets/ are no section's source. generate-report.sh now collects again when that run status is gone, or when bin/, bin/.og/ or assets/ is newer than the summary. The fake-docker run-all case checks both. The CHANGELOG says the shared generated time holds only when the text report renders from step 24's summary. Bead: pgp-bs9u.45 --- CHANGELOG.md | 2 +- scripts/generate-report.sh | 13 ++++++++++--- tests/fake-docker/run-all-partial-failure.sh | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfbcdeace..ae6213999 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source is gone. The text report's generated time is then step 24's, the same as the HTML report's; it was the minute of its own collect. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/scripts/generate-report.sh b/scripts/generate-report.sh index f035ece08..5bc3177b6 100755 --- a/scripts/generate-report.sh +++ b/scripts/generate-report.sh @@ -42,12 +42,16 @@ fi # delta, the logs other than run_status.tsv, and nextflow/, the Nextflow launch # folder with work/), and bin/ (og and the tables it reads there) and assets/. Anything newer, or a # summary that is not there, means collect again. A subfolder newer than the -# summary (other than logs/) had a file added or removed since, and a section's -# source that is gone (its file or its whole folder removed) means the same: -# collect again. +# summary (other than logs/) had a file added or removed since, and so did +# bin/, bin/.og/ or assets/ when it is newer. A section's source that is gone +# (its file or its whole folder removed), or the run status the summary read, +# means the same: collect again. summary_current() { local s="${SAMPLE_DIR}/summary.json" newer src [ -s "$s" ] && [ ! -L "$s" ] || return 1 + if grep -q '^ "status_file": true' "$s"; then + [ -e "${SAMPLE_DIR}/logs/run_status.tsv" ] || return 1 + fi # Each section's "source", at the sections' depth of og's one-space indent. while IFS= read -r src; do case $src in *\\*) return 1 ;; esac @@ -61,6 +65,9 @@ summary_current() { ! -name "${SAMPLE}_charts.json" ! -name "${SAMPLE}_delta.txt" ! -name "${SAMPLE}_delta.html" \ ! -name "${SAMPLE}_delta.json" -print -quit) || return 1 [ -z "$newer" ] || return 1 + newer=$(find "${PGP_ROOT}/bin" "${PGP_ROOT}/bin/.og" "${PGP_ROOT}/assets" -maxdepth 0 -newer "$s" -print -quit 2> /dev/null) \ + || return 1 + [ -z "$newer" ] || return 1 newer=$(find "${PGP_ROOT}/bin" "${PGP_ROOT}/bin/.og" "${PGP_ROOT}/assets" -maxdepth 1 ! -type d -newer "$s" -print -quit 2> /dev/null) \ || return 1 [ -z "$newer" ] diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index ea8d7b30c..00a53e4dc 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -190,6 +190,21 @@ mv "${S}/${src}" "${CASE_WORK}/moved-away" run_expect 0 recollect-folder "${SCRIPTS}/generate-report.sh" sample1 mv "${CASE_WORK}/moved-away" "${S}/${src}" ! rendered_from recollect-folder || miss "generate-report.sh rendered from summary.json, but ${src}/, a section's folder, was removed after it was written" +# The run status the summary read, removed. +grep -q '^ "status_file": true' "${S}/summary.json" || fail "summary.json read no run status" +fresh +mv "${S}/logs/run_status.tsv" "${CASE_WORK}/run_status.away" +run_expect 0 recollect-status "${SCRIPTS}/generate-report.sh" sample1 +mv "${CASE_WORK}/run_status.away" "${S}/logs/run_status.tsv" +! rendered_from recollect-status || miss "generate-report.sh rendered from summary.json, but logs/run_status.tsv was removed after it was written" +# A file removed from assets/ (a table the collect reads, as far as the check +# can tell): assets/ is newer than summary.json. +extra="${REPO_ROOT}/assets/.fake-docker-extra" +: > "$extra" +fresh +rm -f "$extra" +run_expect 0 recollect-asset "${SCRIPTS}/generate-report.sh" sample1 +! rendered_from recollect-asset || miss "generate-report.sh rendered from summary.json, but a file was removed from assets/ after it was written" # --- 3. the same failure on a first run: nothing cached ------------------------------------ awk -F'\t' -v OFS='\t' '{$2 += $3; $3 = 0; print}' "${CASE_WORK}/resume.tsv" > "${CASE_WORK}/first.tsv" From b55093ddf5b8fc3181913bc0ba138e28c9c6a9e5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:02:43 +0100 Subject: [PATCH 087/255] test(roh): require chromosome scope and neutral UPD hints Bead: lin-10 Bead: lin-13 Bead: lin-15 Bead: lin-16 Bead: lin-17 Bead: lic-41 --- .../og-summary/src/collect/group_c/tests.rs | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index 82afd2dc8..b072af7cb 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -223,3 +223,79 @@ fn every_row_of_the_repeat_ranges_table_is_read() { assert_eq!(read, loci); assert!(loci.len() >= 3, "{loci:?}"); } + +// i411-roh-scope: ST rows describe assessed chromosomes, including ones +// with no long run. A run list alone cannot establish whole-genome scope. +fn roh_scope_summary(sites: &[u8], runs: &str) -> Value { + let d = Dir::new("i411-roh-scope"); + let mut body = String::new(); + for c in sites { + body.push_str(&format!("ST\tS\tchr{c}\t40000000\t0\t60\n")); + } + body.push_str(runs); + d.put("S/roh/S_roh.txt", body); + at(&summary(&d, None), &["sections", "roh", "data"]).clone() +} + +#[test] +fn roh_scope_chr20_uses_its_own_unmasked_denominator() { + let r = roh_scope_summary(&[20], "RG\tS\tchr20\t1000001\t22000000\t21000000\n"); + assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.3515)); + assert_eq!(at(&r, &["relationship"]), &Value::Null); + assert_eq!(at(&r, &["upd_flag"]), &Value::Null); + assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(59_744_167)); + assert_eq!(at(&r, &["scope", "autosomes"]), &Value::Array(vec![Value::Str("chr20".into())])); +} + +#[test] +fn roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty() { + let r = roh_scope_summary(&[22, 20, 20], "RG\tS\tchr20\t1000001\t9000000\t8000000\nRG\tS\tchr22\t20000001\t28000000\t8000000\n"); + assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.1718)); + assert_eq!(at(&r, &["relationship"]), &Value::Null); + assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(93_162_635)); + let no_runs = roh_scope_summary(&[20], ""); + assert_eq!(at(&no_runs, &["f_roh"]), &Value::Float(0.0)); + assert_eq!(at(&no_runs, &["relationship"]), &Value::Null); +} + +#[test] +fn roh_scope_sparse_all_autosomes_keep_the_whole_genome_band() { + let sites: Vec = (1..=22).collect(); + let r = roh_scope_summary(&sites, "RG\tS\tchr2\t10000001\t18000000\t8000000\n"); + assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.003)); + assert_eq!(at(&r, &["relationship"]), &Value::Str("no sign of related parents".into())); + assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(2_658_001_522)); + assert_eq!(at(&r, &["scope", "partial"]), &Value::Bool(false)); +} + +#[test] +fn roh_scope_a_segment_only_file_never_establishes_scope() { + let r = roh_scope_summary(&[], "RG\tS\tchr20\t1000001\t22000000\t21000000\n"); + assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.3515)); + assert_eq!(at(&r, &["relationship"]), &Value::Null); + assert_eq!(at(&r, &["upd_flag"]), &Value::Null); + assert_eq!(at(&r, &["scope", "source"]), &Value::Str("segments_only".into())); + let empty = roh_scope_summary(&[], "# empty\n"); + assert_eq!(at(&empty, &["f_roh"]), &Value::Null); +} + +#[test] +fn roh_scope_upd_is_a_region_hint_without_syndrome_names() { + let sites: Vec = (1..=22).collect(); + // Distal chr15 and chr11 runs do not establish the imprinted locus's + // parental origin, but mixed iso/heterodisomy can involve that locus. + for (chrom, start, end, length, region) in [ + (15, 40000001, 101991189, 61991189, "15q11-q13 (SNRPN, UBE3A)"), + (11, 60000001, 100000000, 40000000, "11p15 (H19/IGF2, KCNQ1OT1)"), + (20, 1000001, 22000000, 21000000, "20q13.32 (GNAS)"), + ] { + let r = roh_scope_summary(&sites, &format!("RG\tS\tchr{chrom}\t{start}\t{end}\t{length}\n")); + let u = at(&r, &["upd_flag"]); + assert_eq!(at(u, &["imprinted", "0", "region"]), &Value::Str(region.into())); + assert_eq!(at(u, &["imprinted", "0", "maternal_upd"]), &Value::Str("-".into())); + assert_eq!(at(u, &["imprinted", "0", "paternal_upd"]), &Value::Str("-".into())); + let note = text(at(u, &["note"])); + assert!(note.contains("deletion") && note.contains("acquired") && note.contains("parent")); + assert_eq!(at(&r, &["relationship"]), &Value::Null); + } +} From 2fbbc528274fc8b293faafcb93b1cd00790a6888 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:04:02 +0100 Subject: [PATCH 088/255] test(setup): require clinical data in a default setup Bead: D11 --- scripts/ci/fake-docker/lib.sh | 23 ++++++++++ tests/fake-docker/base-setup.sh | 4 +- tests/fake-docker/setup-clinical-default.sh | 47 +++++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 tests/fake-docker/setup-clinical-default.sh diff --git a/scripts/ci/fake-docker/lib.sh b/scripts/ci/fake-docker/lib.sh index d8a28d811..27b6069dd 100644 --- a/scripts/ci/fake-docker/lib.sh +++ b/scripts/ci/fake-docker/lib.sh @@ -28,6 +28,29 @@ REF_FASTA_MD5=$(fake_body "$REF_FASTA_URL" | fake_md5) REF_FAI_MD5=$(fake_body "${REF_FASTA_URL}.fai" | fake_md5) export REF_FASTA_MD5 REF_FAI_MD5 +# Clinical setup uses real archive layouts, served by the existing fake curl +# and wget. CHECKSUMS is shared by basename, so it lists both VEP archives. +export FAKE_DOWNLOAD_DIR="${CASE_WORK}/downloads" +mkdir -p "$FAKE_DOWNLOAD_DIR" +for rel in 116 115; do + mkdir -p "${CASE_WORK}/cache/homo_sapiens/${rel}_GRCh38" + printf 'fake VEP cache\n' > "${CASE_WORK}/cache/homo_sapiens/${rel}_GRCh38/info.txt" + tar -czf "${FAKE_DOWNLOAD_DIR}/homo_sapiens_vep_${rel}_GRCh38.tar.gz" -C "${CASE_WORK}/cache" "homo_sapiens/${rel}_GRCh38" +done +for rel in 116 115; do + name="homo_sapiens_vep_${rel}_GRCh38.tar.gz" + printf '%s %s\n' "$(sum -r "${FAKE_DOWNLOAD_DIR}/${name}" | awk '{print $1, $2}')" "$name" +done > "${FAKE_DOWNLOAD_DIR}/CHECKSUMS" +mkdir -p "${CASE_WORK}/bundle/data/grch38" +printf 'fake PCGR data\n' > "${CASE_WORK}/bundle/data/grch38/README" +tar -czf "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" -C "${CASE_WORK}/bundle" data +if command -v sha256sum >/dev/null 2>&1; then + PCGR_DATA_SHA256=$(sha256sum "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') +else + PCGR_DATA_SHA256=$(shasum -a 256 "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') +fi +export PCGR_DATA_SHA256 + fail() { echo "ASSERT FAIL: $*" >&2 exit 1 diff --git a/tests/fake-docker/base-setup.sh b/tests/fake-docker/base-setup.sh index d373d9df9..6e5412647 100755 --- a/tests/fake-docker/base-setup.sh +++ b/tests/fake-docker/base-setup.sh @@ -13,7 +13,9 @@ output_lacks setup 'unbound variable' output_has setup 'Setup complete!' for f in reference/GRCh38_no_alt_analysis_set.fasta reference/GRCh38_no_alt_analysis_set.fasta.fai \ - clinvar/clinvar.vcf.gz clinvar/clinvar.vcf.gz.tbi; do + clinvar/clinvar.vcf.gz clinvar/clinvar.vcf.gz.tbi \ + vep_cache/homo_sapiens/116_GRCh38/info.txt vep_cache/homo_sapiens/115_GRCh38/info.txt \ + pcgr_data/20260620/data/grch38/README; do [ -s "${G}/${f}" ] || fail "setup.sh did not download ${f}" done grep -q '^image inspect' "$FAKE_DOCKER_LOG" || fail "setup.sh never checked a docker image" diff --git a/tests/fake-docker/setup-clinical-default.sh b/tests/fake-docker/setup-clinical-default.sh new file mode 100644 index 000000000..4c289cdde --- /dev/null +++ b/tests/fake-docker/setup-clinical-default.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Default clinical downloads, explicit opt-out, disk gates and corrupt input. +# shellcheck source=../../scripts/ci/fake-docker/lib.sh +. "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" +export FETCH_TRIES=1 FETCH_WAIT=0 +G="${CASE_WORK}/without" +run_expect 0 without "${SCRIPTS}/setup.sh" --no-clinical-data "$G" +output_has without 'Setup complete!' +if grep -E '^(wget|curl).*homo_sapiens_vep_|^(wget|curl).*pcgr_ref_data' "$FAKE_DOCKER_LOG"; then + fail '--no-clinical-data downloaded clinical data' +fi + +G="${CASE_WORK}/clinical" +run_expect 0 clinical "${SCRIPTS}/setup.sh" --clinical-data "$G" +for rel in 116 115; do + [ -s "${G}/vep_cache/homo_sapiens/${rel}_GRCh38/info.txt" ] || fail "VEP ${rel} missing" +done +[ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'PCGR bundle missing' +output_lacks clinical 'Phase 1:|Docker Images' +# A second install does no network work, even with too little space. +cp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" +run_expect 0 again "${SCRIPTS}/setup.sh" --clinical-data "$G" +cmp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" || fail 'installed clinical data downloaded again' + +mkdir -p "${CASE_WORK}/low-bin" +cat > "${CASE_WORK}/low-bin/df" <<'DF' +#!/usr/bin/env bash +echo 'Filesystem 1024-blocks Used Available Capacity Mounted on' +echo 'fakefs 41943040 0 41943040 0% /' +DF +chmod +x "${CASE_WORK}/low-bin/df" +G="${CASE_WORK}/low" +PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 0 low "${SCRIPTS}/setup.sh" --clinical-data "$G" +output_has low '56 GB.*5 GB margin' +output_has low '52 GB.*5 GB margin' +[ ! -e "${G}/vep_cache/homo_sapiens/116_GRCh38" ] || fail 'low-space VEP installed' +[ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'small bundle skipped with 40 GB free' + +# The digest belongs to the original fixture. Serve a different valid tarball. +printf 'corrupt\n' >> "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" +G="${CASE_WORK}/corrupt" +run_expect 0 corrupt "${SCRIPTS}/setup.sh" --clinical-data "$G" +output_has corrupt 'sha256 checksum.*expected' +output_has corrupt '\[WARN\].*PCGR' +[ ! -e "${G}/pcgr_data/20260620" ] || fail 'corrupt bundle published' +[ ! -e "${G}/pcgr_data/pcgr_ref_data.20260620.grch38.tgz.part" ] || fail 'bad bundle kept' +[ -s "${G}/vep_cache/homo_sapiens/115_GRCh38/info.txt" ] || fail 'bundle failure stopped remaining install' From b81d49a420c98f236745dcd9998f6a8fb1745ebe Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:04:47 +0100 Subject: [PATCH 089/255] test(pgx): pin the disease-gene drug warnings for carriers, SCN1A and provisional levels Red on main: a single heterozygous urea-cycle hit warns, SCN1A warns, the level reads as a settled CPIC level. Bead: pgx-21 Bead: pgx-22 Bead: pgx-23 Bead: D13 --- .../og-pgx/tests/i411_contraindications.rs | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 rust/crates/og-pgx/tests/i411_contraindications.rs diff --git a/rust/crates/og-pgx/tests/i411_contraindications.rs b/rust/crates/og-pgx/tests/i411_contraindications.rs new file mode 100644 index 000000000..33ce246ba --- /dev/null +++ b/rust/crates/og-pgx/tests/i411_contraindications.rs @@ -0,0 +1,140 @@ +//! The disease-gene drug warnings of `og pgx cpic-report` against the shipped +//! `assets/pgx_contraindications.tsv` (issue #411, pgx-21, pgx-22, pgx-23): +//! +//! - one heterozygous pathogenic variant in a recessive urea-cycle gene +//! (ASL, ASS1, CPS1, NAGS) makes a carrier: a NOTE, never a WARNING; +//! - both copies changed, or two variants in the gene, give the WARNING; +//! - one POLG variant still warns (a known valproate risk); +//! - SCN1A gives nothing: no label or guideline names it; +//! - the CPIC level is printed as the provisional level it is. + +use std::collections::HashSet; + +use og_pgx::sections::contraindication_lines; +use og_pgx::tables::{ClinvarHit, Contraindication, read_contraindications}; + +fn table() -> Vec { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../assets/pgx_contraindications.tsv" + ); + read_contraindications(path).unwrap() +} + +fn hit(gene: &str, pos: &str, alt: &str, genotype: &str) -> ClinvarHit { + ClinvarHit { + chrom: "chr13".to_owned(), + pos: pos.to_owned(), + ref_: "G".to_owned(), + alt: alt.to_owned(), + genotype: genotype.to_owned(), + geneinfo: Some(format!("{gene}:1")), + clnsig: Some("Pathogenic".to_owned()), + } +} + +fn lines(hits: &[ClinvarHit]) -> Vec { + let meds: HashSet = ["valproate".to_owned()].into(); + contraindication_lines(hits, &table(), &meds) +} + +fn warnings(out: &[String]) -> Vec<&str> { + out.iter() + .map(|l| l.trim()) + .filter(|l| l.starts_with("WARNING:")) + .collect() +} + +#[test] +fn one_heterozygous_urea_cycle_hit_is_a_note() { + for gene in ["ASL", "ASS1", "CPS1", "NAGS"] { + let out = lines(&[hit(gene, "40000000", "A", "0/1")]); + assert!(warnings(&out).is_empty(), "{gene}: {out:#?}"); + let note = format!( + " NOTE: valproic acid -- {gene} chr13:40000000 G>A, genotype 0/1, ClinVar Pathogenic; ON THE MEDICATION LIST." + ); + assert_eq!(out.first(), Some(¬e), "{gene}: {out:#?}"); + assert!( + out.iter().any(|l| l.contains("a carrier, not a patient")), + "{gene}: {out:#?}" + ); + } + // A phased or partly missing genotype is still one changed copy. + for gt in ["0|1", "1|0", "./1"] { + let out = lines(&[hit("ASS1", "40000000", "A", gt)]); + assert!(warnings(&out).is_empty(), "{gt}: {out:#?}"); + } +} + +#[test] +fn both_copies_or_two_variants_in_a_urea_cycle_gene_warn() { + for gt in ["1/1", "1|1", "1"] { + let out = lines(&[hit("ASS1", "40000000", "A", gt)]); + assert_eq!(warnings(&out).len(), 1, "{gt}: {out:#?}"); + assert!( + warnings(&out)[0].contains("both copies changed"), + "{gt}: {out:#?}" + ); + } + // Two variants in trans, at two positions or as the two alleles of one + // split multi-allelic site: one WARNING for each. + for (pos2, alt2) in [("40000500", "A"), ("40000000", "C")] { + let out = lines(&[ + hit("CPS1", "40000000", "A", "0/1"), + hit("CPS1", pos2, alt2, "1/0"), + ]); + let w = warnings(&out); + assert_eq!(w.len(), 2, "{pos2}>{alt2}: {out:#?}"); + assert!(w.iter().all(|l| l.contains("2 variants in CPS1")), "{w:#?}"); + assert!(!out.iter().any(|l| l.contains("NOTE:")), "{out:#?}"); + } + // The same variant listed twice (two ClinVar records) is one variant. + let out = lines(&[ + hit("NAGS", "40000000", "A", "0/1"), + hit("NAGS", "40000000", "A", "0/1"), + ]); + assert!(warnings(&out).is_empty(), "{out:#?}"); +} + +#[test] +fn one_polg_or_otc_variant_still_warns() { + for gene in ["POLG", "OTC"] { + let out = lines(&[hit(gene, "89300000", "T", "0/1")]); + assert_eq!( + warnings(&out), + [format!( + "WARNING: valproic acid -- {gene} chr13:89300000 G>T, genotype 0/1, ClinVar Pathogenic; ON THE MEDICATION LIST." + )], + "{out:#?}" + ); + } +} + +#[test] +fn an_scn1a_hit_gives_no_drug_line() { + assert!(!table().iter().any(|r| r.gene == "SCN1A")); + let out = lines(&[hit("SCN1A", "166000000", "T", "0/1")]); + assert!(out.is_empty(), "{out:#?}"); +} + +#[test] +fn every_row_quotes_a_label() { + let bad: Vec = table() + .into_iter() + .filter(|r| !r.label_source.starts_with("https://dailymed.nlm.nih.gov/")) + .map(|r| format!("{} {}", r.gene, r.drug)) + .collect(); + assert!(bad.is_empty(), "rows no label names: {bad:?}"); +} + +#[test] +fn the_cpic_level_reads_as_provisional() { + let out = lines(&[hit("POLG", "89300000", "T", "0/1")]); + assert!( + out.iter().any(|l| l.ends_with( + "(provisional CPIC level A/B: no CPIC guideline covers this pair)" + )), + "{out:#?}" + ); + assert!(!out.iter().any(|l| l.ends_with("(CPIC level A/B)")), "{out:#?}"); +} From 594270158406e0f1f67fc07bcb8d19ae25a06f02 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:05:01 +0100 Subject: [PATCH 090/255] test(live): check inline reads and same-origin viewer framing Bead: pgp-iqon.11 --- tests/test_live_viewer.py | 127 ++++++++++++++++++++++++++++++++++- tests/test_owngenome_live.py | 1 + 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index 2521c816d..afe6ba730 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -33,6 +33,8 @@ Run: python3 tests/test_live_viewer.py """ import http.client +import gzip +import json import importlib.util import os import re @@ -51,6 +53,7 @@ import owngenome_live # noqa: E402 import render_report # noqa: E402 import test_viewer as tv # noqa: E402 +import test_owngenome_live as tl # noqa: E402 PASSWORD = "viewer test password" FAILED = [] @@ -113,7 +116,7 @@ def viewer_checks(port, auth, s, label=""): code, h, body = request(port, "/runs/S/viewer/", auth) csp = h.get("content-security-policy", "") check(f"{label}page: 200 with the viewer's policy, served by the site", - code == 200 and csp == tv.serve.CSP and b"served by the run site" in body + code == 200 and csp == tv.serve.CSP.replace("frame-ancestors 'none'", "frame-ancestors 'self'") and b"served by the run site" in body and b"served from this computer only" not in body, (code, csp, body[:300])) code, _, body = request(port, "/runs/S/viewer/igv.min.js", auth) c2, _, js = request(port, "/runs/S/viewer/viewer.js", auth) @@ -170,6 +173,126 @@ def outside_checks(port, auth, label=""): url == "data/0/my%20ref%231%3F.fasta" and code == 206 and body == b">chr1", (url, code, body)) +def framing_checks(port): + for path in ("/runs/S/viewer/", "/runs/S/viewer/viewer.js", "/runs/S/viewer/config.json", + "/runs/S/viewer/data/2/S_sorted.bam"): + code, h, _ = request(port, path) + csp = h.get("content-security-policy", "") + check("same-origin viewer framing: " + path, + code == 200 and "frame-ancestors 'self'" in csp + and h.get("x-frame-options") == "SAMEORIGIN", (code, h)) + for path in ("/", "/login", "/assets/map.js", "/runs/S/", "/runs/S/state.json", + "/runs/S/report.html", "/not-found"): + _, h, _ = request(port, path) + check("non-viewer framing stays closed: " + path, + "frame-ancestors 'none'" in h.get("content-security-policy", "") + and h.get("x-frame-options") == "DENY", h) + + +def map_reads_checks(srv, work, chrome): + """The real map and viewer, on invented map rows and test_viewer's synthetic reads. + + No product data loader is bypassed: the fixture endpoint supplies the same + gzip document as map.json. The page supplies its existing veil callbacks. + """ + app, port = srv[0].app, srv[1].server_address[1] + policy = {"block": False} + page = ('' + '' + '' + '') + doc = {"v": 1, "people": [{"id": "S"}], "dict": {"veil": ["closed"]}, + "ref": {"chroms": [{"n": "chr1", "len": 60000, "masked": []}], "bands": {}}, + "per": {"S": {"findings": { + "id": ["open", "closed"], "level": ["review", "act"], "gate": ["", ""], + "chrom": ["chr1", "chr1"], "pos": [451, 551], "s": [450, 550], "e": [451, 551], + "gene": ["", ""], "title": ["Synthetic finding", "Closed finding"], + "href": ["#open", "#closed"], "phrase": ["Synthetic row", "Closed row"], "veil": [-1, 0]}}}, + "layers": {"findings": {"state": "landed"}, "roh": {"state": "not_run"}}, "text": {}} + data = gzip.compress(json.dumps(doc).encode()) + + def fixture(req, _): + csp = owngenome_live.CSP + if policy["block"]: + csp = csp.replace("frame-src 'self'; ", "") + return req.send_body(200, "text/html", page, csp=csp) + + app.route("GET", r"/map-1g/", fixture) + app.route("GET", r"/map-1g.json", lambda req, _: req.send_body(200, "application/gzip", data)) + b = tl.Chrome(chrome, f"{work}/chrome-map-1g") + try: + b.call("Emulation.setEmulatedMedia", {"features": [{"name": "prefers-color-scheme", "value": "dark"}, + {"name": "prefers-reduced-motion", "value": "reduce"}]}, b.session) + + def mount(demo=False): + b.go(f"http://127.0.0.1:{port}/map-1g/#chr1:1-49000") + if not b.wait("window.PGPMap && window.PGPUI"): + return False + b.eval("window.closedResult = true; " + "document.documentElement.dataset.mode = " + json.dumps("replay" if demo else "live") + "; " + "window.testMap = PGPMap.create({url:'/map-1g.json'," + "veil:function(id){return id==='closed' && window.closedResult ? 'closed':'open';}," + "known:function(){return true;},now:function(){return true;}});" + "document.body.appendChild(testMap.node);testMap.refresh('one');") + return b.wait("document.querySelector('.gmap[data-level=reads]')") + + def loaded(): + return b.wait("(function(){var f=document.querySelector('.gm-reads iframe');" + "return f && f.contentDocument && f.contentDocument.documentElement.dataset.pgpStatus==='ready';})()", 30) + + check("map: opens at a 49 kb view", mount()) + check("map: under 50 kb offers the reads pill", b.eval("!!document.querySelector('.gm-reads-pill:not([hidden])')")) + if not b.eval("!!document.querySelector('.gm-reads-pill')"): + return + b.eval("document.querySelector('.gm-reads-pill').click()") + check("map: the reads panel loads the viewer in an iframe at the nearest open finding", loaded(), + b.eval("document.querySelector('.gm-reads').textContent")) + check("map: the iframe locus is 1-based, finding plus or minus 50", + b.eval("document.querySelector('.gm-reads iframe').contentWindow.location.hash") == "#chr1:401-501") + check("map: a closed result has no card or reads link", + b.eval("!document.querySelector('.gm-marks [data-id=closed]') && !document.querySelector('.gm-card-reads[data-id=closed]')")) + # A moved view must update the existing frame, with no page reload. + b.eval("window.savedFrame=document.querySelector('.gm-reads iframe');location.hash='chr1:1-49001'") + check("map: moving updates the same iframe to the map's locus", + b.wait("savedFrame===document.querySelector('.gm-reads iframe') && savedFrame.contentWindow.location.hash==='#chr1:1-49001'")) + check("viewer: alignments remain visible at 49 kb", + b.wait("(function(){var d=savedFrame.contentDocument;return d.querySelector('canvas.igv-canvas') && " + "!Array.from(d.querySelectorAll('.igv-zoom-in-notice-container')).some(function(x){return x.offsetHeight>0;});})()")) + b.eval("document.querySelector('.gm-reads-close').dispatchEvent(new KeyboardEvent('keydown',{key:'Escape',bubbles:true}))") + check("map: Escape closes the reads and returns focus without moving the map", + b.eval("document.querySelector('.gm-reads').hidden && document.activeElement===document.querySelector('.gm-canvas') && " + "document.querySelector('.gmap').dataset.view==='chr1:1-49001'")) + b.eval("document.querySelector('.gm-canvas').dispatchEvent(new KeyboardEvent('keydown',{key:'r',bubbles:true}))") + check("map: R reopens the panel", loaded()) + b.eval("document.querySelector('.gm-reads-close').click();document.querySelector('[data-stop=reads]').click()") + check("map: Reads gauge opens a 2 kb view and the panel", b.wait("!document.querySelector('.gm-reads').hidden && " + "document.querySelector('.gmap').dataset.level==='reads'")) + b.eval("document.querySelector('.gm-reads-close').click();document.querySelector('.gm-marks [data-id=open]').focus()") + b.eval("document.querySelector('.gm-card-reads').click()") + check("map: the card opens reads at its finding", b.wait("document.querySelector('.gm-reads iframe').contentWindow.location.hash==='#chr1:401-501'")) + b.eval("window.closedResult=false;testMap.refresh('one');document.querySelector('.gm-marks [data-id=closed]').focus();" + "document.querySelector('.gm-card-reads').click()") + check("map: an opened finding offers reads", b.wait("document.querySelector('.gm-reads iframe').contentWindow.location.hash==='#chr1:501-601'")) + b.eval("window.closedResult=true;testMap.refresh('one')") + check("map: closing a result removes its open reads panel and link", + b.eval("document.querySelector('.gm-reads').hidden && !document.querySelector('.gm-card-reads')")) + + # CI proves the browser assertion detects the original policy mistake. + policy["block"] = True + mount() + b.eval("window.blockedFrames=0;document.addEventListener('securitypolicyviolation',function(e){" + "if(e.violatedDirective==='frame-src')window.blockedFrames++;});document.querySelector('.gm-reads-pill').click()") + check("control: without frame-src the reads iframe is blocked", b.wait("window.blockedFrames>0") and not loaded()) + policy["block"] = False + mount(demo=True) + b.eval("document.querySelector('.gm-reads-pill').click()") + check("map: the static demo explains why it has no viewer", + b.wait("document.querySelector('.gm-reads').textContent.includes('The genome viewer is not in the demo')") + and b.eval("!document.querySelector('.gm-reads iframe')")) + finally: + b.close() + + def mutant_dir(work, name, old, new): """A plugin folder holding bin/owngenome_live_viewer.py with one planted mistake (beside a link to assets/, which the plugin finds from its own folder).""" @@ -269,6 +392,7 @@ def main(): try: p = srv[1].server_address[1] outside_checks(p, {}) + framing_checks(p) # 7. headless Chrome, on the site without a password (loopback) chrome = tv.find_chrome() @@ -290,6 +414,7 @@ def main(): status and status.group(1) == "ready" and tracks and tracks.group(1) == "3", (status and status.group(1), r.stderr[-800:], r.stdout[:600])) check("browser: the policy blocked nothing the page needed", not blocked, blocked and blocked.group(1)) + map_reads_checks(srv, work, chrome) finally: stop(*srv) diff --git a/tests/test_owngenome_live.py b/tests/test_owngenome_live.py index 29f84d7fa..1296d5b5e 100644 --- a/tests/test_owngenome_live.py +++ b/tests/test_owngenome_live.py @@ -841,6 +841,7 @@ def stored_leaks(text): code, _, _ = request(port, "GET", "/nothing/here", headers=auth) bad = [p for p, hdr in RESPONSES if "default-src 'none'" not in hdr.get("content-security-policy", "") or "frame-ancestors 'none'" not in hdr.get("content-security-policy", "") + or hdr.get("x-frame-options") != "DENY" or hdr.get("x-content-type-options") != "nosniff" or hdr.get("cache-control") != "no-store"] check(f"every browser response ({len(RESPONSES)}) carries the policy, nosniff and no-store", not bad, bad) texts = {f: open(os.path.join(ASSETS, f)).read() for f in os.listdir(ASSETS) From cd57a630d32e2e1bb0a643f99b5e0f34d1f17ee6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:05:59 +0100 Subject: [PATCH 091/255] test(freshness): require metadata checks for clinical downloads Bead: D11 --- .../tests/screen_setup_clinical_default.rs | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 rust/crates/og-fresh/tests/screen_setup_clinical_default.rs diff --git a/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs b/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs new file mode 100644 index 000000000..077cd2a27 --- /dev/null +++ b/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs @@ -0,0 +1,53 @@ +//! Clinical links must be live and metadata must agree, without archive GETs. +use og_fresh::http::{Body, Fetch, GitHub, Headers, Request, Want}; +use og_fresh::lookups::{Ctx, DATABASES}; +use og_fresh::pins::Pins; +use og_fresh::selftest::FakeRegistry; +use og_fresh::{Lookup, failed}; +use std::path::Path; +use std::sync::Mutex; + +struct Downloads { + github: GitHub, + missing: bool, + bad_length: bool, + bad_checksum: bool, + calls: Mutex>, +} +impl Fetch for Downloads { + fn github_api(&self) -> &GitHub { &self.github } + fn request(&self, r: &Request<'_>) -> Lookup<(Body, Headers)> { + self.calls.lock().unwrap().push((r.method.into(), r.url.into())); + if self.missing { return failed("HTTP 404: release does not exist"); } + if r.url.ends_with("CHECKSUMS") { + assert_eq!(r.want, Want::Text); + let body = if self.bad_checksum { "123 5 wrong.tar.gz\n".into() } else { + "123 5 homo_sapiens_vep_116_GRCh38.tar.gz\n456 6 homo_sapiens_vep_115_GRCh38.tar.gz\n".into() + }; + Ok((Body::Text(body), Headers::default())) + } else { + assert_eq!(r.method, "HEAD", "large archive body was requested"); + Ok((Body::None, Headers::from_pairs(vec![("Content-Length".into(), if self.bad_length { "1" } else { "7664082467" }.into())]))) + } + } +} +fn check(missing: bool, bad_length: bool, bad_checksum: bool) { + let fetch = Downloads { github: GitHub { api: "https://invalid".into(), token: None }, missing, bad_length, bad_checksum, calls: Mutex::new(vec![]) }; + let registry = FakeRegistry { tags: |_| panic!("registry is not needed"), digests: |_| panic!("registry is not needed"), asked: Mutex::new(vec![]) }; + let pins = Pins::parse("VEP_CACHE_RELEASE=\"116\"\nPCGR_VEP_CACHE_RELEASE=\"115\"\nPCGR_DATA_BUNDLE=\"20260620\"\nPCGR_DATA_BYTES=\"7664082467\"\n"); + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let common = root.join("scripts/lib/common.sh"); + let c = Ctx { fetch: &fetch, registry: ®istry, pins: &pins, versions: Path::new("unused"), setup: Path::new("unused"), common: &common, now: "2026-11-15T00:00:00Z".parse().unwrap() }; + for name in ["VEP cache download", "CPSR cache download", "PCGR bundle download"] { + let (_, lookup) = DATABASES.iter().find(|(n, _)| *n == name).unwrap_or_else(|| panic!("missing liveness row: {name}")); + let got = lookup(&c); + let should_fail = missing || (bad_length && name == "PCGR bundle download") || (bad_checksum && name != "PCGR bundle download"); + assert_eq!(got.is_err(), should_fail, "{name}: {got:?}"); + } + let calls = fetch.calls.lock().unwrap(); + assert!(calls.iter().any(|(_, url)| url.contains("release-115/"))); +} +#[test] fn clinical_links_head_only() { check(false, false, false); } +#[test] fn missing_release_fails() { check(true, false, false); } +#[test] fn changed_bundle_length_fails() { check(false, true, false); } +#[test] fn missing_cache_checksum_fails() { check(false, false, true); } From 73175279dddc5b89b28d45560d0635720f8cf1de Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:06:28 +0100 Subject: [PATCH 092/255] test(bloodtype): pin the record a marker reads when several overlap it Four cases that fail on main, each read through the index and by a full pass: an ABO c.261 insertion after an all-sites reference record or a split SNV record, a later no-call record of the marker allele, a shorter reference record at the A2 deletion marker, and a padded record that spans KEL c.578 with another base there. Bead: trait-40 Bead: trait-41 Bead: trait-29 --- rust/crates/og-bloodtype/tests/report.rs | 208 +++++++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/rust/crates/og-bloodtype/tests/report.rs b/rust/crates/og-bloodtype/tests/report.rs index c025c3e7b..893169aa8 100644 --- a/rust/crates/og-bloodtype/tests/report.rs +++ b/rust/crates/og-bloodtype/tests/report.rs @@ -215,3 +215,211 @@ fn an_index_older_than_its_vcf_is_not_used() { let (_, how) = read_vcf(&gz.to_string_lossy(), read_markers(&markers()).unwrap()).unwrap(); assert_eq!(how, Access::Stream); } + +/// A one-sample VCF record line. +fn line(chrom: &str, pos: i64, r: &str, alt: &str, filt: &str, gt: &str) -> String { + format!("{chrom}\t{pos}\t.\t{r}\t{alt}\t50\t{filt}\t.\tGT\t{gt}\n") +} + +/// `records` (sorted, under 64 KiB) as a plain VCF and as a bgzipped VCF +/// with a `.tbi`: one bin per sequence whose one chunk spans that +/// sequence's lines, the smallest index the reader takes. +fn both_ways(d: &Path, records: &[String]) -> (PathBuf, PathBuf) { + use std::io::Write; + let text = format!("{HEAD}{}", records.concat()); + let plain = d.join("s.vcf"); + std::fs::write(&plain, &text).unwrap(); + let gz = d.join("s.vcf.gz"); + let mut w = og_io::bgzf::Writer::new( + std::fs::File::create(&gz).unwrap(), + std::num::NonZero::new(1).unwrap(), + ) + .unwrap(); + w.write_all(text.as_bytes()).unwrap(); + w.finish().unwrap(); + // Each sequence's (first byte, end byte) in the one block's text. + let mut seqs: Vec<(String, u64, u64)> = Vec::new(); + let mut at = HEAD.len() as u64; + for r in records { + let chrom = r.split('\t').next().unwrap().to_owned(); + let end = at + r.len() as u64; + match seqs.last_mut() { + Some(s) if s.0 == chrom => s.2 = end, + _ => seqs.push((chrom, at, end)), + } + at = end; + } + let names: Vec = seqs + .iter() + .flat_map(|s| s.0.bytes().chain([0])) + .collect(); + let mut tbi: Vec = b"TBI\x01".to_vec(); + let i32s = |v: &mut Vec, xs: &[i32]| xs.iter().for_each(|x| v.extend(x.to_le_bytes())); + // n_ref, VCF preset, columns 1 2 0, meta '#', skip 0, names. + i32s(&mut tbi, &[seqs.len() as i32, 2, 1, 2, 0, b'#' as i32, 0]); + i32s(&mut tbi, &[names.len() as i32]); + tbi.extend(&names); + for (_, beg, end) in &seqs { + // One bin (0, the whole sequence), one chunk, no linear index. + i32s(&mut tbi, &[1, 0, 1]); + tbi.extend(beg.to_le_bytes()); + tbi.extend(end.to_le_bytes()); + i32s(&mut tbi, &[0]); + } + std::fs::write(d.join("s.vcf.gz.tbi"), tbi).unwrap(); + (plain, gz) +} + +/// (status, alt copies, note) of the site `name`, read through the index +/// and by a full pass, which must agree. +fn site_both_ways(d: &Path, records: &[String], name: &str) -> (String, Option, String) { + let (plain, gz) = both_ways(d, records); + let (whole, how) = + read_vcf(&plain.to_string_lossy(), read_markers(&markers()).unwrap()).unwrap(); + assert_eq!(how, Access::Stream); + let (indexed, how) = + read_vcf(&gz.to_string_lossy(), read_markers(&markers()).unwrap()).unwrap(); + assert_eq!(how, Access::Index); + assert_eq!(indexed, whole, "{records:?}"); + let s = &whole[name]; + (s.status.as_str().to_owned(), s.n, s.note.clone()) +} + +#[test] +fn the_abo_insertion_counts_after_a_reference_or_other_record() { + let d = dir("abo261"); + let ins = |gt: &str| line("chr9", 133257521, "T", "TC", "PASS", gt); + for (first, gt, copies) in [ + // An all-sites VCF (bcftools call -m without -v): the reference + // record comes first, then the insertion. + (line("chr9", 133257521, "T", ".", ".", "0/0"), "0/1", 1), + (line("chr9", 133257521, "T", ".", ".", "0/0"), "1/1", 2), + // A gVCF-style reference record. + (line("chr9", 133257521, "T", "<*>", ".", "0/0"), "0/1", 1), + ] { + for recs in [[first.clone(), ins(gt)], [ins(gt), first.clone()]] { + let (status, n, note) = site_both_ways(&d, &recs, "ABO_261"); + assert_eq!( + (status.as_str(), n, note.as_str()), + ("called", Some(copies), ""), + "{recs:?}" + ); + } + } + // A split multi-allelic record (bcftools norm -m-any writes the SNV + // first): the insertion counts and the SNV is named. + let snv = line("chr9", 133257521, "T", "G", "PASS", "1/0"); + for recs in [[snv.clone(), ins("0/1")], [ins("0/1"), snv.clone()]] { + let (status, n, note) = site_both_ways(&d, &recs, "ABO_261"); + assert_eq!((status.as_str(), n), ("called", Some(1)), "{recs:?}"); + assert_eq!(note, "another allele at this position (T>G, genotype 1/0)"); + } + // The whole report: A1/O.01, not O. + let t = run( + &d, + &[line("chr9", 133257521, "T", ".", ".", "0/0"), ins("0/1")], + Some(depth(&d, 2.0, &HashMap::new())), + ); + assert_eq!(t["ABO"][..3], ["A1", "A1/O.01", "high"]); +} + +#[test] +fn a_later_record_of_the_marker_allele_that_is_no_call_wins_over_a_reference_record() { + let d = dir("abo261-nocall"); + let r = line("chr9", 133257521, "T", ".", ".", "0/0"); + for (later, note) in [ + (line("chr9", 133257521, "T", "TC", "PASS", "./."), "genotype ./."), + ( + line("chr9", 133257521, "T", "TC", "LowQual", "0/1"), + "FILTER LowQual", + ), + ] { + for recs in [[r.clone(), later.clone()], [later.clone(), r.clone()]] { + let (status, n, got) = site_both_ways(&d, &recs, "ABO_261"); + assert_eq!( + (status.as_str(), n, got.as_str()), + ("no call", None, note), + "{recs:?}" + ); + } + } +} + +#[test] +fn a_shorter_reference_record_leaves_a_deletion_marker_to_the_reads() { + let d = dir("abo1061"); + let ins = line("chr9", 133257521, "T", "TC", "PASS", "0/1"); + // An all-sites VCF writes `C . 0/0` at the A2 marker (REF CG): it says + // nothing past its own base, so the reads decide, as with no record. + for r in [ + line("chr9", 133255669, "C", ".", ".", "0/0"), + line("chr9", 133255669, "C", "<*>", ".", "0/0"), + ] { + let recs = [r.clone(), ins.clone()]; + assert_eq!( + site_both_ways(&d, &recs, "ABO_1061"), + ("absent".to_owned(), None, String::new()) + ); + let t = run(&d, &recs, Some(depth(&d, 2.0, &HashMap::new()))); + assert_eq!(t["ABO"][..3], ["A1", "A1/O.01", "high"], "{r}"); + // The deletion after it still counts, in either order. + let del = line("chr9", 133255669, "CG", "C", "PASS", "0/1"); + for recs in [[r.clone(), del.clone()], [del.clone(), r.clone()]] { + let (status, n, _) = site_both_ways(&d, &recs, "ABO_1061"); + assert_eq!((status.as_str(), n), ("called", Some(1)), "{recs:?}"); + } + } + // A shorter record with a non-reference call stays a no call. + let (status, _, note) = site_both_ways( + &d, + &[line("chr9", 133255669, "C", "T", "PASS", "0/1")], + "ABO_1061", + ); + assert_eq!( + (status.as_str(), note.as_str()), + ("no call", "REF C at this position, the marker's REF is CG") + ); +} + +#[test] +fn a_record_that_spans_a_marker_with_another_base_there_is_no_call() { + let d = dir("kel578"); + let reads = || Some(depth(&d, 2.0, &HashMap::new())); + // KEL c.578C>T (K) is chr7:142957921 G>A. A padded record one base + // upstream carries the same change (CG>CA), or a deletion of that + // base with it (DeepVariant's CG>C,CA 1/2). + for (r, alt, gt) in [("CG", "CA", "1/1"), ("CG", "C,CA", "1/2"), ("CG", "C", "0/1")] { + let recs = [line("chr7", 142957920, r, alt, "PASS", gt)]; + let (status, n, note) = site_both_ways(&d, &recs, "KEL_578"); + assert_eq!( + (status.as_str(), n, note.as_str()), + ( + "no call", + None, + format!("a record at chr7:142957920 ({r}>{alt}, genotype {gt}) spans this site") + .as_str() + ) + ); + let t = run(&d, &recs, reads()); + assert_eq!(t["Kell"][0], "K/k not determined", "{recs:?}"); + assert_ne!(t["Kell"][2], "high", "{recs:?}"); + } + // Split (bcftools norm -m-any): the deletion before the SNV record + // still makes the SNV's copy count unknown (one copy may be deleted). + let recs = [ + line("chr7", 142957920, "CG", "C", "PASS", "0/1"), + line("chr7", 142957921, "G", "A", "PASS", "0/1"), + ]; + let (status, ..) = site_both_ways(&d, &recs, "KEL_578"); + assert_eq!(status, "no call"); + // A spanning record that keeps the marker's base, or calls reference, + // leaves the site to its own records and the reads. + for recs in [ + [line("chr7", 142957920, "CG", "TG", "PASS", "1/1")], + [line("chr7", 142957920, "CG", "C", "PASS", "0/0")], + [line("chr7", 142957920, "CG", "C", "PASS", "./.")], + ] { + let (status, ..) = site_both_ways(&d, &recs, "KEL_578"); + assert_eq!(status, "absent", "{recs:?}"); + } +} From 5fc18ba0b281c431dfb61ec6e941d7d127d831eb Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:04 +0100 Subject: [PATCH 093/255] docs(licence): say GPL-3.0-or-later wherever OwnGenome states its licence CONTRIBUTING.md granted contributions under GPL-3.0 only; CLAUDE.md, the traits page and the Nextflow page said GPL-3.0. README, docs/index.md and rust/Cargo.toml already said GPL-3.0-or-later. Now every statement says GPL-3.0-or-later, README carries the GPL's or-later notice and the nextflow.config manifest names the licence. No DCO and no SPDX headers. Bead: lic-01 Bead: plat-75 Bead: lic-04 Bead: lic-05 Bead: D7 --- CLAUDE.md | 2 +- CONTRIBUTING.md | 2 +- README.md | 2 +- docs/42-traits.md | 4 ++-- docs/nextflow.md | 2 +- nextflow.config | 1 + 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1b8788d1a..c713c6cfb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -148,7 +148,7 @@ Change its line in `versions.env`, plus the coupled data variable its comment na - All processing is local; genomic data never leaves the machine - Pin tool versions; never use floating tags - Reference genome: GRCh38, NCBI's no-ALT analysis set (`reference/GRCh38_no_alt_analysis_set.fasta`). No aligner here runs ALT-aware, so a reference with ALT contigs gives MAPQ 0 at CYP2D6, the MHC and KIR; `validate-setup.sh` refuses one (unless `ALLOW_ALT_REFERENCE=true`) and refuses a BAM whose @SQ lines differ from the `.fai`. A change of reference means realigning every sample (`docs/realignment.md`) -- License: GPL-3.0 +- License: GPL-3.0-or-later (README, CONTRIBUTING.md, `rust/Cargo.toml` and the `nextflow.config` manifest say the same) ## Tool-Specific Gotchas diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f3382205e..a4929bcb3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -123,4 +123,4 @@ Be respectful. Genomic data is deeply personal. This project exists to empower i ## License -By contributing, you agree that your contributions will be licensed under GPL-3.0. +By contributing, you agree that your contributions will be licensed under GPL-3.0-or-later: the GNU General Public License, version 3 or (at your option) any later version, the same licence as the rest of the repository ([LICENSE](LICENSE), [README](README.md#license)). diff --git a/README.md b/README.md index e3465df5a..e2e53b432 100644 --- a/README.md +++ b/README.md @@ -86,4 +86,4 @@ Your genome data is sensitive personal information. This pipeline runs locally a ## License -[GPL-3.0-or-later](LICENSE) +[GPL-3.0-or-later](LICENSE). OwnGenome is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. Contributions come in under the same terms ([CONTRIBUTING.md](CONTRIBUTING.md#license)). diff --git a/docs/42-traits.md b/docs/42-traits.md index ad3aa25cf..8e85611bc 100644 --- a/docs/42-traits.md +++ b/docs/42-traits.md @@ -6,7 +6,7 @@ Reads your genotype at a small set of well-replicated variants and prints what e ## Why -These are the results most people find easiest to read, and the pipeline had no such section. SNPedia, the usual source, is CC BY-NC-SA, which a GPL-3.0 project cannot copy, so the table here is written from the primary papers instead: each row's reading follows the cited abstract or its open full text, and the alleles follow dbSNP's GRCh38 record. No text comes from SNPedia or any other database. +These are the results most people find easiest to read, and the pipeline had no such section. SNPedia, the usual source, is CC BY-NC-SA, which a GPL-3.0-or-later project cannot copy, so the table here is written from the primary papers instead: each row's reading follows the cited abstract or its open full text, and the alleles follow dbSNP's GRCh38 record. No text comes from SNPedia or any other database. ## Tool @@ -148,4 +148,4 @@ Seconds: bcftools reads the gVCF at the table's positions through its index (36 - Each traits table row is read alone. The genotype rules read several sites together: APOE, the TAS2R38 taster haplotype with read-backed phase, and the six rules of the table above. Two different MC1R red-hair variants on the same or on different copies are still read one by one; their note says so. - SNVs only; the table has no indel and no chrX, chrY or chrM row. - Positions with no genotype (no reads, or a no-call) are listed as not read. On the e2e fixture, a small slice of the HG002 genome, only the table positions inside its slices are read, and the e2e case checks their genotypes against the GIAB truth. -- The licence: the readings, notes and table are this project's own text, GPL-3.0 like the rest of the repository. +- The licence: the readings, notes and table are this project's own text, GPL-3.0-or-later like the rest of the repository. diff --git a/docs/nextflow.md b/docs/nextflow.md index 0f15b62c5..694f952aa 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -451,7 +451,7 @@ Cyrius (CYP2D6 star alleles) is under a non-commercial licence and is not in the ## Relationship to nf-core -This pipeline uses [nf-core](https://nf-co.re/) template patterns and tooling for code quality, but is **not an official nf-core pipeline** (it uses a GPL-3.0 license; nf-core requires MIT). +This pipeline uses [nf-core](https://nf-co.re/) template patterns and tooling for code quality, but is **not an official nf-core pipeline** (it uses the GPL-3.0-or-later license; nf-core requires MIT). Individual modules (PharmCAT, pypgx, slivar) will be contributed to [nf-core/modules](https://github.com/nf-core/modules) under MIT license for use by the broader community. diff --git a/nextflow.config b/nextflow.config index 46f83e521..e50156bf4 100644 --- a/nextflow.config +++ b/nextflow.config @@ -271,6 +271,7 @@ manifest { homePage = 'https://github.com/GeiserX/OwnGenome' description = 'Whole genome sequencing from FASTQ, BAM or VCF to clinical interpretation' mainScript = 'main.nf' + license = 'GPL-3.0-or-later' // Validated on the Nextflow release pinned in versions.env (NEXTFLOW_VERSION) // and on the newest 26.10.x (both legs of .github/workflows/nextflow.yml). // The floor rejects 24.x, whose vcfanno strict-parser regression breaks this From da97715cad505a4ecea8fa4ed173293911cdf2df Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:04 +0100 Subject: [PATCH 094/255] docs(acmg): cite ACMG SF v3.3 and say the rule text is a short quotation docs/resources.md named ACMG SF v3.2 with 81 genes; it now names v3.3, 84 genes and the Lee 2025 DOI. The asset header and the clinical filter page say that table_text holds short cited quotations of a factual reporting rule and that "(See text)" points to the DCM section. A new og-summary test shows that # lines, at the top or between rows, do not change the table read from acmg_sf.tsv. Bead: lic-06 Bead: clin-43 Bead: D2 --- assets/acmg_sf.tsv | 7 +++-- docs/23-clinical-filter.md | 2 ++ docs/resources.md | 2 +- rust/crates/og-summary/src/collect/assets.rs | 32 ++++++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) diff --git a/assets/acmg_sf.tsv b/assets/acmg_sf.tsv index 9d7d27e6e..66c1fed5b 100644 --- a/assets/acmg_sf.tsv +++ b/assets/acmg_sf.tsv @@ -28,8 +28,11 @@ # pseudodeficiency allele is set aside; a hypomorph allele counts, but the finding says # partial, not profound. "." for none # sf_version the ACMG SF list version that added the gene (Table 1) -# table_text Table 1's "Variants to report" text -# note what Table 1's footnotes add, shown with the finding; "." +# table_text Table 1's "Variants to report" text, quoted as the paper writes it: short cited +# quotations of a factual reporting rule (six phrases, the longest 44 characters), with +# the citation above. "(See text)" points to the paper's section on the dilated +# cardiomyopathy (DCM) genes +# note what Table 1's footnotes add, in our own words, shown with the finding; "." gene hgnc_id inheritance report_if variants sv_cn sv_regions severity sf_version table_text note APC HGNC:583 AD any . no . . 1.0 All P and LP . RET HGNC:9967 AD any . no . . 1.0 All P and LP . diff --git a/docs/23-clinical-filter.md b/docs/23-clinical-filter.md index 66f03986c..c2b3088a5 100644 --- a/docs/23-clinical-filter.md +++ b/docs/23-clinical-filter.md @@ -88,6 +88,8 @@ The Nextflow `CLINICAL_FILTER` module applies the same tiers, ranking, frequency Both reports list, beside the clinical filter's counts, the secondary findings in the 84 genes of the ACMG SF v3.3 list (Lee et al., Genet Med 2025;27(8):101454; the genes in which the ACMG recommends reporting pathogenic variants found by chance, such as BRCA1, LDLR, MYBPC3 or TTN): the step 6 ClinVar hits in those genes, the clinical filter's records with a HIGH-impact consequence on the ranked transcript in them, and step 5's AnnotSV class 4 and 5 structural variants in the genes whose rule allows them. Each gene's rule from Table 1 of the list is a row of `assets/acmg_sf.tsv`: one variant in a dominant gene, two in a recessive one (MUTYH, BTD, ATP7B and others; a homozygote, or two variants of unknown phase), HFE only as homozygous p.C282Y, GLA and OTC also in a heterozygous female, ABCD1 hemizygous, homozygous or with two heterozygous variants. A variant the rule does not report is listed apart with the reason; a pseudodeficiency allele is not counted, and a BTD finding with the p.Asp444His hypomorph says partial, not profound. Of the reported HIGH-impact records the reports count as "rare HIGH impact" only the rare ones with a sound call; a weak call or an unknown frequency is listed with its mark and counted apart, and a common one is left out. The list version is `ACMG_SF_VERSION` in `bin/collect_summary.py`. It is a list to review with a clinician: a rare HIGH-impact variant is not a ClinVar classification. [Interpreting results](interpreting-results.md#carrier-status-is-not-disease) gives every rule. Step 17 (CPSR) reports the ACMG secondary findings with its own classification. +The `table_text` column of `assets/acmg_sf.tsv` quotes each gene's rule from Table 1 of the paper (doi:10.1016/j.gim.2025.101454), such as "All P and LP" or "P and LP (2 variants)". summary.json gives that text as the rule of each finding, and the reports use it in the reason a variant is not reported. These are short cited quotations of a factual reporting rule: six phrases, the longest 44 characters. "(See text)" after a rule points to the paper's section on the dilated cardiomyopathy (DCM) genes. The `note` column says what the table's footnotes add, in our own words. + ## Runtime 2.5 to 6.5 minutes, measured in three 30-40x WGS runs ([runtime](hardware-requirements.md#runtime-per-step)) (I/O-bound, reading the large VEP VCF) diff --git a/docs/resources.md b/docs/resources.md index e34492214..d90ddc14f 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -26,7 +26,7 @@ Free resources for understanding your genomic data, organized from most accessib ### Variant Interpretation - **[ClinGen](https://www.clinicalgenome.org/)** — The Clinical Genome Resource. Maintains gene-disease validity assessments and variant interpretation standards. Useful for understanding how variants are classified. -- **[ACMG SF v3.2 Gene List](https://www.nature.com/articles/s41436-023-02171-w)** — The 81 genes that ACMG recommends reporting incidental findings for. Note: CPSR uses its own cancer-focused panels (Genomics England PanelApp), not the ACMG SF list — CPSR covers 500+ cancer genes but omits non-cancer ACMG genes (cardiac, metabolic). A negative CPSR result does not constitute a complete ACMG incidental-findings screen. +- **[ACMG SF v3.3 Gene List](https://doi.org/10.1016/j.gim.2025.101454)** — The 84 genes that ACMG recommends reporting incidental findings for (Lee et al., Genet Med 2025;27(8):101454, doi:10.1016/j.gim.2025.101454). This is the list the pipeline's secondary-findings section applies (`assets/acmg_sf.tsv`). Note: CPSR uses its own cancer-focused panels (Genomics England PanelApp), not the ACMG SF list — CPSR covers 500+ cancer genes but omits non-cancer ACMG genes (cardiac, metabolic). A negative CPSR result does not constitute a complete ACMG incidental-findings screen. - **[MitoMap](https://www.mitomap.org/)** — The human mitochondrial genome database, with the curated disease status of mtDNA variants. Step 20 already puts ClinVar's record and gnomAD v3.1's frequency beside each call; MITOMAP is a manual lookup in a browser, as it answers scripted requests with a 403. - **[GeneReviews](https://www.ncbi.nlm.nih.gov/books/NBK1116/)** — Expert-authored, peer-reviewed descriptions of genetic conditions. If you find a pathogenic variant, the GeneReviews entry for the associated condition is the best single resource for understanding it. diff --git a/rust/crates/og-summary/src/collect/assets.rs b/rust/crates/og-summary/src/collect/assets.rs index a71eeb613..0c80fa8f1 100644 --- a/rust/crates/og-summary/src/collect/assets.rs +++ b/rust/crates/og-summary/src/collect/assets.rs @@ -294,4 +294,36 @@ mod tests { assert!(parse_alleles(Some("chr1:5:A"), false).is_err()); assert!(parse_alleles(None, true).unwrap().is_empty()); } + + /// The `#` lines of `acmg_sf.tsv` (its source, its columns, the note + /// on the quoted Table 1 text) are not table rows: the shipped file, + /// the same rows with no `#` line, and the same rows with more `#` + /// lines among them read as one table. + #[test] + fn acmg_header_lines_do_not_change_the_table() { + let repo = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../assets"); + let shipped = std::fs::read_to_string(format!("{repo}/acmg_sf.tsv")).unwrap(); + assert!(shipped.lines().any(|l| l.starts_with('#'))); + let rows: Vec<&str> = shipped + .split_inclusive('\n') + .filter(|l| !l.starts_with('#')) + .collect(); + let bare = rows.concat(); + let mut extra = String::from("# a header line\n"); + for (i, l) in rows.iter().enumerate() { + extra.push_str(l); + if i % 20 == 1 { + extra.push_str("# a line between rows\tAPC\tHGNC:583\n"); + } + } + let read = |tag: &str, body: &str| { + let d = crate::testutil::Dir::new(tag); + d.put("acmg_sf.tsv", body); + format!("{:?}", acmg_sf_rules(&d.0.display().to_string()).unwrap()) + }; + let want = format!("{:?}", acmg_sf_rules(repo).unwrap()); + assert_eq!(acmg_sf_rules(repo).unwrap().len(), 84); + assert_eq!(read("acmg-bare", &bare), want); + assert_eq!(read("acmg-extra", &extra), want); + } } From 7875c956f01af70cd103917e6599e67f8b2a151a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:04 +0100 Subject: [PATCH 095/255] docs(calling): name the source of the GRCh38 PAR intervals assets/par_grch38.bed stays without a header, because some readers take every line as an interval; the variant calling page names the Genome Reference Consortium's GRCh38 page and the coordinates. Bead: lic-19 --- docs/03-variant-calling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/03-variant-calling.md b/docs/03-variant-calling.md index 14c281141..2a4c23fe0 100644 --- a/docs/03-variant-calling.md +++ b/docs/03-variant-calling.md @@ -84,7 +84,7 @@ DeepVariant writes its outputs under `.part` names, and the script renames them ## Sex chromosomes -A male sample has one X and one Y. Outside the pseudoautosomal regions (PARs, the ends of X and Y that pair with each other), a heterozygous call on chrX or chrY cannot be real. With `male`, DeepVariant calls chrX and chrY haploid outside the PARs listed in `assets/par_grch38.bed` (the GRCh38 PAR1 and PAR2 of chrX and chrY), so those calls are homozygous. `female` and no sex call every contig diploid; no sex prints a note saying so. `run-all.sh` passes the sex it was given. +A male sample has one X and one Y. Outside the pseudoautosomal regions (PARs, the ends of X and Y that pair with each other), a heterozygous call on chrX or chrY cannot be real. With `male`, DeepVariant calls chrX and chrY haploid outside the PARs listed in `assets/par_grch38.bed` (the GRCh38 PAR1 and PAR2 of chrX and chrY), so those calls are homozygous. The PAR coordinates come from the [Genome Reference Consortium's GRCh38 page](https://www.ncbi.nlm.nih.gov/grc/human): PAR1 is chrX and chrY 10,001-2,781,479; PAR2 is chrX 155,701,383-156,030,895 and chrY 56,887,903-57,217,415 (1-based). The BED holds the same intervals with 0-based starts. It has no header line, because some of its readers take every line as an interval. `female` and no sex call every contig diploid; no sex prints a note saying so. `run-all.sh` passes the sex it was given. The alternative callers (03a GATK, 03b FreeBayes, 03d Octopus) call every contig diploid, chrX and chrY of a male sample too. Clair3 (03e, long reads) takes the same `male` or `female` argument (`--gender`). From d58805eb2b4ad403de09b42e9ec7b34c0f7137c2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:05 +0100 Subject: [PATCH 096/255] docs(changelog): note the licence and ACMG citation fixes Bead: lic-01 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..7dcc474de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The contribution terms and the guides now state one licence for OwnGenome: GPL-3.0-or-later. CONTRIBUTING.md granted contributions under GPL-3.0 only, and CLAUDE.md, the traits page and the Nextflow page said GPL-3.0; they now say GPL-3.0-or-later like README and `rust/Cargo.toml`. README carries the GPL's or-later notice, and the `nextflow.config` manifest names the licence. No output changes. + - The resources page named ACMG SF v3.2 with 81 genes; it now names v3.3 with 84 genes and the 2025 paper, the list `assets/acmg_sf.tsv` applies. + - `assets/acmg_sf.tsv` and the clinical filter page say that the `table_text` column holds short cited quotations of Table 1's reporting rule, and that "(See text)" points to the paper's section on the DCM genes. A new `og-summary` test shows that `#` lines do not change the table read from the file. + - The variant calling page names the Genome Reference Consortium as the source of the PAR intervals in `assets/par_grch38.bed`. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. From 207283a63039b37e027677018eb9d9ceee9c8542 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:06 +0100 Subject: [PATCH 097/255] fix(clinvar): keep ClinVar's condition on each hit Step 06 copied only ID, GENEINFO, CLNSIG and CLNREVSTAT onto each hit, so later steps knew the gene but not the disease ClinVar asserted. Both mirrors now also copy CLNDN and CLNDISDB when the ClinVar file's header defines them (every release does); a hand-built file without them runs as before, with a notice. The hits TSV and the output contract stay as they are. Part of #411 Bead: i411-hits-keep-condition --- CHANGELOG.md | 2 ++ docs/06-clinvar-screen.md | 6 +++--- modules/local/clinvar_screen/main.nf | 15 +++++++++++++-- scripts/06-clinvar-screen.sh | 17 +++++++++++++++-- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..2f5e6340c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Each ClinVar hit now keeps the condition ClinVar names for it. Step 06 copied only ClinVar's ID, gene, significance and review status onto each hit, so a later step knew the gene but not the disease: a RET hit could be Hirschsprung disease or MEN2. The hits VCF (`clinvar/_clinvar_hits.vcf`) now also carries `CLNDN`, ClinVar's disease name, and `CLNDISDB`, its disease database ids, in both the bash step and the Nextflow module. Two INFO fields are added; nothing is renamed or removed, and `clinvar/_clinvar_hits.tsv` is unchanged. A hand-built ClinVar file that does not define the two fields still runs, without them and with a notice. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/docs/06-clinvar-screen.md b/docs/06-clinvar-screen.md index 9a608b5ac..3a8e5873a 100644 --- a/docs/06-clinvar-screen.md +++ b/docs/06-clinvar-screen.md @@ -53,7 +53,7 @@ In the Nextflow pipeline the same floor is `ext.min_gq` and `ext.min_dp` of `CLI 3. Filters the sample VCF to PASS records. If the VCF has no PASS record at all (callers that leave FILTER as `.`), it uses `-f .,PASS` instead and prints a notice saying so. If no record is left, the step stops with an error. 4. Splits and left-aligns the sample the same way. `bcftools isec` matches only identical REF/ALT, so a pathogenic allele inside a multiallelic record (genotype `1/2`) is found only after this split. The split writes the other alt allele as missing, not as the reference (`bcftools norm --multi-overlaps .`): a `1/2` record gives `1/.` and `./1`, where the default would give `1/0` and `0/1`. Both copies of such a call carry an alt allele, so neither part may read as het. A poly-T tract is where this shows most: a call of 5 T on one copy and 9 T on the other, against a reference of 7, is one `1/2` record, and its 5 T allele is a ClinVar hit. The reports word such a hit "one copy, other copy not called ref (1/.)", never het; the ACMG and carrier lists count it as one copy of two. This step does not size the tract: a short-read call of a homopolymer length can be off by one or two bases, so check the reads. 5. Builds the combined alleles of neighbouring records. A caller can write one complex allele, a delins, as two records a few bases apart on the same haplotype, for example a deletion and an insertion. Matched one by one, the parts can hit two single-submitter ClinVar records and miss the better reviewed record ClinVar holds for the whole allele. `bcftools norm -m +any` joins only records at the same position, so it cannot build that allele. `og clinvar join-adjacent` pairs two records when the second starts at most 10 bases after the first ends (records that overlap are never paired) and their genotypes allow both ALT alleles on one haplotype: phased with the same genotype in the same phase set (`FORMAT/PS`), or unphased with the same genotype (`0/1` and `1/0` count as the same het call; an unphased call with a missing allele such as `./1` is never paired). A phased call with a missing allele, such as `1|.`, has its ALT on a known haplotype: it pairs with a record in the same phase set whose ALT sits on that haplotype (`1|0` or `1|.`), and the combined record keeps `.` on the haplotype where either part is missing. That `.` means the copy was not called there; when the part comes from a split multiallelic record, it holds the other alt allele. Two unphased het calls can also sit on different haplotypes, and the genotype cannot tell which, so the notes of a combined allele and its parts built from them end with "unphased, so the parts may sit on different haplotypes". The combined record runs from the first part's start to the second part's end; `og clinvar join-adjacent` takes the bases between the parts from the reference (an uncompressed FASTA with its `.fai`), and `bcftools norm` trims and left-aligns it the way ClinVar was. The gene is not checked here, because the sample VCF has no gene annotation yet: a combined allele only counts when ClinVar holds a record for it. The step prints how many combined alleles it built. Only pairs are built: an allele split into three or more records is matched by its pairs only. -6. Keeps the sample's records and combined alleles whose allele is in ClinVar (`bcftools isec -n=2 -w1`), copies ClinVar's `ID`, `GENEINFO`, `CLNSIG` and `CLNREVSTAT` onto them (`bcftools annotate --pair-logic exact`), and keeps only those whose genotype carries an ALT allele (`bcftools view -i 'GT~"[1-9]"'`, a non-zero allele index anywhere in the genotype). The match is by allele, so without this a `0/0` or `./.` record, the `0/0` half of a split multiallelic record and a reference-only row with ALT `.` would be listed as hits. A half call such as `./1` carries the ALT allele and is a hit; `GT="alt"` would drop it. The filter runs after the split, where it sees each allele's own genotype. No command line goes into the hits file's header (`--no-version`), so ClinVar's path on your machine stays out of it. +6. Keeps the sample's records and combined alleles whose allele is in ClinVar (`bcftools isec -n=2 -w1`), copies ClinVar's `ID`, `GENEINFO`, `CLNSIG`, `CLNREVSTAT`, `CLNDN` and `CLNDISDB` onto them (`bcftools annotate --pair-logic exact`), and keeps only those whose genotype carries an ALT allele (`bcftools view -i 'GT~"[1-9]"'`, a non-zero allele index anywhere in the genotype). The match is by allele, so without this a `0/0` or `./.` record, the `0/0` half of a split multiallelic record and a reference-only row with ALT `.` would be listed as hits. A half call such as `./1` carries the ALT allele and is a hit; `GT="alt"` would drop it. The filter runs after the split, where it sees each allele's own genotype. No command line goes into the hits file's header (`--no-version`), so ClinVar's path on your machine stays out of it. `CLNDN` is the condition ClinVar names for the allele and `CLNDISDB` its ids in disease databases such as MONDO, OMIM and MedGen. A gene can cause more than one disease, so a later step reads these to know which one ClinVar asserted. Every ClinVar release defines both. `bcftools annotate` stops on a tag the ClinVar file does not define, so a hand-built ClinVar file without them gives hits without them, and the step prints a notice. 7. Marks the hits that belong together. A matched combined allele is a hit of its own and carries `INFO/PGP_JOINED_FROM`, its two parts as `CHROM:POS:REF:ALT`. Each of its parts that is a hit too keeps its row and gets `INFO/PGP_PART_OF`, the combined record it belongs to, so the reader sees one allele with the combined record's own review status, not two unrelated hits. Two hits in one gene that could be one allele by the rule of item 5 but matched no combined record each get `INFO/PGP_ONE_ALLELE`, the other hit: possibly one allele, check ClinVar for a combined record. 8. Writes a call floor other than the default into the hits file's header, for example `##PGP_CALL_FLOOR=GQ=20,DP=8` (below). At the default the file has no such line and reads as GQ 30 and DP 10. 9. Writes the always-report list (below): the sample's call at every site of `assets/deep_intronic_sites.tsv`. @@ -100,8 +100,8 @@ The [runtime table](hardware-requirements.md#runtime-per-step) gives the measure | File | Description | |---|---| -| `clinvar/${SAMPLE}_clinvar_hits.vcf` | **The hits: your records that carry a ClinVar Pathogenic/Likely_pathogenic allele, with ClinVar's ID, gene (`GENEINFO`), significance (`CLNSIG`) and review status (`CLNREVSTAT`). Your genotype, GQ and DP are in the sample column, and a `##PGP_CALL_FLOOR` header line holds the call floor of a run that changed it.** A combined allele of two neighbouring records is a record of its own, with `PGP_JOINED_FROM`; its parts carry `PGP_PART_OF`, and two hits that may be one allele carry `PGP_ONE_ALLELE` | -| `clinvar/${SAMPLE}_clinvar_hits.tsv` | The same hits, one row each: `chrom`, `pos`, `ref`, `alt`, `genotype`, `clinvar_id`, `geneinfo`, `clnsig`, `clnrevstat` | +| `clinvar/${SAMPLE}_clinvar_hits.vcf` | **The hits: your records that carry a ClinVar Pathogenic/Likely_pathogenic allele, with ClinVar's ID, gene (`GENEINFO`), significance (`CLNSIG`), review status (`CLNREVSTAT`) and condition (`CLNDN`, with its disease database ids in `CLNDISDB`). Your genotype, GQ and DP are in the sample column, and a `##PGP_CALL_FLOOR` header line holds the call floor of a run that changed it.** A combined allele of two neighbouring records is a record of its own, with `PGP_JOINED_FROM`; its parts carry `PGP_PART_OF`, and two hits that may be one allele carry `PGP_ONE_ALLELE` | +| `clinvar/${SAMPLE}_clinvar_hits.tsv` | The same hits, one row each: `chrom`, `pos`, `ref`, `alt`, `genotype`, `clinvar_id`, `geneinfo`, `clnsig`, `clnrevstat`. The condition is in the hits VCF only | | `clinvar/${SAMPLE}_clinvar_cdna_check.tsv` | The cDNA read check, one row per hit: `chrom`, `pos`, `ref`, `alt`, `alt_reads`, `alt_clipped`, `ref_reads`, `ref_clipped` (the reads with each allele, and how many of them are clipped at an exon edge), `cdna_check` (`flagged`, `clear` or `not_checked`), `cdna_note` (the flag text or the reason it was not checked) and `construct_site` (the gene and name of a known lab-construct site, empty otherwise) | | `clinvar/${SAMPLE}_clinvar_cdna_scan.tsv` | The cDNA scan: a `# cdna_scan:` line, then one row per listed locus: `chrom`, `start` and `end` (the first junction's exon end and the last junction's exon start, 1-based), `junctions`, `reads` (distinct reads, so a read across two junctions counts once) and `junction_list` (`end-start:reads` per junction) | | `clinvar/${SAMPLE}_clinvar_deep_intronic.tsv` | The always-report list, one row per site: the site's `chrom`, `pos`, `ref`, `alt`, `gene`, `hgvs` and `clinvar_id`, then `call` and `detail` | diff --git a/modules/local/clinvar_screen/main.nf b/modules/local/clinvar_screen/main.nf index 65f006875..f8241cedb 100644 --- a/modules/local/clinvar_screen/main.nf +++ b/modules/local/clinvar_screen/main.nf @@ -6,7 +6,8 @@ PASS variants against the ClinVar pathogenic subset. Writes _clinvar_hits.vcf: the sample's matching records the person carries (genotype with an ALT allele) with ClinVar's ID, GENEINFO, CLNSIG and - CLNREVSTAT copied on; _clinvar_hits.tsv: the same hits, one row each; + CLNREVSTAT copied on, and ClinVar's condition (CLNDN, CLNDISDB) when the + ClinVar file defines it; _clinvar_hits.tsv: the same hits, one row each; and prints the hits per ClinVar review-status star tier. Both files are read only on the contigs they share with each other and @@ -177,10 +178,20 @@ process CLINVAR_SCREEN { # Last, the tag pass marks each part of a matched combined allele # (PGP_PART_OF), and two hits in one gene that can be one allele but # matched no combined record (PGP_ONE_ALLELE). + # The condition goes on too, as scripts/06 copies it: CLNDN (ClinVar's + # disease name) and CLNDISDB (its disease database ids), so a later step + # can tell which of a gene's diseases ClinVar asserted. bcftools annotate + # stops on a tag the ClinVar file does not define, so each is copied only + # when the file's header defines it (every ClinVar release does). + clinvar_cols="ID,INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT\$(bcftools view -h "\${CLINVAR_NORM}" | awk '/^##INFO= '$(cpath "${HITS}.part")' rm -f '$(cpath "${HITS}.untagged")' From 3c8f08110964e316fa24d7723b2811d43e92d814 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:07:07 +0100 Subject: [PATCH 098/255] test(clinvar): fake-docker case for the condition on step 06 hits Runs scripts/06 against a fake docker with a ClinVar header that defines CLNDN and CLNDISDB and one that does not, and runs the module's column line against the same two headers. Red on main in both halves. Part of #411 Bead: i411-hits-keep-condition --- tests/fake-docker/i411-hits-keep-condition.sh | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 tests/fake-docker/i411-hits-keep-condition.sh diff --git a/tests/fake-docker/i411-hits-keep-condition.sh b/tests/fake-docker/i411-hits-keep-condition.sh new file mode 100755 index 000000000..851bc9dec --- /dev/null +++ b/tests/fake-docker/i411-hits-keep-condition.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Step 06 copies ClinVar's condition onto each hit, in both mirrors: +# - scripts/06-clinvar-screen.sh: the bcftools annotate that writes the hits +# VCF copies INFO/CLNDN and INFO/CLNDISDB with ID, GENEINFO, CLNSIG and +# CLNREVSTAT when ClinVar's header defines them (every ClinVar release +# does). Without them a later step knows the gene but not which of its +# diseases ClinVar asserted for the allele; +# - a hand-built ClinVar file without them still runs: bcftools annotate +# stops on a tag the file does not define, so they are left out, with a +# notice; +# - modules/local/clinvar_screen/main.nf builds the same column list from +# the same header and hands it to its own bcftools annotate. +# shellcheck source=../../scripts/ci/fake-docker/lib.sh +. "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" + +export GENOME_DIR="${CASE_WORK}/genome" +seed_reference "$GENOME_DIR" +seed_clinvar "$GENOME_DIR" +seed_sample "$GENOME_DIR" sample1 +use_output_hook + +# ClinVar's own header lines for the two tags, from a ClinVar release. +CLNDN_LINE="##INFO=" +CLNDISDB_LINE='##INFO=' +BASE_HEADER='##fileformat=VCFv4.1 +##INFO= +##INFO= +##INFO=' +CHROM_LINE=$(printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO') + +# The answers step 06 reads from bcftools: one contig with records on both +# sides, a PASS record, record counts, and ClinVar's header from +# ${CASE_WORK}/clinvar-header.txt. Every call also gets its outputs created. +cat > "${CASE_WORK}/hook-step06" <<'HOOK' +#!/usr/bin/env bash +"${CASE_WORK:?}/hook-outputs" "$@" || exit $? +shift +case "$*" in + *"index -s "*) printf 'chr1\t248956422\t3\n' ;; + *"index -n "*) echo 3 ;; + *"view -H -f PASS "*) echo 1 ;; + *"view -h "*clinvar_pathogenic_chr.norm.vcf.gz*) cat "${CASE_WORK}/clinvar-header.txt" ;; + *"view -h "*) printf '##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tsample1\n' ;; +esac +exit 0 +HOOK +chmod +x "${CASE_WORK}/hook-step06" +export FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/hook-step06" + +# The step stops later, at an output the fake docker does not write (an -o +# relative to -w), so its exit code is not checked: the annotate call +# comes before it. +# annotate_cols: the -c list of the hits VCF's bcftools annotate in the docker log. +annotate_cols() { + awk '/^run / && /bcftools annotate / && /pair-logic exact/' "$FAKE_DOCKER_LOG" \ + | grep -oE -- "-c[^ ]* [^ ]*INFO/CLNREVSTAT[^ ]*" | head -n 1 +} + +# --- scripts/06: a ClinVar release defines both tags ------------------------------ +printf '%s\n%s\n%s\n%s\n' "$BASE_HEADER" "$CLNDN_LINE" "$CLNDISDB_LINE" "$CHROM_LINE" > "${CASE_WORK}/clinvar-header.txt" +: > "$FAKE_DOCKER_LOG" +run_rc step06-release "${SCRIPTS}/06-clinvar-screen.sh" sample1 +COLS=$(annotate_cols) +echo "annotate columns: ${COLS:-none}" +[ -n "$COLS" ] || fail "step 06 ran no bcftools annotate for the hits VCF" +grep -qF 'ID,INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT,INFO/CLNDN,INFO/CLNDISDB' <<<"$COLS" \ + || fail "step 06 does not copy ClinVar's condition (INFO/CLNDN, INFO/CLNDISDB) onto the hits: ${COLS}" +output_lacks step06-release 'does not define CLNDN' + +# --- scripts/06: a hand-built ClinVar file without them -------------------------- +printf '%s\n%s\n' "$BASE_HEADER" "$CHROM_LINE" > "${CASE_WORK}/clinvar-header.txt" +rm -rf "${GENOME_DIR}/sample1/clinvar" +: > "$FAKE_DOCKER_LOG" +run_rc step06-handbuilt "${SCRIPTS}/06-clinvar-screen.sh" sample1 +COLS=$(annotate_cols) +echo "annotate columns: ${COLS:-none}" +[ -n "$COLS" ] || fail "step 06 ran no bcftools annotate for a ClinVar file without CLNDN" +if grep -q 'CLNDN\|CLNDISDB' <<<"$COLS"; then + fail "step 06 asks bcftools annotate for CLNDN or CLNDISDB, which this ClinVar file does not define (bcftools stops): ${COLS}" +fi +grep -qF 'ID,INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT' <<<"$COLS" || fail "step 06 lost ClinVar's ID, gene, significance or review status: ${COLS}" +output_has step06-handbuilt 'does not define CLNDN and CLNDISDB' + +# --- the Nextflow module: the same columns, from the same header ------------------ +# Its clinvar_cols line, with the Groovy escapes undone, runs with a bcftools +# that prints the header; its annotate must take that list. +MOD="${REPO_ROOT}/modules/local/clinvar_screen/main.nf" +LINE=$(awk '/^process CLINVAR_SCREEN \{/ { on = 1 } on && /^ *clinvar_cols=/ { print; exit }' "$MOD") +[ -n "$LINE" ] || fail "the ClinVar screen module builds no clinvar_cols list for bcftools annotate" +LINE=$(sed -e 's/\\\$/$/g' -e 's/\\\\/\\/g' <<<"$LINE") +# shellcheck disable=SC2016 # the module's text, Groovy escape included +awk '/^process CLINVAR_SCREEN \{/ { on = 1 } on && /bcftools annotate .*pair-logic exact/ { getline; print; exit }' "$MOD" \ + | grep -qF -- '-c "\${clinvar_cols}"' || fail "the module's bcftools annotate does not take clinvar_cols" +# module_cols HEADER_FILE: the list the module's line builds from that header. +module_cols() { + # shellcheck disable=SC2034 # read by the module's line + CLINVAR_NORM=clinvar.norm.vcf.gz HEADER=$1 bash -c 'set -euo pipefail; bcftools() { cat "$HEADER"; }; eval "$1"; printf "%s" "$clinvar_cols"' _ "$LINE" +} +printf '%s\n%s\n%s\n%s\n' "$BASE_HEADER" "$CLNDISDB_LINE" "$CLNDN_LINE" "$CHROM_LINE" > "${CASE_WORK}/release-header.txt" +printf '%s\n%s\n' "$BASE_HEADER" "$CHROM_LINE" > "${CASE_WORK}/handbuilt-header.txt" +GOT=$(module_cols "${CASE_WORK}/release-header.txt") +[ "$GOT" = 'ID,INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT,INFO/CLNDN,INFO/CLNDISDB' ] \ + || fail "the module copies '${GOT}' from a ClinVar release, want ClinVar's condition too (INFO/CLNDN, INFO/CLNDISDB)" +GOT=$(module_cols "${CASE_WORK}/handbuilt-header.txt") +[ "$GOT" = 'ID,INFO/GENEINFO,INFO/CLNSIG,INFO/CLNREVSTAT' ] \ + || fail "the module copies '${GOT}' from a ClinVar file without CLNDN and CLNDISDB, want only ID, GENEINFO, CLNSIG and CLNREVSTAT" +echo "both mirrors copy ClinVar's condition when ClinVar defines it: PASS" From cd801724a91799a62122ae5dbc36155497654054 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:08:47 +0100 Subject: [PATCH 099/255] fix(roh): stream chromosome scope and emit neutral region hints Use assessed autosomes for the length denominator. Require per-site evidence on all 22 autosomes for genome-wide labels. Embed the one attributed imprinting asset and keep legacy parent keys as placeholders. Bead: lin-10 Bead: lin-13 Bead: lin-15 Bead: lin-16 Bead: lin-17 Bead: lic-41 --- assets/imprinted_regions.tsv | 21 +- .../og-summary/src/collect/group_c/roh.rs | 296 +++++++++--------- .../og-summary/src/collect/group_c/tests.rs | 63 +++- 3 files changed, 217 insertions(+), 163 deletions(-) diff --git a/assets/imprinted_regions.tsv b/assets/imprinted_regions.tsv index c71bb24dd..b55146765 100644 --- a/assets/imprinted_regions.tsv +++ b/assets/imprinted_regions.tsv @@ -1,14 +1,17 @@ -# Imprinting disorders caused by uniparental disomy (UPD), one row per chromosome or region. -# Read by og summary collect (IMPRINTED_REGIONS in rust/crates/og-summary holds the same rows; its -# tests check that the two agree) to name what a run of homozygosity over most of one chromosome could mean. -# start and end: GRCh38, 0-based start, exclusive end, the UCSC cytoBand.hg38.txt bands of the region; -# empty for a row that applies to the whole chromosome. A row with a region applies when a run on the -# flagged chromosome overlaps it. "-" is no known imprinting disorder for UPD from that parent. -# Overview: Eggermann et al., Imprinting disorders, Nat Rev Dis Primers 2023 (PMID 37386011). +# Imprinted chromosomes and regions. Factual reference rows, curated by OwnGenome. +# Sole source: embedded by og summary collect at build time. The UPD hint emits region and PMID only; +# maternal_upd and paternal_upd remain reference facts here, never a diagnosis from a ROH match. +# start and end are retained for format compatibility and are empty: the hint is chromosome-wide. +# Mixed iso/heterodisomy may leave the imprinted locus outside the observed homozygous runs. +# Region labels are broad cytobands, not diagnostic locus coordinates or a specialist-reviewed assay. +# Overview: Eggermann et al., Imprinting disorders, Nat Rev Dis Primers 9, 33 (2023). +# https://doi.org/10.1038/s41572-023-00443-4 (PMID 37386011). +# Mechanism: Robinson, BioEssays 22:452-459 (2000), https://pubmed.ncbi.nlm.nih.gov/10797485/. +# Per-row PMIDs support the parent-of-origin reference facts; names and symbols are factual labels. chrom start end region maternal_upd paternal_upd pmid chr6 6q24 (PLAGL1, HYMAI) - Transient neonatal diabetes mellitus (6q24-related) 20301706 chr7 7p12 (GRB10), 7q32 (MEST) Silver-Russell syndrome - 27585961 -chr11 0 22000000 11p15 (H19/IGF2, KCNQ1OT1) Silver-Russell syndrome Beckwith-Wiedemann syndrome 29377879;27585961 +chr11 11p15 (H19/IGF2, KCNQ1OT1) Silver-Russell syndrome Beckwith-Wiedemann syndrome 29377879;27585961 chr14 14q32.2 (DLK1-MEG3) Temple syndrome Kagami-Ogata syndrome 24891339;26377239 -chr15 20500000 33400000 15q11-q13 (SNRPN, UBE3A) Prader-Willi syndrome Angelman syndrome 22237428;27615419 +chr15 15q11-q13 (SNRPN, UBE3A) Prader-Willi syndrome Angelman syndrome 22237428;27615419 chr20 20q13.32 (GNAS) Mulchandani-Bhoj-Conlin syndrome (growth failure) Pseudohypoparathyroidism type 1B 26248010;29959430 diff --git a/rust/crates/og-summary/src/collect/group_c/roh.rs b/rust/crates/og-summary/src/collect/group_c/roh.rs index f0fb24dc4..2724c416d 100644 --- a/rust/crates/og-summary/src/collect/group_c/roh.rs +++ b/rust/crates/og-summary/src/collect/group_c/roh.rs @@ -51,6 +51,54 @@ pub const MASKED_BANDS: [(&str, i128, i128); 25] = [ /// GRCh38's chr1 to chr22 (UCSC hg38.chrom.sizes). pub const AUTOSOME_BP: i128 = 2_875_001_522; + +// UCSC hg38.chrom.sizes, in chromosome order. ST rows tell us which +// chromosome lengths belong in this run's denominator, not callable bases. +const AUTOSOME_LENGTHS: [i128; 22] = [ + 248_956_422, + 242_193_529, + 198_295_559, + 190_214_555, + 181_538_259, + 170_805_979, + 159_345_973, + 145_138_636, + 138_394_717, + 133_797_422, + 135_086_622, + 133_275_309, + 114_364_328, + 107_043_718, + 101_991_189, + 90_338_345, + 83_257_441, + 80_373_285, + 58_617_616, + 64_444_167, + 46_709_983, + 50_818_468, +]; + +fn autosome_index(chrom: &str) -> Option { + is_autosome(chrom).then(|| chrom[3..].parse::().unwrap() - 1) +} + +fn covered_unmasked_bp(covered: &[bool; 22]) -> i128 { + covered + .iter() + .enumerate() + .filter(|(_, yes)| **yes) + .map(|(i, _)| { + let chrom = format!("chr{}", i + 1); + AUTOSOME_LENGTHS[i] + - MASKED_BANDS + .iter() + .filter(|(c, ..)| *c == chrom) + .map(|(_, a, b)| b - a) + .sum::() + }) + .sum() +} /// The runs F_ROH and the one-chromosome flag count (docs/11-roh-analysis.md). pub const ROH_MIN_BP: f64 = 5e6; /// One chromosome carrying most of the long runs: at least this much of it @@ -123,87 +171,31 @@ pub fn roh_band(f: f64) -> &'static str { .map_or("parent and child, full siblings (F 1/4)", |b| b.1) } -/// assets/imprinted_regions.tsv, the same rows (a test keeps the two -/// equal): chrom, the region's span (none for the whole chromosome), the -/// region, the maternal and paternal UPD syndromes and the PMIDs. -/// chrom, span, then four text fields. -type Imprinted = ( - &'static str, - Option<(i128, i128)>, - &'static str, - &'static str, - &'static str, - &'static str, -); +/// One source for region names and their citations. Embed the shipped +/// asset so a staged report needs no second runtime file or copied table. +const IMPRINTED_TSV: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../assets/imprinted_regions.tsv" +)); -pub const IMPRINTED_REGIONS: [Imprinted; 6] = [ - ( - "chr6", - None, - "6q24 (PLAGL1, HYMAI)", - "-", - "Transient neonatal diabetes mellitus (6q24-related)", - "20301706", - ), - ( - "chr7", - None, - "7p12 (GRB10), 7q32 (MEST)", - "Silver-Russell syndrome", - "-", - "27585961", - ), - ( - "chr11", - Some((0, 22_000_000)), - "11p15 (H19/IGF2, KCNQ1OT1)", - "Silver-Russell syndrome", - "Beckwith-Wiedemann syndrome", - "29377879;27585961", - ), - ( - "chr14", - None, - "14q32.2 (DLK1-MEG3)", - "Temple syndrome", - "Kagami-Ogata syndrome", - "24891339;26377239", - ), - ( - "chr15", - Some((20_500_000, 33_400_000)), - "15q11-q13 (SNRPN, UBE3A)", - "Prader-Willi syndrome", - "Angelman syndrome", - "22237428;27615419", - ), - ( - "chr20", - None, - "20q13.32 (GNAS)", - "Mulchandani-Bhoj-Conlin syndrome (growth failure)", - "Pseudohypoparathyroidism type 1B", - "26248010;29959430", - ), -]; - -/// The imprinted regions a flagged chromosome joins: every row of the -/// whole chromosome, and a row with a region only when a run overlaps it. -fn imprinted(chrom: &str, runs: &[(i128, i128)]) -> Value { +/// A chromosome hint cannot determine parental origin or diagnose a +/// syndrome. The two legacy keys stay as placeholders for old readers. +fn imprinted(chrom: &str) -> Value { Value::Array( - IMPRINTED_REGIONS - .iter() - .filter(|r| r.0 == chrom) - .filter(|r| { - r.1.is_none_or(|(a, b)| runs.iter().any(|&(s, e)| overlap(a, b, s, e) > 0)) - }) - .map(|r| { - Obj::new() - .set("region", s(r.2)) - .set("maternal_upd", s(r.3)) - .set("paternal_upd", s(r.4)) - .set("pmid", s(r.5)) - .val() + IMPRINTED_TSV + .lines() + .filter(|line| !line.starts_with('#')) + .skip(1) + .filter_map(|line| { + let f: Vec<&str> = line.split('\t').collect(); + (f.len() == 7 && f[0] == chrom).then(|| { + Obj::new() + .set("region", s(f[3])) + .set("maternal_upd", s("-")) + .set("paternal_upd", s("-")) + .set("pmid", s(f[6])) + .val() + }) }) .collect(), ) @@ -228,11 +220,25 @@ pub fn section(c: &Ctx<'_>) -> Out { let (mut total, mut largest, mut masked) = (0.0_f64, 0.0_f64, 0.0_f64); let (mut segs, mut sex_segs) = (0usize, 0usize); let mut big = Vec::new(); - // chrom -> the bases of its long runs, and the runs (start, end) - let mut per_chrom: indexmap::IndexMap)> = - indexmap::IndexMap::new(); - for line in lines(&p.path)? { - let c = pycompat::text::split(&line, None, None); + // chrom -> the bases of its long runs + let mut per_chrom: indexmap::IndexMap = indexmap::IndexMap::new(); + let (mut assessed, mut observed) = ([false; 22], [false; 22]); + let mut reader = crate::text_lines(&p.path)?; + let mut line = String::new(); + loop { + line.clear(); + if !reader.read_line(&mut line).map_err(|e| e.to_string())? { + break; + } + let c: Vec<&str> = line.split_whitespace().collect(); + if c.len() >= 6 + && c[0] == "ST" + && let Some(i) = autosome_index(c[2]) + && crate::int(c[3]).is_ok_and(|pos| pos > 0 && pos <= AUTOSOME_LENGTHS[i]) + && matches!(c[4], "0" | "1") + { + assessed[i] = true; + } if c.len() < 6 || c[0] != "RG" { continue; } @@ -257,6 +263,7 @@ pub fn section(c: &Ctx<'_>) -> Out { if !is_autosome(c[2]) { continue; } + observed[autosome_index(c[2]).unwrap()] = true; let inside = length.min(masked_overlap(c[2], start, end) as f64); masked += inside; length -= inside; @@ -268,8 +275,7 @@ pub fn section(c: &Ctx<'_>) -> Out { largest = largest.max(length); if length >= ROH_MIN_BP { let pc = per_chrom.entry(c[2].to_owned()).or_default(); - pc.0 += length; - pc.1.push((start, end)); + *pc += length; let mut row = Obj::new() .set("region", s(&format!("{}:{}-{}", c[2], c[3], c[4]))) .set("mb", r1(length / 1e6, 1)); @@ -279,19 +285,30 @@ pub fn section(c: &Ctx<'_>) -> Out { big.push(row.val()); } } - let long_bp: f64 = per_chrom.values().fold(0.0, |a, v| a + v.0); - let f_roh = long_bp / autosome_unmasked_bp() as f64; + let long_bp: f64 = per_chrom.values().sum(); + let has_sites = assessed.iter().any(|yes| *yes); + // Old -O r files omit ST rows. Their runs give a lower bound on + // chromosome scope, never evidence for a genome-wide interpretation. + let covered = std::array::from_fn(|i| assessed[i] || observed[i]); + let denominator = covered_unmasked_bp(&covered); + let complete = has_sites + && assessed.iter().all(|yes| *yes) + && denominator >= autosome_unmasked_bp() + && observed + .iter() + .zip(assessed) + .all(|(run, site)| !run || site); + let f_roh = (denominator > 0).then(|| long_bp / denominator as f64); let mut upd = Value::Null; - // The first chromosome of the most bases, as max() picks. - let top = - per_chrom.iter().fold( - None::<(&String, &(f64, Vec<(i128, i128)>))>, - |best, kv| match best { - Some(b) if b.1.0 >= kv.1.0 => Some(b), - _ => Some(kv), - }, - ); - if let Some((chrom, (bp, runs))) = top + // Retain the first chromosome on ties, as before. + let top = per_chrom + .iter() + .fold(None::<(&String, &f64)>, |best, kv| match best { + Some(b) if b.1 >= kv.1 => Some(b), + _ => Some(kv), + }); + if complete + && let Some((chrom, bp)) = top && *bp >= UPD_MIN_BP && *bp > UPD_SHARE * long_bp { @@ -299,11 +316,12 @@ pub fn section(c: &Ctx<'_>) -> Out { .set("chrom", s(chrom)) .set("mb", r1(bp / 1e6, 1)) .set("share", r1(bp / long_bp, 2)) - .set("imprinted", imprinted(chrom, runs)) + .set("imprinted", imprinted(chrom)) + .set("note", s("Long chromosome-focused ROH; investigate possible isodisomy. A deletion, acquired loss of heterozygosity or shared ancestry can give this pattern. Copy neutrality and parental origin are not established; check copy-number results and parents' genotypes. Heterodisomy can leave no ROH.")) .val(); } - let relationship = if upd == Value::Null { - s(roh_band(f_roh)) + let relationship = if complete && upd == Value::Null { + s(roh_band(f_roh.unwrap_or(0.0))) } else { Value::Null }; @@ -315,9 +333,20 @@ pub fn section(c: &Ctx<'_>) -> Out { .set("masked_mb", r1(masked / 1e6, 1)) .set("sex_chromosome_segments", count(sex_segs)) .set("long_mb", r1(long_bp / 1e6, 1)) - .set("f_roh", r1(f_roh, 4)) + .set("f_roh", f_roh.map_or(Value::Null, |f| r1(f, 4))) .set("relationship", relationship) - .set("upd_flag", upd); + .set("upd_flag", upd) + .set("scope", Obj::new() + .set("source", s(if has_sites { "sites" } else { "segments_only" })) + .set("autosomes", Value::Array(covered.iter().enumerate() + .filter(|(_, yes)| **yes).map(|(i, _)| s(&format!("chr{}", i + 1))).collect())) + .set("unmasked_bp", Value::Int(denominator)) + .set("partial", Value::Bool(!complete)) + .set("note", s(if complete { + "Sites on all 22 autosomes; marker presence does not establish callable breadth or copy neutrality." + } else { + "Partial or unverified chromosome scope: no relationship band or UPD flag. F_ROH uses only the listed chromosomes, not callable bases." + })).val()); Ok(Some((p, data.val()))) } @@ -443,47 +472,30 @@ mod tests { use super::*; use crate::testutil::Dir; - /// assets/imprinted_regions.tsv holds the rows of [`IMPRINTED_REGIONS`] - /// (tests/test_collect_summary.py checked the Python copy the same way). #[test] - fn imprinted_regions_are_the_asset_rows() { - let path = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../../assets/imprinted_regions.tsv" - ); - let text = std::fs::read_to_string(path).unwrap(); - let mut lines = text.lines().filter(|l| !l.starts_with('#')); - let head: Vec<&str> = lines.next().unwrap().split('\t').collect(); - let col = |name: &str| head.iter().position(|h| *h == name).unwrap(); - let num = |x: &str| (!x.is_empty()).then(|| x.parse::().unwrap()); - let rows: Vec<_> = lines - .map(|l| { - let f: Vec<&str> = l.split('\t').collect(); - let span = num(f[col("start")]).zip(num(f[col("end")])); - ( - f[col("chrom")].to_owned(), - span, - f[col("region")].to_owned(), - f[col("maternal_upd")].to_owned(), - f[col("paternal_upd")].to_owned(), - f[col("pmid")].to_owned(), - ) - }) - .collect(); - let want: Vec<_> = IMPRINTED_REGIONS - .iter() - .map(|r| { - ( - r.0.to_owned(), - r.1, - r.2.to_owned(), - r.3.to_owned(), - r.4.to_owned(), - r.5.to_owned(), - ) - }) + fn imprinted_asset_is_the_only_table() { + let rows: Vec<_> = IMPRINTED_TSV + .lines() + .filter(|l| !l.starts_with('#')) + .skip(1) .collect(); - assert_eq!(rows, want); + assert_eq!(rows.len(), 6); + for chrom in ["chr6", "chr7", "chr11", "chr14", "chr15", "chr20"] { + let Value::Array(hints) = imprinted(chrom) else { + panic!() + }; + assert_eq!(hints.len(), 1); + let row = rows + .iter() + .find(|line| line.starts_with(&format!("{chrom}\t"))) + .unwrap(); + let f: Vec<_> = row.split('\t').collect(); + assert_eq!(f.len(), 7); + assert!(f[6].split(';').all(|pmid| pmid.parse::().is_ok())); + assert!(f[1].is_empty() && f[2].is_empty()); + } + assert_eq!(AUTOSOME_LENGTHS.iter().sum::(), AUTOSOME_BP); + assert_eq!(covered_unmasked_bp(&[true; 22]), autosome_unmasked_bp()); } #[test] diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index b072af7cb..4f34ef10f 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -244,13 +244,19 @@ fn roh_scope_chr20_uses_its_own_unmasked_denominator() { assert_eq!(at(&r, &["relationship"]), &Value::Null); assert_eq!(at(&r, &["upd_flag"]), &Value::Null); assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(59_744_167)); - assert_eq!(at(&r, &["scope", "autosomes"]), &Value::Array(vec![Value::Str("chr20".into())])); + assert_eq!( + at(&r, &["scope", "autosomes"]), + &Value::Array(vec![Value::Str("chr20".into())]) + ); } #[test] fn roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty() { - let r = roh_scope_summary(&[22, 20, 20], "RG\tS\tchr20\t1000001\t9000000\t8000000\nRG\tS\tchr22\t20000001\t28000000\t8000000\n"); - assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.1718)); + let r = roh_scope_summary( + &[22, 20, 20], + "RG\tS\tchr20\t1000001\t9000000\t8000000\nRG\tS\tchr22\t20000001\t28000000\t8000000\n", + ); + assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.1717)); assert_eq!(at(&r, &["relationship"]), &Value::Null); assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(93_162_635)); let no_runs = roh_scope_summary(&[20], ""); @@ -263,8 +269,14 @@ fn roh_scope_sparse_all_autosomes_keep_the_whole_genome_band() { let sites: Vec = (1..=22).collect(); let r = roh_scope_summary(&sites, "RG\tS\tchr2\t10000001\t18000000\t8000000\n"); assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.003)); - assert_eq!(at(&r, &["relationship"]), &Value::Str("no sign of related parents".into())); - assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(2_658_001_522)); + assert_eq!( + at(&r, &["relationship"]), + &Value::Str("no sign of related parents".into()) + ); + assert_eq!( + at(&r, &["scope", "unmasked_bp"]), + &Value::Int(2_658_001_522) + ); assert_eq!(at(&r, &["scope", "partial"]), &Value::Bool(false)); } @@ -274,7 +286,10 @@ fn roh_scope_a_segment_only_file_never_establishes_scope() { assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.3515)); assert_eq!(at(&r, &["relationship"]), &Value::Null); assert_eq!(at(&r, &["upd_flag"]), &Value::Null); - assert_eq!(at(&r, &["scope", "source"]), &Value::Str("segments_only".into())); + assert_eq!( + at(&r, &["scope", "source"]), + &Value::Str("segments_only".into()) + ); let empty = roh_scope_summary(&[], "# empty\n"); assert_eq!(at(&empty, &["f_roh"]), &Value::Null); } @@ -285,15 +300,39 @@ fn roh_scope_upd_is_a_region_hint_without_syndrome_names() { // Distal chr15 and chr11 runs do not establish the imprinted locus's // parental origin, but mixed iso/heterodisomy can involve that locus. for (chrom, start, end, length, region) in [ - (15, 40000001, 101991189, 61991189, "15q11-q13 (SNRPN, UBE3A)"), - (11, 60000001, 100000000, 40000000, "11p15 (H19/IGF2, KCNQ1OT1)"), + ( + 15, + 40000001, + 101991189, + 61991189, + "15q11-q13 (SNRPN, UBE3A)", + ), + ( + 11, + 60000001, + 100000000, + 40000000, + "11p15 (H19/IGF2, KCNQ1OT1)", + ), (20, 1000001, 22000000, 21000000, "20q13.32 (GNAS)"), ] { - let r = roh_scope_summary(&sites, &format!("RG\tS\tchr{chrom}\t{start}\t{end}\t{length}\n")); + let r = roh_scope_summary( + &sites, + &format!("RG\tS\tchr{chrom}\t{start}\t{end}\t{length}\n"), + ); let u = at(&r, &["upd_flag"]); - assert_eq!(at(u, &["imprinted", "0", "region"]), &Value::Str(region.into())); - assert_eq!(at(u, &["imprinted", "0", "maternal_upd"]), &Value::Str("-".into())); - assert_eq!(at(u, &["imprinted", "0", "paternal_upd"]), &Value::Str("-".into())); + assert_eq!( + at(u, &["imprinted", "0", "region"]), + &Value::Str(region.into()) + ); + assert_eq!( + at(u, &["imprinted", "0", "maternal_upd"]), + &Value::Str("-".into()) + ); + assert_eq!( + at(u, &["imprinted", "0", "paternal_upd"]), + &Value::Str("-".into()) + ); let note = text(at(u, &["note"])); assert!(note.contains("deletion") && note.contains("acquired") && note.contains("parent")); assert_eq!(at(&r, &["relationship"]), &Value::Null); From 0de4fd2b5696e0cd548b1892fcfeaabfea0b9618 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:12:53 +0100 Subject: [PATCH 100/255] fix(archive): verify complete read content before removing BAM Preserve explicit MD/NM tags and compare order-independent checksums of all records and tags in archive and restore. Keep the source on failed checks. Bead: plat-62 --- CHANGELOG.md | 2 + assets/output_contract.tsv | 2 + docs/34-cram-archive.md | 31 +++++--- modules/local/cram_archive/main.nf | 38 +++++++-- scripts/34-cram-archive.sh | 44 +++++++++-- scripts/ci/e2e-case-paths.tsv | 1 + scripts/ci/e2e-case-times.tsv | 1 + tests/e2e/qc-3-cram-checksum.sh | 115 ++++++++++++++++++++++++++++ tests/fake-docker/qc-steps-33-34.sh | 31 +++++++- 9 files changed, 241 insertions(+), 24 deletions(-) create mode 100755 tests/e2e/qc-3-cram-checksum.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..4e3a75bfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- CRAM archiving now compares full-content samtools checksums before `--delete-bam` can remove the BAM. Matching read counts alone could accept changed or unreadable read data. Archive and restore in bash and Nextflow now check all records, alignment fields and tags, independent of order. Explicit MD/NM tags are preserved and missing tags stay missing. A mismatch or checksum failure keeps the source and removes the partial output. The step adds `.cram.checksum` and `.bam.checksum` reports beside the alignments; Nextflow publishes the CRAM report. Existing output names and formats stay the same. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/assets/output_contract.tsv b/assets/output_contract.tsv index 1d567bf17..16f876eac 100644 --- a/assets/output_contract.tsv +++ b/assets/output_contract.tsv @@ -57,6 +57,8 @@ file {sample}/aligned/{sample}_sorted.bam.bai 02 ALIGN_MINIMAP2 both index stabl file {sample}/aligned/{sample}_sorted.cram 34 CRAM_ARCHIVE both cram experimental - - file {sample}/aligned/{sample}_sorted.cram.crai 34 CRAM_ARCHIVE both index experimental - - file {sample}/aligned/{sample}_sorted.cram.flagstat 34 CRAM_ARCHIVE both bin experimental - - +file {sample}/aligned/{sample}_sorted.cram.checksum 34 CRAM_ARCHIVE both text experimental - - +file {sample}/aligned/{sample}_sorted.bam.checksum 34 - bash text experimental - - file {sample}/ancestry/{sample}_ancestry.tsv 25 PRS_SUMMARY both tsv stable - - file {sample}/ancestry_fine/{sample}_ancestry_fine.tsv 26b ANCESTRY_FINE_REPORT both tsv experimental - - file {sample}/ancestry_fine/{sample}_ancestry_fine.txt 26b ANCESTRY_FINE_REPORT nextflow text experimental - - diff --git a/docs/34-cram-archive.md b/docs/34-cram-archive.md index cefcf3d42..d4d74b062 100644 --- a/docs/34-cram-archive.md +++ b/docs/34-cram-archive.md @@ -1,20 +1,20 @@ # Step 34: CRAM Archive ## What This Does -Writes the sample's alignments as CRAM beside the BAM, checks that the CRAM holds exactly the same reads, and, only when you ask and only after that check passed, deletes the BAM. `--restore` writes the BAM back from the CRAM. +Writes the sample's alignments as CRAM beside the BAM and checks their read content. It deletes the BAM only when you ask, after the checks pass. `--restore` writes the BAM back from the CRAM. ## Why -A 30x genome keeps 80 to 120 GB of BAM after the analysis is done. A CRAM stores each read as its difference from the reference and is about half that size or less (on the CI fixture, 68 MB of CRAM for 133 MB of BAM), with nothing lost: every read, base quality, flag and tag is still there. Most people keep their alignments to rerun a step when a tool or database improves; a CRAM keeps that option for half the disk. +A 30x genome keeps 80 to 120 GB of BAM after the analysis is done. A CRAM stores each read as its difference from the reference and is about half that size or less. The CI fixture has 68 MB of CRAM for 133 MB of BAM. Read content is checked before the original can be deleted. Most people keep their alignments to rerun a step when a tool or database improves. A CRAM keeps that option for half the disk. The price is the reference: a CRAM can only be read with a FASTA that holds the same sequences as the one it was written with. Keep `reference/GRCh38_no_alt_analysis_set.fasta` (or your `REF_FASTA`) as long as you keep the CRAM. samtools checks each contig's MD5 when it reads a CRAM and stops with an error on a different file, so a wrong reference cannot give silently wrong reads. ## Tool -- **samtools** (`view -C`, `index`, `quickcheck`, `flagstat`) +- **samtools** (`view -C`, `index`, `quickcheck`, `flagstat`, `checksum`) ## Docker Image - `SAMTOOLS_IMAGE` -Pinned in `versions.env`; [Image versions](versions.md) lists the current tag. +Pinned in `versions.env`. [Image versions](versions.md) lists the current tag. ## Command ```bash @@ -25,20 +25,26 @@ Pinned in `versions.env`; [Image versions](versions.md) lists the current tag. The script runs, in the samtools container: ```bash -samtools view -C --reference "${REF_FASTA}" -o aligned/${SAMPLE}_sorted.part.cram aligned/${SAMPLE}_sorted.bam +samtools view -C --reference "${REF_FASTA}" --output-fmt-option store_md=1 --output-fmt-option store_nm=1 -o aligned/${SAMPLE}_sorted.part.cram aligned/${SAMPLE}_sorted.bam samtools quickcheck -v aligned/${SAMPLE}_sorted.part.cram samtools index aligned/${SAMPLE}_sorted.part.cram samtools flagstat aligned/${SAMPLE}_sorted.bam samtools flagstat --input-fmt-option reference="${REF_FASTA}" aligned/${SAMPLE}_sorted.part.cram +samtools checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' aligned/${SAMPLE}_sorted.bam +samtools checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' --input-fmt-option reference="${REF_FASTA}" --input-fmt-option decode_md=0 aligned/${SAMPLE}_sorted.part.cram ``` -The check is that the CRAM passes `samtools quickcheck` (a CRAM cut short has no end-of-file block) and that the two `flagstat` outputs are equal line for line: the same number of reads, mapped, paired, properly paired, duplicates and secondary and supplementary alignments. Only then does the CRAM lose its `.part` name. When the check fails, the CRAM is removed, the BAM is kept and the step exits non-zero, also with `--delete-bam`. An older CRAM is always written again rather than trusted, because the BAM may have been aligned again since. +The CRAM must pass `samtools quickcheck`, the two `flagstat` reports must match, and the content checksums must match. The checksum reads every record, including secondary and supplementary alignments. It covers read names, sequences, qualities, flags, positions, CIGARs, mate fields and all auxiliary tags. A changed base quality or tag can leave `flagstat` counts unchanged and still fail this check. A decoding error also fails the step. -`--restore` does the same in reverse: the BAM is written under a `.part` name, checked against the CRAM, and moved into place. It refuses to write over a BAM that exists. +The checksums use samtools' order-independent aggregation. The comparison ignores the filename header and sorts the report rows, so read and tag order do not affect it. samtools sanitizes format-level differences, such as an unmapped read's MAPQ and equivalent CIGARs. Explicit `MD` and `NM` tags are stored in the CRAM. They are not regenerated during the check or restore, so missing tags stay missing. These CRC-based content checks detect conversion faults. They are not a cryptographic identity proof. See the [samtools 1.24 checksum manual](https://www.htslib.org/doc/1.24/samtools-checksum.html). -One archive or restore of a sample runs at a time. Archive, `--delete-bam` and `--restore` all take the same lock, `aligned/${SAMPLE}_sorted.lock`, with `flock` before they read or write any of the sample's files. A second run on the same sample, in another terminal or as a Nextflow `CRAM_ARCHIVE` task on that BAM, stops at once with `ERROR: another archive or restore of is running` and changes nothing; run it again when the first has finished. Without the lock, two runs would delete or overwrite the `.part` files the other is writing and checking, and `--delete-bam` could remove a BAM the other run is still reading. The lock goes when the run that holds it ends, however it ends (an error, Ctrl-C or `kill -9`), so it is never left behind; the empty `.lock` file stays and means nothing on its own. `flock` comes with util-linux on Linux; on macOS install it with `brew install flock`, or the step warns and runs without the lock. +Only after all checks pass does the CRAM lose its `.part` name. A mismatch or a failed or empty checksum command removes the partial CRAM, keeps the BAM and its index, and exits non-zero. This also applies with `--delete-bam`. An older published CRAM stays in place if a new attempt fails. Every archive attempt writes a fresh CRAM from the current BAM before it can delete anything. The checksum adds a full read of each file to the archive and restore checks. -In Nextflow, `cram_archive` in `--tools` runs the same conversion and check (`CRAM_ARCHIVE`) for every BAM of the run, except the rows given as CRAM, which have one already, and publishes `//aligned/_sorted.cram`. It never deletes anything: delete the BAM yourself once the CRAM is there, or run this script on it. It takes the same lock, next to the BAM the samplesheet row names (`.lock`), and fails the task while this script holds it. That exclusion between this script and a `CRAM_ARCHIVE` task holds on Linux, where the container shares the host kernel; on macOS with Docker Desktop only two runs of this script exclude each other. When it cannot write the lock file there, it warns and reads the BAM without the lock. +`--restore` does the same in reverse: the BAM is written under a `.part` name, checked against the CRAM, and moved into place. It refuses to write over a BAM that exists. A failed check removes the partial BAM and keeps the CRAM. + +One archive or restore of a sample runs at a time. Archive, `--delete-bam` and `--restore` all take the same lock, `aligned/${SAMPLE}_sorted.lock`, with `flock` before they read or write any of the sample's files. A second run on the same sample, from another terminal or a Nextflow `CRAM_ARCHIVE` task, stops at once with `ERROR: another archive or restore of is running`. It changes nothing. Run it again when the first has finished. Without the lock, two runs would delete or overwrite the `.part` files the other is writing and checking, and `--delete-bam` could remove a BAM the other run is still reading. The lock goes when its run ends, including on an error, Ctrl-C or `kill -9`. The empty `.lock` file stays and means nothing on its own. `flock` comes with util-linux on Linux. On macOS install it with `brew install flock`, or the step warns and runs without the lock. + +In Nextflow, `cram_archive` in `--tools` runs the same conversion and check (`CRAM_ARCHIVE`) for every BAM of the run, except the rows given as CRAM, which have one already, and publishes `//aligned/_sorted.cram`. It never deletes anything: delete the BAM yourself once the CRAM is there, or run this script on it. It takes the same lock, next to the BAM the samplesheet row names (`.lock`), and fails the task while this script holds it. This script and a `CRAM_ARCHIVE` task exclude each other on Linux, where the container shares the host kernel. On macOS with Docker Desktop, only two runs of this script exclude each other. When it cannot write the lock file there, it warns and reads the BAM without the lock. ## Output Files | File | Description | @@ -46,6 +52,7 @@ In Nextflow, `cram_archive` in `--tools` runs the same conversion and check (`CR | `aligned/${SAMPLE}_sorted.cram` | The alignments | | `aligned/${SAMPLE}_sorted.cram.crai` | Its index | | `aligned/${SAMPLE}_sorted.cram.flagstat`, `aligned/${SAMPLE}_sorted.bam.flagstat` | The two `flagstat` outputs the check compared | +| `aligned/${SAMPLE}_sorted.cram.checksum`, `aligned/${SAMPLE}_sorted.bam.checksum` | Normalized content checksum reports, without the filename header. Nextflow publishes only the CRAM report. | | `aligned/${SAMPLE}_sorted.lock` | The empty file the lock is taken on | ## Rerunning steps from a CRAM @@ -57,9 +64,9 @@ In Nextflow, `cram_archive` in `--tools` runs the same conversion and check (`CR | VCF-only steps | Every one: steps 06, 07, 11, 12, 13, 14, 17, 22, 23, 25, 26, 27, 30 and 31 read the VCF, which this step never touches | ## Runtime -Writing a CRAM reads the whole BAM and compresses it again; the check reads both files once more. On CI's fixture slice it takes seconds; a whole genome has not been timed here. +Writing a CRAM reads the whole BAM and compresses it again. The checks read both files for flagstat and again for the checksum. On a synthetic 100,000-read BAM in the pinned image, three trials took 4.44 to 4.99 seconds with 44.1 to 45.3 MiB peak RSS. The count-only checks took 3.39 to 3.99 seconds with 36.3 to 36.5 MiB. These measurements use one thread under amd64 emulation and exclude container startup. A whole genome has not been timed here. ## Notes -- Keep the reference FASTA (and its `.fai`) with the CRAM. A FASTA decodes the CRAM only when every contig the CRAM uses has the same sequence; keeping the file you wrote it with is the sure way. +- Keep the reference FASTA (and its `.fai`) with the CRAM. A FASTA decodes the CRAM only when every contig the CRAM uses has the same sequence. Keep the file you wrote it with. - The CRAM is written with samtools' default CRAM version and compression, which every current htslib, Picard and GATK read. -- A CRAM made elsewhere (nf-core/sarek writes them) can go straight into the Nextflow samplesheet; for the bash steps, `--restore` needs it at `aligned/${SAMPLE}_sorted.cram` with its `.crai`. +- A CRAM made elsewhere (nf-core/sarek writes them) can go straight into the Nextflow samplesheet. For the bash steps, `--restore` needs it at `aligned/${SAMPLE}_sorted.cram` with its `.crai`. diff --git a/modules/local/cram_archive/main.nf b/modules/local/cram_archive/main.nf index a845b997c..c9b96c270 100644 --- a/modules/local/cram_archive/main.nf +++ b/modules/local/cram_archive/main.nf @@ -4,7 +4,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CRAM_ARCHIVE writes _sorted.cram (+ .crai) beside the published BAM and fails the task unless the CRAM passes `samtools quickcheck` and its - `samtools flagstat` equals the BAM's line for line. It never deletes the + `samtools flagstat` and full-content checksums equal the BAM's. It never deletes the BAM: delete it yourself once the CRAM is there, or use scripts/34-cram-archive.sh --delete-bam. A CRAM can only be read with the reference it was written with: keep that FASTA. It takes the same lock as @@ -32,6 +32,7 @@ process CRAM_ARCHIVE { output: tuple val(meta), path("${meta.id}_sorted.cram"), path("${meta.id}_sorted.cram.crai"), emit: cram tuple val(meta), path("${meta.id}_sorted.cram.flagstat"), emit: flagstat + tuple val(meta), path("${meta.id}_sorted.cram.checksum"), emit: checksum path "versions.yml", emit: versions when: @@ -56,7 +57,7 @@ process CRAM_ARCHIVE { else echo "WARNING: could not take the lock \$lock; nothing stops scripts/34-cram-archive.sh from changing ${bam} during this task." >&2 fi - samtools view -@ ${task.cpus} -C --reference ${reference} -o ${meta.id}_sorted.cram ${bam} + samtools view -@ ${task.cpus} -C --reference ${reference} --output-fmt-option store_md=1 --output-fmt-option store_nm=1 -o ${meta.id}_sorted.cram ${bam} samtools index -@ ${task.cpus} ${meta.id}_sorted.cram samtools quickcheck -v ${meta.id}_sorted.cram samtools flagstat -@ ${task.cpus} ${bam} > bam.flagstat @@ -66,7 +67,20 @@ process CRAM_ARCHIVE { echo "ERROR: the CRAM of ${meta.id} does not hold the same reads as its BAM (flagstat above: < BAM, > CRAM)" >&2 exit 1 fi - echo "${meta.id}: BAM \$(du -k ${bam} | cut -f1) kB, CRAM \$(du -k ${meta.id}_sorted.cram | cut -f1) kB, flagstat equal" + # Match every record field and tag, independent of read/tag/group order. + # MD/NM are stored explicitly and never regenerated while checking. + # pipefail must propagate decoder failures through tail and sort. + set -o pipefail + checksum_rows() { + samtools checksum -@ ${task.cpus} -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "\$@" | tail -n +2 | LC_ALL=C sort + } + checksum_rows ${bam} > bam.checksum + checksum_rows --input-fmt-option reference=${reference} --input-fmt-option decode_md=0 ${meta.id}_sorted.cram > ${meta.id}_sorted.cram.checksum + if ! diff bam.checksum ${meta.id}_sorted.cram.checksum >&2; then + echo "ERROR: the CRAM content of ${meta.id} differs from its BAM" >&2 + exit 1 + fi + echo "${meta.id}: BAM \$(du -k ${bam} | cut -f1) kB, CRAM \$(du -k ${meta.id}_sorted.cram | cut -f1) kB, flagstat and content checksums equal" cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -76,7 +90,7 @@ process CRAM_ARCHIVE { stub: """ - touch ${meta.id}_sorted.cram ${meta.id}_sorted.cram.crai ${meta.id}_sorted.cram.flagstat + touch ${meta.id}_sorted.cram ${meta.id}_sorted.cram.crai ${meta.id}_sorted.cram.flagstat ${meta.id}_sorted.cram.checksum cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -103,7 +117,7 @@ process CRAM_TO_BAM { script: """ - samtools view -@ ${task.cpus} -b --reference ${reference} -o ${meta.id}_from_cram.bam ${cram} + samtools view -@ ${task.cpus} -b --reference ${reference} --input-fmt-option decode_md=0 -o ${meta.id}_from_cram.bam ${cram} samtools index -@ ${task.cpus} ${meta.id}_from_cram.bam samtools quickcheck -v ${meta.id}_from_cram.bam samtools flagstat -@ ${task.cpus} --input-fmt-option reference=${reference} ${cram} > cram.flagstat @@ -113,6 +127,20 @@ process CRAM_TO_BAM { exit 1 fi + # Match every record field and tag, independent of read/tag/group order. + # MD/NM are stored explicitly and never regenerated while checking. + # pipefail must propagate decoder failures through tail and sort. + set -o pipefail + checksum_rows() { + samtools checksum -@ ${task.cpus} -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "\$@" | tail -n +2 | LC_ALL=C sort + } + checksum_rows --input-fmt-option reference=${reference} --input-fmt-option decode_md=0 ${cram} > cram.checksum + checksum_rows ${meta.id}_from_cram.bam > bam.checksum + if ! diff cram.checksum bam.checksum >&2; then + echo "ERROR: the restored BAM content differs from ${cram}" >&2 + exit 1 + fi + cat <<-END_VERSIONS > versions.yml "${task.process}": samtools: ${task.container.replaceFirst(/^[^:@]+[:@]/, '')} diff --git a/scripts/34-cram-archive.sh b/scripts/34-cram-archive.sh index f3144ed30..0b50a0e3b 100755 --- a/scripts/34-cram-archive.sh +++ b/scripts/34-cram-archive.sh @@ -11,8 +11,8 @@ # A CRAM stores each read as its difference from the reference, so it can only # be read with the same reference FASTA it was written with: keep that file # (REF_FASTA) as long as you keep the CRAM. The check is that the CRAM passes -# `samtools quickcheck` and that `samtools flagstat` of the CRAM and of the BAM -# agree line for line (the same number of reads, mapped, paired, duplicates). +# `samtools quickcheck`, matching `flagstat` counts and matching full-content +# `samtools checksum` reports (all records, fields and auxiliary tags). # The BAM is deleted only with --delete-bam, only after that check passed in # this run, and never when it failed. --restore writes the BAM back the same # way, checked against the CRAM, for the bash steps, which read the BAM (the @@ -89,6 +89,27 @@ same_reads() { fi } +# The checksum reads every block, unlike flagstat. Use all record fields and +# tags, including secondary/supplementary records, with order-independent +# aggregation. Normalize the format-level repairs CRAM makes to unmapped +# records and CIGARs. Keep explicit MD/NM in the encoder and disable their +# generation on decode, so missing tags stay missing. Only the filename +# header is removed; sorting makes read-group row order immaterial too. +checksum_rows() { + local -a ref=() + case "$1" in *.cram) ref=(--input-fmt-option "reference=${REF_FASTA_C}" --input-fmt-option decode_md=0) ;; esac + run_in --cpus "$THREADS" --memory 2g "$SAMTOOLS_IMAGE" \ + samtools checksum -@ "$THREADS" -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' \ + ${ref[@]+"${ref[@]}"} "$1" | tail -n +2 | LC_ALL=C sort +} +content_checksum() { atomic_out "$2" checksum_rows "$1"; } +same_content() { + if [ ! -s "$1" ] || [ ! -s "$2" ] || ! diff "$1" "$2" >&2; then + echo " content checksum differs or is missing" >&2 + return 1 + fi +} + if [ "$MODE" = --restore ]; then for f in "$CRAM" "${CRAM}.crai"; do [ -f "$f" ] || { echo "ERROR: File not found: ${f}" >&2; exit 1; } @@ -100,13 +121,13 @@ if [ "$MODE" = --restore ]; then echo "Writing ${BAM} from ${CRAM}..." # fail_restore WHY: remove the partial BAM and its index, then stop. fail_restore() { - rm -f "${ALN}/${SAMPLE}_sorted.part.bam" "${ALN}/${SAMPLE}_sorted.part.bam.bai" "${ALN}/${SAMPLE}_sorted.part.bam.flagstat" + rm -f "${ALN}/${SAMPLE}_sorted.part.bam" "${ALN}/${SAMPLE}_sorted.part.bam.bai" "${ALN}/${SAMPLE}_sorted.part.bam.flagstat" "${ALN}/${SAMPLE}_sorted.part.bam.checksum" echo "ERROR: $1 Nothing was kept." >&2 exit 1 } rm -f "${ALN}/${SAMPLE}_sorted.part.bam" "${ALN}/${SAMPLE}_sorted.part.bam.bai" run_in --cpus "$THREADS" --memory 4g "$SAMTOOLS_IMAGE" \ - samtools view -@ "$THREADS" -b --reference "$REF_FASTA_C" \ + samtools view -@ "$THREADS" -b --reference "$REF_FASTA_C" --input-fmt-option decode_md=0 \ -o "${C}.part.bam" "${C}.cram" || fail_restore "samtools could not write the BAM from the CRAM." run_in --cpus "$THREADS" --memory 2g "$SAMTOOLS_IMAGE" \ samtools index -@ "$THREADS" "${C}.part.bam" || fail_restore "samtools could not index the BAM." @@ -115,6 +136,11 @@ if [ "$MODE" = --restore ]; then run_in "$SAMTOOLS_IMAGE" samtools quickcheck -v "${C}.part.bam" || fail_restore "The BAM fails samtools quickcheck." same_reads "${ALN}/${SAMPLE}_sorted.part.bam.flagstat" "${ALN}/${SAMPLE}_sorted.cram.flagstat" \ || fail_restore "The BAM written from the CRAM does not hold the same reads." + content_checksum "${C}.cram" "${CRAM}.checksum" || fail_restore "samtools checksum failed on the CRAM." + content_checksum "${C}.part.bam" "${ALN}/${SAMPLE}_sorted.part.bam.checksum" || fail_restore "samtools checksum failed on the BAM." + same_content "${ALN}/${SAMPLE}_sorted.part.bam.checksum" "${CRAM}.checksum" \ + || fail_restore "The restored BAM content differs from the CRAM." + mv -f "${ALN}/${SAMPLE}_sorted.part.bam.checksum" "${BAM}.checksum" mv -f "${ALN}/${SAMPLE}_sorted.part.bam.bai" "${BAM}.bai" mv -f "${ALN}/${SAMPLE}_sorted.part.bam" "$BAM" rm -f "${ALN}/${SAMPLE}_sorted.part.bam.flagstat" @@ -133,14 +159,15 @@ done # trusted: the BAM may have been aligned again since. PART="${ALN}/${SAMPLE}_sorted.part.cram" discard() { - rm -f "$PART" "${PART}.crai" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" + rm -f "$PART" "${PART}.crai" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" "${ALN}/${SAMPLE}_sorted.part.cram.checksum" echo "ERROR: $1 The CRAM was removed; the BAM is kept." >&2 exit 1 } -rm -f "$PART" "${PART}.crai" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" +rm -f "$PART" "${PART}.crai" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" "${ALN}/${SAMPLE}_sorted.part.cram.checksum" echo "Writing ${CRAM}..." run_in --cpus "$THREADS" --memory 4g "$SAMTOOLS_IMAGE" \ samtools view -@ "$THREADS" -C --reference "$REF_FASTA_C" \ + --output-fmt-option store_md=1 --output-fmt-option store_nm=1 \ -o "${C}.part.cram" "${C}.bam" || discard "samtools could not write the CRAM." echo "Checking the CRAM against the BAM..." @@ -152,6 +179,11 @@ flagstat "${C}.bam" "${ALN}/${SAMPLE}_sorted.bam.flagstat" || discard "samtools flagstat "${C}.part.cram" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" || discard "samtools flagstat failed on the CRAM." same_reads "${ALN}/${SAMPLE}_sorted.bam.flagstat" "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" \ || discard "The CRAM does not hold the same reads as the BAM." +content_checksum "${C}.bam" "${BAM}.checksum" || discard "samtools checksum failed on the BAM." +content_checksum "${C}.part.cram" "${PART}.checksum" || discard "samtools checksum failed on the CRAM." +same_content "${BAM}.checksum" "${PART}.checksum" \ + || discard "The CRAM content differs from the BAM." +mv -f "${PART}.checksum" "${CRAM}.checksum" mv -f "${PART}.crai" "${CRAM}.crai" mv -f "$PART" "$CRAM" mv -f "${ALN}/${SAMPLE}_sorted.part.cram.flagstat" "${ALN}/${SAMPLE}_sorted.cram.flagstat" diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 7f5b093d3..9ea0dd95d 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -180,3 +180,4 @@ y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh rust-z2e-readers bin/og tests/e2e/lib.sh scripts/ci/image-smoke.sh scripts/ci/nf-trace-requests.tsv rust-c2b-traits assets/traits.tsv assets/genotype_rules.tsv bin/og modules/local/traits/* scripts/42-traits.sh +qc-3-cram-checksum modules/local/cram_archive/* scripts/34-cram-archive.sh scripts/lib/common.sh diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv index 5c80940c7..066ae31fb 100644 --- a/scripts/ci/e2e-case-times.tsv +++ b/scripts/ci/e2e-case-times.tsv @@ -179,3 +179,4 @@ y-str-1 shard 42 - y-tree-current-1-placement shard 21 nextflow-hardening-2-nf rust-z2e-readers shard 1 - rust-c2b-traits shard 2 - +qc-3-cram-checksum shard 40 - diff --git a/tests/e2e/qc-3-cram-checksum.sh b/tests/e2e/qc-3-cram-checksum.sh new file mode 100755 index 000000000..e1df0c207 --- /dev/null +++ b/tests/e2e/qc-3-cram-checksum.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Synthetic read content, including secondary/supplementary reads and tags. +# A quality, sequence or custom-tag change leaves flagstat unchanged. Each +# must stop --delete-bam and restore before a partial output is published. +. "$(dirname "$0")/lib.sh" +export GENOME_DIR="${CASE_TMP}/genome" +export REF_FASTA="${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" +C=checksum +A="${GENOME_DIR}/${C}/aligned" +rm -rf "$A" +mkdir -p "${GENOME_DIR}/reference" "$A" "${CASE_TMP}/fault" +printf '>chr1\n%s\n' "$(printf 'ACGT%.0s' {1..100})" > "$REF_FASTA" +# Include correct and incorrect MD/NM, missing MD/NM, a custom tag, a reverse +# read, a secondary read, a supplementary read, and placed/unplaced unmapped +# reads. Unmapped MAPQ is normalized by samtools' checksum sanitizer. +printf '@HD\tVN:1.6\tSO:coordinate\n@SQ\tSN:chr1\tLN:400\n@RG\tID:rg1\tSM:synthetic\n' > "${A}/in.sam" +for flag in 0 16 256 2048; do + printf 'read%s\t%s\tchr1\t1\t60\t4M\t*\t0\t0\tACGT\tIIII\tMD:Z:4\tNM:i:0\tXZ:Z:kept\tRG:Z:rg1\n' "$flag" "$flag" +done >> "${A}/in.sam" +printf 'wrong-md\t0\tchr1\t5\t60\t4M\t*\t0\t0\tACGT\tIIII\tMD:Z:0A3\tNM:i:1\nunmapped-placed\t4\tchr1\t9\t30\t*\t*\t0\t0\tACGT\tIIII\nunmapped\t4\t*\t0\t0\t*\t*\t0\t0\tACGT\tIIII\n' >> "${A}/in.sam" +sam faidx reference/GRCh38_no_alt_analysis_set.fasta +sam view -b -o "${C}/aligned/original.bam" "${C}/aligned/in.sam" +sam index "${C}/aligned/original.bam" + +# After a real conversion, rewrite its output with one changed field. Both +# commands succeed and all read flags/counts stay the same. For restore the +# same wrapper changes the partial BAM instead. +cat > "${CASE_TMP}/fault/docker" <<'WRAP' +#!/usr/bin/env bash +set -euo pipefail +self_dir="$(cd "$(dirname "$0")" && pwd)" +IFS=: read -r -a dirs <<< "$PATH" +keep=(); real="" +for d in "${dirs[@]}"; do + [ "$d" = "$self_dir" ] && continue + keep+=("$d") + if [ -z "$real" ] && [ -x "${d}/docker" ]; then real="${d}/docker"; fi +done +PATH=$(IFS=:; echo "${keep[*]}"); export PATH +[ -n "$real" ] || exit 127 +"$real" "$@" +args=" $* " +case "$args" in + *" samtools view "*" -o "*".part.${FAULT_FORMAT} "*) ;; + *) exit 0 ;; +esac +out=$(grep -oE "/genome/[^ ]+\.part\.${FAULT_FORMAT}" <<< "$args" | head -n 1) +"$real" run --rm -i -u "$(id -u):$(id -g)" -v "${GENOME_DIR}:/genome" "$SAMTOOLS_IMAGE" \ + bash -euo pipefail -c ' + out=$1; fault=$2; ref=/genome/reference/GRCh38_no_alt_analysis_set.fasta + samtools view -T "$ref" --input-fmt-option decode_md=0 "$out" | + awk -F "\t" -v OFS="\t" -v fault="$fault" '\'' + fault == "reorder" && NR <= 4 { + # All four share one coordinate, so reversing them stays sorted. + t=$NF; $NF=$(NF-1); $(NF-1)=t + rows[NR]=$0 + if (NR == 4) for (j=4; j>=1; j--) print rows[j] + next + } + NR == 1 && fault == "quality" { $11 = "H" substr($11, 2) } + NR == 3 && fault == "sequence" { $10 = "T" substr($10, 2) } + NR == 4 && fault == "tag" { for (i=12; i<=NF; i++) if ($i ~ /^XZ:/) $i="XZ:Z:changed" } + { print } + '\'' > /tmp/changed.sam + samtools view -H "$out" > /tmp/changed-header.sam + cat /tmp/changed-header.sam /tmp/changed.sam > /tmp/changed-all.sam + if [[ $out == *.cram ]]; then + samtools view -C -T "$ref" --output-fmt-option store_md=1 --output-fmt-option store_nm=1 -o "$out" /tmp/changed-all.sam + else + samtools view -b -o "$out" /tmp/changed-all.sam + fi + echo "fault: $fault in $out" >&2 + ' bash "$out" "$FAULT" +WRAP +chmod +x "${CASE_TMP}/fault/docker" +export SAMTOOLS_IMAGE + +seed_bam() { + cp -f "${A}/original.bam" "${A}/${C}_sorted.bam" + cp -f "${A}/original.bam.bai" "${A}/${C}_sorted.bam.bai" +} +for fault in quality sequence tag; do + seed_bam + FAULT=$fault FAULT_FORMAT=cram PATH="${CASE_TMP}/fault:${PATH}" run_step 34-cram-archive.sh "$C" --delete-bam + check "${fault}: fault injected" has "^fault: ${fault} " "$(cat "$STEP_LOG")" + check "${fault}: archive refuses changed content" test "$STEP_RC" -ne 0 + check "${fault}: BAM is unchanged" cmp -s "${A}/original.bam" "${A}/${C}_sorted.bam" + check "${fault}: BAM index stays" test -s "${A}/${C}_sorted.bam.bai" + check "${fault}: no partial CRAM" test ! -e "${A}/${C}_sorted.part.cram" + check "${fault}: no published CRAM" test ! -e "${A}/${C}_sorted.cram" +done +seed_bam +FAULT=reorder FAULT_FORMAT=cram PATH="${CASE_TMP}/fault:${PATH}" run_step 34-cram-archive.sh "$C" --delete-bam +check "reordered read/tag control was injected" has "^fault: reorder " "$(cat "$STEP_LOG")" +check_step_exit 34-cram-archive.sh +check "faithful archive deletes BAM" test ! -e "${A}/${C}_sorted.bam" +check "archive keeps checksum evidence" test -s "${A}/${C}_sorted.cram.checksum" +for fault in quality sequence tag; do + rm -f "${A}/${C}_sorted.bam" "${A}/${C}_sorted.bam.bai" + FAULT=$fault FAULT_FORMAT=bam PATH="${CASE_TMP}/fault:${PATH}" run_step 34-cram-archive.sh "$C" --restore + check "${fault}: restore fault injected" has "^fault: ${fault} " "$(cat "$STEP_LOG")" + check "${fault}: restore refuses changed content" test "$STEP_RC" -ne 0 + check "${fault}: restore leaves no BAM" test ! -e "${A}/${C}_sorted.bam" + check "${fault}: restore leaves no partial BAM" test ! -e "${A}/${C}_sorted.part.bam" +done +rm -f "${A}/${C}_sorted.bam" "${A}/${C}_sorted.bam.bai" +run_step 34-cram-archive.sh "$C" --restore +check_step_exit 34-cram-archive.sh +check "faithful restore writes BAM and index" test -s "${A}/${C}_sorted.bam" -a -s "${A}/${C}_sorted.bam.bai" +# No generated MD/NM tags are added and all explicit tags survive, independent +# of tag order. checksum sanitizes only format-level normalization. +sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/original.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/original.checksum" +sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/${C}_sorted.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/restored.checksum" +check "round trip preserves read content and explicit tags" cmp -s "${CASE_TMP}/original.checksum" "${CASE_TMP}/restored.checksum" +finish diff --git a/tests/fake-docker/qc-steps-33-34.sh b/tests/fake-docker/qc-steps-33-34.sh index fdcf53e2c..4350e5ca4 100755 --- a/tests/fake-docker/qc-steps-33-34.sh +++ b/tests/fake-docker/qc-steps-33-34.sh @@ -12,7 +12,7 @@ # reruns VerifyBamID2 with --DisableSanityCheck and records it; any other # VerifyBamID2 failure, or missing data, stops the step. # Step 34: a CRAM whose flagstat differs from the BAM's, or that fails -# quickcheck, is removed and the BAM kept, also with --delete-bam; the BAM is +# quickcheck or its content checksum fails, is removed and the BAM kept; the BAM is # deleted only after the check passed; --restore refuses to write over a BAM # and leaves no partial BAM when it fails; archive and restore both stop # while another run holds the sample's lock, and the lock goes with its run. @@ -86,6 +86,15 @@ case "$*" in "samtools quickcheck"*) for a in "$@"; do last=$a; done if [ "${FAKE_QUICKCHECK_FAIL:-}" = "${last##*.}" ]; then echo "${last} had no EOF block" >&2; exit 1; fi ;; + "samtools checksum"*) + for a in "$@"; do last=$a; done + case "${FAKE_CHECKSUM_FAIL:-}:$last" in + cram:*.cram|bam:*.bam) echo "checksum: decode failed" >&2; exit 1 ;; + esac + [ "${FAKE_CHECKSUM_EMPTY:-}" != "${last##*.}" ] || exit 0 + sum=good + case "$last" in *.cram) sum=${FAKE_CRAM_CONTENT:-good} ;; esac + printf '# Checksum for file: %s\nall all 1000 %s\n' "$last" "$sum" ;; "samtools flagstat"*) for a in "$@"; do last=$a; done n=1000 @@ -173,6 +182,17 @@ FAKE_QUICKCHECK_FAIL=cram run_expect 1 cram-truncated "${SCRIPTS}/34-cram-archiv output_has cram-truncated 'fails samtools quickcheck' [ -s "${A}/sample1_sorted.bam" ] || fail "the BAM was deleted although the CRAM failed quickcheck" +FAKE_CRAM_CONTENT=changed run_expect 1 cram-content "${SCRIPTS}/34-cram-archive.sh" sample1 --delete-bam +output_has cram-content 'CRAM content differs from the BAM' +[ -s "${A}/sample1_sorted.bam" ] && [ -e "${A}/sample1_sorted.bam.bai" ] || fail "a content mismatch deleted the BAM or index" +[ ! -e "${A}/sample1_sorted.part.cram" ] && [ ! -e "${A}/sample1_sorted.part.cram.checksum" ] || fail "a content mismatch left partial files" +for format in bam cram; do + FAKE_CHECKSUM_FAIL=$format run_expect 1 "cram-checksum-fails-$format" "${SCRIPTS}/34-cram-archive.sh" sample1 --delete-bam + output_has "cram-checksum-fails-$format" 'samtools checksum failed' + FAKE_CHECKSUM_EMPTY=$format run_expect 1 "cram-checksum-empty-$format" "${SCRIPTS}/34-cram-archive.sh" sample1 --delete-bam + [ -s "${A}/sample1_sorted.bam" ] || fail "an empty or failed checksum deleted the BAM" +done + run_expect 0 cram-keep "${SCRIPTS}/34-cram-archive.sh" sample1 [ -s "${A}/sample1_sorted.cram" ] && [ -e "${A}/sample1_sorted.cram.crai" ] || fail "no checked CRAM" [ -s "${A}/sample1_sorted.bam" ] || fail "the BAM was deleted without --delete-bam" @@ -214,6 +234,15 @@ rm -f "${A}/sample1_sorted.bam" "${A}/sample1_sorted.bam.bai" FAKE_CRAM_READS=999 run_expect 1 cram-restore-differs "${SCRIPTS}/34-cram-archive.sh" sample1 --restore [ ! -e "${A}/sample1_sorted.bam" ] && [ ! -e "${A}/sample1_sorted.part.bam" ] || fail "--restore kept a BAM that does not match the CRAM" +FAKE_CRAM_CONTENT=changed run_expect 1 cram-restore-content "${SCRIPTS}/34-cram-archive.sh" sample1 --restore +output_has cram-restore-content 'restored BAM content differs from the CRAM' +[ ! -e "${A}/sample1_sorted.bam" ] && [ ! -e "${A}/sample1_sorted.part.bam" ] && [ ! -e "${A}/sample1_sorted.part.bam.checksum" ] || fail "a restore content mismatch left a BAM or partial checksum" +for format in bam cram; do + FAKE_CHECKSUM_FAIL=$format run_expect 1 "restore-checksum-fails-$format" "${SCRIPTS}/34-cram-archive.sh" sample1 --restore + FAKE_CHECKSUM_EMPTY=$format run_expect 1 "restore-checksum-empty-$format" "${SCRIPTS}/34-cram-archive.sh" sample1 --restore + [ ! -e "${A}/sample1_sorted.bam" ] && [ ! -e "${A}/sample1_sorted.part.bam" ] || fail "a restore with no checksum left a BAM" +done + FAKE_VIEW_FAIL=1 run_expect 1 cram-restore-write-fails "${SCRIPTS}/34-cram-archive.sh" sample1 --restore output_has cram-restore-write-fails 'samtools could not write the BAM from the CRAM' [ ! -e "${A}/sample1_sorted.part.bam" ] || fail "a failed --restore left its partial BAM" From ba002b007be67ae7e303adc3ac2d01f612c835f3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:16:02 +0100 Subject: [PATCH 101/255] feat(live): open the run reads inline from the genome map Bead: pgp-iqon.11 --- CHANGELOG.md | 2 + assets/live/map.css | 24 ++++++ assets/live/map.js | 141 ++++++++++++++++++++++++++++++++--- assets/viewer/serve.py | 2 + assets/viewer/viewer.js | 5 ++ bin/owngenome_live.py | 6 +- bin/owngenome_live_viewer.py | 2 +- docs/viewer.md | 8 +- 8 files changed, 176 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..9382a20d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer; Esc closes it and returns focus to the map. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/assets/live/map.css b/assets/live/map.css index 67cf5fe34..ad413123e 100644 --- a/assets/live/map.css +++ b/assets/live/map.css @@ -69,6 +69,30 @@ .gm-zoom { display: inline-flex; border: 1px solid var(--hair-2); border-radius: 999px; overflow: hidden; } .gm-zoom[hidden] { display: none; } .gm-zoom button { border: 0; border-radius: 0; background: transparent; color: var(--ink-2); padding: 6px 14px; font-size: 14px; line-height: 1; } +/* The reads take the table's place, while the map keeps its camera above them. */ +.gm-reads-pill { position: absolute; top: 0; right: 8px; border: 1px solid var(--brass-dim); border-radius: 999px; + color: var(--brass); background: var(--page); padding: 6px 12px; font-size: 12px; } +.gm-reads-hint { color: var(--brass); font-size: 12px; margin: 6px 0; } +.gm-reads { box-sizing: border-box; height: 330px; display: flex; flex-direction: column; min-width: 0; + margin-top: 10px; border: 1px solid var(--brass-dim); border-radius: 8px; overflow: hidden; background: var(--paper); } +.gm-reads[hidden], .gm-reads-pill[hidden], .gm-reads-hint[hidden] { display: none; } +.gm-reads-head { display: flex; flex-wrap: wrap; align-items: center; gap: 6px 12px; padding: 8px 10px; font-size: 12px; } +.gm-reads-head b { font: 400 19px var(--serif); } +.gm-reads-head button { padding: 4px 8px; font-size: 12px; } +.gm-reads-place { overflow-wrap: anywhere; color: var(--brass); } +.gm-reads-who { margin: 0; padding: 0 10px 6px; font-size: 12px; color: var(--ink-2); } +.gm-reads-body { flex: 1; min-height: 0; padding: 0 8px; font-size: 13px; overflow: auto; } +.gm-reads-frame { display: block; width: 100%; height: 100%; border: 0; background: #fff; } +@media (prefers-reduced-motion: no-preference) { + .gm-reads { animation: gm-reads-in 180ms ease-out; } + @keyframes gm-reads-in { from { transform: translateY(20px); opacity: 0; } to { transform: translateY(0); opacity: 1; } } +} +@media (max-width: 520px) { + .gm-reads { height: 300px; } + .gm-gauge { max-width: 100%; overflow-x: auto; } + .gm-reads-head { gap: 4px 8px; padding: 6px; } + .gm-reads-head b { font-size: 16px; } +} /* Search (/): one box over gene names, band names, the findings drawn and a typed place. */ .gm-find { position: relative; max-width: 420px; } .gm-find-box { width: 100%; box-sizing: border-box; background: var(--paper); color: var(--ink); border: 1px solid var(--hair-2); diff --git a/assets/live/map.js b/assets/live/map.js index d207f45ec..0af553b75 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -148,7 +148,8 @@ var stopC = el('button', { type: 'button', 'data-stop': 'chromosome' }, 'Chromosome'); var stopR = el('button', { type: 'button', 'data-stop': 'region' }, 'Region'); var stopN = el('button', { type: 'button', 'data-stop': 'gene' }, 'Gene'); - add(gauge, stopG, stopC, stopR, stopN); + var stopReads = el('button', { type: 'button', 'data-stop': 'reads' }, 'Reads'); + add(gauge, stopG, stopC, stopR, stopN, stopReads); // Zoom buttons for an open chromosome: the touch path to what the wheel and the + and - keys do. var zoom = el('div', { 'class': 'gm-zoom', role: 'group', 'aria-label': 'Zoom', hidden: 'hidden' }); var zoomOut = el('button', { type: 'button', 'aria-label': 'Zoom out', 'data-zoom': 'out' }, '−'); @@ -168,14 +169,23 @@ var canvas = el('canvas', { 'class': 'gm-canvas', tabindex: '0', role: 'img', 'aria-describedby': 'gm-narration' }); var tip = el('div', { 'class': 'gm-tip', role: 'tooltip', hidden: 'hidden' }); var toast = el('div', { 'class': 'gm-toast', role: 'status', hidden: 'hidden' }); - add(stage, canvas, tip, toast); + var readsPill = el('button', { type: 'button', 'class': 'gm-reads-pill', hidden: 'hidden', 'aria-controls': 'gm-reads', 'aria-expanded': 'false' }, 'See the reads · R'); + var readsHint = el('p', { 'class': 'gm-reads-hint', hidden: 'hidden' }, 'Under 50 kb: the reads are one key away · R'); + add(stage, canvas, tip, toast, readsPill); + var reads = el('section', { 'class': 'gm-reads', id: 'gm-reads', 'aria-label': 'Reads', hidden: 'hidden' }); + var readsPlace = el('span', { 'class': 'mono gm-reads-place' }); + var readsFull = el('a', { target: '_blank', rel: 'noopener' }, 'Open full viewer ↗'); + var readsClose = button('Back to the map · Esc', closeReads, 'gm-reads-close'); + var readsWho = el('p', { 'class': 'gm-reads-who' }); + var readsBody = el('div', { 'class': 'gm-reads-body' }); + add(reads, add(el('div', { 'class': 'gm-reads-head' }), el('b', {}, 'Reads'), readsPlace, readsFull, readsClose), readsWho, readsBody); // The marks drawn, as links: the keyboard and screen-reader path to each finding (Tab steps through them). var marks = el('ul', { 'class': 'gm-marks', 'aria-label': 'Findings on the map' }); var fallback = el('div', { 'class': 'gm-fallback', hidden: 'hidden' }); // The whole-genome strip over an open chromosome, and the In view table under the map (strip(), table()). var strip = el('canvas', { 'class': 'gm-strip', role: 'img', hidden: 'hidden' }); var inview = el('details', { 'class': 'gm-inview' }); - add(main, head, strip, stage, inview, marks, fallback); + add(main, head, strip, stage, readsHint, reads, inview, marks, fallback); var margin = el('aside', { 'class': 'gm-margin' }); var mkick = el('div', { 'class': 'fig-label' }, 'The whole genome'); var mline = el('div', { 'class': 'gm-mline mono' }); @@ -206,6 +216,7 @@ var hatch = null; // The finding whose card the margin shows (an id of drawn() only), and the mark the keyboard is on. var sel = null, focusId = null; + var readsFrame = null, readsId = null, readsView = '', readsLocus = ''; // The gene files, fetched once each: per chromosome when a view first reaches the region level, the // names when the search box gets its first key. Reference data, the same for every reader. var genes = {}, names = null; @@ -559,7 +570,11 @@ ctx.beginPath(); ctx.moveTo(wa, L.over + 12); ctx.lineTo(g, L.ruler); ctx.moveTo(wz, L.over + 12); ctx.lineTo(g + W, L.ruler); ctx.stroke(); ctx.globalAlpha = k; // The ruler, in Mb. - ctx.strokeStyle = 'rgba(255,255,255,.14)'; + if (level() === 'reads') { + ctx.fillStyle = 'rgba(200,171,120,.12)'; + ctx.fillRect(g, L.ruler - 2, W, 23); + } + ctx.strokeStyle = level() === 'reads' ? BRASS : 'rgba(255,255,255,.14)'; ctx.lineWidth = 1; ctx.beginPath(); ctx.moveTo(g, L.ruler); ctx.lineTo(g + W, L.ruler); ctx.stroke(); var stp = niceStep(v[2], W); @@ -1011,7 +1026,10 @@ stopG.setAttribute('aria-pressed', String(lv === 'genome')); stopC.setAttribute('aria-pressed', String(lv === 'chromosome')); stopR.setAttribute('aria-pressed', String(lv === 'region')); - stopN.setAttribute('aria-pressed', String(close_(lv))); + stopN.setAttribute('aria-pressed', String(lv === 'gene')); + stopReads.setAttribute('aria-pressed', String(lv === 'reads')); + readsPill.hidden = readsHint.hidden = lv !== 'reads'; + syncReads(); zoom.hidden = lv === 'genome'; crumb.textContent = ''; crumb.hidden = lv === 'genome'; // the gauge already says Genome @@ -1190,9 +1208,101 @@ add(card, facts); if (r.phrase) { add(card, el('p', { 'class': 'gm-card-phrase' }, r.phrase)); } var foot = el('p', { 'class': 'gm-card-foot' }); - add(foot, el('a', { href: r.href }, 'The finding’s page ›'), ' ', button('Close the card', function () { pick(null); canvas.focus({ preventScroll: true }); })); + var rb = button('See the reads', function () { openReads(r); }, 'gm-card-reads'); + rb.setAttribute('data-id', r.id); + add(foot, el('a', { href: r.href }, 'The finding’s page ›'), ' ', rb, ' ', button('Close the card', function () { pick(null); canvas.focus({ preventScroll: true }); })); add(card, foot); } + + // --- reads: only the existing same-origin viewer loads tracks and byte ranges --- + // Every candidate comes from drawn(), after the page's own veil callbacks. No result body is fetched here. + function readsBase() { + var run = document.documentElement.dataset.run || M.id; + return '/runs/' + encodeURIComponent(run) + '/viewer/'; + } + function exactView() { + var v = view(), ch = chrom(); + return ch.n + ':' + Math.max(1, Math.round(v[0] + 1)) + '-' + Math.min(ch.len, Math.round(v[1])); + } + function findingLocus(r) { + var ch = M.chroms[M.by[r.chrom]], p = r.pos || Math.round((r.s + r.e) / 2) + 1; + return r.chrom + ':' + Math.max(1, p - 50) + '-' + Math.min(ch.len, p + 50); + } + function nearestInView() { + var v = view(), ch = chrom(), best = null, dist = Infinity; + drawn().forEach(function (r) { + if (r.chrom !== ch.n || r.e <= v[0] || r.s >= v[1]) { return; } + var d = Math.abs((r.s + r.e) / 2 - cam.c); + if (d < dist) { best = r; dist = d; } + }); + return best; + } + function setReadsLocus(locus) { + if (locus === readsLocus) { return; } + readsLocus = locus; + readsPlace.textContent = locus; + readsFull.href = readsBase() + '#' + locus; + if (readsFrame) { + // Changing only the fragment lets viewer.js search without reloading igv.js. + readsFrame.src = readsFull.href; + } + } + function closeReads() { + if (reads.hidden) { return; } + reads.hidden = true; + readsBody.textContent = ''; // stop reads requests, and discard a result that can close again + readsFrame = null; readsId = null; readsLocus = ''; readsView = ''; + readsPill.setAttribute('aria-expanded', 'false'); + inview.hidden = !!failure; + canvas.focus({ preventScroll: true }); + } + function syncReads() { + if (reads.hidden) { return; } + if (failure || (readsId && !rowOf(readsId))) { closeReads(); return; } + var v = exactView(); + if (v !== readsView) { readsView = v; setReadsLocus(v); } + } + function openReads(r) { + if (!M || failure || (r && !rowOf(r.id))) { return; } + if (!r && level() !== 'reads') { say('Zoom under 50 kb to see the reads, or open a finding’s card'); return; } + r = r || nearestInView(); + readsId = r ? r.id : null; + reads.hidden = false; + inview.hidden = true; + readsPill.setAttribute('aria-expanded', 'true'); + readsView = exactView(); + readsWho.textContent = M.label + ' · run ' + (document.documentElement.dataset.run || M.id) + + (r ? ' · ' + (r.gene || r.title) + (r.hgvs ? ' ' + r.hgvs : '') : ''); + var demo = document.documentElement.dataset.mode === 'replay'; + readsFull.hidden = demo; + if (demo) { + readsBody.textContent = 'The genome viewer is not in the demo'; + } else if (!readsFrame) { + readsBody.textContent = ''; + var frame = el('iframe', { title: 'Genome viewer reads', 'class': 'gm-reads-frame' }); + readsFrame = frame; + frame.addEventListener('load', function () { + if (readsFrame !== frame) { return; } + var doc = frame.contentDocument; + if (!doc) { return; } + // An unavailable viewer answers with its reason as plain text. Display that reason in the panel. + if (!doc.documentElement.hasAttribute('data-pgp-status')) { + readsBody.textContent = 'The viewer is off on this site: ' + doc.body.textContent.trim(); + readsFrame = null; + readsFull.hidden = true; + return; + } + doc.addEventListener('keydown', function (e) { + if (e.key === 'Escape') { e.preventDefault(); closeReads(); } + }); + }); + readsLocus = ''; // a new frame needs its initial URL even when reopening the same place + } + setReadsLocus(r ? findingLocus(r) : readsView); + if (readsFrame && !readsFrame.parentNode) { add(readsBody, readsFrame); } + readsClose.focus({ preventScroll: true }); + } + readsPill.addEventListener('click', function () { openReads(); }); // The finding nearest the view when none of this chromosome's is in it: the closest one on this // chromosome, else the first on the next chromosome that has one, in reading order. Null when one is in view. function nearest() { @@ -1520,7 +1630,7 @@ var open = level() !== 'genome', span = Math.exp(tgt.ls); var done = true; if (e.key === 'Escape') { - if (sel) { pick(null); } else if (open) { stepOut(); } else { done = false; } + if (!reads.hidden) { closeReads(); } else if (sel) { pick(null); } else if (open) { stepOut(); } else { done = false; } } else if (e.key === '0') { if (open) { closeChrom(true); } else { done = false; } } else if (e.key === '[' || e.key === ']') { var i = (open ? tgt.ci : (e.key === ']' ? -1 : M.chroms.length)) + (e.key === ']' ? 1 : -1); @@ -1548,7 +1658,10 @@ if (!M || failure || e.altKey || e.ctrlKey || e.metaKey || !fig.isConnected) { return; } var t = e.target, tag = t && t.tagName; if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || (t && t.isContentEditable)) { return; } - if (e.key === '/') { e.preventDefault(); findBox.focus(); } + if (e.defaultPrevented) { return; } + if (e.key === 'Escape' && !reads.hidden) { e.preventDefault(); closeReads(); } + else if (e.key === 'r' || e.key === 'R') { e.preventDefault(); openReads(); } + else if (e.key === '/') { e.preventDefault(); findBox.focus(); } else if (e.key === 't' || e.key === 'T') { e.preventDefault(); inview.open = !inview.open; } else if (e.key === 'l' || e.key === 'L') { e.preventDefault(); @@ -1581,6 +1694,13 @@ stopC.addEventListener('click', function () { stopTo('chromosome'); }); stopR.addEventListener('click', function () { stopTo('region'); }); stopN.addEventListener('click', function () { stopTo('gene'); }); + stopReads.addEventListener('click', function () { + stopTo('gene'); + openChrom(tgt.ci, tgt.c, 2000, true); + // The target is already at reads level; finish the camera motion before selecting its nearest row. + cam.ci = tgt.ci; cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; + openReads(); + }); nearBtn.addEventListener('click', function () { goNearest(); }); window.addEventListener('hashchange', function () { readHash(false); }); window.addEventListener('popstate', function () { readHash(false); }); @@ -1943,6 +2063,8 @@ // --- loading --- function fail(why, words) { failure = why; + closeReads(); + readsHint.hidden = true; // The frame and the margin the last refresh scheduled would draw the old document again: drop them. // These lines and the failure guards of draw(), describe() and schedule() cover for each other, so no one // of them can go red alone: the failed-reload check undoes them together. @@ -2004,7 +2126,7 @@ failure = null; stage.hidden = false; marks.hidden = false; - inview.hidden = false; + inview.hidden = !reads.hidden; fallback.hidden = true; hatch = hatch || pattern(0.16); fig.setAttribute('data-pgp-map-parse-ms', String(Math.round(x.parse))); @@ -2043,6 +2165,7 @@ return; } if (!M) { return; } + syncReads(); cacheKey = ''; hideTip(); // a tooltip of a result closed since leaves the page drawCard(); // and so does the card of one diff --git a/assets/viewer/serve.py b/assets/viewer/serve.py index 435c9daeb..fc7d1624d 100755 --- a/assets/viewer/serve.py +++ b/assets/viewer/serve.py @@ -38,6 +38,8 @@ CSP = ("default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self' data: blob:; " "style-src 'self' 'unsafe-inline'; font-src 'self' data:; worker-src 'self' blob:; " "base-uri 'none'; form-action 'none'; frame-ancestors 'none'") +# Only the live site's viewer can be embedded by another page of that site. +LIVE_CSP = CSP.replace("frame-ancestors 'none'", "frame-ancestors 'self'") LOCUS_RE = re.compile(r"^([A-Za-z0-9._-]+)(?::([0-9][0-9,]*)(?:-([0-9][0-9,]*))?)?$") diff --git a/assets/viewer/viewer.js b/assets/viewer/viewer.js index 295234cc0..aadfcf506 100644 --- a/assets/viewer/viewer.js +++ b/assets/viewer/viewer.js @@ -43,6 +43,11 @@ return; } const cfg = await resp.json(); + cfg.tracks.forEach(function (track) { + if (track.type === "alignment") { + track.visibilityWindow = 50000; + } + }); document.getElementById("sample").textContent = cfg.sample; document.title = cfg.sample + " - Genome viewer"; const browser = await igv.createBrowser(document.getElementById("igv"), { diff --git a/bin/owngenome_live.py b/bin/owngenome_live.py index 1b3edbcf0..a712e43a4 100755 --- a/bin/owngenome_live.py +++ b/bin/owngenome_live.py @@ -78,7 +78,7 @@ FAIL_WINDOW_SECONDS = 15 * 60 LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"} CSP = ("default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; img-src 'self' data:; " - "form-action 'self'; base-uri 'none'; frame-ancestors 'none'") + "form-action 'self'; base-uri 'none'; frame-src 'self'; frame-ancestors 'none'") # A run's own HTML (the report, CPSR, PharmCAT, MultiQC) carries inline styles and, for third-party # pages, scripts and outside links. It is sent sandboxed: no script runs, nothing is fetched from # outside, and it keeps its styles. allow-same-origin lets its links reach the run's other files @@ -915,12 +915,12 @@ def client(self): pass return peer - def headers_common(self, csp=CSP): + def headers_common(self, csp=CSP, frame_options="DENY"): self.send_header("Content-Security-Policy", csp) self.send_header("X-Content-Type-Options", "nosniff") self.send_header("Referrer-Policy", "no-referrer") self.send_header("Cache-Control", "no-store") - self.send_header("X-Frame-Options", "DENY") + self.send_header("X-Frame-Options", frame_options) def send_body(self, code, ctype, body, headers=(), csp=CSP): if isinstance(body, str): diff --git a/bin/owngenome_live_viewer.py b/bin/owngenome_live_viewer.py index 7edbf1bae..e7891bb49 100644 --- a/bin/owngenome_live_viewer.py +++ b/bin/owngenome_live_viewer.py @@ -108,7 +108,7 @@ def head(code, ctype, length, extra=()): req.send_header("Cross-Origin-Resource-Policy", "same-origin") for k, v in extra: req.send_header(k, v) - req.headers_common(serve.CSP) + req.headers_common(serve.LIVE_CSP, frame_options="SAMEORIGIN") req.end_headers() return head diff --git a/docs/viewer.md b/docs/viewer.md index 5c5170550..8b30532c8 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -80,7 +80,13 @@ Then open `http://127.0.0.1:8765/` on your computer. The report's links work the ## From the live site -`scripts/owngenome-live.sh` starts the [live site](live-site.md) with the viewer: each run has it at `/runs//viewer/`, behind the site's password, and the report the site serves points its Viewer links there. `bin/owngenome_live_viewer.py` does it with the same page, the same Content-Security-Policy and the same files as `owngenome-view.sh` picks by default (the run folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and `REF_FASTA`). The `READS`, `VCF`, `SV_VCF`, `ALIGN_DIR` and `VCF_DIR` overrides do not apply there, and a track whose file or index is a symlink out of `GENOME_DIR` is left out. The first start downloads igv.js the same way. `OWNGENOME_LIVE_VIEWER=off` starts the site without it. +`scripts/owngenome-live.sh` starts the [live site](live-site.md) with the viewer: each run has it at `/runs//viewer/`, behind the site's password, and the report the site serves points its Viewer links there. `bin/owngenome_live_viewer.py` uses the same page and files as `owngenome-view.sh` picks by default (the run folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and `REF_FASTA`). The `READS`, `VCF`, `SV_VCF`, `ALIGN_DIR` and `VCF_DIR` overrides do not apply there, and a track whose file or index is a symlink out of `GENOME_DIR` is left out. The first start downloads igv.js the same way. `OWNGENOME_LIVE_VIEWER=off` starts the site without it. + +On the genome map, zoom below 50 kb and press **R** or select **See the reads**. The **Reads** gauge opens a 2 kb view. A finding's map card also offers **See the reads**, centred on its position with 50 bases on each side. The panel otherwise starts at the nearest open finding in view, or at the view itself. Moving the map updates the viewer's locus. Alignments are visible up to 50 kb. + +The panel takes the place of the In view table. Press **Esc**, including from inside the viewer, or select **Back to the map** to close it. Focus returns to the map at the same view. **Open full viewer** opens that locus in a new tab. A closed result has no reads button, and closing an opened result removes its reads panel. When the viewer is off, the panel shows the server's reason. The static demo explains that it has no viewer. + +The live viewer allows framing only by the same origin: it sends `frame-ancestors 'self'` and `X-Frame-Options: SAMEORIGIN`. Live pages allow `frame-src 'self'`. Other routes keep `frame-ancestors 'none'` and `X-Frame-Options: DENY`. The standalone local viewer keeps its original policy, which blocks all framing. This differs from the local viewer in one way that matters: the reads now cross the network, to whatever browser logged in. Only the byte ranges igv.js asks for travel, and only after the login, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS), or keep the site on loopback and forward it over ssh. Opening the viewer writes one row in the site's access log; the byte ranges write none. From 80137d4f8d947ed02b1b0beb0bd82e65ab639aa9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:16:23 +0100 Subject: [PATCH 102/255] test(recheck): red first for a failed lookup read as nothing newer, and a replayed shell string Adds the typed call reader (parse_call) without using it yet, and the tests that the next commit turns green: a lookup failure must not hide a newer ClinVar file here or print 'Nothing here is newer', and --rerun must refuse a command that is not a run-all.sh call. Bead: plat-25 Bead: plat-26 --- rust/crates/og-recheck/src/rerun.rs | 200 ++++++++++++++- rust/crates/og-recheck/src/tests.rs | 372 ++++++++++++++++++++++++++-- 2 files changed, 544 insertions(+), 28 deletions(-) diff --git a/rust/crates/og-recheck/src/rerun.rs b/rust/crates/og-recheck/src/rerun.rs index 26c27e091..78c9f242a 100644 --- a/rust/crates/og-recheck/src/rerun.rs +++ b/rust/crates/og-recheck/src/rerun.rs @@ -121,12 +121,206 @@ pub enum Rerun { Refused(u8, String), } +/// One shell word of a recorded command. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Token { + /// A word: its text, and how many of its first bytes stood unquoted. + Word(String, usize), + /// The `&&` between the `cd` and the call. + And, +} + +/// Bytes `printf %q` leaves unquoted anywhere in a word, and that mean +/// nothing to the shell there. `#` and `~` do too, except first in a word. +fn plain(b: u8, first: bool) -> bool { + b.is_ascii_alphanumeric() + || b"%+-./:=@_,".contains(&b) + || (!first && (b == b'#' || b == b'~')) + || b >= 0x80 +} + +/// The text of a `$'...'` quote from just after its `$'`; `None` for an +/// escape `printf %q` never writes. Moves `i` past the closing quote. +fn ansi_c(s: &[u8], i: &mut usize, out: &mut Vec) -> Option<()> { + loop { + let b = *s.get(*i)?; + *i += 1; + match b { + b'\'' => return Some(()), + b'\\' => { + let e = *s.get(*i)?; + *i += 1; + out.push(match e { + b'a' => 7, + b'b' => 8, + b'e' | b'E' => 27, + b'f' => 12, + b'n' => b'\n', + b'r' => b'\r', + b't' => b'\t', + b'v' => 11, + b'\\' | b'\'' | b'"' | b'?' => e, + b'0'..=b'7' => { + let mut v = u32::from(e - b'0'); + for _ in 0..2 { + match s.get(*i) { + Some(d @ b'0'..=b'7') => { + v = v * 8 + u32::from(d - b'0'); + *i += 1; + } + _ => break, + } + } + u8::try_from(v).ok()? + } + b'x' => { + let mut v = 0u32; + let mut n = 0; + while n < 2 + && let Some(d) = s.get(*i).and_then(|d| char::from(*d).to_digit(16)) + { + v = v * 16 + d; + *i += 1; + n += 1; + } + if n == 0 { + return None; + } + u8::try_from(v).ok()? + } + _ => return None, + }); + } + _ => out.push(b), + } + } +} + +/// The words of a command as `run-all.sh` writes it with `printf %q`: +/// plain bytes, `\x`, `'...'` and `$'...'`, words split on spaces and tabs, +/// and the `&&` operator. `None` for anything else a shell would act on (a +/// `;`, a pipe, a redirect, an expansion, a glob, a double quote, a line +/// break) and for text that is not UTF-8. +fn words(cmd: &str) -> Option> { + let s = cmd.as_bytes(); + let (mut out, mut i) = (Vec::new(), 0); + while i < s.len() { + if matches!(s[i], b' ' | b'\t') { + i += 1; + continue; + } + if s[i..].starts_with(b"&&") && s.get(i + 2).is_none_or(|b| matches!(*b, b' ' | b'\t')) { + out.push(Token::And); + i += 2; + continue; + } + let (mut w, mut lead, mut quoted) = (Vec::new(), 0, false); + while i < s.len() && !matches!(s[i], b' ' | b'\t') { + let b = s[i]; + if b == b'\\' { + let ch = cmd.get(i + 1..)?.chars().next()?; + if ch == '\n' { + return None; + } + let mut buf = [0u8; 4]; + w.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes()); + i += 1 + ch.len_utf8(); + quoted = true; + } else if b == b'\'' { + let end = s[i + 1..].iter().position(|&c| c == b'\'')? + i + 1; + w.extend_from_slice(&s[i + 1..end]); + i = end + 1; + quoted = true; + } else if b == b'$' && s.get(i + 1) == Some(&b'\'') { + i += 2; + ansi_c(s, &mut i, &mut w)?; + quoted = true; + } else if plain(b, w.is_empty() && !quoted) { + w.push(b); + i += 1; + if !quoted { + lead = w.len(); + } + } else { + return None; + } + } + out.push(Token::Word(String::from_utf8(w).ok()?, lead)); + } + Some(out) +} + +/// A recorded `run-all.sh` call as typed parts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Call { + /// The directory it was run from. + pub dir: String, + /// The switches it set, as `NAME`, value. + pub env: Vec<(String, String)>, + /// The `run-all.sh` it ran. + pub script: String, + /// Its arguments. + pub args: Vec, +} + +/// `meta command` as a typed call, or `None` when it is not the shape +/// `run-all.sh` writes: `cd DIR && [NAME=VALUE ...] .../scripts/run-all.sh +/// ARG ...`, every `NAME` one of `switches`. Nothing of it reaches a +/// shell. +pub fn parse_call(cmd: &str, switches: &str) -> Option { + let names: Vec<&str> = switches.split_whitespace().collect(); + let mut t = words(cmd)?.into_iter(); + let mut word = || match t.next() { + Some(Token::Word(w, lead)) => Some((w, lead)), + _ => None, + }; + let (cd, lead) = word()?; + if cd != "cd" || lead != 2 { + return None; + } + let (dir, _) = word()?; + if !matches!(t.next(), Some(Token::And)) { + return None; + } + let mut env = Vec::new(); + let script = loop { + let (w, lead) = match t.next() { + Some(Token::Word(w, lead)) => (w, lead), + _ => return None, + }; + match w.find('=') { + Some(eq) if eq < lead && names.contains(&&w[..eq]) => { + env.push((w[..eq].to_owned(), w[eq + 1..].to_owned())); + } + _ => break w, + } + }; + if !script.ends_with("/scripts/run-all.sh") { + return None; + } + let args = t + .map(|x| match x { + Token::Word(w, _) => Some(w), + Token::And => None, + }) + .collect::>>()?; + Some(Call { + dir, + env, + script, + args, + }) +} + /// Replays the last `run-all.sh` command of the sample (`meta command` of /// `logs/run_status.tsv`) with the same switches: the switch variables the /// run read (`meta switches`) are dropped from the environment, so the -/// command's own assignments decide. One `--rerun` per sample at a time: -/// `recheck/rerun.lock` is held from the running check through the end of -/// the replay. Records the rerun in `state` and the log. +/// command's own assignments decide. The command is read as a typed call +/// ([`parse_call`]) and `run-all.sh` is started by `bash` with its +/// arguments one by one; a command of any other shape is refused and +/// nothing runs. One `--rerun` per sample at a time: `recheck/rerun.lock` +/// is held from the running check through the end of the replay. Records +/// the rerun in `state` and the log. pub fn rerun( sample: &str, sample_dir: &Path, diff --git a/rust/crates/og-recheck/src/tests.rs b/rust/crates/og-recheck/src/tests.rs index 9d465286d..fb1fd6bea 100644 --- a/rust/crates/og-recheck/src/tests.rs +++ b/rust/crates/og-recheck/src/tests.rs @@ -4,7 +4,7 @@ use super::*; use crate::answers::{Offline, Online, PGS_API}; -use crate::rerun::{compare, launch_dir_command}; +use crate::rerun::{Call, compare, launch_dir_command, parse_call}; use og_fresh::http::GitHub; use og_fresh::pins::Pins; use std::fs; @@ -219,6 +219,27 @@ fn s(p: &Path) -> &str { p.to_str().unwrap() } +fn chmod_x(path: &Path) { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); +} + +/// A stand-in `scripts/run-all.sh` beside the genome folder that runs +/// `body`, recorded in `logs/run_status.tsv` the way `run-all.sh` records +/// itself: `cd` to the launch folder, then the script and `S1 male`. +fn stub_run_all(g: &Genome, body: &str) { + let launch = g.root.parent().unwrap().join("launch"); + let script = launch.join("scripts/run-all.sh"); + fs::create_dir_all(script.parent().unwrap()).unwrap(); + fs::write(&script, format!("#!/bin/sh\n{body}\n")).unwrap(); + chmod_x(&script); + let cmd = format!("cd '{}' && '{}' S1 male", s(&launch), s(&script)); + let st = fs::read_to_string(g.status()) + .unwrap() + .replace("echo replayed", &cmd); + fs::write(g.status(), st).unwrap(); +} + #[test] fn clinvar_newer_upstream_is_refresh_available_with_06_and_its_downstream() { let g = genome("cv", Opt::default()); @@ -568,20 +589,11 @@ fn after_rerun_the_check_is_taken_again_on_the_manifest_the_rerun_wrote() { .collect(); fs::write(&newman, text).unwrap(); let g = genome("after-g", Opt::default()); - let cmd = format!("cp '{}' '{}'", s(&newman), s(&g.manifest())); - let g = Genome { - root: { - let st = fs::read_to_string(g.status()) - .unwrap() - .replace("echo replayed", &cmd); - fs::write(g.status(), st).unwrap(); - gz( - &g.root.join("clinvar/clinvar_pathogenic_chr.vcf.gz"), - "##fileformat=VCFv4.1\n##fileDate=20261005\n#CHROM\n", - ); - g.root - }, - }; + stub_run_all(&g, &format!("cp '{}' '{}'", s(&newman), s(&g.manifest()))); + gz( + &g.root.join("clinvar/clinvar_pathogenic_chr.vcf.gz"), + "##fileformat=VCFv4.1\n##fileDate=20261005\n#CHROM\n", + ); let off = fix("offline.json"); let (rc, out, _) = cli(&[ "--sample", @@ -698,11 +710,7 @@ fn refresh_clinvar_holds_the_lock_while_setup_refreshes_and_lets_it_go_before_th .status() .unwrap(); let g = genome("refresh-held-g", Opt::default()); - let replay = probe(s(&g.root.join(CLINVAR_LOCK)), &seen, 20); - let st = fs::read_to_string(g.status()) - .unwrap() - .replace("echo replayed", &replay); - fs::write(g.status(), st).unwrap(); + stub_run_all(&g, &probe(s(&g.root.join(CLINVAR_LOCK)), &seen, 20)); let off = fix("offline.json"); let (rc, out, err) = cli(&[ "--sample", @@ -1049,11 +1057,7 @@ fn a_lookup_that_fails_after_the_replay_makes_the_command_exit_1() { text += "data\tpgs:PGS000099\tpgs_id=PGS000099;HmPOS_date=2022-07-29\n"; fs::write(&newman, text).unwrap(); let g = genome("after-fail-g", Opt::default()); - let cmd = format!("cp '{}' '{}'", s(&newman), s(&g.manifest())); - let st = fs::read_to_string(g.status()) - .unwrap() - .replace("echo replayed", &cmd); - fs::write(g.status(), st).unwrap(); + stub_run_all(&g, &format!("cp '{}' '{}'", s(&newman), s(&g.manifest()))); let off = fix("offline.json"); let (rc, out, _) = cli(&[ "--sample", @@ -1068,3 +1072,321 @@ fn a_lookup_that_fails_after_the_replay_makes_the_command_exit_1() { assert!(state.contains("pgs:PGS000099") && state.contains("\"class\": \"error\"")); assert_eq!(rc, 1, "{out}"); } + +/// Every lookup through the online client at a port where nothing listens: +/// a refused connection, not an HTTP error. +struct Refused(Online, u16); + +impl Answers for Refused { + fn clinvar(&self) -> og_fresh::Lookup> { + let url = format!("http://127.0.0.1:{}/clinvar", self.1); + og_fresh::http::Fetch::text(&self.0.client, &url).map(Some) + } + fn pinned(&self, key: &str) -> og_fresh::Lookup> { + let url = format!("http://127.0.0.1:{}/{key}", self.1); + og_fresh::http::Fetch::text(&self.0.client, &url).map(Some) + } + fn pgs(&self, id: &str) -> og_fresh::Lookup> { + self.0.pgs(id) + } + fn release(&self) -> og_fresh::Lookup { + self.0.release() + } + fn hosts(&self) -> Vec { + self.0.hosts() + } + fn offline(&self) -> bool { + false + } +} + +fn printed(c: &Check) -> String { + let mut o = Vec::new(); + print_check(c, &mut o).unwrap(); + String::from_utf8(o).unwrap() +} + +#[test] +fn a_failed_lookup_never_prints_nothing_newer_and_keeps_a_newer_file_here_rerun_ready() { + let g = genome( + "cv-down", + Opt { + installed_clinvar: Some("20261101"), + ..Opt::default() + }, + ); + let c = run_check( + &g, + &answers(&[( + "clinvar_file_date", + r#"{"error": "no answer (Could not resolve host)"}"#, + )]), + &this_pins(), + Some("3.66.0"), + ); + let cv = by_item(&c)["clinvar_file_date"]; + assert_eq!(cv.class, Class::RerunReady, "{cv:?}"); + assert!(cv.error.as_deref().unwrap().contains("resolve host")); + assert_eq!(c.rerun_ready_steps, CLINVAR_STEPS); + assert_eq!(c.errors(), 1); + let out = printed(&c); + assert!(!out.contains("Nothing here is newer"), "{out}"); + assert!(out.contains("the lookup failed: no answer"), "{out}"); + assert!(out.contains("A rerun would redo steps 06,"), "{out}"); + assert!(out.contains("1 of 7 lookups failed"), "{out}"); +} + +#[test] +fn when_every_lookup_is_refused_the_check_says_it_could_not_check() { + let port = { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + l.local_addr().unwrap().port() + }; + let g = genome("all-refused", Opt::default()); + let r = Refused(online(port), port); + let c = run_check(&g, &r, &this_pins(), Some("3.66.0")); + for i in c + .items + .iter() + .filter(|i| i.error.is_some() || i.upstream.is_some()) + { + assert_eq!(i.class, Class::Error, "{i:?}"); + } + assert!(c.release.is_err()); + assert_eq!(c.errors(), 7); + assert!(c.affected_steps.is_empty()); + let out = printed(&c); + assert!(!out.contains("Nothing here is newer"), "{out}"); + assert!( + out.contains("Could not check: all 7 lookups failed"), + "{out}" + ); + // The command: every answer an error, exit 1, the same closing line. + let p = tmp("all-err").join("all-err.json"); + fs::write( + &p, + r#"{"clinvar_file_date": {"error": "no answer"}, "VEP_CACHE_RELEASE": {"error": "no answer"}, + "PCGR_DATA_BUNDLE": {"error": "no answer"}, "PYPGX_BUNDLE_VERSION": {"error": "no answer"}, + "PHARMCAT_IMAGE": {"error": "no answer"}, "hla_database": {"error": "no answer"}, + "pgs:PGS000001": {"error": "no answer"}, "release": {"error": "no answer"}}"#, + ) + .unwrap(); + let (rc, out, _) = cli(&[ + "--sample", + "S1", + "--genome-dir", + s(&g.root), + "--offline", + s(&p), + ]); + assert_eq!(rc, 1, "{out}"); + assert!(!out.contains("Nothing here is newer"), "{out}"); + assert!( + out.contains("Could not check: all 7 lookups failed"), + "{out}" + ); + // Control: with every answer given, nothing newer is said. + let g = genome("all-answered", Opt::default()); + let c = run_check( + &g, + &answers(&[ + ("clinvar_file_date", r#"{"upstream": "20260901"}"#), + ("VEP_CACHE_RELEASE", r#"{"upstream": "116"}"#), + ]), + &this_pins(), + Some("3.66.0"), + ); + assert_eq!(c.errors(), 0); + let out = printed(&c); + assert!(out.contains("Nothing here is newer"), "{out}"); +} + +#[test] +fn a_recorded_command_is_read_as_a_typed_call() { + // As bash 5's printf %q writes them: a space, a comma, a tab, a quote, + // an empty word, a non-ASCII letter in a C locale. + let cmd = "cd /data/my\\ run && TOOLS=deepvariant\\,pharmcat NXF_VER=25.04.6 \ + /opt/og/scripts/run-all.sh S1 male --publish_dir out\\ dir/x\\;y it\\'s $'a\\tb' '' \ + $'\\303\\251' a~b a#b"; + let call = parse_call(cmd, "GENOME_DIR TOOLS NXF_VER").unwrap(); + assert_eq!( + call, + Call { + dir: "/data/my run".into(), + env: vec![ + ("TOOLS".into(), "deepvariant,pharmcat".into()), + ("NXF_VER".into(), "25.04.6".into()), + ], + script: "/opt/og/scripts/run-all.sh".into(), + args: [ + "S1", + "male", + "--publish_dir", + "out dir/x;y", + "it's", + "a\tb", + "", + "\u{e9}", + "a~b", + "a#b" + ] + .map(String::from) + .to_vec(), + } + ); + // Anything else a shell would act on is no call. + let base = "cd /d && /o/scripts/run-all.sh S1 male"; + for bad in [ + "touch /tmp/X && echo hi".to_owned(), + format!("{base}; rm -rf /tmp/X"), + format!("{base} ; rm -rf /tmp/X"), + format!("{base} && rm -rf /tmp/X"), + format!("{base} | tee x"), + format!("{base} > x"), + format!("{base} $(id)"), + format!("{base} `id`"), + format!("{base} \"quoted\""), + format!("{base} *"), + format!("{base} $HOME"), + format!("{base} #comment"), + format!("{base} ~/x"), + format!("{base} 'open"), + format!("{base} $'\\u00e9'"), + "cd /d && OTHER=1 /o/scripts/run-all.sh S1 male".to_owned(), + "cd /d && /o/scripts/evil.sh S1 male".to_owned(), + "cd /d /o/scripts/run-all.sh S1 male".to_owned(), + "\\cd /d && /o/scripts/run-all.sh S1 male".to_owned(), + "cd /d && 'TOOLS=x' /o/scripts/run-all.sh S1".to_owned(), + ] { + assert_eq!(parse_call(&bad, "TOOLS"), None, "{bad}"); + } +} + +#[test] +fn rerun_refuses_a_command_that_is_not_a_run_all_call_and_runs_nothing() { + let off = fix("offline.json"); + let base = tmp("inject"); + let victim = base.join("victim"); + let made = base.join("made"); + for (name, cmd) in [ + ("touch", format!("touch '{}' && echo not-run-all", s(&made))), + ( + "suffix", + format!( + "cd '{}' && '{}/scripts/run-all.sh' S1 male; rm -f '{}'", + s(&base), + s(&base), + s(&victim) + ), + ), + ] { + fs::write(&victim, "").unwrap(); + let g = genome(&format!("inject-{name}"), Opt::default()); + let st = fs::read_to_string(g.status()) + .unwrap() + .replace("echo replayed", &cmd); + fs::write(g.status(), st).unwrap(); + let (rc, out, err) = cli(&[ + "--sample", + "S1", + "--genome-dir", + s(&g.root), + "--offline", + s(&off), + "--rerun", + ]); + assert_eq!(rc, 2, "{name}: {out}{err}"); + assert!(err.contains("not a run-all.sh call"), "{err}"); + assert!(!out.contains("Rerunning"), "{out}"); + assert!(!made.exists(), "{name}: the command ran"); + assert!(victim.exists(), "{name}: the suffix ran"); + let log = fs::read_to_string(g.sample().join("recheck/log.tsv")).unwrap(); + assert!(!log.contains("\trerun-start\t"), "{log}"); + } + // A call whose launch folder is gone is refused too. + let g = genome("inject-gone", Opt::default()); + let st = fs::read_to_string(g.status()).unwrap().replace( + "echo replayed", + &format!( + "cd '{}/gone' && '{}/scripts/run-all.sh' S1 male", + s(&base), + s(&base) + ), + ); + fs::write(g.status(), st).unwrap(); + let (rc, _, err) = cli(&[ + "--sample", + "S1", + "--genome-dir", + s(&g.root), + "--offline", + s(&off), + "--rerun", + ]); + assert_eq!(rc, 2, "{err}"); + assert!(err.contains("is gone"), "{err}"); +} + +#[test] +fn rerun_starts_run_all_with_the_recorded_arguments_one_by_one() { + let g = genome("typed", Opt::default()); + let launch = g.root.parent().unwrap().join("launch dir"); + let script = launch.join("scripts/run-all.sh"); + fs::create_dir_all(script.parent().unwrap()).unwrap(); + let seen = g.root.parent().unwrap().join("seen.txt"); + fs::write( + &script, + format!( + "{{ printf '%s\\n' \"$PWD\" \"${{TOOLS-unset}}\" \"${{GENOME_DIR-unset}}\" \"$#\"; \ + for a in \"$@\"; do printf '[%s]\\n' \"$a\"; done; }} > '{}'\n", + s(&seen) + ), + ) + .unwrap(); + chmod_x(&script); + let q = |p: &Path| s(p).replace(' ', "\\ "); + let cmd = format!( + "cd {} && TOOLS=deepvariant\\,pharmcat {} S1 male --publish_dir out\\ dir it\\'s $'a\\tb' '' x\\;\\ touch\\ y", + q(&launch), + q(&script) + ); + let st = fs::read_to_string(g.status()) + .unwrap() + .replace("echo replayed", &cmd); + fs::write(g.status(), st).unwrap(); + let off = fix("offline.json"); + let (rc, out, err) = cli(&[ + "--sample", + "S1", + "--genome-dir", + s(&g.root), + "--offline", + s(&off), + "--rerun", + ]); + assert_eq!(rc, 0, "{out}{err}"); + let got = fs::read_to_string(&seen).unwrap(); + let pwd = launch.canonicalize().unwrap(); + let first = got.lines().next().unwrap(); + assert!(Path::new(first).canonicalize().unwrap() == pwd, "{got}"); + let rest: Vec<&str> = got.lines().skip(1).collect(); + assert_eq!( + rest, + [ + "deepvariant,pharmcat", + // GENOME_DIR is a recorded switch the command does not set: dropped. + "unset", + "8", + "[S1]", + "[male]", + "[--publish_dir]", + "[out dir]", + "[it's]", + "[a\tb]", + "[]", + "[x; touch y]", + ], + "{got}" + ); + assert!(!launch.join("y").exists()); +} From ad9606554f06bee128f9da7bac642fefd0b28974 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:18:08 +0100 Subject: [PATCH 103/255] fix(freshness): verify clinical download links and metadata Bead: D11 --- rust/crates/og-fresh/src/lookups.rs | 74 ++++++++++ .../tests/screen_setup_clinical_default.rs | 135 +++++++++++++++--- versions.env | 2 + 3 files changed, 195 insertions(+), 16 deletions(-) diff --git a/rust/crates/og-fresh/src/lookups.rs b/rust/crates/og-fresh/src/lookups.rs index 70a6e37bb..14511e2a8 100644 --- a/rust/crates/og-fresh/src/lookups.rs +++ b/rust/crates/og-fresh/src/lookups.rs @@ -223,6 +223,77 @@ pub fn pcgr(c: &Ctx<'_>) -> Lookup { }) } +/// Verify a pinned cache URL and its entry in Ensembl's small CHECKSUMS file. +/// Large archives are only read with HEAD, never downloaded by freshness. +fn cache_download(c: &Ctx<'_>, pin: &str, label: &str) -> Lookup { + let release = c.pins.need(pin)?; + let url = vep_cache_template(c.common)?.replace("%s", release); + c.fetch.head(&url)?; + let (base, name) = url.rsplit_once('/').expect("cache URL has a basename"); + let checksums = c.fetch.text(&format!("{base}/CHECKSUMS"))?; + let listed = checksums.lines().any(|line| { + let fields: Vec<_> = line.split_whitespace().collect(); + fields.len() == 3 + && fields[2] == name + && fields[0].parse::().is_ok() + && fields[1].parse::().is_ok_and(|n| n > 0) + }); + if !listed { + return failed(format!("{name} has no usable entry in {base}/CHECKSUMS")); + } + Ok(Found { + row: row( + label, + release.into(), + "URL live; CHECKSUMS entry present".into(), + false, + String::new, + ), + upstream: Upstream::ReportOnly, + }) +} + +/// The release used by steps 13, 30, 23 and 31. +pub fn vep_download(c: &Ctx<'_>) -> Lookup { + cache_download(c, "VEP_CACHE_RELEASE", "VEP cache download") +} + +/// The separate release used inside CPSR's image. +pub fn cpsr_download(c: &Ctx<'_>) -> Lookup { + cache_download(c, "PCGR_VEP_CACHE_RELEASE", "CPSR cache download") +} + +/// The pinned PCGR bundle URL and size. SHA-256 is checked by setup, which +/// downloads the file; freshness deliberately reads only its headers. +pub fn pcgr_download(c: &Ctx<'_>) -> Lookup { + let bundle = c.pins.need("PCGR_DATA_BUNDLE")?; + let expected = c + .pins + .need("PCGR_DATA_BYTES")? + .parse::() + .map_err(|_| LookupError::Shape("PCGR_DATA_BYTES is not a byte count".into()))?; + let url = format!("https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.{bundle}.grch38.tgz"); + let headers = c.fetch.head(&url)?; + let size = headers + .get("Content-Length") + .and_then(|s| s.parse::().ok()); + if expected == 0 || size != Some(expected) { + return failed(format!( + "{url}: Content-Length {size:?}, expected {expected}" + )); + } + Ok(Found { + row: row( + "PCGR bundle download", + bundle.into(), + format!("URL live; {expected} bytes"), + false, + String::new, + ), + upstream: Upstream::ReportOnly, + }) +} + pub fn pypgx(c: &Ctx<'_>) -> Lookup { let image_tag = tag_of(c.pins, "PYPGX_IMAGE")?; let held = image_tag.split("--").next().unwrap_or("").to_owned(); @@ -667,6 +738,9 @@ pub type LookupFn = fn(&Ctx<'_>) -> Lookup; pub const DATABASES: &[(&str, LookupFn)] = &[ ("Ensembl / VEP cache", ensembl), ("PCGR / CPSR", pcgr), + ("VEP cache download", vep_download), + ("CPSR cache download", cpsr_download), + ("PCGR bundle download", pcgr_download), ("pypgx", pypgx), ("PharmCAT", pharmcat), ("IPD-IMGT/HLA", imgt), diff --git a/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs b/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs index 077cd2a27..0564a0ace 100644 --- a/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs +++ b/rust/crates/og-fresh/tests/screen_setup_clinical_default.rs @@ -15,39 +15,142 @@ struct Downloads { calls: Mutex>, } impl Fetch for Downloads { - fn github_api(&self) -> &GitHub { &self.github } + fn github_api(&self) -> &GitHub { + &self.github + } fn request(&self, r: &Request<'_>) -> Lookup<(Body, Headers)> { - self.calls.lock().unwrap().push((r.method.into(), r.url.into())); - if self.missing { return failed("HTTP 404: release does not exist"); } + self.calls + .lock() + .unwrap() + .push((r.method.into(), r.url.into())); + if self.missing { + return failed("HTTP 404: release does not exist"); + } if r.url.ends_with("CHECKSUMS") { assert_eq!(r.want, Want::Text); - let body = if self.bad_checksum { "123 5 wrong.tar.gz\n".into() } else { + let body = if self.bad_checksum { + "123 5 wrong.tar.gz\n".into() + } else { "123 5 homo_sapiens_vep_116_GRCh38.tar.gz\n456 6 homo_sapiens_vep_115_GRCh38.tar.gz\n".into() }; Ok((Body::Text(body), Headers::default())) } else { assert_eq!(r.method, "HEAD", "large archive body was requested"); - Ok((Body::None, Headers::from_pairs(vec![("Content-Length".into(), if self.bad_length { "1" } else { "7664082467" }.into())]))) + Ok(( + Body::None, + Headers::from_pairs(vec![( + "Content-Length".into(), + if self.bad_length { "1" } else { "7664082467" }.into(), + )]), + )) } } } fn check(missing: bool, bad_length: bool, bad_checksum: bool) { - let fetch = Downloads { github: GitHub { api: "https://invalid".into(), token: None }, missing, bad_length, bad_checksum, calls: Mutex::new(vec![]) }; - let registry = FakeRegistry { tags: |_| panic!("registry is not needed"), digests: |_| panic!("registry is not needed"), asked: Mutex::new(vec![]) }; - let pins = Pins::parse("VEP_CACHE_RELEASE=\"116\"\nPCGR_VEP_CACHE_RELEASE=\"115\"\nPCGR_DATA_BUNDLE=\"20260620\"\nPCGR_DATA_BYTES=\"7664082467\"\n"); + let fetch = Downloads { + github: GitHub { + api: "https://invalid".into(), + token: None, + }, + missing, + bad_length, + bad_checksum, + calls: Mutex::new(vec![]), + }; + let registry = FakeRegistry { + tags: |_| panic!("registry is not needed"), + digests: |_| panic!("registry is not needed"), + asked: Mutex::new(vec![]), + }; + let pins = Pins::parse( + "VEP_CACHE_RELEASE=\"116\"\nPCGR_VEP_CACHE_RELEASE=\"115\"\nPCGR_DATA_BUNDLE=\"20260620\"\nPCGR_DATA_BYTES=\"7664082467\"\n", + ); let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); let common = root.join("scripts/lib/common.sh"); - let c = Ctx { fetch: &fetch, registry: ®istry, pins: &pins, versions: Path::new("unused"), setup: Path::new("unused"), common: &common, now: "2026-11-15T00:00:00Z".parse().unwrap() }; - for name in ["VEP cache download", "CPSR cache download", "PCGR bundle download"] { - let (_, lookup) = DATABASES.iter().find(|(n, _)| *n == name).unwrap_or_else(|| panic!("missing liveness row: {name}")); + let c = Ctx { + fetch: &fetch, + registry: ®istry, + pins: &pins, + versions: Path::new("unused"), + setup: Path::new("unused"), + common: &common, + now: "2026-11-15T00:00:00Z".parse().unwrap(), + }; + for name in [ + "VEP cache download", + "CPSR cache download", + "PCGR bundle download", + ] { + let (_, lookup) = DATABASES + .iter() + .find(|(n, _)| *n == name) + .unwrap_or_else(|| panic!("missing liveness row: {name}")); let got = lookup(&c); - let should_fail = missing || (bad_length && name == "PCGR bundle download") || (bad_checksum && name != "PCGR bundle download"); + let should_fail = missing + || (bad_length && name == "PCGR bundle download") + || (bad_checksum && name != "PCGR bundle download"); assert_eq!(got.is_err(), should_fail, "{name}: {got:?}"); } let calls = fetch.calls.lock().unwrap(); assert!(calls.iter().any(|(_, url)| url.contains("release-115/"))); } -#[test] fn clinical_links_head_only() { check(false, false, false); } -#[test] fn missing_release_fails() { check(true, false, false); } -#[test] fn changed_bundle_length_fails() { check(false, true, false); } -#[test] fn missing_cache_checksum_fails() { check(false, false, true); } +#[test] +fn clinical_links_head_only() { + check(false, false, false); +} +#[test] +fn missing_release_fails() { + check(true, false, false); +} +#[test] +fn changed_bundle_length_fails() { + check(false, true, false); +} +#[test] +fn missing_cache_checksum_fails() { + check(false, false, true); +} + +#[test] +#[ignore = "public network: archive HEADs and small Ensembl CHECKSUMS only"] +fn clinical_downloads_live() { + use og_fresh::http::Client; + use og_fresh::registry::Online; + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let versions = root.join("versions.env"); + let setup = root.join("scripts/setup.sh"); + let common = root.join("scripts/lib/common.sh"); + let client = Client::default(); + let registry = Online(&client); + let pins = Pins::read(&versions).unwrap(); + let c = Ctx { + fetch: &client, + registry: ®istry, + pins: &pins, + versions: &versions, + setup: &setup, + common: &common, + now: "2026-11-15T00:00:00Z".parse().unwrap(), + }; + for name in [ + "VEP cache download", + "CPSR cache download", + "PCGR bundle download", + ] { + let (_, lookup) = DATABASES.iter().find(|(n, _)| *n == name).unwrap(); + let found = lookup(&c).unwrap(); + println!("{}: {}", found.row.item, found.row.upstream); + } + let mut bad_pins = pins.clone(); + bad_pins + .values + .insert("VEP_CACHE_RELEASE".into(), "99999999".into()); + let bad = Ctx { + pins: &bad_pins, + ..c + }; + assert!( + og_fresh::lookups::vep_download(&bad).is_err(), + "nonexistent release accepted" + ); +} diff --git a/versions.env b/versions.env index 304024130..018f5c605 100644 --- a/versions.env +++ b/versions.env @@ -190,6 +190,8 @@ VEP_CACHE_RELEASE="116" # with PCGR_IMAGE. PCGR_VEP_CACHE_RELEASE="115" PCGR_DATA_BUNDLE="20260620" +# Exact archive size from the server; SHA-256 is measured from the full download. +PCGR_DATA_BYTES="7664082467" # pypgx-bundle git tag (step 32): must equal the version of PYPGX_IMAGE. PYPGX_BUNDLE_VERSION="0.26.0" # Cyrius (step 21, opt-in; legacy: no automated bumps) is installed by From cd1d2d82e0dc87977df8b06d996003a1fe054f40 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:18:19 +0100 Subject: [PATCH 104/255] fix(e2e): collect with og summary collect where two cases still ran the old CLI align-read-count-gate-1 and vendor-vcf-intake-2-nf-report ran bin/collect_summary.py as a command, which no longer has one: it exited 0 and wrote nothing. Also point stale comments at og summary collect and og traits check. --- CHANGELOG.md | 2 +- assets/genotype_rules.tsv | 5 +++-- bin/render_report.py | 4 ++-- scripts/17-cpsr.sh | 2 +- tests/e2e/align-read-count-gate-1.sh | 4 ++-- tests/e2e/vendor-vcf-intake-2-nf-report.sh | 12 ++++++------ 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58cac8762..969285415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: +- The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/assets/genotype_rules.tsv b/assets/genotype_rules.tsv index bd99e77bf..c4dd1b1e1 100644 --- a/assets/genotype_rules.tsv +++ b/assets/genotype_rules.tsv @@ -3,8 +3,9 @@ # Hand-curated from the primary papers in the pmids column; no text is taken from SNPedia or any # other database. GRCh38: ref is the GRCh38 base and alt the other allele the rule reads, both on # the plus strand as in dbSNP's GRCh38 record, also for a gene on the minus strand. -# scripts/ci/check-rule-alleles.sh checks every site against dbSNP; bin/traits_report.py -# --check-rules checks the rest (tests/test_traits_report.py runs both). +# scripts/ci/check-rule-alleles.sh checks every site against dbSNP; og traits check --table +# assets/traits.tsv --rules assets/genotype_rules.tsv checks the rest (tests/test_traits_report.py +# runs both). # Four kinds of line, tab separated, each with its own columns: # rule gene trait needs_phase ancestry note pmids # site rsid chrom pos ref alt (one per site, in the order the haplotypes use) diff --git a/bin/render_report.py b/bin/render_report.py index e949a759b..530cee487 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -127,7 +127,7 @@ def ancient_lines(d): def marker_state(m): - """A copy-aware marker's state in a few words (collect_summary.read_marker_table): + """A copy-aware marker's state in a few words (og summary collect, group_c/cn.rs read_marker_table): present or uncertain with the reads and the copies that carry it, not seen with the reads, too little depth, or not assessed.""" st = m.get("status") or "not assessed" @@ -2027,7 +2027,7 @@ def hemi(x): def two_hit_lines(t): - """The genes a called deletion and a ClinVar hit both reach (collect_summary.two_hits), and which deletion + """The genes a called deletion and a ClinVar hit both reach (og summary collect, two_hits.rs), and which deletion callers were read; none for a summary written before the rule.""" if not t: return [] diff --git a/scripts/17-cpsr.sh b/scripts/17-cpsr.sh index c2414a29b..fd3f0a9f9 100755 --- a/scripts/17-cpsr.sh +++ b/scripts/17-cpsr.sh @@ -65,7 +65,7 @@ CPSR_ID=$(sh "${PGP_ROOT}/bin/cpsr_sample_id" "$SAMPLE") # (CPSR_CLASSIFICATION), so 2.2's --classify_all is gone (2.3 refuses it). # The final class (CLASSIFICATION, source in ASSERTION_AUTHORITY) is # ClinVar's unless ClinVar has no record or a conflicted one: the default -# --clinvar_trust_level 0. The reports (bin/collect_summary.py) count the +# --clinvar_trust_level 0. The reports (og summary collect) count the # final class and list every row where CPSR's own class and ClinVar's # disagree on P/LP. diff --git a/tests/e2e/align-read-count-gate-1.sh b/tests/e2e/align-read-count-gate-1.sh index 0fe8e1dcb..3398a3cc0 100755 --- a/tests/e2e/align-read-count-gate-1.sh +++ b/tests/e2e/align-read-count-gate-1.sh @@ -50,7 +50,7 @@ TT="${T}/qc/${T}_read_count.tsv" check_eq "input_reads equals fastp's total_reads" "$(val input_reads "$TT")" "$FASTP" check_eq "the BAM's primary records equal fastp's total_reads" "$(primary "${T}/aligned/${T}_sorted.bam")" "$FASTP" check_eq "read_count_check" "$(val read_count_check "$TT")" ok -python3 "${REPO}/bin/collect_summary.py" --sample "$T" --sample-dir "${G}/${T}" --out "${CASE_TMP}/T.json" >/dev/null +"${REPO}/bin/og" summary collect --sample "$T" --sample-dir "${G}/${T}" --assets "${REPO}/assets" --out "${CASE_TMP}/T.json" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/T.json" -o "${CASE_TMP}/T.txt" >/dev/null check "the text report shows both counts" \ has "^ Alignment: ${FASTP} reads in, ${FASTP} primary records in the BAM\$" "$(cat "${CASE_TMP}/T.txt" 2>/dev/null)" @@ -141,7 +141,7 @@ check_eq "the published table: primary_records" "$(val primary_records "${R}/qc/ check_eq "the published table: missing_reads" "$(val missing_reads "${R}/qc/${X}_read_count.tsv")" 8000 check_eq "the published BAM's primary records" "$(primary "${R}/aligned/${X}_sorted.bam")" 24000 check "the table is not published beside the BAM" test ! -e "${G}/${R}/aligned/${X}_read_count.tsv" -python3 "${REPO}/bin/collect_summary.py" --sample "$X" --sample-dir "${G}/${R}" --out "${CASE_TMP}/X.json" >/dev/null +"${REPO}/bin/og" summary collect --sample "$X" --sample-dir "${G}/${R}" --assets "${REPO}/assets" --out "${CASE_TMP}/X.json" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/X.json" -o "${CASE_TMP}/X.txt" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/X.json" -o "${CASE_TMP}/X.html" >/dev/null check "the text report flags the missing reads" \ diff --git a/tests/e2e/vendor-vcf-intake-2-nf-report.sh b/tests/e2e/vendor-vcf-intake-2-nf-report.sh index 936730d4c..a3598ec60 100755 --- a/tests/e2e/vendor-vcf-intake-2-nf-report.sh +++ b/tests/e2e/vendor-vcf-intake-2-nf-report.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # A VCF-first Nextflow run (no BAM) with the tools a vendor-VCF user picks: # - the HTML report shows ROH, the mito haplogroup and CPIC, with the same -# ROH numbers and haplogroup bin/collect_summary.py (the bash report) reads +# ROH numbers and haplogroup og summary collect (the bash report) reads # from the same files, and it starts after the steps it shows; # - multiqc is selected but has nothing to read: the log says so in one line; # - the completion message prints, with no onComplete handler error; @@ -35,18 +35,18 @@ check "the report exists" test -s "$HTML" for h in 'Runs of Homozygosity' 'Mitochondrial Haplogroup' 'CPIC Drug Recommendations'; do check_eq "report card '${h}' is filled" "$(html_stat "$HTML" "$h" Status)" Complete done -python3 "${REPO}/bin/collect_summary.py" --sample "$SAMPLE" --sample-dir "$R" --out "${CASE_TMP}/summary.json" \ - > "${CASE_TMP}/summary.log" 2>&1 || cat "${CASE_TMP}/summary.log" +"${REPO}/bin/og" summary collect --sample "$SAMPLE" --sample-dir "$R" --assets "${REPO}/assets" \ + --out "${CASE_TMP}/summary.json" > "${CASE_TMP}/summary.log" 2>&1 || cat "${CASE_TMP}/summary.log" want() { python3 -c "import json, sys; d = json.load(open('${CASE_TMP}/summary.json'))['sections']$1; print($2)" 2>/dev/null; } ROH_TOTAL=$(want "['roh']['data']['total_mb']" "'%.1f MB' % d") ROH_LARGEST=$(want "['roh']['data']['largest_mb']" "'%.1f MB' % d") HG=$(want "['haplogroup']['data']['haplogroup']" "d") -echo "collect_summary.py: ROH total ${ROH_TOTAL:-?}, largest ${ROH_LARGEST:-?}, haplogroup ${HG:-?}" -check "collect_summary.py read the ROH file" test -n "$ROH_TOTAL" +echo "og summary collect: ROH total ${ROH_TOTAL:-?}, largest ${ROH_LARGEST:-?}, haplogroup ${HG:-?}" +check "og summary collect read the ROH file" test -n "$ROH_TOTAL" check_eq "ROH total matches the bash report's" "$(html_stat "$HTML" 'Runs of Homozygosity' 'ROH total')" "$ROH_TOTAL" check_eq "ROH largest segment matches the bash report's" \ "$(html_stat "$HTML" 'Runs of Homozygosity' 'ROH largest segment')" "$ROH_LARGEST" -check "collect_summary.py read the haplogroup" test -n "$HG" +check "og summary collect read the haplogroup" test -n "$HG" check_eq "haplogroup matches the bash report's" "$(html_stat "$HTML" 'Mitochondrial Haplogroup' Haplogroup)" "$HG" # The report waits for the steps it shows. From 08322f4c83e05e698bbf2b45d424e4aca424d528 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:19:05 +0100 Subject: [PATCH 105/255] fix(archaic): read a site with no call as unknown unless a gVCF reference block covers it og archaic genotypes wrote every table site with no VCF record as 0 (REF/REF), so a stretch with no reads could score as a long archaic segment. It now takes --gvcf and reads such a site as the reference only inside a confident reference block (FILTER PASS or ., GT all 0, GQ and depth of 1 or more); every other one goes to IBDmix as unknown (9). The span and the site count run over the sites read. Bead: lin-01 --- rust/crates/og-ancestry/src/archaic.rs | 259 +++++++++++++++-- .../tests/i411_archaic_callable.rs | 266 ++++++++++++++++++ 2 files changed, 506 insertions(+), 19 deletions(-) create mode 100644 rust/crates/og-ancestry/tests/i411_archaic_callable.rs diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 0dbd7db0c..f31e29c0b 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -4,9 +4,11 @@ //! IBDmix compares a modern genome with an archaic one site by site and //! adds up a log-odds score (LOD) that a stretch of the modern genome shares //! a recent ancestor with the archaic genome. It takes each site's allele -//! frequency from the modern people in its input, and one genome alone -//! gives no frequency, so `genotypes` adds stand-in columns whose genotypes -//! carry the 1000 Genomes allele count of the chosen population. +//! frequency from the modern people in its input, your column included, +//! and one genome alone gives no usable frequency, so `genotypes` adds 100 +//! stand-in columns that carry round(AF x 200) ALT alleles of the chosen +//! 1000 Genomes population: about the size of one 1000 Genomes group, as +//! Chen et al. ran IBDmix once per group with the scored person in it. //! //! - `table` (setup): one GRCh37 chromosome of the archaic table, a //! sites-only VCF that Picard lifts to GRCh38. The 1000 Genomes SNVs and @@ -17,7 +19,11 @@ //! position, with every call outside the 1000 Genomes strict mask marked //! masked. //! - `genotypes` (step 43): IBDmix's genotype file of each chromosome, each -//! archaic genome's mask of masked positions, and the spans table. +//! archaic genome's mask of masked positions, and the spans table. A table +//! site with no record in the VCF is the reference (0) only where a +//! confident reference block of the gVCF (`--gvcf`) covers it, and unknown +//! (9) otherwise, so a gap in coverage never reads as a match; the span +//! runs over the sites read. //! - `report` (step 43): the segments at LOD 4 and 50 kb or longer, merged //! into Neanderthal and Denisovan ones, the shares, the ten longest and //! their protein-coding genes. @@ -53,7 +59,9 @@ const TABLE_HEADER: [&str; 13] = [ ]; /// Table code: MASKED plus the ALT count, a call outside its genome's mask. const MASKED: u8 = 10; -/// Stand-in modern people: 200 alleles, about one 1000 Genomes population. +/// Stand-in modern people: 200 alleles, about one 1000 Genomes group. Each +/// carries round(AF x 200) ALT alleles in all, so an AF at or under 0.0025 +/// is 0 and one at or over 0.9975 is 200. const PSEUDO: i64 = 100; /// Chen et al. 2020: LOD 4 and 50 kb. const MIN_LOD: f64 = 4.0; @@ -856,6 +864,182 @@ impl Calls { } } +/// One record of the gVCF: its chromosome, the positions it covers and +/// whether it is a confident reference call there. +struct Block { + chrom: String, + start: i64, + end: i64, + reference: bool, +} + +/// Whether the one sample of a gVCF record is a confident reference call: +/// FILTER PASS or `.`, every allele of GT `0`, and GQ and depth (MIN_DP, +/// else DP) of 1 or more where the record gives them. A GQ 0 or depth 0 +/// block is a stretch without reads, as HaplotypeCaller writes it. +fn confident_reference(c: &[&str]) -> bool { + if c[6] != "PASS" && c[6] != "." { + return false; + } + let keys: Vec<&str> = c[8].split(':').collect(); + let values: Vec<&str> = c[9].split(':').collect(); + let field = |name: &str| { + keys.iter() + .position(|k| *k == name) + .and_then(|i| values.get(i).copied()) + .filter(|v| *v != ".") + }; + let Some(gt) = field("GT") else { + return false; + }; + if !gt.split(['/', '|']).all(|a| a == "0") { + return false; + } + let at_least_one = |v: Option<&str>| v.is_none_or(|v| v.parse::().is_ok_and(|x| x >= 1.0)); + at_least_one(field("GQ")) && at_least_one(field("MIN_DP").or_else(|| field("DP"))) +} + +/// The confident reference calls of a one-sample gVCF, read alongside the +/// table in the VCF's chromosome order: a table site with no record in the +/// VCF is the reference (0) when such a call covers it, and unknown (9) +/// otherwise. Only the next record and the furthest end of the calls so +/// far are kept; a chromosome passed over is remembered by name, so a gVCF +/// in another chromosome order than the VCF stops the step instead of +/// reading as no calls. +struct Callable { + text: Text, + path: String, + pending: Option, + /// The chromosomes read so far, in file order. + seen: Vec, + chrom: String, + /// The last position the confident calls so far cover on `chrom`. + covered: i64, +} + +impl Callable { + fn open(path: &str) -> Out { + let mut text = Text::open(path)?; + loop { + let Some(raw) = text.raw()? else { + return die(format!( + "{path} has no #CHROM header line: step 43 reads a one-sample gVCF" + )); + }; + if raw.starts_with("##") { + continue; + } + if !raw.starts_with('#') { + return die(format!( + "{path} has no #CHROM header line: step 43 reads a one-sample gVCF" + )); + } + let n = raw.trim_end_matches('\n').split('\t').count(); + if n != 10 { + return die(format!( + "{path} holds {} samples; step 43 reads a one-sample gVCF \ + (keep one with bcftools view -s ).", + n.saturating_sub(9) + )); + } + break; + } + let mut callable = Callable { + text, + path: path.to_owned(), + pending: None, + seen: Vec::new(), + chrom: String::new(), + covered: 0, + }; + callable.pending = callable.read()?; + Ok(callable) + } + + /// The next record, checked to come after the one before it. + fn read(&mut self) -> Out> { + loop { + let Some(raw) = self.text.raw()? else { + return Ok(None); + }; + let line = raw.strip_suffix('\n').unwrap_or(raw); + if line.is_empty() || line.starts_with('#') { + continue; + } + let c: Vec<&str> = line.split('\t').collect(); + if c.len() < 10 { + continue; + } + let start = need_int(c[1])?; + let end = c[7] + .split(';') + .find_map(|kv| kv.strip_prefix("END=")) + .map(need_int) + .transpose()? + .unwrap_or(start + c[3].chars().count().max(1) as i64 - 1); + let path = &self.path; + match &self.pending { + Some(p) if p.chrom == c[0] => { + if start < p.start { + return usage(format!( + "{path} is not sorted: {}:{start} comes after {}", + c[0], p.start + )); + } + } + _ if self.seen.iter().any(|x| x == c[0]) => { + return usage(format!( + "{path} is not sorted: {} comes back after another chromosome", + c[0] + )); + } + _ => self.seen.push(c[0].to_owned()), + } + let reference = confident_reference(&c); + return Ok(Some(Block { + chrom: c[0].to_owned(), + start, + end, + reference, + })); + } + } + + /// Move to `chrom`, the next table chromosome of the VCF, reading past + /// the records of the chromosomes before it. + fn start(&mut self, chrom: &str) -> Out<()> { + let here = self.pending.as_ref().is_some_and(|b| b.chrom == chrom); + if !here && self.seen.iter().any(|x| x == chrom) { + return usage(format!( + "{} lists {chrom} before a chromosome the VCF lists first; \ + give the gVCF and the VCF in one chromosome order", + self.path + )); + } + while self.pending.as_ref().is_some_and(|b| b.chrom != chrom) { + self.pending = self.read()?; + } + self.chrom = chrom.to_owned(); + self.covered = 0; + Ok(()) + } + + /// Whether a confident reference call covers `pos` on the current + /// chromosome; positions are asked in ascending order. + fn covers(&mut self, pos: i64) -> Out { + while let Some(b) = &self.pending { + if b.chrom != self.chrom || b.start > pos { + break; + } + if b.reference && b.end > self.covered { + self.covered = b.end; + } + self.pending = self.read()?; + } + Ok(self.covered >= pos) + } +} + /// `chrN.tsv.gz` with N from 1 to 22: the chromosome's name. fn table_chrom(name: &str) -> Option<&str> { let chrom = name.strip_suffix(".tsv.gz")?; @@ -941,7 +1125,9 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { vcf: Vcf::new(Text::open(vcf_path)?), pending: None, }; + let mut callable = a.get("--gvcf").map(Callable::open).transpose()?; let mut spans: Vec = Vec::new(); + let mut unread = 0u64; let mut done: Vec = Vec::new(); let mut pending = calls.next()?; while let Some(first) = &pending { @@ -959,17 +1145,22 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { table, &mut calls, &mut pending, + callable.as_mut(), &head, &pseudo, af_col, two_n, out_dir, )? { - Written::Rows(span, files) => { + Written::Rows(span, not_read, files) => { spans.push(span); + unread += not_read; staged.extend(files); } - Written::Empty(path) => stale.push(path), + Written::Empty(path, not_read) => { + unread += not_read; + stale.push(path); + } } } else { while pending.as_ref().is_some_and(|r| r.chrom == chrom) { @@ -982,7 +1173,8 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { let mut sorted = names.clone(); sorted.sort_unstable(); return die(format!( - "{vcf_path} has no records on a chromosome of the archaic table ({})", + "{vcf_path} has no site read on a chromosome of the archaic table ({}): \ + no call there, and no reference block of the gVCF over a table site", sorted.join(", ") )); } @@ -1013,7 +1205,11 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { } } let sites: u64 = spans.iter().map(|s| s.3).sum(); - let _ = writeln!(err, "{} chromosomes, {sites} sites", spans.len()); + let _ = writeln!( + err, + "{} chromosomes, {sites} sites read, {unread} not read (unknown to IBDmix)", + spans.len() + ); Ok(()) } @@ -1023,12 +1219,13 @@ type Span = (String, i64, i64, u64, u64, u64); /// What one chromosome leaves to publish when the whole run has passed. enum Written { - /// Rows were written: the span row, and the staged genotype file and - /// masks with their final paths. - Rows(Span, Vec<(PathBuf, AtomicFile)>), - /// No row was written: a genotype file of this name from an earlier - /// run is stale and goes. - Empty(PathBuf), + /// Sites were read: the span row, the count of rows not read, and the + /// staged genotype file and masks with their final paths. + Rows(Span, u64, Vec<(PathBuf, AtomicFile)>), + /// No site was read (no row, or only unknown ones): a genotype file of + /// this name from an earlier run is stale and goes. The count of rows + /// not read. + Empty(PathBuf, u64), } /// IBDmix's genotype file of one chromosome, and each archaic genome's @@ -1039,6 +1236,7 @@ fn write_chrom( table: &str, calls: &mut Calls, pending: &mut Option, + mut callable: Option<&mut Callable>, head: &str, pseudo: &[String], af_col: usize, @@ -1059,7 +1257,13 @@ fn write_chrom( TABLE_HEADER.join(" ") )); } + if let Some(c) = callable.as_deref_mut() { + c.start(chrom)?; + } + // n: the rows read (code 0, 1 or 2), first and last their positions; + // unread: the rows written as unknown (9). let (mut n, mut first, mut last, mut dropped, mut carried) = (0u64, 0i64, 0i64, 0u64, 0u64); + let mut unread = 0u64; // Per archaic genome, the runs of written rows whose call is masked: // [first, last] positions, closed at the next unmasked row. let mask_paths = @@ -1114,7 +1318,19 @@ fn write_chrom( .as_ref() .filter(|r| r.chrom == chrom && r.pos == pos) { - None => 0, + // No record: the reference only where a confident reference + // call of the gVCF covers the site; otherwise nothing was read + // there, which IBDmix must see as unknown, not as a match. + None => match callable.as_deref_mut() { + Some(c) => { + if c.covers(pos)? { + 0 + } else { + 9 + } + } + None => 9, + }, Some(record) => { let call = &record.call; if call.r#ref != r || call.alt.as_ref().is_some_and(|x| x != alt) { @@ -1157,6 +1373,10 @@ fn write_chrom( } row += &format!("\t{code}\t{}\n", pseudo[ac]); gz.write_all(row.as_bytes()).map_err(werr)?; + if code == 9 { + unread += 1; + continue; + } if n == 0 { first = pos; } @@ -1170,7 +1390,7 @@ fn write_chrom( } if n == 0 { drop(gz); - return Ok(Written::Empty(path)); + return Ok(Written::Empty(path, unread)); } let gz = gz.finish().map_err(werr)?; let mut files = vec![(path, gz)]; @@ -1184,6 +1404,7 @@ fn write_chrom( } Ok(Written::Rows( (chrom.to_owned(), first, last, n, carried, dropped), + unread, files, )) } @@ -1599,8 +1820,8 @@ const STRICT: Spec = Spec { }; const GENOTYPES: Spec = Spec { verb: "genotypes", - usage: "usage: og archaic genotypes --table-dir TABLE_DIR --vcf VCF [--pop POP] [--pseudo PSEUDO] --out-dir OUT_DIR\n", - options: &["--table-dir", "--vcf", "--pop", "--pseudo", "--out-dir"], + usage: "usage: og archaic genotypes --table-dir TABLE_DIR --vcf VCF [--gvcf GVCF] [--pop POP] [--pseudo PSEUDO] --out-dir OUT_DIR\n", + options: &["--table-dir", "--vcf", "--gvcf", "--pop", "--pseudo", "--out-dir"], required: &["--table-dir", "--vcf", "--out-dir"], }; const REPORT: Spec = Spec { diff --git a/rust/crates/og-ancestry/tests/i411_archaic_callable.rs b/rust/crates/og-ancestry/tests/i411_archaic_callable.rs new file mode 100644 index 000000000..4f319b623 --- /dev/null +++ b/rust/crates/og-ancestry/tests/i411_archaic_callable.rs @@ -0,0 +1,266 @@ +//! `og archaic genotypes` reads a table site with no VCF record as the +//! reference only where a confident reference block of the gVCF covers it. +//! Before this, every such site was written as 0 (REF/REF), so a stretch +//! with no reads at all could score as a long archaic segment. + +use flate2::Compression; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use og_ancestry::archaic::cli; +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +const TABLE_HEAD: &str = + "chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tAF_ALL\tAF_AFR\tAF_AMR\tAF_EAS\tAF_EUR\tAF_SAS\n"; +const VCF_HEAD: &str = + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n"; + +/// A fresh folder for one test, with an empty `table/`. +fn dir(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-i411-archaic-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(d.join("table")).unwrap(); + d +} + +/// One chromosome of the table: an A>G row at each position, the three +/// archaic genomes 0/0 and the ALT at 0.95 in every population, the shape +/// where a site read as the reference matches the archaic genomes and +/// rarely the modern people. +fn table(d: &Path, chrom: &str, positions: impl IntoIterator) { + let mut text = String::from(TABLE_HEAD); + for p in positions { + text += &format!("{chrom}\t{p}\tA\tG\t0\t0\t0\t0.95\t0.95\t0.95\t0.95\t0.95\t0.95\n"); + } + let path = d.join(format!("table/{chrom}.tsv.gz")); + let mut gz = GzEncoder::new(std::fs::File::create(path).unwrap(), Compression::new(1)); + gz.write_all(text.as_bytes()).unwrap(); + gz.finish().unwrap(); +} + +/// A reference block over `from`-`to` with these FILTER and FORMAT values. +fn block(chrom: &str, from: i64, to: i64, filter: &str, format: &str, values: &str) -> String { + format!("{chrom}\t{from}\t.\tA\t<*>\t0\t{filter}\tEND={to}\t{format}\t{values}\n") +} + +/// `og archaic genotypes` with these VCF records and, when given, these gVCF +/// records: the exit code and stderr. +fn run(d: &Path, vcf: &str, gvcf: Option<&str>) -> (u8, String) { + std::fs::write(d.join("s.vcf"), format!("{VCF_HEAD}{vcf}")).unwrap(); + let mut args: Vec = [ + "genotypes", + "--table-dir", + d.join("table").to_str().unwrap(), + "--vcf", + d.join("s.vcf").to_str().unwrap(), + "--pseudo", + "2", + "--out-dir", + d.join("gt").to_str().unwrap(), + ] + .map(str::to_owned) + .to_vec(); + if let Some(g) = gvcf { + std::fs::write(d.join("s.g.vcf"), format!("{VCF_HEAD}{g}")).unwrap(); + args.extend(["--gvcf".to_owned(), d.join("s.g.vcf").to_str().unwrap().to_owned()]); + } + let mut err = Vec::new(); + let code = cli(&args, &mut Vec::new(), &mut err); + (code, String::from_utf8(err).unwrap()) +} + +/// Your code (the SAMPLE column) at each position of a chromosome's +/// genotype file. +fn codes(d: &Path, chrom: &str) -> BTreeMap { + let mut text = String::new(); + GzDecoder::new(std::fs::File::open(d.join(format!("gt/{chrom}.gt.gz"))).unwrap()) + .read_to_string(&mut text) + .unwrap(); + text.lines() + .skip(1) + .map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + (c[1].parse().unwrap(), c[7].to_owned()) + }) + .collect() +} + +fn spans(d: &Path) -> Vec> { + std::fs::read_to_string(d.join("gt/spans.tsv")) + .unwrap() + .lines() + .skip(1) + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect() +} + +/// The issue's case: 201 table sites from 100,000 to 300,000 with no +/// record and no reference block over them, and 51 more covered by one. +/// The uncovered sites go to IBDmix as unknown (9), so they cannot add up +/// to a segment, and the span runs over the covered ones only. +#[test] +fn a_gap_in_coverage_is_unknown_not_the_reference() { + let d = dir("gap"); + table(&d, "chr22", (100_000..=300_000).step_by(1000).chain((301_000..=351_000).step_by(1000))); + let vcf = "chr22\t50000\t.\tC\tT\t50\tPASS\t.\tGT\t0/1\n"; + let gvcf = block("chr22", 1, 99_999, ".", "GT:GQ:MIN_DP", "0/0:50:30") + + &block("chr22", 300_500, 360_000, ".", "GT:GQ:MIN_DP", "0/0:50:30"); + let (code, err) = run(&d, vcf, Some(&gvcf)); + assert_eq!(code, 0, "{err}"); + let c = codes(&d, "chr22"); + let gap: Vec<&String> = c.range(100_000..=300_000).map(|(_, v)| v).collect(); + assert_eq!(gap.len(), 201); + assert!(gap.iter().all(|v| *v == "9"), "the gap must be unknown: {gap:?}"); + assert!(c.range(301_000..).all(|(_, v)| v == "0"), "a covered site is the reference"); + assert_eq!(spans(&d), [["chr22", "301000", "351000", "51", "0", "0"]]); + assert!(err.contains("51 sites read, 201 not read"), "{err}"); + let _ = std::fs::remove_dir_all(&d); +} + +/// Without a gVCF nothing says a site with no record was read: it is +/// unknown, and a VCF with no call at any table site has nothing to analyse. +#[test] +fn without_a_gvcf_a_site_with_no_record_is_unknown() { + let d = dir("no-gvcf"); + table(&d, "chr22", (100_000..=300_000).step_by(1000)); + let (code, err) = run(&d, "chr22\t50000\t.\tC\tT\t50\tPASS\t.\tGT\t0/1\n", None); + assert_eq!(code, 1, "{err}"); + assert!(err.contains("no site read"), "{err}"); + assert!(!d.join("gt/chr22.gt.gz").exists()); + // A VCF with a record at every site (an all-sites VCF) needs no gVCF. + let all: String = (100_000..=300_000) + .step_by(1000) + .map(|p| format!("chr22\t{p}\t.\tA\tG\t50\tPASS\t.\tGT\t0/0\n")) + .collect(); + let (code, err) = run(&d, &all, None); + assert_eq!(code, 0, "{err}"); + assert!(codes(&d, "chr22").values().all(|v| v == "0")); + let _ = std::fs::remove_dir_all(&d); +} + +/// Which gVCF records count as a confident reference call, and that a VCF +/// record always wins over the gVCF. +#[test] +fn only_a_confident_reference_block_reads_as_the_reference() { + let d = dir("confident"); + table(&d, "chr22", (1..=12).map(|i| i * 1000)); + let gvcf = [ + block("chr22", 1000, 1000, ".", "GT:GQ:MIN_DP", "0/0:20:10"), // reference + block("chr22", 2000, 2000, "PASS", "GT", "0|0"), // reference, phased, no GQ + block("chr22", 3000, 3000, ".", "GT:GQ:MIN_DP", "0/0:0:10"), // GQ 0 + block("chr22", 4000, 4000, ".", "GT:GQ:MIN_DP", "0/0:30:0"), // no reads + block("chr22", 5000, 5000, ".", "GT:GQ:DP", "0/0:30:0"), // no reads (DP) + block("chr22", 6000, 6000, "RefCall", "GT:GQ", "0/0:30"), // filtered + block("chr22", 7000, 7000, ".", "GT:GQ", "./.:0"), // no call + "chr22\t8000\t.\tA\tG,<*>\t30\tPASS\t.\tGT:GQ\t0/1:30\n".to_owned(), // a variant + block("chr22", 9000, 9000, ".", "GT:GQ:MIN_DP", "0:20:10"), // haploid reference + // 10000 is in no record; 11000 is a VCF call outside every block. + block("chr22", 12000, 12000, ".", "GT:GQ:MIN_DP", "0/0:.:."), // missing values + ] + .concat(); + let vcf = "chr22\t11000\t.\tA\tG\t50\tPASS\t.\tGT\t1/1\n"; + let (code, err) = run(&d, vcf, Some(&gvcf)); + assert_eq!(code, 0, "{err}"); + let got: Vec<(i64, String)> = codes(&d, "chr22").into_iter().collect(); + let want: Vec<(i64, String)> = [ + (1000, "0"), + (2000, "0"), + (3000, "9"), + (4000, "9"), + (5000, "9"), + (6000, "9"), + (7000, "9"), + (8000, "9"), + (9000, "0"), + (10000, "9"), + (11000, "2"), + (12000, "0"), + ] + .map(|(p, c)| (p, c.to_owned())) + .to_vec(); + assert_eq!(got, want); + assert_eq!(spans(&d), [["chr22", "1000", "12000", "5", "1", "0"]]); + let _ = std::fs::remove_dir_all(&d); +} + +/// A long block covers the sites up to its END, also past a shorter record +/// that starts inside it. +#[test] +fn a_site_is_covered_up_to_the_furthest_end() { + let d = dir("overlap"); + table(&d, "chr22", [1000, 1500, 2500, 3500]); + let gvcf = block("chr22", 900, 3000, ".", "GT:GQ", "0/0:40") + + &block("chr22", 1200, 1300, ".", "GT:GQ", "./.:0"); + let (code, err) = run(&d, "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n", Some(&gvcf)); + assert_eq!(code, 0, "{err}"); + let c = codes(&d, "chr22"); + assert_eq!(c[&1500], "0"); + assert_eq!(c[&2500], "0"); + assert_eq!(c[&3500], "9"); + let _ = std::fs::remove_dir_all(&d); +} + +/// The gVCF is read alongside the VCF: the chromosomes the table lacks are +/// read past, and a gVCF in another chromosome order, or unsorted, stops +/// the step (exit 2) instead of reading as no reference calls. +#[test] +fn the_gvcf_is_read_in_the_vcf_order() { + let d = dir("order"); + table(&d, "chr21", [1000, 2000]); + table(&d, "chr22", [1000, 2000]); + let vcf = "chr21\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\nchr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n"; + let chr21 = block("chr21", 1, 5000, ".", "GT:GQ", "0/0:40"); + let chr22 = block("chr22", 1, 5000, ".", "GT:GQ", "0/0:40"); + let other = block("chrM", 1, 100, ".", "GT:GQ", "0/0:40"); + let (code, err) = run(&d, vcf, Some(&format!("{other}{chr21}{other_x}{chr22}", other_x = block("chrX", 1, 9, ".", "GT", "0/0")))); + assert_eq!(code, 0, "{err}"); + assert_eq!(codes(&d, "chr21")[&2000], "0"); + assert_eq!(codes(&d, "chr22")[&2000], "0"); + + let (code, err) = run(&d, vcf, Some(&format!("{chr22}{chr21}"))); + assert_eq!(code, 2, "{err}"); + assert!(err.contains("one chromosome order"), "{err}"); + + let back = format!("{chr21}{chr22}{}", block("chr21", 9000, 9100, ".", "GT", "0/0")); + let (code, err) = run(&d, vcf, Some(&back)); + assert_eq!(code, 2, "{err}"); + assert!(err.contains("comes back"), "{err}"); + + let unsorted = format!("{}{chr21}{chr22}", block("chr21", 3000, 3100, ".", "GT", "0/0")); + let (code, err) = run(&d, vcf, Some(&unsorted)); + assert_eq!(code, 2, "{err}"); + assert!(err.contains("is not sorted"), "{err}"); + let _ = std::fs::remove_dir_all(&d); +} + +/// A gVCF of two samples, or one without a header line, is refused. +#[test] +fn a_gvcf_of_one_sample_only() { + let d = dir("samples"); + table(&d, "chr22", [1000]); + let vcf = "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n"; + std::fs::write(d.join("s.vcf"), format!("{VCF_HEAD}{vcf}")).unwrap(); + let two = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n"; + for (gvcf, want) in [(two, "holds 2 samples"), ("chr22\t1\t.\tA\t<*>\t0\t.\tEND=9\tGT\t0/0\n", "no #CHROM header")] { + std::fs::write(d.join("x.g.vcf"), gvcf).unwrap(); + let args: Vec = [ + "genotypes", + "--table-dir", + d.join("table").to_str().unwrap(), + "--vcf", + d.join("s.vcf").to_str().unwrap(), + "--gvcf", + d.join("x.g.vcf").to_str().unwrap(), + "--out-dir", + d.join("gt").to_str().unwrap(), + ] + .map(str::to_owned) + .to_vec(); + let mut err = Vec::new(); + assert_eq!(cli(&args, &mut Vec::new(), &mut err), 1); + let err = String::from_utf8(err).unwrap(); + assert!(err.contains(want), "{err}"); + } + let _ = std::fs::remove_dir_all(&d); +} From baced2682d4a0bb0cfd801aaa0e2762e2878c8f8 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:19:39 +0100 Subject: [PATCH 106/255] build(rust): give every planned port its own crate, module and verb Adds the crates og-render, og-live, og-couple and og-admix, and every verb the 1.0.0 plan adds as an arm that calls its own module. Each such module is a stub that prints "og : not ported yet" and exits 2, so a port only replaces a module's body and never edits main.rs, the workspace member list or a crate's module list. A cargo test lists every planned verb and fails when one stops being routed. Bead: D1 Bead: D14 --- rust/Cargo.lock | 19 +++ rust/Cargo.toml | 6 +- rust/crates/og-admix/Cargo.toml | 12 ++ rust/crates/og-admix/src/lib.rs | 9 + rust/crates/og-admix/src/paint.rs | 2 + rust/crates/og-admix/src/project.rs | 2 + rust/crates/og-ancestry/src/compare.rs | 19 +++ rust/crates/og-ancestry/src/freqs.rs | 13 ++ rust/crates/og-ancestry/src/lib.rs | 17 ++ rust/crates/og-ancestry/src/paint.rs | 13 ++ rust/crates/og-ancestry/src/project.rs | 13 ++ rust/crates/og-ancestry/src/simulate.rs | 13 ++ rust/crates/og-couple/Cargo.toml | 12 ++ rust/crates/og-couple/src/genes.rs | 13 ++ rust/crates/og-couple/src/lib.rs | 8 + rust/crates/og-couple/src/screen.rs | 13 ++ rust/crates/og-dev/src/build/acmg_lof.rs | 13 ++ rust/crates/og-dev/src/build/mod.rs | 8 +- rust/crates/og-dev/src/build/roh_af.rs | 13 ++ rust/crates/og-dev/src/demo/ancestry_paint.rs | 2 + rust/crates/og-dev/src/demo/clinical.rs | 2 + .../og-dev/src/demo/clinical_clinvar.rs | 2 + rust/crates/og-dev/src/demo/family.rs | 1 + rust/crates/og-dev/src/demo/lineage.rs | 2 + rust/crates/og-dev/src/demo/mod.rs | 23 +++ rust/crates/og-dev/src/demo/pgx.rs | 2 + rust/crates/og-dev/src/demo/qc.rs | 2 + rust/crates/og-dev/src/demo/sv.rs | 2 + rust/crates/og-dev/src/demo/traits.rs | 1 + rust/crates/og-dev/src/demo_events.rs | 13 ++ rust/crates/og-dev/src/docs_checks.rs | 12 ++ rust/crates/og-dev/src/everything.rs | 13 ++ rust/crates/og-dev/src/lib.rs | 27 ++- rust/crates/og-dev/src/licences.rs | 13 ++ rust/crates/og-dev/src/params_doc.rs | 12 ++ rust/crates/og-dev/src/python_gate.rs | 13 ++ rust/crates/og-dev/src/steps_check.rs | 13 ++ rust/crates/og-dev/src/stub_http.rs | 13 ++ rust/crates/og-io/src/lib.rs | 3 + rust/crates/og-io/src/tabix_write.rs | 13 ++ rust/crates/og-live/Cargo.toml | 12 ++ rust/crates/og-live/src/auth.rs | 1 + rust/crates/og-live/src/charts.rs | 1 + rust/crates/og-live/src/csp.rs | 2 + rust/crates/og-live/src/demo.rs | 13 ++ rust/crates/og-live/src/family.rs | 12 ++ rust/crates/og-live/src/headlines.rs | 2 + rust/crates/og-live/src/ingest.rs | 2 + rust/crates/og-live/src/lib.rs | 30 ++++ rust/crates/og-live/src/map.rs | 2 + rust/crates/og-live/src/ops.rs | 2 + rust/crates/og-live/src/plugins.rs | 1 + rust/crates/og-live/src/recheck.rs | 2 + rust/crates/og-live/src/results/mod.rs | 12 ++ rust/crates/og-live/src/runs.rs | 2 + rust/crates/og-live/src/search.rs | 1 + rust/crates/og-live/src/server.rs | 13 ++ rust/crates/og-live/src/state.rs | 1 + rust/crates/og-live/src/static_files.rs | 2 + rust/crates/og-live/src/veil.rs | 2 + rust/crates/og-live/src/viewer.rs | 13 ++ rust/crates/og-live/src/widgets.rs | 2 + rust/crates/og-map/src/crossovers.rs | 2 + rust/crates/og-map/src/family.rs | 2 + rust/crates/og-map/src/layers/archaic.rs | 2 + rust/crates/og-map/src/layers/clinvar.rs | 2 + rust/crates/og-map/src/layers/mca.rs | 2 + rust/crates/og-map/src/layers/mod.rs | 5 + rust/crates/og-map/src/layers/paint.rs | 2 + rust/crates/og-map/src/layers/sv.rs | 2 + rust/crates/og-map/src/lib.rs | 5 + rust/crates/og-map/src/live.rs | 2 + rust/crates/og-map/src/svg.rs | 12 ++ rust/crates/og-map/src/threads.rs | 2 + rust/crates/og-pedigree/src/layout.rs | 2 + rust/crates/og-pedigree/src/lib.rs | 4 + rust/crates/og-render/Cargo.toml | 12 ++ rust/crates/og-render/src/charts/mod.rs | 13 ++ rust/crates/og-render/src/index.rs | 13 ++ rust/crates/og-render/src/lib.rs | 13 ++ rust/crates/og-render/src/render.rs | 13 ++ rust/crates/og-render/src/sections/acmg_sf.rs | 2 + .../crates/og-render/src/sections/ancestry.rs | 2 + .../og-render/src/sections/ancestry_fine.rs | 2 + rust/crates/og-render/src/sections/ancient.rs | 2 + rust/crates/og-render/src/sections/annotsv.rs | 2 + rust/crates/og-render/src/sections/apoe.rs | 1 + rust/crates/og-render/src/sections/archaic.rs | 2 + .../og-render/src/sections/bloodtype.rs | 2 + .../og-render/src/sections/callability.rs | 2 + rust/crates/og-render/src/sections/carrier.rs | 2 + rust/crates/og-render/src/sections/cdna.rs | 1 + .../og-render/src/sections/cdna_scan.rs | 2 + rust/crates/og-render/src/sections/chip.rs | 1 + rust/crates/og-render/src/sections/clinvar.rs | 2 + .../og-render/src/sections/cn_intervals.rs | 2 + rust/crates/og-render/src/sections/comphet.rs | 2 + rust/crates/og-render/src/sections/couple.rs | 2 + .../crates/og-render/src/sections/coverage.rs | 2 + rust/crates/og-render/src/sections/cpsr.rs | 1 + rust/crates/og-render/src/sections/cyp21a2.rs | 2 + .../og-render/src/sections/deep_intronic.rs | 2 + rust/crates/og-render/src/sections/dosage.rs | 2 + .../crates/og-render/src/sections/exomiser.rs | 2 + rust/crates/og-render/src/sections/g2.rs | 1 + .../og-render/src/sections/haplogroups.rs | 2 + rust/crates/og-render/src/sections/hba.rs | 1 + rust/crates/og-render/src/sections/hla.rs | 1 + .../og-render/src/sections/locityper.rs | 2 + rust/crates/og-render/src/sections/lpa.rs | 1 + rust/crates/og-render/src/sections/mei.rs | 1 + rust/crates/og-render/src/sections/mito.rs | 1 + rust/crates/og-render/src/sections/mocha.rs | 2 + rust/crates/og-render/src/sections/mod.rs | 54 ++++++ .../crates/og-render/src/sections/paralogs.rs | 2 + .../og-render/src/sections/parascopy.rs | 2 + rust/crates/og-render/src/sections/pgx.rs | 1 + rust/crates/og-render/src/sections/prs.rs | 1 + rust/crates/og-render/src/sections/qc.rs | 1 + rust/crates/og-render/src/sections/rfc1.rs | 1 + rust/crates/og-render/src/sections/roh.rs | 1 + rust/crates/og-render/src/sections/run.rs | 1 + rust/crates/og-render/src/sections/smn1.rs | 1 + rust/crates/og-render/src/sections/str.rs | 1 + .../og-render/src/sections/sv_callers.rs | 2 + .../crates/og-render/src/sections/telomere.rs | 2 + rust/crates/og-render/src/sections/traits.rs | 2 + .../crates/og-render/src/sections/two_hits.rs | 2 + .../crates/og-render/src/sections/variants.rs | 2 + rust/crates/og-render/src/sections/viral.rs | 2 + rust/crates/og-render/src/sections/vntyper.rs | 2 + rust/crates/og-render/src/sections/y_tree.rs | 2 + rust/crates/og-render/src/sections/ystr.rs | 1 + rust/crates/og-setup/src/lib.rs | 5 +- rust/crates/og-setup/src/revel.rs | 12 ++ rust/crates/og-setup/src/roh_af.rs | 12 ++ rust/crates/og-steps/src/lib.rs | 3 + rust/crates/og-steps/src/longread.rs | 13 ++ rust/crates/og-steps/src/sv.rs | 2 + .../crates/og-steps/src/sv/consensus_check.rs | 13 ++ rust/crates/og-steps/src/sv/survivor_prep.rs | 13 ++ rust/crates/og-summary/src/collect.rs | 1 + .../og-summary/src/collect/acmg_card.rs | 2 + rust/crates/og-vcf/src/lib.rs | 1 + rust/crates/og-vcf/src/shift.rs | 20 +++ rust/og/Cargo.toml | 3 + rust/og/src/main.rs | 90 ++++++++++ rust/og/tests/port_scaffold.rs | 158 ++++++++++++++++++ 148 files changed, 1123 insertions(+), 4 deletions(-) create mode 100644 rust/crates/og-admix/Cargo.toml create mode 100644 rust/crates/og-admix/src/lib.rs create mode 100644 rust/crates/og-admix/src/paint.rs create mode 100644 rust/crates/og-admix/src/project.rs create mode 100644 rust/crates/og-ancestry/src/compare.rs create mode 100644 rust/crates/og-ancestry/src/freqs.rs create mode 100644 rust/crates/og-ancestry/src/paint.rs create mode 100644 rust/crates/og-ancestry/src/project.rs create mode 100644 rust/crates/og-ancestry/src/simulate.rs create mode 100644 rust/crates/og-couple/Cargo.toml create mode 100644 rust/crates/og-couple/src/genes.rs create mode 100644 rust/crates/og-couple/src/lib.rs create mode 100644 rust/crates/og-couple/src/screen.rs create mode 100644 rust/crates/og-dev/src/build/acmg_lof.rs create mode 100644 rust/crates/og-dev/src/build/roh_af.rs create mode 100644 rust/crates/og-dev/src/demo/ancestry_paint.rs create mode 100644 rust/crates/og-dev/src/demo/clinical.rs create mode 100644 rust/crates/og-dev/src/demo/clinical_clinvar.rs create mode 100644 rust/crates/og-dev/src/demo/family.rs create mode 100644 rust/crates/og-dev/src/demo/lineage.rs create mode 100644 rust/crates/og-dev/src/demo/mod.rs create mode 100644 rust/crates/og-dev/src/demo/pgx.rs create mode 100644 rust/crates/og-dev/src/demo/qc.rs create mode 100644 rust/crates/og-dev/src/demo/sv.rs create mode 100644 rust/crates/og-dev/src/demo/traits.rs create mode 100644 rust/crates/og-dev/src/demo_events.rs create mode 100644 rust/crates/og-dev/src/docs_checks.rs create mode 100644 rust/crates/og-dev/src/everything.rs create mode 100644 rust/crates/og-dev/src/licences.rs create mode 100644 rust/crates/og-dev/src/params_doc.rs create mode 100644 rust/crates/og-dev/src/python_gate.rs create mode 100644 rust/crates/og-dev/src/steps_check.rs create mode 100644 rust/crates/og-dev/src/stub_http.rs create mode 100644 rust/crates/og-io/src/tabix_write.rs create mode 100644 rust/crates/og-live/Cargo.toml create mode 100644 rust/crates/og-live/src/auth.rs create mode 100644 rust/crates/og-live/src/charts.rs create mode 100644 rust/crates/og-live/src/csp.rs create mode 100644 rust/crates/og-live/src/demo.rs create mode 100644 rust/crates/og-live/src/family.rs create mode 100644 rust/crates/og-live/src/headlines.rs create mode 100644 rust/crates/og-live/src/ingest.rs create mode 100644 rust/crates/og-live/src/lib.rs create mode 100644 rust/crates/og-live/src/map.rs create mode 100644 rust/crates/og-live/src/ops.rs create mode 100644 rust/crates/og-live/src/plugins.rs create mode 100644 rust/crates/og-live/src/recheck.rs create mode 100644 rust/crates/og-live/src/results/mod.rs create mode 100644 rust/crates/og-live/src/runs.rs create mode 100644 rust/crates/og-live/src/search.rs create mode 100644 rust/crates/og-live/src/server.rs create mode 100644 rust/crates/og-live/src/state.rs create mode 100644 rust/crates/og-live/src/static_files.rs create mode 100644 rust/crates/og-live/src/veil.rs create mode 100644 rust/crates/og-live/src/viewer.rs create mode 100644 rust/crates/og-live/src/widgets.rs create mode 100644 rust/crates/og-map/src/crossovers.rs create mode 100644 rust/crates/og-map/src/family.rs create mode 100644 rust/crates/og-map/src/layers/archaic.rs create mode 100644 rust/crates/og-map/src/layers/clinvar.rs create mode 100644 rust/crates/og-map/src/layers/mca.rs create mode 100644 rust/crates/og-map/src/layers/paint.rs create mode 100644 rust/crates/og-map/src/layers/sv.rs create mode 100644 rust/crates/og-map/src/live.rs create mode 100644 rust/crates/og-map/src/svg.rs create mode 100644 rust/crates/og-map/src/threads.rs create mode 100644 rust/crates/og-pedigree/src/layout.rs create mode 100644 rust/crates/og-render/Cargo.toml create mode 100644 rust/crates/og-render/src/charts/mod.rs create mode 100644 rust/crates/og-render/src/index.rs create mode 100644 rust/crates/og-render/src/lib.rs create mode 100644 rust/crates/og-render/src/render.rs create mode 100644 rust/crates/og-render/src/sections/acmg_sf.rs create mode 100644 rust/crates/og-render/src/sections/ancestry.rs create mode 100644 rust/crates/og-render/src/sections/ancestry_fine.rs create mode 100644 rust/crates/og-render/src/sections/ancient.rs create mode 100644 rust/crates/og-render/src/sections/annotsv.rs create mode 100644 rust/crates/og-render/src/sections/apoe.rs create mode 100644 rust/crates/og-render/src/sections/archaic.rs create mode 100644 rust/crates/og-render/src/sections/bloodtype.rs create mode 100644 rust/crates/og-render/src/sections/callability.rs create mode 100644 rust/crates/og-render/src/sections/carrier.rs create mode 100644 rust/crates/og-render/src/sections/cdna.rs create mode 100644 rust/crates/og-render/src/sections/cdna_scan.rs create mode 100644 rust/crates/og-render/src/sections/chip.rs create mode 100644 rust/crates/og-render/src/sections/clinvar.rs create mode 100644 rust/crates/og-render/src/sections/cn_intervals.rs create mode 100644 rust/crates/og-render/src/sections/comphet.rs create mode 100644 rust/crates/og-render/src/sections/couple.rs create mode 100644 rust/crates/og-render/src/sections/coverage.rs create mode 100644 rust/crates/og-render/src/sections/cpsr.rs create mode 100644 rust/crates/og-render/src/sections/cyp21a2.rs create mode 100644 rust/crates/og-render/src/sections/deep_intronic.rs create mode 100644 rust/crates/og-render/src/sections/dosage.rs create mode 100644 rust/crates/og-render/src/sections/exomiser.rs create mode 100644 rust/crates/og-render/src/sections/g2.rs create mode 100644 rust/crates/og-render/src/sections/haplogroups.rs create mode 100644 rust/crates/og-render/src/sections/hba.rs create mode 100644 rust/crates/og-render/src/sections/hla.rs create mode 100644 rust/crates/og-render/src/sections/locityper.rs create mode 100644 rust/crates/og-render/src/sections/lpa.rs create mode 100644 rust/crates/og-render/src/sections/mei.rs create mode 100644 rust/crates/og-render/src/sections/mito.rs create mode 100644 rust/crates/og-render/src/sections/mocha.rs create mode 100644 rust/crates/og-render/src/sections/mod.rs create mode 100644 rust/crates/og-render/src/sections/paralogs.rs create mode 100644 rust/crates/og-render/src/sections/parascopy.rs create mode 100644 rust/crates/og-render/src/sections/pgx.rs create mode 100644 rust/crates/og-render/src/sections/prs.rs create mode 100644 rust/crates/og-render/src/sections/qc.rs create mode 100644 rust/crates/og-render/src/sections/rfc1.rs create mode 100644 rust/crates/og-render/src/sections/roh.rs create mode 100644 rust/crates/og-render/src/sections/run.rs create mode 100644 rust/crates/og-render/src/sections/smn1.rs create mode 100644 rust/crates/og-render/src/sections/str.rs create mode 100644 rust/crates/og-render/src/sections/sv_callers.rs create mode 100644 rust/crates/og-render/src/sections/telomere.rs create mode 100644 rust/crates/og-render/src/sections/traits.rs create mode 100644 rust/crates/og-render/src/sections/two_hits.rs create mode 100644 rust/crates/og-render/src/sections/variants.rs create mode 100644 rust/crates/og-render/src/sections/viral.rs create mode 100644 rust/crates/og-render/src/sections/vntyper.rs create mode 100644 rust/crates/og-render/src/sections/y_tree.rs create mode 100644 rust/crates/og-render/src/sections/ystr.rs create mode 100644 rust/crates/og-setup/src/revel.rs create mode 100644 rust/crates/og-setup/src/roh_af.rs create mode 100644 rust/crates/og-steps/src/longread.rs create mode 100644 rust/crates/og-steps/src/sv/consensus_check.rs create mode 100644 rust/crates/og-steps/src/sv/survivor_prep.rs create mode 100644 rust/crates/og-summary/src/collect/acmg_card.rs create mode 100644 rust/crates/og-vcf/src/shift.rs create mode 100644 rust/og/tests/port_scaffold.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 0dbef91fe..6bf5d012d 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -603,16 +603,19 @@ dependencies = [ "og-chip", "og-clinvar", "og-cn", + "og-couple", "og-dev", "og-filters", "og-fresh", "og-intervals", "og-io", + "og-live", "og-lookup", "og-map", "og-pgx", "og-qc", "og-recheck", + "og-render", "og-reports", "og-runlog", "og-setup", @@ -622,6 +625,10 @@ dependencies = [ "pycompat", ] +[[package]] +name = "og-admix" +version = "0.1.0" + [[package]] name = "og-ancestry" version = "0.1.0" @@ -687,6 +694,10 @@ dependencies = [ "pycompat", ] +[[package]] +name = "og-couple" +version = "0.1.0" + [[package]] name = "og-dev" version = "0.1.0" @@ -738,6 +749,10 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "og-live" +version = "0.1.0" + [[package]] name = "og-lookup" version = "0.1.0" @@ -806,6 +821,10 @@ dependencies = [ "shlex 1.3.0", ] +[[package]] +name = "og-render" +version = "0.1.0" + [[package]] name = "og-reports" version = "0.1.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 2fa060ac3..416ee70c9 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "3" -members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-qc", "crates/og-clinvar", "crates/og-pgx", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-setup", "crates/og-chip", "crates/og-cn", "crates/og-bloodtype", "crates/og-ancestry", "crates/og-pedigree", "crates/og-reports", "crates/og-summary", "crates/og-lookup", "crates/og-fresh", "crates/og-recheck", "og"] +members = ["crates/pycompat", "crates/og-io", "crates/og-vcf", "crates/og-bam", "crates/og-map", "crates/og-tsv", "crates/og-intervals", "crates/og-dev", "crates/og-qc", "crates/og-clinvar", "crates/og-pgx", "crates/og-filters", "crates/og-nf", "crates/og-runlog", "crates/og-steps", "crates/og-setup", "crates/og-chip", "crates/og-cn", "crates/og-bloodtype", "crates/og-ancestry", "crates/og-pedigree", "crates/og-reports", "crates/og-summary", "crates/og-lookup", "crates/og-fresh", "crates/og-recheck", "crates/og-render", "crates/og-live", "crates/og-couple", "crates/og-admix", "og"] [workspace.package] version = "0.1.0" @@ -35,6 +35,10 @@ og-summary = { path = "crates/og-summary" } og-lookup = { path = "crates/og-lookup" } og-fresh = { path = "crates/og-fresh" } og-recheck = { path = "crates/og-recheck" } +og-render = { path = "crates/og-render" } +og-live = { path = "crates/og-live" } +og-couple = { path = "crates/og-couple" } +og-admix = { path = "crates/og-admix" } indexmap = "2" flate2 = "1" memchr = "2" diff --git a/rust/crates/og-admix/Cargo.toml b/rust/crates/og-admix/Cargo.toml new file mode 100644 index 000000000..784875681 --- /dev/null +++ b/rust/crates/og-admix/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "og-admix" +description = "What the fine-ancestry engines share: our own local-ancestry painting and the projection of ancestry fractions" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/rust/crates/og-admix/src/lib.rs b/rust/crates/og-admix/src/lib.rs new file mode 100644 index 000000000..bf0a44088 --- /dev/null +++ b/rust/crates/og-admix/src/lib.rs @@ -0,0 +1,9 @@ +//! What the fine-ancestry engines of step 26b share: our own +//! local-ancestry painting ([`paint`]) and the projection of ancestry +//! fractions on the panel's frequencies ([`project`]). +//! +//! Empty until their units of the 1.0.0 plan land. The crate exists now so +//! those units add code without editing the same files. + +pub mod paint; +pub mod project; diff --git a/rust/crates/og-admix/src/paint.rs b/rust/crates/og-admix/src/paint.rs new file mode 100644 index 000000000..1fc0f571c --- /dev/null +++ b/rust/crates/og-admix/src/paint.rs @@ -0,0 +1,2 @@ +//! Local-ancestry painting, our own engine in place of RFMix. Empty until +//! its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-admix/src/project.rs b/rust/crates/og-admix/src/project.rs new file mode 100644 index 000000000..39710102e --- /dev/null +++ b/rust/crates/og-admix/src/project.rs @@ -0,0 +1,2 @@ +//! Ancestry fractions projected on the panel's frequencies, in place of +//! ADMIXTURE. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-ancestry/src/compare.rs b/rust/crates/og-ancestry/src/compare.rs new file mode 100644 index 000000000..426305ebe --- /dev/null +++ b/rust/crates/og-ancestry/src/compare.rs @@ -0,0 +1,19 @@ +//! `og archaic compare` and `og archaic genomes`: step 43's archaic totals +//! beside published calls, which `bin/archaic_compare.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, each verb says +//! so on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og archaic compare ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn compare_cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og archaic compare: not ported yet"); + 2 +} + +/// `og archaic genomes ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn genomes_cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og archaic genomes: not ported yet"); + 2 +} diff --git a/rust/crates/og-ancestry/src/freqs.rs b/rust/crates/og-ancestry/src/freqs.rs new file mode 100644 index 000000000..5a88a4dc8 --- /dev/null +++ b/rust/crates/og-ancestry/src/freqs.rs @@ -0,0 +1,13 @@ +//! `og ancestry-fine freqs`: each reference group's allele frequencies, +//! counted from the panel. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og ancestry-fine freqs ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og ancestry-fine freqs: not ported yet"); + 2 +} diff --git a/rust/crates/og-ancestry/src/lib.rs b/rust/crates/og-ancestry/src/lib.rs index 74febd042..5ae862961 100644 --- a/rust/crates/og-ancestry/src/lib.rs +++ b/rust/crates/og-ancestry/src/lib.rs @@ -10,6 +10,10 @@ //! - [`chroms`]: the chromosomes long enough for RFMix to paint. //! - [`report`]: the fractions table, the painting (BED and PNG) and the //! plain-text summary. +//! - [`freqs`], [`project`], [`paint`], [`simulate`]: `og ancestry-fine +//! freqs|project|paint|simulate`, the engines that replace ADMIXTURE and +//! RFMix, and [`compare`]: `og archaic compare|genomes`. Not ported yet: +//! each says so and exits 2. //! //! Each command writes what `bin/ancestry_fine_report.py` wrote (checked by //! `tests/parity`), with three deliberate changes: every output is written @@ -25,10 +29,15 @@ pub mod ancient; pub mod archaic; pub mod chroms; +pub mod compare; +pub mod freqs; pub mod order; +pub mod paint; pub mod panel; +pub mod project; pub mod report; pub mod sample_bed; +pub mod simulate; use std::fmt; use std::io; @@ -557,6 +566,14 @@ pub fn cli(args: &[String], out: &mut dyn io::Write, err: &mut dyn io::Write) -> return 2; }; let rest = &args[1..]; + // The engines read their own command lines, outside the Spec table. + match verb.as_str() { + "freqs" => return freqs::cli(rest, out, err), + "project" => return project::cli(rest, out, err), + "paint" => return paint::cli(rest, out, err), + "simulate" => return simulate::cli(rest, out, err), + _ => {} + } let (spec, run): (&Spec, Run) = match verb.as_str() { "panel-map" => (&panel::PANEL_MAP, panel::panel_map_main), "check-map" => (&panel::CHECK_MAP, panel::check_map_main), diff --git a/rust/crates/og-ancestry/src/paint.rs b/rust/crates/og-ancestry/src/paint.rs new file mode 100644 index 000000000..10e8df969 --- /dev/null +++ b/rust/crates/og-ancestry/src/paint.rs @@ -0,0 +1,13 @@ +//! `og ancestry-fine paint`: the local-ancestry painting, in place of +//! RFMix. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og ancestry-fine paint ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og ancestry-fine paint: not ported yet"); + 2 +} diff --git a/rust/crates/og-ancestry/src/project.rs b/rust/crates/og-ancestry/src/project.rs new file mode 100644 index 000000000..ff633027b --- /dev/null +++ b/rust/crates/og-ancestry/src/project.rs @@ -0,0 +1,13 @@ +//! `og ancestry-fine project`: a person's ancestry fractions projected on +//! the panel's frequencies, in place of ADMIXTURE. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og ancestry-fine project ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og ancestry-fine project: not ported yet"); + 2 +} diff --git a/rust/crates/og-ancestry/src/simulate.rs b/rust/crates/og-ancestry/src/simulate.rs new file mode 100644 index 000000000..bccf46ec0 --- /dev/null +++ b/rust/crates/og-ancestry/src/simulate.rs @@ -0,0 +1,13 @@ +//! `og ancestry-fine simulate`: simulated admixed genomes that check the +//! painting and the projection. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og ancestry-fine simulate ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og ancestry-fine simulate: not ported yet"); + 2 +} diff --git a/rust/crates/og-couple/Cargo.toml b/rust/crates/og-couple/Cargo.toml new file mode 100644 index 000000000..7ece1392f --- /dev/null +++ b/rust/crates/og-couple/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "og-couple" +description = "og couple screen|genes: step 39's couple carrier screen" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/rust/crates/og-couple/src/genes.rs b/rust/crates/og-couple/src/genes.rs new file mode 100644 index 000000000..22511fb6a --- /dev/null +++ b/rust/crates/og-couple/src/genes.rs @@ -0,0 +1,13 @@ +//! `og couple genes`: the gene list of step 39's screen, which +//! `bin/couple_report.py genes` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og couple genes ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og couple genes: not ported yet"); + 2 +} diff --git a/rust/crates/og-couple/src/lib.rs b/rust/crates/og-couple/src/lib.rs new file mode 100644 index 000000000..0ae5dbe68 --- /dev/null +++ b/rust/crates/og-couple/src/lib.rs @@ -0,0 +1,8 @@ +//! Step 39, the couple carrier screen: `og couple screen` ([`screen`]) and +//! `og couple genes` ([`genes`]). +//! +//! Not ported yet: each verb says so and exits 2 until its unit of the +//! 1.0.0 plan lands. + +pub mod genes; +pub mod screen; diff --git a/rust/crates/og-couple/src/screen.rs b/rust/crates/og-couple/src/screen.rs new file mode 100644 index 000000000..d86757436 --- /dev/null +++ b/rust/crates/og-couple/src/screen.rs @@ -0,0 +1,13 @@ +//! `og couple screen`: step 39's couple carrier screen, which +//! `bin/couple_report.py screen` runs today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og couple screen ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og couple screen: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/build/acmg_lof.rs b/rust/crates/og-dev/src/build/acmg_lof.rs new file mode 100644 index 000000000..7cab0cd81 --- /dev/null +++ b/rust/crates/og-dev/src/build/acmg_lof.rs @@ -0,0 +1,13 @@ +//! `og dev build acmg-lof`: the loss-of-function column of +//! `assets/acmg_sf.tsv`, built from pinned ClinGen scores. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev build acmg-lof ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev build acmg-lof: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/build/mod.rs b/rust/crates/og-dev/src/build/mod.rs index d8e9c5931..f0471e546 100644 --- a/rust/crates/og-dev/src/build/mod.rs +++ b/rust/crates/og-dev/src/build/mod.rs @@ -11,6 +11,8 @@ //! `assets/callability_genes.tsv` (`scripts/ci/build-callability.sh`). //! - `og dev build sv-rule-regions` ([`sv_rules`]): //! `assets/sv_rule_regions.tsv` (`scripts/ci/build-sv-rule-regions.sh`). +//! - `og dev build roh-af` ([`roh_af`]) and `og dev build acmg-lof` +//! ([`acmg_lof`]): not ported yet, each says so and exits 2. //! //! Exit 0 when the file was written (or `--check` found it right); 1 when //! `--check` found it different, or when the callability or SV rule @@ -19,9 +21,11 @@ //! exited; 2 for a usage error or an input that cannot be read, and for //! every cn-intervals input the rules refuse, as its Python builder exited. +pub mod acmg_lof; pub mod callability; pub mod cn_intervals; mod gtf; +pub mod roh_af; pub mod sv_rules; use og_io::{Lines, Utf8}; @@ -29,7 +33,7 @@ use std::fmt; use std::io::{self, BufRead, Write}; use std::path::Path; -const USAGE: &str = "usage: og dev build cn-intervals --clingen BED --segdups BED --gaps BED --satellites BED --parascopy TXT (--catalog FILE [--check] | --stdout)\n og dev build callability --work DIR --repo DIR --out FILE\n og dev build sv-rule-regions --work DIR --out FILE\n"; +const USAGE: &str = "usage: og dev build cn-intervals --clingen BED --segdups BED --gaps BED --satellites BED --parascopy TXT (--catalog FILE [--check] | --stdout)\n og dev build callability --work DIR --repo DIR --out FILE\n og dev build sv-rule-regions --work DIR --out FILE\n og dev build roh-af [args]\n og dev build acmg-lof [args]\n"; /// GRCh38's primary chromosomes in their sort order: chr1-22, X, Y. pub(crate) const CHROMS: [&str; 24] = [ @@ -194,6 +198,8 @@ pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { Some("cn-intervals") => cn_intervals::cli(&args[1..], out, err), Some("callability") => callability::cli(&args[1..], err), Some("sv-rule-regions") => sv_rules::cli(&args[1..], err), + Some("roh-af") => Ok(roh_af::cli(&args[1..], out, err)), + Some("acmg-lof") => Ok(acmg_lof::cli(&args[1..], out, err)), Some("-h" | "--help") => { let _ = out.write_all(USAGE.as_bytes()); return 0; diff --git a/rust/crates/og-dev/src/build/roh_af.rs b/rust/crates/og-dev/src/build/roh_af.rs new file mode 100644 index 000000000..a07b95d42 --- /dev/null +++ b/rust/crates/og-dev/src/build/roh_af.rs @@ -0,0 +1,13 @@ +//! `og dev build roh-af`: the ROH allele-frequency panel, built from a +//! pinned gnomAD release. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev build roh-af ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev build roh-af: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/demo/ancestry_paint.rs b/rust/crates/og-dev/src/demo/ancestry_paint.rs new file mode 100644 index 000000000..49494c61e --- /dev/null +++ b/rust/crates/og-dev/src/demo/ancestry_paint.rs @@ -0,0 +1,2 @@ +//! The demo sample's ancestry painting. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-dev/src/demo/clinical.rs b/rust/crates/og-dev/src/demo/clinical.rs new file mode 100644 index 000000000..62a29a193 --- /dev/null +++ b/rust/crates/og-dev/src/demo/clinical.rs @@ -0,0 +1,2 @@ +//! The demo sample's secondary findings. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-dev/src/demo/clinical_clinvar.rs b/rust/crates/og-dev/src/demo/clinical_clinvar.rs new file mode 100644 index 000000000..d7cbe0aff --- /dev/null +++ b/rust/crates/og-dev/src/demo/clinical_clinvar.rs @@ -0,0 +1,2 @@ +//! The demo sample's ClinVar results. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-dev/src/demo/family.rs b/rust/crates/og-dev/src/demo/family.rs new file mode 100644 index 000000000..5732c8364 --- /dev/null +++ b/rust/crates/og-dev/src/demo/family.rs @@ -0,0 +1 @@ +//! The demo family. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-dev/src/demo/lineage.rs b/rust/crates/og-dev/src/demo/lineage.rs new file mode 100644 index 000000000..b2b79df2a --- /dev/null +++ b/rust/crates/og-dev/src/demo/lineage.rs @@ -0,0 +1,2 @@ +//! The demo sample's ancestry and lineage. Empty until its unit of the +//! 1.0.0 plan lands. diff --git a/rust/crates/og-dev/src/demo/mod.rs b/rust/crates/og-dev/src/demo/mod.rs new file mode 100644 index 000000000..1ff961bca --- /dev/null +++ b/rust/crates/og-dev/src/demo/mod.rs @@ -0,0 +1,23 @@ +//! `og dev demo-sample`: the made-up demo sample the docs, the pictures and +//! the live demo use, which `tests/demo/make_demo_sample.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +pub mod ancestry_paint; +pub mod clinical; +pub mod clinical_clinvar; +pub mod family; +pub mod lineage; +pub mod pgx; +pub mod qc; +pub mod sv; +pub mod traits; + +use std::io::Write; + +/// `og dev demo-sample ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev demo-sample: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/demo/pgx.rs b/rust/crates/og-dev/src/demo/pgx.rs new file mode 100644 index 000000000..f8cad997b --- /dev/null +++ b/rust/crates/og-dev/src/demo/pgx.rs @@ -0,0 +1,2 @@ +//! The demo sample's pharmacogenomics. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-dev/src/demo/qc.rs b/rust/crates/og-dev/src/demo/qc.rs new file mode 100644 index 000000000..5520120bf --- /dev/null +++ b/rust/crates/og-dev/src/demo/qc.rs @@ -0,0 +1,2 @@ +//! The demo sample's quality tables. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-dev/src/demo/sv.rs b/rust/crates/og-dev/src/demo/sv.rs new file mode 100644 index 000000000..41d8a3e6c --- /dev/null +++ b/rust/crates/og-dev/src/demo/sv.rs @@ -0,0 +1,2 @@ +//! The demo sample's structural variants and repeats. Empty until its unit +//! of the 1.0.0 plan lands. diff --git a/rust/crates/og-dev/src/demo/traits.rs b/rust/crates/og-dev/src/demo/traits.rs new file mode 100644 index 000000000..fae44a955 --- /dev/null +++ b/rust/crates/og-dev/src/demo/traits.rs @@ -0,0 +1 @@ +//! The demo sample's traits. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-dev/src/demo_events.rs b/rust/crates/og-dev/src/demo_events.rs new file mode 100644 index 000000000..7ff3320b0 --- /dev/null +++ b/rust/crates/og-dev/src/demo_events.rs @@ -0,0 +1,13 @@ +//! `og dev demo-events`: the replayed event log of the live demo, which +//! `tests/demo/make_demo_events.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev demo-events ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev demo-events: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/docs_checks.rs b/rust/crates/og-dev/src/docs_checks.rs new file mode 100644 index 000000000..e4593ad08 --- /dev/null +++ b/rust/crates/og-dev/src/docs_checks.rs @@ -0,0 +1,12 @@ +//! `og dev docs-check`: the checks that keep the docs true to the code. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev docs-check ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev docs-check: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/everything.rs b/rust/crates/og-dev/src/everything.rs new file mode 100644 index 000000000..be8e755e5 --- /dev/null +++ b/rust/crates/og-dev/src/everything.rs @@ -0,0 +1,13 @@ +//! `og dev everything-check`: `assets/everything.tsv` against every tool, +//! switch and installer. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev everything-check ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev everything-check: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index a5f776c46..d2f16ba86 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -33,32 +33,48 @@ //! found, against every pin in the repository. //! - `og dev build cn-intervals|callability|sv-rule-regions` ([`build`]): //! rebuild a committed asset table from its pinned downloads. +//! - Not ported yet, each says so and exits 2: `og dev demo-sample` +//! ([`demo`]), `og dev demo-events` ([`demo_events`]), `og dev params-doc` +//! ([`params_doc`]), `og dev python-gate` ([`python_gate`]), +//! `og dev everything-check` ([`everything`]), `og dev licences` +//! ([`licences`]), `og dev docs-check` ([`docs_checks`]), +//! `og dev stub-http` ([`stub_http`]) and `og dev steps-check` +//! ([`steps_check`]). pub mod build; pub mod contract; +pub mod demo; +pub mod demo_events; +pub mod docs_checks; +pub mod everything; pub mod evidence; pub mod fixture; pub mod helpers; pub mod images; +pub mod licences; mod lookaround; pub mod nf_trace; +pub mod params_doc; pub mod parity; pub mod personal_data; pub mod pgs; pub mod pharmcat; +pub mod python_gate; pub mod remnants; pub mod renovate; pub mod report_only; pub mod schema; pub mod shlex; pub mod smoke; +pub mod steps_check; +pub mod stub_http; pub mod versions; pub mod versions_env; mod zip; use std::io::Write; -const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf [args]\n og dev renovate-check LOG [--pins FILE] [--repo DIR]\n og dev helpers --list|--static|--images|--orad-calls [--root DIR]\n og dev helpers --orad --help-file FILE [--orad-version V] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions [args]\n og dev scan personal-data|images [args]\n og dev versions check-run|expand|coupled [args]\n og dev pgs trait|label-match|metric-match [args]\n\n og dev pharmcat diplotypes REPORT [--called|--called-lines]\n og dev nf-trace requests|attempts TRACE [options]\n og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args]"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf [args]\n og dev renovate-check LOG [--pins FILE] [--repo DIR]\n og dev helpers --list|--static|--images|--orad-calls [--root DIR]\n og dev helpers --orad --help-file FILE [--orad-version V] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions|roh-af|acmg-lof [args]\n og dev scan personal-data|images [args]\n og dev versions check-run|expand|coupled [args]\n og dev pgs trait|label-match|metric-match [args]\n\n og dev pharmcat diplotypes REPORT [--called|--called-lines]\n og dev nf-trace requests|attempts TRACE [options]\n og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args]\n og dev demo-sample [args]\n og dev demo-events [args]\n og dev params-doc [args]\n og dev python-gate [args]\n og dev everything-check [args]\n og dev licences [args]\n og dev docs-check [args]\n og dev stub-http [args]\n og dev steps-check [args]"; /// Python's `str(OSError)`: `[Errno 2] No such file or directory: 'path'`. pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { @@ -182,6 +198,15 @@ pub fn main(args: &[String]) -> u8 { Some("smoke") => run(smoke::cli, &args[1..]), Some("build") => run(build::cli, &args[1..]), Some("helpers") => run(helpers::cli, &args[1..]), + Some("demo-sample") => run(demo::cli, &args[1..]), + Some("demo-events") => run(demo_events::cli, &args[1..]), + Some("params-doc") => run(params_doc::cli, &args[1..]), + Some("python-gate") => run(python_gate::cli, &args[1..]), + Some("everything-check") => run(everything::cli, &args[1..]), + Some("licences") => run(licences::cli, &args[1..]), + Some("docs-check") => run(docs_checks::cli, &args[1..]), + Some("stub-http") => run(stub_http::cli, &args[1..]), + Some("steps-check") => run(steps_check::cli, &args[1..]), Some("-h" | "--help") => { print!("{USAGE}"); 0 diff --git a/rust/crates/og-dev/src/licences.rs b/rust/crates/og-dev/src/licences.rs new file mode 100644 index 000000000..58319e4e5 --- /dev/null +++ b/rust/crates/og-dev/src/licences.rs @@ -0,0 +1,13 @@ +//! `og dev licences`: the licences of the data and code OwnGenome ships, +//! and the notices of the libraries in the binary. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev licences ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev licences: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/params_doc.rs b/rust/crates/og-dev/src/params_doc.rs new file mode 100644 index 000000000..c4cd72b4e --- /dev/null +++ b/rust/crates/og-dev/src/params_doc.rs @@ -0,0 +1,12 @@ +//! `og dev params-doc`: every schema parameter against its row in the docs. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev params-doc ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev params-doc: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/python_gate.rs b/rust/crates/og-dev/src/python_gate.rs new file mode 100644 index 000000000..bbd69d72f --- /dev/null +++ b/rust/crates/og-dev/src/python_gate.rs @@ -0,0 +1,13 @@ +//! `og dev python-gate`: every Python file, heredoc and `python3 -c` call +//! against the allow list. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev python-gate ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev python-gate: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/steps_check.rs b/rust/crates/og-dev/src/steps_check.rs new file mode 100644 index 000000000..0d162ab6d --- /dev/null +++ b/rust/crates/og-dev/src/steps_check.rs @@ -0,0 +1,13 @@ +//! `og dev steps-check`: `assets/steps.tsv` against `run-all.sh`, `main.nf` +//! and the modules. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev steps-check ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev steps-check: not ported yet"); + 2 +} diff --git a/rust/crates/og-dev/src/stub_http.rs b/rust/crates/og-dev/src/stub_http.rs new file mode 100644 index 000000000..9d7ce06ee --- /dev/null +++ b/rust/crates/og-dev/src/stub_http.rs @@ -0,0 +1,13 @@ +//! `og dev stub-http`: a local HTTP server with scripted failures, for the +//! shell tests of download retries. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og dev stub-http ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og dev stub-http: not ported yet"); + 2 +} diff --git a/rust/crates/og-io/src/lib.rs b/rust/crates/og-io/src/lib.rs index 1943c18be..d5d3d46fa 100644 --- a/rust/crates/og-io/src/lib.rs +++ b/rust/crates/og-io/src/lib.rs @@ -11,9 +11,12 @@ //! BGZF writer that compresses on many cores. //! - [`tabix`]: the lines of a region of a bgzipped file, through its //! `.tbi` or `.csi` index, as `tabix FILE seq:beg-end` prints them. +//! - [`tabix_write`]: `og vcf index`, a `.tbi` for a BGZF VCF (not ported +//! yet). pub mod bgzf; pub mod tabix; +pub mod tabix_write; use flate2::bufread::GzDecoder; use std::fs::{self, File}; diff --git a/rust/crates/og-io/src/tabix_write.rs b/rust/crates/og-io/src/tabix_write.rs new file mode 100644 index 000000000..e395a1d78 --- /dev/null +++ b/rust/crates/og-io/src/tabix_write.rs @@ -0,0 +1,13 @@ +//! `og vcf index`: a `.tbi` index for a BGZF VCF, which step 38 writes +//! today with `pysam.tabix_index`. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og vcf index ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og vcf index: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/Cargo.toml b/rust/crates/og-live/Cargo.toml new file mode 100644 index 000000000..1e1e7ad65 --- /dev/null +++ b/rust/crates/og-live/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "og-live" +description = "og live serve|results|family|demo and og view serve: the live site, its pages and the reads viewer" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/rust/crates/og-live/src/auth.rs b/rust/crates/og-live/src/auth.rs new file mode 100644 index 000000000..9123ad3d5 --- /dev/null +++ b/rust/crates/og-live/src/auth.rs @@ -0,0 +1 @@ +//! Sign-in for the live site. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/charts.rs b/rust/crates/og-live/src/charts.rs new file mode 100644 index 000000000..63070ae95 --- /dev/null +++ b/rust/crates/og-live/src/charts.rs @@ -0,0 +1 @@ +//! The live site's charts. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/csp.rs b/rust/crates/og-live/src/csp.rs new file mode 100644 index 000000000..a3a75eb2d --- /dev/null +++ b/rust/crates/og-live/src/csp.rs @@ -0,0 +1,2 @@ +//! The live site's Content-Security-Policy. Empty until its unit of the +//! 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/demo.rs b/rust/crates/og-live/src/demo.rs new file mode 100644 index 000000000..94096e9cb --- /dev/null +++ b/rust/crates/og-live/src/demo.rs @@ -0,0 +1,13 @@ +//! `og live demo`: the public demo as static pages, which +//! `tests/demo/build_live_demo.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og live demo ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og live demo: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/src/family.rs b/rust/crates/og-live/src/family.rs new file mode 100644 index 000000000..ab31e08b1 --- /dev/null +++ b/rust/crates/og-live/src/family.rs @@ -0,0 +1,12 @@ +//! `og live family`: the live site's family page. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og live family ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og live family: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/src/headlines.rs b/rust/crates/og-live/src/headlines.rs new file mode 100644 index 000000000..8757bbc4d --- /dev/null +++ b/rust/crates/og-live/src/headlines.rs @@ -0,0 +1,2 @@ +//! The headlines of a results page. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-live/src/ingest.rs b/rust/crates/og-live/src/ingest.rs new file mode 100644 index 000000000..24f2b8b32 --- /dev/null +++ b/rust/crates/og-live/src/ingest.rs @@ -0,0 +1,2 @@ +//! Runs taken into the live site. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-live/src/lib.rs b/rust/crates/og-live/src/lib.rs new file mode 100644 index 000000000..acd150a9c --- /dev/null +++ b/rust/crates/og-live/src/lib.rs @@ -0,0 +1,30 @@ +//! The live site: its server (`og live serve`, [`server`]), each person's +//! results page (`og live results`, [`results`]), the family page +//! (`og live family`, [`family`]), the public demo as static pages +//! (`og live demo`, [`demo`]) and the reads viewer (`og view serve`, +//! [`viewer`]). +//! +//! Not ported yet: each verb says so and exits 2 until its unit of the +//! 1.0.0 plan lands. Every module those units name is declared here now, +//! so they add code without editing the same files. + +pub mod auth; +pub mod charts; +pub mod csp; +pub mod demo; +pub mod family; +pub mod headlines; +pub mod ingest; +pub mod map; +pub mod ops; +pub mod plugins; +pub mod recheck; +pub mod results; +pub mod runs; +pub mod search; +pub mod server; +pub mod state; +pub mod static_files; +pub mod veil; +pub mod viewer; +pub mod widgets; diff --git a/rust/crates/og-live/src/map.rs b/rust/crates/og-live/src/map.rs new file mode 100644 index 000000000..288f8a36f --- /dev/null +++ b/rust/crates/og-live/src/map.rs @@ -0,0 +1,2 @@ +//! The genome map on the live site. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-live/src/ops.rs b/rust/crates/og-live/src/ops.rs new file mode 100644 index 000000000..0b4d3ea7b --- /dev/null +++ b/rust/crates/og-live/src/ops.rs @@ -0,0 +1,2 @@ +//! The live site's run progress and estimates. Empty until its unit of the +//! 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/plugins.rs b/rust/crates/og-live/src/plugins.rs new file mode 100644 index 000000000..e6939bce0 --- /dev/null +++ b/rust/crates/og-live/src/plugins.rs @@ -0,0 +1 @@ +//! The live site's plugins. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/recheck.rs b/rust/crates/og-live/src/recheck.rs new file mode 100644 index 000000000..9c9f1bd55 --- /dev/null +++ b/rust/crates/og-live/src/recheck.rs @@ -0,0 +1,2 @@ +//! The recheck view of the live site. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-live/src/results/mod.rs b/rust/crates/og-live/src/results/mod.rs new file mode 100644 index 000000000..072f085f0 --- /dev/null +++ b/rust/crates/og-live/src/results/mod.rs @@ -0,0 +1,12 @@ +//! `og live results`: each person's results page, which +//! `bin/live_results.py` writes today. +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og live results ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og live results: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/src/runs.rs b/rust/crates/og-live/src/runs.rs new file mode 100644 index 000000000..2b7502103 --- /dev/null +++ b/rust/crates/og-live/src/runs.rs @@ -0,0 +1,2 @@ +//! The runs the live site knows. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-live/src/search.rs b/rust/crates/og-live/src/search.rs new file mode 100644 index 000000000..a4e302f07 --- /dev/null +++ b/rust/crates/og-live/src/search.rs @@ -0,0 +1 @@ +//! The live site's search. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/server.rs b/rust/crates/og-live/src/server.rs new file mode 100644 index 000000000..b15cb7cb8 --- /dev/null +++ b/rust/crates/og-live/src/server.rs @@ -0,0 +1,13 @@ +//! `og live serve`: the live site's server, which `bin/owngenome_live.py` +//! runs today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og live serve ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og live serve: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/src/state.rs b/rust/crates/og-live/src/state.rs new file mode 100644 index 000000000..abe3b071f --- /dev/null +++ b/rust/crates/og-live/src/state.rs @@ -0,0 +1 @@ +//! The live site's run state. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/static_files.rs b/rust/crates/og-live/src/static_files.rs new file mode 100644 index 000000000..71d0806aa --- /dev/null +++ b/rust/crates/og-live/src/static_files.rs @@ -0,0 +1,2 @@ +//! The live site's static files. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-live/src/veil.rs b/rust/crates/og-live/src/veil.rs new file mode 100644 index 000000000..be15c34c5 --- /dev/null +++ b/rust/crates/og-live/src/veil.rs @@ -0,0 +1,2 @@ +//! What a results page keeps closed until it is opened. Empty until its +//! unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-live/src/viewer.rs b/rust/crates/og-live/src/viewer.rs new file mode 100644 index 000000000..c3b9d3847 --- /dev/null +++ b/rust/crates/og-live/src/viewer.rs @@ -0,0 +1,13 @@ +//! `og view serve`: igv.js and one sample's reads and calls on 127.0.0.1, +//! which `assets/viewer/serve.py` serves today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og view serve ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og view serve: not ported yet"); + 2 +} diff --git a/rust/crates/og-live/src/widgets.rs b/rust/crates/og-live/src/widgets.rs new file mode 100644 index 000000000..c3d21640f --- /dev/null +++ b/rust/crates/og-live/src/widgets.rs @@ -0,0 +1,2 @@ +//! The widgets of a results page. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-map/src/crossovers.rs b/rust/crates/og-map/src/crossovers.rs new file mode 100644 index 000000000..5f18284e1 --- /dev/null +++ b/rust/crates/og-map/src/crossovers.rs @@ -0,0 +1,2 @@ +//! The crossovers a child's genome map draws. Empty until its unit of the +//! 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/family.rs b/rust/crates/og-map/src/family.rs new file mode 100644 index 000000000..7f4cf1052 --- /dev/null +++ b/rust/crates/og-map/src/family.rs @@ -0,0 +1,2 @@ +//! The family view of the genome map. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-map/src/layers/archaic.rs b/rust/crates/og-map/src/layers/archaic.rs new file mode 100644 index 000000000..960675be8 --- /dev/null +++ b/rust/crates/og-map/src/layers/archaic.rs @@ -0,0 +1,2 @@ +//! The archaic segments layer of the genome map. Empty until its unit of +//! the 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/layers/clinvar.rs b/rust/crates/og-map/src/layers/clinvar.rs new file mode 100644 index 000000000..cec5af38b --- /dev/null +++ b/rust/crates/og-map/src/layers/clinvar.rs @@ -0,0 +1,2 @@ +//! The ClinVar layer of the genome map. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-map/src/layers/mca.rs b/rust/crates/og-map/src/layers/mca.rs new file mode 100644 index 000000000..c94e144ba --- /dev/null +++ b/rust/crates/og-map/src/layers/mca.rs @@ -0,0 +1,2 @@ +//! The mosaic chromosome layer of the genome map. Empty until its unit of +//! the 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/layers/mod.rs b/rust/crates/og-map/src/layers/mod.rs index 29eeca48b..f19761991 100644 --- a/rust/crates/og-map/src/layers/mod.rs +++ b/rust/crates/og-map/src/layers/mod.rs @@ -1,3 +1,8 @@ //! One module per layer of the map. +pub mod archaic; +pub mod clinvar; +pub mod mca; +pub mod paint; pub mod roh; +pub mod sv; diff --git a/rust/crates/og-map/src/layers/paint.rs b/rust/crates/og-map/src/layers/paint.rs new file mode 100644 index 000000000..ac5b24a59 --- /dev/null +++ b/rust/crates/og-map/src/layers/paint.rs @@ -0,0 +1,2 @@ +//! The ancestry painting layer of the genome map. Empty until its unit of +//! the 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/layers/sv.rs b/rust/crates/og-map/src/layers/sv.rs new file mode 100644 index 000000000..1bc051f9e --- /dev/null +++ b/rust/crates/og-map/src/layers/sv.rs @@ -0,0 +1,2 @@ +//! The structural-variant layer of the genome map. Empty until its unit of +//! the 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/lib.rs b/rust/crates/og-map/src/lib.rs index 6769b8592..cc4643ef6 100644 --- a/rust/crates/og-map/src/lib.rs +++ b/rust/crates/og-map/src/lib.rs @@ -7,15 +7,20 @@ //! runs-of-homozygosity layer. The gene files (`genes`) are built once per //! GENCODE release, apart from any run. +pub mod crossovers; pub mod doc; pub mod facts; +pub mod family; pub mod findings; pub mod genes; pub mod layers; +pub mod live; pub mod predicate; pub mod reference; pub mod stamp; +pub mod svg; pub mod text; +pub mod threads; pub mod veil; use facts::{Facts, Sex}; diff --git a/rust/crates/og-map/src/live.rs b/rust/crates/og-map/src/live.rs new file mode 100644 index 000000000..cc371e081 --- /dev/null +++ b/rust/crates/og-map/src/live.rs @@ -0,0 +1,2 @@ +//! The genome map's answers to the live site. Empty until its unit of the +//! 1.0.0 plan lands. diff --git a/rust/crates/og-map/src/svg.rs b/rust/crates/og-map/src/svg.rs new file mode 100644 index 000000000..182b42da1 --- /dev/null +++ b/rust/crates/og-map/src/svg.rs @@ -0,0 +1,12 @@ +//! `og map svg`: the report's genome figure, drawn from the map model. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og map svg ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og map svg: not ported yet"); + 2 +} diff --git a/rust/crates/og-map/src/threads.rs b/rust/crates/og-map/src/threads.rs new file mode 100644 index 000000000..1c7a79568 --- /dev/null +++ b/rust/crates/og-map/src/threads.rs @@ -0,0 +1,2 @@ +//! The inherited threads of the family genome map. Empty until its unit of +//! the 1.0.0 plan lands. diff --git a/rust/crates/og-pedigree/src/layout.rs b/rust/crates/og-pedigree/src/layout.rs new file mode 100644 index 000000000..d2bd31b80 --- /dev/null +++ b/rust/crates/og-pedigree/src/layout.rs @@ -0,0 +1,2 @@ +//! The drawing layout of a pedigree. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-pedigree/src/lib.rs b/rust/crates/og-pedigree/src/lib.rs index e05f68c02..16bcb0804 100644 --- a/rust/crates/og-pedigree/src/lib.rs +++ b/rust/crates/og-pedigree/src/lib.rs @@ -14,6 +14,10 @@ //! - [`Par`]: the pseudoautosomal regions of a BED file. //! //! A problem is a [`Die`]: `ERROR: ` lines on stderr and an exit code. +//! [`layout`], the drawing layout of a pedigree, is empty until its port +//! lands. + +pub mod layout; use std::collections::{HashMap, HashSet}; use std::fs::File; diff --git a/rust/crates/og-render/Cargo.toml b/rust/crates/og-render/Cargo.toml new file mode 100644 index 000000000..d785aae58 --- /dev/null +++ b/rust/crates/og-render/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "og-render" +description = "og report render|index|charts: the HTML and text reports, the run's search index and the chart specs, from summary.json" +version.workspace = true +edition.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] + +[lints] +workspace = true diff --git a/rust/crates/og-render/src/charts/mod.rs b/rust/crates/og-render/src/charts/mod.rs new file mode 100644 index 000000000..814b431b6 --- /dev/null +++ b/rust/crates/og-render/src/charts/mod.rs @@ -0,0 +1,13 @@ +//! `og report charts`: the chart specs the HTML report embeds and the live +//! site serves, which `bin/report_charts.py` builds today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og report charts ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og report charts: not ported yet"); + 2 +} diff --git a/rust/crates/og-render/src/index.rs b/rust/crates/og-render/src/index.rs new file mode 100644 index 000000000..4a9025aab --- /dev/null +++ b/rust/crates/og-render/src/index.rs @@ -0,0 +1,13 @@ +//! `og report index`: the search index of one run, every gene symbol and +//! rsID the summary names, which `bin/results_index.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og report index ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og report index: not ported yet"); + 2 +} diff --git a/rust/crates/og-render/src/lib.rs b/rust/crates/og-render/src/lib.rs new file mode 100644 index 000000000..c4d9f7301 --- /dev/null +++ b/rust/crates/og-render/src/lib.rs @@ -0,0 +1,13 @@ +//! The reports of one run, from its `summary.json`: the HTML and text +//! reports (`og report render`, [`render`]), the run's search index +//! (`og report index`, [`index`]), the chart specs (`og report charts`, +//! [`charts`]) and the report's sections, one module each ([`sections`]). +//! +//! Not ported yet: each verb says so and exits 2 until its unit of the +//! 1.0.0 plan lands. The crate exists now so those units add code without +//! editing the same files. + +pub mod charts; +pub mod index; +pub mod render; +pub mod sections; diff --git a/rust/crates/og-render/src/render.rs b/rust/crates/og-render/src/render.rs new file mode 100644 index 000000000..a0d37c7d5 --- /dev/null +++ b/rust/crates/og-render/src/render.rs @@ -0,0 +1,13 @@ +//! `og report render`: the HTML and text reports from summary.json, which +//! `bin/render_report.py` writes today. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og report render ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og report render: not ported yet"); + 2 +} diff --git a/rust/crates/og-render/src/sections/acmg_sf.rs b/rust/crates/og-render/src/sections/acmg_sf.rs new file mode 100644 index 000000000..70669f6e2 --- /dev/null +++ b/rust/crates/og-render/src/sections/acmg_sf.rs @@ -0,0 +1,2 @@ +//! The report section `acmg_sf`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/ancestry.rs b/rust/crates/og-render/src/sections/ancestry.rs new file mode 100644 index 000000000..f882faadb --- /dev/null +++ b/rust/crates/og-render/src/sections/ancestry.rs @@ -0,0 +1,2 @@ +//! The report section `ancestry`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/ancestry_fine.rs b/rust/crates/og-render/src/sections/ancestry_fine.rs new file mode 100644 index 000000000..d87e67915 --- /dev/null +++ b/rust/crates/og-render/src/sections/ancestry_fine.rs @@ -0,0 +1,2 @@ +//! The report section `ancestry_fine`. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-render/src/sections/ancient.rs b/rust/crates/og-render/src/sections/ancient.rs new file mode 100644 index 000000000..0bcd3409a --- /dev/null +++ b/rust/crates/og-render/src/sections/ancient.rs @@ -0,0 +1,2 @@ +//! The report section `ancient`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/annotsv.rs b/rust/crates/og-render/src/sections/annotsv.rs new file mode 100644 index 000000000..b9d5acde1 --- /dev/null +++ b/rust/crates/og-render/src/sections/annotsv.rs @@ -0,0 +1,2 @@ +//! The report section `annotsv`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/apoe.rs b/rust/crates/og-render/src/sections/apoe.rs new file mode 100644 index 000000000..de95c5197 --- /dev/null +++ b/rust/crates/og-render/src/sections/apoe.rs @@ -0,0 +1 @@ +//! The report section `apoe`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/archaic.rs b/rust/crates/og-render/src/sections/archaic.rs new file mode 100644 index 000000000..4295ec431 --- /dev/null +++ b/rust/crates/og-render/src/sections/archaic.rs @@ -0,0 +1,2 @@ +//! The report section `archaic`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/bloodtype.rs b/rust/crates/og-render/src/sections/bloodtype.rs new file mode 100644 index 000000000..afc1017a2 --- /dev/null +++ b/rust/crates/og-render/src/sections/bloodtype.rs @@ -0,0 +1,2 @@ +//! The report section `bloodtype`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/callability.rs b/rust/crates/og-render/src/sections/callability.rs new file mode 100644 index 000000000..b91918804 --- /dev/null +++ b/rust/crates/og-render/src/sections/callability.rs @@ -0,0 +1,2 @@ +//! The report section `callability`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/carrier.rs b/rust/crates/og-render/src/sections/carrier.rs new file mode 100644 index 000000000..98e47f503 --- /dev/null +++ b/rust/crates/og-render/src/sections/carrier.rs @@ -0,0 +1,2 @@ +//! The report section `carrier`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/cdna.rs b/rust/crates/og-render/src/sections/cdna.rs new file mode 100644 index 000000000..bd3715784 --- /dev/null +++ b/rust/crates/og-render/src/sections/cdna.rs @@ -0,0 +1 @@ +//! The report section `cdna`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/cdna_scan.rs b/rust/crates/og-render/src/sections/cdna_scan.rs new file mode 100644 index 000000000..722f7f3a5 --- /dev/null +++ b/rust/crates/og-render/src/sections/cdna_scan.rs @@ -0,0 +1,2 @@ +//! The report section `cdna_scan`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/chip.rs b/rust/crates/og-render/src/sections/chip.rs new file mode 100644 index 000000000..b84420658 --- /dev/null +++ b/rust/crates/og-render/src/sections/chip.rs @@ -0,0 +1 @@ +//! The report section `chip`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/clinvar.rs b/rust/crates/og-render/src/sections/clinvar.rs new file mode 100644 index 000000000..9876ec648 --- /dev/null +++ b/rust/crates/og-render/src/sections/clinvar.rs @@ -0,0 +1,2 @@ +//! The report section `clinvar`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/cn_intervals.rs b/rust/crates/og-render/src/sections/cn_intervals.rs new file mode 100644 index 000000000..11bb963e1 --- /dev/null +++ b/rust/crates/og-render/src/sections/cn_intervals.rs @@ -0,0 +1,2 @@ +//! The report section `cn_intervals`. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-render/src/sections/comphet.rs b/rust/crates/og-render/src/sections/comphet.rs new file mode 100644 index 000000000..df853c692 --- /dev/null +++ b/rust/crates/og-render/src/sections/comphet.rs @@ -0,0 +1,2 @@ +//! The report section `comphet`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/couple.rs b/rust/crates/og-render/src/sections/couple.rs new file mode 100644 index 000000000..cf30455af --- /dev/null +++ b/rust/crates/og-render/src/sections/couple.rs @@ -0,0 +1,2 @@ +//! The report section `couple`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/coverage.rs b/rust/crates/og-render/src/sections/coverage.rs new file mode 100644 index 000000000..f4023fc69 --- /dev/null +++ b/rust/crates/og-render/src/sections/coverage.rs @@ -0,0 +1,2 @@ +//! The report section `coverage`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/cpsr.rs b/rust/crates/og-render/src/sections/cpsr.rs new file mode 100644 index 000000000..4db6327f4 --- /dev/null +++ b/rust/crates/og-render/src/sections/cpsr.rs @@ -0,0 +1 @@ +//! The report section `cpsr`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/cyp21a2.rs b/rust/crates/og-render/src/sections/cyp21a2.rs new file mode 100644 index 000000000..3699dbddb --- /dev/null +++ b/rust/crates/og-render/src/sections/cyp21a2.rs @@ -0,0 +1,2 @@ +//! The report section `cyp21a2`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/deep_intronic.rs b/rust/crates/og-render/src/sections/deep_intronic.rs new file mode 100644 index 000000000..8bda20f8a --- /dev/null +++ b/rust/crates/og-render/src/sections/deep_intronic.rs @@ -0,0 +1,2 @@ +//! The report section `deep_intronic`. Empty until its unit of the 1.0.0 +//! plan lands. diff --git a/rust/crates/og-render/src/sections/dosage.rs b/rust/crates/og-render/src/sections/dosage.rs new file mode 100644 index 000000000..49d9dbfa1 --- /dev/null +++ b/rust/crates/og-render/src/sections/dosage.rs @@ -0,0 +1,2 @@ +//! The report section `dosage`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/exomiser.rs b/rust/crates/og-render/src/sections/exomiser.rs new file mode 100644 index 000000000..2a2b69ea8 --- /dev/null +++ b/rust/crates/og-render/src/sections/exomiser.rs @@ -0,0 +1,2 @@ +//! The report section `exomiser`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/g2.rs b/rust/crates/og-render/src/sections/g2.rs new file mode 100644 index 000000000..6f24ec5e0 --- /dev/null +++ b/rust/crates/og-render/src/sections/g2.rs @@ -0,0 +1 @@ +//! The report section `g2`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/haplogroups.rs b/rust/crates/og-render/src/sections/haplogroups.rs new file mode 100644 index 000000000..8f28f8f32 --- /dev/null +++ b/rust/crates/og-render/src/sections/haplogroups.rs @@ -0,0 +1,2 @@ +//! The report section `haplogroups`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/hba.rs b/rust/crates/og-render/src/sections/hba.rs new file mode 100644 index 000000000..d3f347f6a --- /dev/null +++ b/rust/crates/og-render/src/sections/hba.rs @@ -0,0 +1 @@ +//! The report section `hba`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/hla.rs b/rust/crates/og-render/src/sections/hla.rs new file mode 100644 index 000000000..79b3352a3 --- /dev/null +++ b/rust/crates/og-render/src/sections/hla.rs @@ -0,0 +1 @@ +//! The report section `hla`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/locityper.rs b/rust/crates/og-render/src/sections/locityper.rs new file mode 100644 index 000000000..1e0636177 --- /dev/null +++ b/rust/crates/og-render/src/sections/locityper.rs @@ -0,0 +1,2 @@ +//! The report section `locityper`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/lpa.rs b/rust/crates/og-render/src/sections/lpa.rs new file mode 100644 index 000000000..087b3e19b --- /dev/null +++ b/rust/crates/og-render/src/sections/lpa.rs @@ -0,0 +1 @@ +//! The report section `lpa`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/mei.rs b/rust/crates/og-render/src/sections/mei.rs new file mode 100644 index 000000000..f1c507eb1 --- /dev/null +++ b/rust/crates/og-render/src/sections/mei.rs @@ -0,0 +1 @@ +//! The report section `mei`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/mito.rs b/rust/crates/og-render/src/sections/mito.rs new file mode 100644 index 000000000..e6f32d585 --- /dev/null +++ b/rust/crates/og-render/src/sections/mito.rs @@ -0,0 +1 @@ +//! The report section `mito`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/mocha.rs b/rust/crates/og-render/src/sections/mocha.rs new file mode 100644 index 000000000..b56dd3a80 --- /dev/null +++ b/rust/crates/og-render/src/sections/mocha.rs @@ -0,0 +1,2 @@ +//! The report section `mocha`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/mod.rs b/rust/crates/og-render/src/sections/mod.rs new file mode 100644 index 000000000..8ce9d5696 --- /dev/null +++ b/rust/crates/og-render/src/sections/mod.rs @@ -0,0 +1,54 @@ +//! The report's sections, one module each, so the units that port them +//! never edit the same file. + +pub mod acmg_sf; +pub mod ancestry; +pub mod ancestry_fine; +pub mod ancient; +pub mod annotsv; +pub mod apoe; +pub mod archaic; +pub mod bloodtype; +pub mod callability; +pub mod carrier; +pub mod cdna; +pub mod cdna_scan; +pub mod chip; +pub mod clinvar; +pub mod cn_intervals; +pub mod comphet; +pub mod couple; +pub mod coverage; +pub mod cpsr; +pub mod cyp21a2; +pub mod deep_intronic; +pub mod dosage; +pub mod exomiser; +pub mod g2; +pub mod haplogroups; +pub mod hba; +pub mod hla; +pub mod locityper; +pub mod lpa; +pub mod mei; +pub mod mito; +pub mod mocha; +pub mod paralogs; +pub mod parascopy; +pub mod pgx; +pub mod prs; +pub mod qc; +pub mod rfc1; +pub mod roh; +pub mod run; +pub mod smn1; +pub mod str; +pub mod sv_callers; +pub mod telomere; +pub mod traits; +pub mod two_hits; +pub mod variants; +pub mod viral; +pub mod vntyper; +pub mod y_tree; +pub mod ystr; diff --git a/rust/crates/og-render/src/sections/paralogs.rs b/rust/crates/og-render/src/sections/paralogs.rs new file mode 100644 index 000000000..fffdfd985 --- /dev/null +++ b/rust/crates/og-render/src/sections/paralogs.rs @@ -0,0 +1,2 @@ +//! The report section `paralogs`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/parascopy.rs b/rust/crates/og-render/src/sections/parascopy.rs new file mode 100644 index 000000000..e688a9b01 --- /dev/null +++ b/rust/crates/og-render/src/sections/parascopy.rs @@ -0,0 +1,2 @@ +//! The report section `parascopy`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/pgx.rs b/rust/crates/og-render/src/sections/pgx.rs new file mode 100644 index 000000000..0591b1e47 --- /dev/null +++ b/rust/crates/og-render/src/sections/pgx.rs @@ -0,0 +1 @@ +//! The report section `pgx`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/prs.rs b/rust/crates/og-render/src/sections/prs.rs new file mode 100644 index 000000000..3d832baa8 --- /dev/null +++ b/rust/crates/og-render/src/sections/prs.rs @@ -0,0 +1 @@ +//! The report section `prs`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/qc.rs b/rust/crates/og-render/src/sections/qc.rs new file mode 100644 index 000000000..3292a6ba8 --- /dev/null +++ b/rust/crates/og-render/src/sections/qc.rs @@ -0,0 +1 @@ +//! The report section `qc`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/rfc1.rs b/rust/crates/og-render/src/sections/rfc1.rs new file mode 100644 index 000000000..aff217dcf --- /dev/null +++ b/rust/crates/og-render/src/sections/rfc1.rs @@ -0,0 +1 @@ +//! The report section `rfc1`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/roh.rs b/rust/crates/og-render/src/sections/roh.rs new file mode 100644 index 000000000..4adf29175 --- /dev/null +++ b/rust/crates/og-render/src/sections/roh.rs @@ -0,0 +1 @@ +//! The report section `roh`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/run.rs b/rust/crates/og-render/src/sections/run.rs new file mode 100644 index 000000000..e23cc28cf --- /dev/null +++ b/rust/crates/og-render/src/sections/run.rs @@ -0,0 +1 @@ +//! The report section `run`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/smn1.rs b/rust/crates/og-render/src/sections/smn1.rs new file mode 100644 index 000000000..eb4742abe --- /dev/null +++ b/rust/crates/og-render/src/sections/smn1.rs @@ -0,0 +1 @@ +//! The report section `smn1`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/str.rs b/rust/crates/og-render/src/sections/str.rs new file mode 100644 index 000000000..047c968c6 --- /dev/null +++ b/rust/crates/og-render/src/sections/str.rs @@ -0,0 +1 @@ +//! The report section `str`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-render/src/sections/sv_callers.rs b/rust/crates/og-render/src/sections/sv_callers.rs new file mode 100644 index 000000000..13956ab3f --- /dev/null +++ b/rust/crates/og-render/src/sections/sv_callers.rs @@ -0,0 +1,2 @@ +//! The report section `sv_callers`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/telomere.rs b/rust/crates/og-render/src/sections/telomere.rs new file mode 100644 index 000000000..0c43f4b3f --- /dev/null +++ b/rust/crates/og-render/src/sections/telomere.rs @@ -0,0 +1,2 @@ +//! The report section `telomere`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/traits.rs b/rust/crates/og-render/src/sections/traits.rs new file mode 100644 index 000000000..a087281ea --- /dev/null +++ b/rust/crates/og-render/src/sections/traits.rs @@ -0,0 +1,2 @@ +//! The report section `traits`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/two_hits.rs b/rust/crates/og-render/src/sections/two_hits.rs new file mode 100644 index 000000000..695939b43 --- /dev/null +++ b/rust/crates/og-render/src/sections/two_hits.rs @@ -0,0 +1,2 @@ +//! The report section `two_hits`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/variants.rs b/rust/crates/og-render/src/sections/variants.rs new file mode 100644 index 000000000..60ac2c9c6 --- /dev/null +++ b/rust/crates/og-render/src/sections/variants.rs @@ -0,0 +1,2 @@ +//! The report section `variants`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/viral.rs b/rust/crates/og-render/src/sections/viral.rs new file mode 100644 index 000000000..bb9ffa672 --- /dev/null +++ b/rust/crates/og-render/src/sections/viral.rs @@ -0,0 +1,2 @@ +//! The report section `viral`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/vntyper.rs b/rust/crates/og-render/src/sections/vntyper.rs new file mode 100644 index 000000000..d3a89d2e8 --- /dev/null +++ b/rust/crates/og-render/src/sections/vntyper.rs @@ -0,0 +1,2 @@ +//! The report section `vntyper`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/y_tree.rs b/rust/crates/og-render/src/sections/y_tree.rs new file mode 100644 index 000000000..892584a1c --- /dev/null +++ b/rust/crates/og-render/src/sections/y_tree.rs @@ -0,0 +1,2 @@ +//! The report section `y_tree`. Empty until its unit of the 1.0.0 plan +//! lands. diff --git a/rust/crates/og-render/src/sections/ystr.rs b/rust/crates/og-render/src/sections/ystr.rs new file mode 100644 index 000000000..7e7922e39 --- /dev/null +++ b/rust/crates/og-render/src/sections/ystr.rs @@ -0,0 +1 @@ +//! The report section `ystr`. Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-setup/src/lib.rs b/rust/crates/og-setup/src/lib.rs index 1a74193bb..f7fe0a98b 100644 --- a/rust/crates/og-setup/src/lib.rs +++ b/rust/crates/og-setup/src/lib.rs @@ -7,9 +7,12 @@ //! - [`genetic_map`] (`og setup genetic-map`): the PLINK GRCh38 genetic map //! per chromosome for steps 26b and 51, written atomically. //! -//! [`zip`] reads the archives the last two take. +//! [`zip`] reads the archives the last two take. [`revel`] (`og setup +//! revel`) and [`roh_af`] (`og setup roh-af`) are not ported yet. pub mod genetic_map; +pub mod revel; +pub mod roh_af; pub mod strchive; pub mod unzip; pub mod zip; diff --git a/rust/crates/og-setup/src/revel.rs b/rust/crates/og-setup/src/revel.rs new file mode 100644 index 000000000..114fb1cb8 --- /dev/null +++ b/rust/crates/og-setup/src/revel.rs @@ -0,0 +1,12 @@ +//! `og setup revel`: the REVEL GRCh38 score table that vcfanno reads. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og setup revel ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og setup revel: not ported yet"); + 2 +} diff --git a/rust/crates/og-setup/src/roh_af.rs b/rust/crates/og-setup/src/roh_af.rs new file mode 100644 index 000000000..21219064f --- /dev/null +++ b/rust/crates/og-setup/src/roh_af.rs @@ -0,0 +1,12 @@ +//! `og setup roh-af`: the allele-frequency panel that the ROH step reads. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og setup roh-af ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og setup roh-af: not ported yet"); + 2 +} diff --git a/rust/crates/og-steps/src/lib.rs b/rust/crates/og-steps/src/lib.rs index fae338970..2993b4349 100644 --- a/rust/crates/og-steps/src/lib.rs +++ b/rust/crates/og-steps/src/lib.rs @@ -17,10 +17,13 @@ //! check and its two tables. //! - [`sv`]: `og sv add-callers|postfilter`, step 05's SV calls into //! AnnotSV and the checks of its class 4 and 5 calls. +//! - [`longread`]: `og longread clair3-model`, the Clair3 model for the +//! basecaller (not ported yet). pub mod constraint; pub mod consumer; pub mod family; +pub mod longread; pub mod paraphase; pub mod pgx_rules; pub mod prs; diff --git a/rust/crates/og-steps/src/longread.rs b/rust/crates/og-steps/src/longread.rs new file mode 100644 index 000000000..f3dcb97bf --- /dev/null +++ b/rust/crates/og-steps/src/longread.rs @@ -0,0 +1,13 @@ +//! `og longread clair3-model`: the Clair3 model that matches the basecaller +//! of a long-read sample. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og longread clair3-model ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn clair3_model_cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og longread clair3-model: not ported yet"); + 2 +} diff --git a/rust/crates/og-steps/src/sv.rs b/rust/crates/og-steps/src/sv.rs index 39ff04891..622dacdd9 100644 --- a/rust/crates/og-steps/src/sv.rs +++ b/rust/crates/og-steps/src/sv.rs @@ -15,7 +15,9 @@ //! ([`OverlapIndex`]). pub mod add_callers; +pub mod consensus_check; pub mod postfilter; +pub mod survivor_prep; use std::collections::{BTreeMap, HashMap}; diff --git a/rust/crates/og-steps/src/sv/consensus_check.rs b/rust/crates/og-steps/src/sv/consensus_check.rs new file mode 100644 index 000000000..a6c528d46 --- /dev/null +++ b/rust/crates/og-steps/src/sv/consensus_check.rs @@ -0,0 +1,13 @@ +//! `og sv consensus-check`: each record SURVIVOR merged checked for type, +//! strands and overlap, with its coordinates computed again. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og sv consensus-check ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og sv consensus-check: not ported yet"); + 2 +} diff --git a/rust/crates/og-steps/src/sv/survivor_prep.rs b/rust/crates/og-steps/src/sv/survivor_prep.rs new file mode 100644 index 000000000..4f17375a3 --- /dev/null +++ b/rust/crates/og-steps/src/sv/survivor_prep.rs @@ -0,0 +1,13 @@ +//! `og sv survivor-prep`: one caller's structural variants made ready for +//! SURVIVOR's merge in step 22. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so +//! on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og sv survivor-prep ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og sv survivor-prep: not ported yet"); + 2 +} diff --git a/rust/crates/og-summary/src/collect.rs b/rust/crates/og-summary/src/collect.rs index 4bbc9529f..2c87bff83 100644 --- a/rust/crates/og-summary/src/collect.rs +++ b/rust/crates/og-summary/src/collect.rs @@ -27,6 +27,7 @@ //! then `unreadable` with a note that names the file. pub mod acmg; +pub mod acmg_card; pub mod assets; pub mod carrier; pub mod clinical; diff --git a/rust/crates/og-summary/src/collect/acmg_card.rs b/rust/crates/og-summary/src/collect/acmg_card.rs new file mode 100644 index 000000000..9506a9040 --- /dev/null +++ b/rust/crates/og-summary/src/collect/acmg_card.rs @@ -0,0 +1,2 @@ +//! The summary section of the ACMG card, computed from the step 06 hits. +//! Empty until its unit of the 1.0.0 plan lands. diff --git a/rust/crates/og-vcf/src/lib.rs b/rust/crates/og-vcf/src/lib.rs index e8532bd5c..6f6994459 100644 --- a/rust/crates/og-vcf/src/lib.rs +++ b/rust/crates/og-vcf/src/lib.rs @@ -27,6 +27,7 @@ mod gt; mod header; pub mod rank; mod record; +pub mod shift; pub use csq::{CsqLayout, csq_field}; pub use gt::Gt; diff --git a/rust/crates/og-vcf/src/shift.rs b/rust/crates/og-vcf/src/shift.rs new file mode 100644 index 000000000..21f70ddf5 --- /dev/null +++ b/rust/crates/og-vcf/src/shift.rs @@ -0,0 +1,20 @@ +//! `og setup chrm-shift` and `og vcf shift-back`: the chrM reference +//! rotated by 8000 bp for the shifted-reference pass, and the calls on it +//! mapped back to chrM's own positions. +//! +//! Not ported yet: until its unit of the 1.0.0 plan lands, each verb says +//! so on stderr and exits 2, before reading any argument. + +use std::io::Write; + +/// `og setup chrm-shift ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn chrm_shift_cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og setup chrm-shift: not ported yet"); + 2 +} + +/// `og vcf shift-back ARGS`: not ported yet, so one line on stderr and exit 2. +pub fn shift_back_cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let _ = writeln!(err, "og vcf shift-back: not ported yet"); + 2 +} diff --git a/rust/og/Cargo.toml b/rust/og/Cargo.toml index e401700b1..dc40567a0 100644 --- a/rust/og/Cargo.toml +++ b/rust/og/Cargo.toml @@ -30,6 +30,9 @@ og-summary.workspace = true og-lookup.workspace = true og-fresh.workspace = true og-recheck.workspace = true +og-render.workspace = true +og-live.workspace = true +og-couple.workspace = true [features] # BAM reading (og-bam, on a statically linked htslib written in C) is on by diff --git a/rust/og/src/main.rs b/rust/og/src/main.rs index 7603ed025..c20fad8a1 100644 --- a/rust/og/src/main.rs +++ b/rust/og/src/main.rs @@ -25,16 +25,24 @@ fn tree_hash() -> &'static str { const USAGE: &str = "usage: og [args] og map [args] + og map svg [args] og run state|tasks|eta|headroom|delta [args] og recheck --sample S [--genome-dir DIR] [--offline FILE] [--rerun [--refresh-clinvar]] | --affected KEYS | --launch-dir DIR og setup mask-fasta SRC BED DEST og setup unzip ZIP DEST og setup genetic-map --zip ZIP --out DIR [--rfmix] CHROM... + og setup chrm-shift [args] + og setup revel [args] + og setup roh-af [args] og vcf csq-rank [options] + og vcf index [args] + og vcf shift-back [args] og table constraint-join --gene-col NAME [--constrained] CONSTRAINT TABLE og sv svlen-fix|inv-filter [args] og sv add-callers [args] og sv postfilter [args] + og sv survivor-prep [args] + og sv consensus-check [args] og pgx cpic-report|step36-outside [args] og pgx refblock-fill|mito-merge|cyp2d6-bed|cyp2d6-measure|cyp2d6-check [args] og pgx outside-calls|hla-concordance|pypgx-summary [args] @@ -51,6 +59,16 @@ const USAGE: &str = "usage: og [args] og prs table [args] og mito annotate|deletions|copies [args] og summary collect [--only {a,b,c}] [args] + og report render [args] + og report index [args] + og report charts [args] + og live serve [args] + og live results [args] + og live family [args] + og live demo [args] + og view serve [args] + og couple screen [args] + og couple genes [args] og roh summary --sample SAMPLE --roh ROH [--out OUT] og chip prepare|report|check-table [args] og cn bed|call|intervals|check-bam|marker-regions|marker-read [args] @@ -62,7 +80,13 @@ const USAGE: &str = "usage: og [args] og family ibd [args] og consumer build-sites|export|check [args] og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report [args] + og ancestry-fine freqs [args] + og ancestry-fine project [args] + og ancestry-fine paint [args] + og ancestry-fine simulate [args] og archaic table|strict|genotypes|report [args] + og archaic compare [args] + og archaic genomes [args] og ancient select|lift|person-geno|align|report [args] og vntyper report [args] og locityper report [args] @@ -78,6 +102,7 @@ const USAGE: &str = "usage: og [args] og y yleaf-positions --data DATA OUT og exomiser prepare [args] og exomiser report [args] + og longread clair3-model [args] og lookup query|resolve|report [args] og dev parity-compare|contract|schema|report-only|remnants [args] og dev scan personal-data|images [args] @@ -93,6 +118,17 @@ const USAGE: &str = "usage: og [args] og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args] og dev helpers [args] og dev build cn-intervals|callability|sv-rule-regions [args] + og dev build roh-af [args] + og dev build acmg-lof [args] + og dev demo-sample [args] + og dev demo-events [args] + og dev params-doc [args] + og dev python-gate [args] + og dev everything-check [args] + og dev licences [args] + og dev docs-check [args] + og dev stub-http [args] + og dev steps-check [args] og dev freshness [--out FILE] [--sections a,b] | --update-issue BODY --repo O/N | --self-test og version og --tree-hash @@ -105,6 +141,15 @@ fn usage_error(msg: &str) -> ExitCode { ExitCode::from(2) } +/// Runs a verb that writes to stdout and stderr; returns its exit code. +fn io(f: fn(&[String], &mut dyn Write, &mut dyn Write) -> u8, args: &[String]) -> ExitCode { + ExitCode::from(f( + args, + &mut std::io::stdout().lock(), + &mut std::io::stderr().lock(), + )) +} + /// A BAM tool called on a build without the `bam` feature (the macOS build /// for a live host, rust/og.sh): it says so and exits 2, before any input /// is read, so a caller never mistakes it for a result. @@ -159,11 +204,15 @@ fn main() -> ExitCode { } } } + "map" if rest.first().is_some_and(|v| v == "svg") => io(og_map::svg::cli, &rest[1..]), "map" => map::run(rest, tree_hash()), "setup" => match rest.first().map(String::as_str) { Some("mask-fasta") => ExitCode::from(og_intervals::mask::cli(&rest[1..])), Some("unzip") => ExitCode::from(og_setup::unzip::cli(&rest[1..])), Some("genetic-map") => ExitCode::from(og_setup::genetic_map::cli(&rest[1..])), + Some("chrm-shift") => io(og_vcf::shift::chrm_shift_cli, &rest[1..]), + Some("revel") => io(og_setup::revel::cli, &rest[1..]), + Some("roh-af") => io(og_setup::roh_af::cli, &rest[1..]), Some(other) => usage_error(&format!("unknown setup tool '{other}'")), None => usage_error("the following arguments are required: setup tool"), }, @@ -181,6 +230,8 @@ fn main() -> ExitCode { "consumer" => ExitCode::from(og_steps::consumer::cli(rest)), "vcf" => match rest.first().map(String::as_str) { Some("csq-rank") => ExitCode::from(og_vcf::rank::cli(&rest[1..])), + Some("index") => io(og_io::tabix_write::cli, &rest[1..]), + Some("shift-back") => io(og_vcf::shift::shift_back_cli, &rest[1..]), Some(other) => usage_error(&format!("unknown vcf tool '{other}'")), None => usage_error("the following arguments are required: vcf tool"), }, @@ -194,6 +245,8 @@ fn main() -> ExitCode { Some("inv-filter") => ExitCode::from(og_filters::inv::cli(&rest[1..])), Some("add-callers") => ExitCode::from(og_steps::sv::add_callers::cli(&rest[1..])), Some("postfilter") => ExitCode::from(og_steps::sv::postfilter::cli(&rest[1..])), + Some("survivor-prep") => io(og_steps::sv::survivor_prep::cli, &rest[1..]), + Some("consensus-check") => io(og_steps::sv::consensus_check::cli, &rest[1..]), Some(other) => usage_error(&format!("unknown sv tool '{other}'")), None => usage_error("the following arguments are required: sv tool"), }, @@ -285,6 +338,12 @@ fn main() -> ExitCode { &mut std::io::stdout().lock(), &mut std::io::stderr().lock(), )), + "archaic" if rest.first().is_some_and(|v| v == "compare") => { + io(og_ancestry::compare::compare_cli, &rest[1..]) + } + "archaic" if rest.first().is_some_and(|v| v == "genomes") => { + io(og_ancestry::compare::genomes_cli, &rest[1..]) + } "archaic" => ExitCode::from(og_ancestry::archaic::cli( rest, &mut std::io::stdout().lock(), @@ -345,6 +404,37 @@ fn main() -> ExitCode { )), _ => usage_error("og summary takes collect"), }, + "report" => match rest.first().map(String::as_str) { + Some("render") => io(og_render::render::cli, &rest[1..]), + Some("index") => io(og_render::index::cli, &rest[1..]), + Some("charts") => io(og_render::charts::cli, &rest[1..]), + Some(other) => usage_error(&format!("unknown report tool '{other}'")), + None => usage_error("the following arguments are required: report tool"), + }, + "live" => match rest.first().map(String::as_str) { + Some("serve") => io(og_live::server::cli, &rest[1..]), + Some("results") => io(og_live::results::cli, &rest[1..]), + Some("family") => io(og_live::family::cli, &rest[1..]), + Some("demo") => io(og_live::demo::cli, &rest[1..]), + Some(other) => usage_error(&format!("unknown live tool '{other}'")), + None => usage_error("the following arguments are required: live tool"), + }, + "view" => match rest.first().map(String::as_str) { + Some("serve") => io(og_live::viewer::cli, &rest[1..]), + Some(other) => usage_error(&format!("unknown view tool '{other}'")), + None => usage_error("the following arguments are required: view tool"), + }, + "couple" => match rest.first().map(String::as_str) { + Some("screen") => io(og_couple::screen::cli, &rest[1..]), + Some("genes") => io(og_couple::genes::cli, &rest[1..]), + Some(other) => usage_error(&format!("unknown couple tool '{other}'")), + None => usage_error("the following arguments are required: couple tool"), + }, + "longread" => match rest.first().map(String::as_str) { + Some("clair3-model") => io(og_steps::longread::clair3_model_cli, &rest[1..]), + Some(other) => usage_error(&format!("unknown longread tool '{other}'")), + None => usage_error("the following arguments are required: longread tool"), + }, "roh" => match rest.first().map(String::as_str) { Some("summary") => ExitCode::from(og_summary::collect::group_c::roh::cli( &rest[1..], diff --git a/rust/og/tests/port_scaffold.rs b/rust/og/tests/port_scaffold.rs new file mode 100644 index 000000000..da22c7cbc --- /dev/null +++ b/rust/og/tests/port_scaffold.rs @@ -0,0 +1,158 @@ +//! Every verb the 1.0.0 plan adds is registered in `og`: listed on its own +//! line of `og --help` and routed to its module, never to the "unknown" +//! error of a router. A port replaces a stub's body, so this test keeps +//! holding after it; a verb that a later merge drops turns it red. +//! +//! While a verb is still a stub it must say exactly `og : not ported +//! yet` on stderr and exit 2, so no caller mistakes it for a result. + +use std::io::Read; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// Every verb the 1.0.0 plan adds, as its words after `og`. +const PLANNED: &[&str] = &[ + "report render", + "report index", + "report charts", + "live serve", + "live results", + "live family", + "live demo", + "view serve", + "couple screen", + "couple genes", + "vcf index", + "vcf shift-back", + "setup chrm-shift", + "setup revel", + "setup roh-af", + "sv survivor-prep", + "sv consensus-check", + "archaic compare", + "archaic genomes", + "ancestry-fine freqs", + "ancestry-fine project", + "ancestry-fine paint", + "ancestry-fine simulate", + "longread clair3-model", + "map svg", + "dev demo-sample", + "dev demo-events", + "dev params-doc", + "dev python-gate", + "dev everything-check", + "dev licences", + "dev docs-check", + "dev stub-http", + "dev steps-check", + "dev build roh-af", + "dev build acmg-lof", +]; + +/// An option no verb takes, so a ported verb stops at its argument check +/// instead of doing its work. +const PROBE: &str = "--og-scaffold-probe"; + +/// Runs `og ARGS` with no stdin; the exit code and stderr. A verb still +/// running after 30 s is killed and fails the test. +fn og(args: &[&str]) -> (Option, String) { + let mut child = Command::new(env!("CARGO_BIN_EXE_og")) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .expect("og starts"); + let mut stderr = child.stderr.take().expect("piped stderr"); + let reader = std::thread::spawn(move || { + let mut text = String::new(); + stderr.read_to_string(&mut text).map(|_| text) + }); + let start = Instant::now(); + let status = loop { + if let Some(status) = child.try_wait().expect("og waits") { + break status; + } + if start.elapsed() > Duration::from_secs(30) { + let _ = child.kill(); + let _ = child.wait(); + panic!("og {} {PROBE} still ran after 30 s", args.join(" ")); + } + std::thread::sleep(Duration::from_millis(20)); + }; + let text = reader.join().expect("reader").expect("utf-8 stderr"); + (status.code(), text) +} + +#[test] +fn every_planned_verb_has_its_own_usage_line() { + let out = Command::new(env!("CARGO_BIN_EXE_og")) + .arg("--help") + .output() + .expect("og --help runs"); + assert!(out.status.success()); + let usage = String::from_utf8(out.stdout).expect("utf-8 usage"); + let lines: Vec<&str> = usage + .lines() + .map(|l| l.trim_start_matches("usage:").trim()) + .collect(); + let missing: Vec<&&str> = PLANNED + .iter() + .filter(|verb| { + let want = format!("og {verb}"); + !lines + .iter() + .any(|l| l == &want || l.starts_with(&format!("{want} "))) + }) + .collect(); + assert!(missing.is_empty(), "no usage line for: {missing:?}"); +} + +#[test] +fn every_planned_verb_is_routed_and_a_stub_says_so() { + let mut failures = Vec::new(); + for verb in PLANNED { + let words: Vec<&str> = verb.split(' ').collect(); + let mut args = words.clone(); + args.push(PROBE); + let (code, stderr) = og(&args); + let last = words[words.len() - 1]; + let tool = words[words.len() - 2]; + let refusals = [ + format!("unknown tool '{tool}'"), + format!("unknown {tool} tool '{last}'"), + format!("unknown verb '{last}'"), + format!("unknown builder '{last}'"), + format!("invalid choice: '{last}'"), + ]; + if let Some(r) = refusals.iter().find(|r| stderr.contains(r.as_str())) { + failures.push(format!("og {verb}: not routed ({r})")); + } else if stderr.contains("not ported yet") { + let want = format!("og {verb}: not ported yet\n"); + if stderr != want || code != Some(2) { + failures.push(format!( + "og {verb}: a stub must print {want:?} and exit 2, got {stderr:?} and {code:?}" + )); + } + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn an_unplanned_verb_is_still_refused() { + // The control for the test above: the router's refusal is what it looks for. + let (code, stderr) = og(&["sv", "no-such-verb", PROBE]); + assert_eq!(code, Some(2)); + assert!( + stderr.contains("unknown sv tool 'no-such-verb'"), + "{stderr}" + ); + let (code, stderr) = og(&["dev", "build", "no-such-builder", PROBE]); + assert_eq!(code, Some(2)); + assert!( + stderr.contains("unknown builder 'no-such-builder'"), + "{stderr}" + ); +} From abc792a15a375c7d08db6f2a0282d91932bef9c9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:22:03 +0100 Subject: [PATCH 107/255] fix(live): show viewer setup failures inside the reads panel Extend the synthetic reference to verify the 49 kb alignment window. Check camera changes, veil closure, Escape in the frame, and the original frame-src failure. Bead: pgp-iqon.11 --- bin/owngenome_live_viewer.py | 6 +- tests/test_live_viewer.py | 104 ++++++++++++++++++++++++++--------- 2 files changed, 82 insertions(+), 28 deletions(-) diff --git a/bin/owngenome_live_viewer.py b/bin/owngenome_live_viewer.py index e7891bb49..dddc65f25 100644 --- a/bin/owngenome_live_viewer.py +++ b/bin/owngenome_live_viewer.py @@ -15,8 +15,8 @@ (assets/viewer/serve.py sample_files): the run folder's BAM or CRAM, its VCF and its SV VCF, each with its index, and the reference. A track whose file or index lies outside GENOME_DIR (a symlink out of it) is left out. The page and the files carry the -viewer's Content-Security-Policy (serve.CSP): the page talks to this site -only. The report the site serves (report.html) gets its viewer links pointed +viewer's Content-Security-Policy (serve.LIVE_CSP): the page talks to this site +only and allows same-origin framing. The report the site serves (report.html) gets its viewer links pointed here. Opening the page writes one access-log row; the byte ranges igv.js reads write none. @@ -137,7 +137,7 @@ def viewer(req, m): why = ready() if why: req.log("viewer", run, status=404) - return req.deny(404, why) + return send(req, 404, "text/plain; charset=utf-8", why) try: if page: with open(os.path.join(VIEWER_DIR, "index.html")) as f: diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index afe6ba730..803d7e33f 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -25,6 +25,9 @@ 7. in a headless Chrome (skipped without Chrome, except in CI) the page, loaded from the site, starts igv.js (status ready, 3 tracks) and the policy blocks nothing. + 8. the map opens a same-origin reads iframe, follows the camera, hides + closed findings and returns focus on Escape; a planted page policy + without frame-src blocks the iframe. Other routes keep DENY. Checks 2, 3 and 5 are also run against planted mistakes (a viewer that serves any file of the run folder, a report filter that does nothing, a viewer that compares the still-quoted name, one that serves an index out of GENOME_DIR) @@ -33,6 +36,7 @@ Run: python3 tests/test_live_viewer.py """ import http.client +import base64 import gzip import json import importlib.util @@ -197,7 +201,8 @@ def map_reads_checks(srv, work, chrome): """ app, port = srv[0].app, srv[1].server_address[1] policy = {"block": False} - page = ('' + page = ('' + '' '' '' '') @@ -221,20 +226,31 @@ def fixture(req, _): app.route("GET", r"/map-1g.json", lambda req, _: req.send_body(200, "application/gzip", data)) b = tl.Chrome(chrome, f"{work}/chrome-map-1g") try: + visit = 0 + + def js(expr): + reply = b.call("Runtime.evaluate", {"expression": expr, "returnByValue": True}, b.session) + if reply.get("exceptionDetails"): + raise RuntimeError(str(reply["exceptionDetails"])) + return reply.get("result", {}).get("value") + b.call("Emulation.setEmulatedMedia", {"features": [{"name": "prefers-color-scheme", "value": "dark"}, {"name": "prefers-reduced-motion", "value": "reduce"}]}, b.session) + b.call("Emulation.setDeviceMetricsOverride", {"width": 1440, "height": 900, "deviceScaleFactor": 1, "mobile": False}, b.session) def mount(demo=False): - b.go(f"http://127.0.0.1:{port}/map-1g/#chr1:1-49000") - if not b.wait("window.PGPMap && window.PGPUI"): + nonlocal visit + visit += 1 + b.go(f"http://127.0.0.1:{port}/map-1g/?visit={visit}#chr1:1-49000") + if not b.wait("!!(window.PGPMap && window.PGPUI)"): return False - b.eval("window.closedResult = true; " + js("window.closedResult = true;document.documentElement.dataset.run='S'; " "document.documentElement.dataset.mode = " + json.dumps("replay" if demo else "live") + "; " "window.testMap = PGPMap.create({url:'/map-1g.json'," "veil:function(id){return id==='closed' && window.closedResult ? 'closed':'open';}," "known:function(){return true;},now:function(){return true;}});" "document.body.appendChild(testMap.node);testMap.refresh('one');") - return b.wait("document.querySelector('.gmap[data-level=reads]')") + return b.wait("!!document.querySelector('.gmap[data-level=reads]')") def loaded(): return b.wait("(function(){var f=document.querySelector('.gm-reads iframe');" @@ -256,9 +272,21 @@ def loaded(): check("map: moving updates the same iframe to the map's locus", b.wait("savedFrame===document.querySelector('.gm-reads iframe') && savedFrame.contentWindow.location.hash==='#chr1:1-49001'")) check("viewer: alignments remain visible at 49 kb", - b.wait("(function(){var d=savedFrame.contentDocument;return d.querySelector('canvas.igv-canvas') && " - "!Array.from(d.querySelectorAll('.igv-zoom-in-notice-container')).some(function(x){return x.offsetHeight>0;});})()")) - b.eval("document.querySelector('.gm-reads-close').dispatchEvent(new KeyboardEvent('keydown',{key:'Escape',bubbles:true}))") + b.wait("(function(){var v=savedFrame.contentDocument.querySelector('.igv-viewport[data-track-type=alignment]');" + "var n=v && v.querySelector('.igv-zoom-in-notice-container');" + "return !!(v && v.querySelector('canvas') && n && n.style.display!=='flex');})()")) + if os.environ.get("MAP_SCREENS"): + os.makedirs(os.environ["MAP_SCREENS"], exist_ok=True) + for width, height in ((1440, 900), (390, 844)): + b.call("Emulation.setDeviceMetricsOverride", {"width": width, "height": height, + "deviceScaleFactor": 1, "mobile": False}, b.session) + b.eval("document.querySelector('.gm-reads').scrollIntoView({block:'center'})") + png = b.call("Page.captureScreenshot", {"format": "png"}, b.session)["data"] + with open(os.path.join(os.environ["MAP_SCREENS"], f"map-reads-{width}.png"), "wb") as out: + out.write(base64.b64decode(png)) + b.call("Emulation.setDeviceMetricsOverride", {"width": 1440, "height": 900, + "deviceScaleFactor": 1, "mobile": False}, b.session) + b.eval("savedFrame.contentDocument.dispatchEvent(new KeyboardEvent('keydown',{key:'Escape',bubbles:true}))") check("map: Escape closes the reads and returns focus without moving the map", b.eval("document.querySelector('.gm-reads').hidden && document.activeElement===document.querySelector('.gm-canvas') && " "document.querySelector('.gmap').dataset.view==='chr1:1-49001'")) @@ -267,22 +295,39 @@ def loaded(): b.eval("document.querySelector('.gm-reads-close').click();document.querySelector('[data-stop=reads]').click()") check("map: Reads gauge opens a 2 kb view and the panel", b.wait("!document.querySelector('.gm-reads').hidden && " "document.querySelector('.gmap').dataset.level==='reads'")) - b.eval("document.querySelector('.gm-reads-close').click();document.querySelector('.gm-marks [data-id=open]').focus()") - b.eval("document.querySelector('.gm-card-reads').click()") + js("document.querySelector('.gm-reads-close').click();location.hash='chr1:1-2000'") + b.wait("document.querySelector('.gmap').dataset.view==='chr1:1-2000'") + js("document.querySelector('.gm-marks [data-id=open]').focus();document.querySelector('.gm-card-reads').click()") check("map: the card opens reads at its finding", b.wait("document.querySelector('.gm-reads iframe').contentWindow.location.hash==='#chr1:401-501'")) - b.eval("window.closedResult=false;testMap.refresh('one');document.querySelector('.gm-marks [data-id=closed]').focus();" + js("window.closedResult=false;testMap.refresh('one');document.querySelector('.gm-marks [data-id=closed]').focus();" "document.querySelector('.gm-card-reads').click()") check("map: an opened finding offers reads", b.wait("document.querySelector('.gm-reads iframe').contentWindow.location.hash==='#chr1:501-601'")) b.eval("window.closedResult=true;testMap.refresh('one')") check("map: closing a result removes its open reads panel and link", b.eval("document.querySelector('.gm-reads').hidden && !document.querySelector('.gm-card-reads')")) + # No finding in view: use the view itself, and no viewer flags: show the server's reason. + b.eval("location.hash='chr1:10000-59000'") + b.wait("document.querySelector('.gmap').dataset.view==='chr1:10000-59000'") + fasta = srv[0].cfg.viewer_fasta + srv[0].cfg.viewer_fasta = None + try: + b.eval("document.querySelector('.gm-reads-pill').click()") + check("map: a missing viewer shows the server's reason, without a broken frame", + b.wait("document.querySelector('.gm-reads').textContent.includes('not set up')") + and b.eval("!document.querySelector('.gm-reads iframe')")) + finally: + srv[0].cfg.viewer_fasta = fasta + # CI proves the browser assertion detects the original policy mistake. policy["block"] = True mount() b.eval("window.blockedFrames=0;document.addEventListener('securitypolicyviolation',function(e){" "if(e.violatedDirective==='frame-src')window.blockedFrames++;});document.querySelector('.gm-reads-pill').click()") - check("control: without frame-src the reads iframe is blocked", b.wait("window.blockedFrames>0") and not loaded()) + check("control: without frame-src the reads iframe is blocked", b.wait("window.blockedFrames>0") and b.eval( + "(function(){var f=document.querySelector('.gm-reads iframe');return !f || !f.contentDocument || " + "f.contentDocument.documentElement.dataset.pgpStatus!=='ready';})()"), + b.eval("document.documentElement.dataset.pgpBlocked")) policy["block"] = False mount(demo=True) b.eval("document.querySelector('.gm-reads-pill').click()") @@ -313,7 +358,19 @@ def main(): proc = None try: gd = f"{work}/genome" - tv.make_sample(gd) + _, contigs = tv.make_sample(gd) + # A 49 kb alignment check needs a reference longer than the original 3 kb sample. + contigs[0] = ("chr1", contigs[0][1] + "ACGT" * 14250) + fasta = f"{gd}/reference/synthetic.fasta" + with open(fasta, "w") as fa, open(fasta + ".fai", "w") as fai: + offset = 0 + for name, seq in contigs: + offset += len(name) + 2 + fai.write(f"{name}\t{len(seq)}\t{offset}\t60\t61\n") + fa.write(f">{name}\n" + "\n".join(seq[i:i + 60] for i in range(0, len(seq), 60)) + "\n") + offset += len(seq) + (len(seq) + 59) // 60 + tv.write_bam(f"{gd}/S/aligned/S_sorted.bam", contigs, + [(0, p, contigs[0][1][p:p + 50]) for p in range(400, 700, 10)]) s = f"{gd}/S" os.makedirs(f"{s}/nextflow") # the site lists a folder with nextflow/ as a run html, _ = tv.findings_report(work) @@ -402,18 +459,15 @@ def main(): if ci: FAILED.append("no Chrome in CI") else: - r = subprocess.run([chrome, "--headless=new", "--disable-gpu", "--no-sandbox", "--no-first-run", - "--no-default-browser-check", f"--user-data-dir={work}/chrome", - "--virtual-time-budget=20000", "--dump-dom", - f"http://127.0.0.1:{p}/runs/S/viewer/#chr1:451"], - capture_output=True, text=True, timeout=180) - status = re.search(r'data-pgp-status="([^"]*)"', r.stdout) - tracks = re.search(r'data-pgp-tracks="([^"]*)"', r.stdout) - blocked = re.search(r'data-pgp-blocked="([^"]*)"', r.stdout) - check("browser: the page from the site starts igv.js with the run's 3 tracks", - status and status.group(1) == "ready" and tracks and tracks.group(1) == "3", - (status and status.group(1), r.stderr[-800:], r.stdout[:600])) - check("browser: the policy blocked nothing the page needed", not blocked, blocked and blocked.group(1)) + b = tl.Chrome(chrome, f"{work}/chrome") + try: + b.go(f"http://127.0.0.1:{p}/runs/S/viewer/#chr1:451") + check("browser: the page from the site starts igv.js with the run's 3 tracks", + b.wait("document.documentElement.dataset.pgpStatus==='ready' && document.documentElement.dataset.pgpTracks==='3'"), + b.eval("document.documentElement.dataset.pgpStatus")) + check("browser: the policy blocked nothing the page needed", not b.eval("document.documentElement.dataset.pgpBlocked")) + finally: + b.close() map_reads_checks(srv, work, chrome) finally: stop(*srv) From 079b864e1488e163dbeeac90534c2a47af32c132 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:22:36 +0100 Subject: [PATCH 108/255] fix(pgx): stop warning against valproate for one recessive hit, and drop the SCN1A drug warnings One heterozygous pathogenic variant in ASL, ASS1, CPS1 or NAGS makes a carrier: the line is now a NOTE that says so and stays out of the summary. Both copies changed, or two variants in the gene, still warn. The rule lives in a new Report_if column of the contraindication table. POLG, OTC and HPRT1 keep warning on one variant. The SCN1A carbamazepine and phenytoin rows go: no label or guideline names the gene, as for ABL2. Each drug line prints its CPIC level as provisional, which every pair in the table is. The pgx-parse parity cases that read the table get goldens: the oracle cannot read the new column. Bead: pgx-21 Bead: pgx-22 Bead: pgx-23 Bead: D13 --- CHANGELOG.md | 2 + assets/pgx_contraindications.tsv | 40 +++++---- docs/27-cpic-lookup.md | 4 +- modules/local/cpic_lookup/main.nf | 6 +- rust/crates/og-pgx/src/cli.rs | 8 +- rust/crates/og-pgx/src/sections.rs | 86 ++++++++++++++++--- rust/crates/og-pgx/src/tables.rs | 52 +++++++++-- .../og-pgx/tests/i411_contraindications.rs | 11 ++- scripts/27-cpic-lookup.sh | 6 +- .../pgx-inhibitor-rows-1-contraindication.sh | 20 +++-- .../CONTRA_BAD_HITS_cpic_recommendations.txt | 37 ++++++++ .../golden/pgx-parse/contra-bad-hits/stderr | 1 + .../CONTRA_BAD_TABLE_cpic_recommendations.txt | 36 ++++++++ .../golden/pgx-parse/contra-bad-table/stderr | 1 + .../CONTRA_NO_HITS_cpic_recommendations.txt | 37 ++++++++ .../golden/pgx-parse/contra-no-hits/stderr | 0 .../out/CONTRA_NONE_cpic_recommendations.txt | 36 ++++++++ .../golden/pgx-parse/contra-none/stderr | 0 .../CONTRA_ON_LIST_cpic_recommendations.txt | 53 ++++++++++++ .../golden/pgx-parse/contra-on-list/stderr | 0 .../out/CONTRA_PLP_cpic_recommendations.txt | 46 ++++++++++ .../parity/golden/pgx-parse/contra-plp/stderr | 0 .../CONTRA_SHORT_ROW_cpic_recommendations.txt | 36 ++++++++ .../golden/pgx-parse/contra-short-row/stderr | 0 tests/test_cpic_parser.py | 17 ++-- 25 files changed, 472 insertions(+), 63 deletions(-) create mode 100644 tests/parity/golden/pgx-parse/contra-bad-hits/out/CONTRA_BAD_HITS_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-bad-hits/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-bad-table/out/CONTRA_BAD_TABLE_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-bad-table/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-no-hits/out/CONTRA_NO_HITS_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-no-hits/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-none/out/CONTRA_NONE_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-none/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-on-list/out/CONTRA_ON_LIST_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-on-list/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-plp/out/CONTRA_PLP_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-plp/stderr create mode 100644 tests/parity/golden/pgx-parse/contra-short-row/out/CONTRA_SHORT_ROW_cpic_recommendations.txt create mode 100644 tests/parity/golden/pgx-parse/contra-short-row/stderr diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..05209c768 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- A single pathogenic variant in a recessive urea cycle gene (ASL, ASS1, CPS1 or NAGS) no longer gives a valproate `WARNING:` in step 27's drug contraindications. One copy makes a carrier, so the line is now a `NOTE:` that says so, and it stays out of the summary. A genotype with both copies changed, or two variants in the gene, still warns, and the line says which; the hits are not phased, so two variants on one copy still warn. POLG, OTC and HPRT1 warn on one variant, as before. The table `assets/pgx_contraindications.tsv` gets a `Report_if` column (`any` or `biallelic`) that holds this rule. The SCN1A rows for carbamazepine and phenytoin are gone: no drug label or guideline names SCN1A, and its gain-of-function variants make a gene-wide warning wrong. Each drug line now prints its CPIC level as provisional ("provisional CPIC level B: no CPIC guideline covers this pair"), which is what CPIC says of every pair in the table. Part of #411. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/assets/pgx_contraindications.tsv b/assets/pgx_contraindications.tsv index f9c882a2f..0d6165987 100644 --- a/assets/pgx_contraindications.tsv +++ b/assets/pgx_contraindications.tsv @@ -5,24 +5,32 @@ # The rows are the CPIC gene-drug pairs at level A, B or A/B whose gene is a # disease gene outside PharmCAT and whose drug the variant makes risky # (CPIC API pair_view, queried 2026-11-03; Pair_source is the query for the -# gene). Left out: NAGS with carglumic acid (the treatment, not a risk), and -# ABL2 with valproic acid (no label or guideline names it). +# gene). Every pair here is provisional in CPIC (provisional true, no CPIC +# guideline): a gene with a CPIC guideline is one PharmCAT calls. A row needs +# a drug label that names the gene's disorder. Left out: NAGS with carglumic +# acid (the treatment, not a risk), ABL2 with valproic acid, and SCN1A with +# carbamazepine and phenytoin (no label or guideline names them; SCN1A also +# has gain-of-function disease variants, for which a warning against these +# drugs does not follow). # Gene: the HGNC symbol, as ClinVar's GENEINFO writes it. # Drug: the drug as CPIC names it. # Names: the first words a medication list may use for it, comma-separated, # lower case; a drug on the list is marked in the report. -# CPIC_level: the pair's CPIC level. -# Note: what the label says, or for a pair no label names, what CPIC says. -# Label_source: the FDA label (DailyMed) the note quotes, or '-' when none -# names the gene. +# CPIC_level: the pair's CPIC level, printed as provisional. +# Note: what the label says. +# Label_source: the FDA label (DailyMed) the note quotes. +# Report_if: when a hit gives a WARNING. any: one variant does (POLG, where +# one variant is a known valproate risk, and the X-linked OTC and HPRT1, +# where a male has one copy and a female carrier can be affected). +# biallelic: the disorder is recessive, so the WARNING needs both copies +# changed (a homozygous or hemizygous genotype) or two variants in the gene; +# one heterozygous variant makes a carrier and gives a NOTE instead. # tests/test_cpic_parser.py checks the shape of every row. -Gene Drug Names CPIC_level Note Pair_source Label_source -POLG valproic acid valproic,valproate,divalproex A/B The valproate label contraindicates it in people with mitochondrial disorders caused by POLG mutations, and in children under two suspected of a POLG-related disorder: valproate can cause acute liver failure in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.POLG https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -ASL valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.ASL https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -ASS1 valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.ASS1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -CPS1 valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.CPS1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -NAGS valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.NAGS https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -OTC valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.OTC https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f -HPRT1 mycophenolic acid mycophenolic,mycophenolate B The mycophenolate label says to avoid it in people with a hereditary HGPRT deficiency, such as Lesch-Nyhan or Kelley-Seegmiller syndrome. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.HPRT1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=37241e87-4af4-4dc3-a1aa-ea6f20d8dc40 -SCN1A carbamazepine carbamazepine B CPIC lists SCN1A with carbamazepine at level B, with no CPIC guideline and no label that names the gene: read its ClinPGx clinical annotations. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.SCN1A - -SCN1A phenytoin phenytoin B CPIC lists SCN1A with phenytoin at level B, with no CPIC guideline and no label that names the gene: read its ClinPGx clinical annotations. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.SCN1A - +Gene Drug Names CPIC_level Note Pair_source Label_source Report_if +POLG valproic acid valproic,valproate,divalproex A/B The valproate label contraindicates it in people with mitochondrial disorders caused by POLG mutations, and in children under two suspected of a POLG-related disorder: valproate can cause acute liver failure in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.POLG https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f any +ASL valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.ASL https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f biallelic +ASS1 valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.ASS1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f biallelic +CPS1 valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.CPS1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f biallelic +NAGS valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.NAGS https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f biallelic +OTC valproic acid valproic,valproate,divalproex B The valproate label contraindicates it in people with a urea cycle disorder: valproate can cause hyperammonemic encephalopathy in them. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.OTC https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f any +HPRT1 mycophenolic acid mycophenolic,mycophenolate B The mycophenolate label says to avoid it in people with a hereditary HGPRT deficiency, such as Lesch-Nyhan or Kelley-Seegmiller syndrome. https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.HPRT1 https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=37241e87-4af4-4dc3-a1aa-ea6f20d8dc40 any diff --git a/docs/27-cpic-lookup.md b/docs/27-cpic-lookup.md index c65813c3f..180009f50 100644 --- a/docs/27-cpic-lookup.md +++ b/docs/27-cpic-lookup.md @@ -44,7 +44,7 @@ MEDICATIONS=/path/to/medications.txt ./scripts/27-cpic-lookup.sh your_name 5. With the pypgx summary it writes the comparison table and, for a gene PharmCAT reports as not called or ambiguous but pypgx called (CYP2D6 is the usual one), a warning in the recommendations naming every drug PharmCAT links to that gene. When pypgx's call is normal the line is a note instead. 6. With step 36's consensus table it writes a section "Calls From Other Tools": each gene passed to PharmCAT as an outside call, from which tool, and whether PharmCAT's report shows it as one (`callSource` `OUTSIDE`); and each gene held back, with the reason and what each caller said. For a held-back CYP2D6 it names the drugs CYP2D6 affects and says no guidance is given for them, and the pypgx-only warning of point 5 is not printed for it: one caller's call is what the consensus refused. Genes PharmCAT reports as outside calls carry `[outside call]` in the gene results. An outside call of CYP2D6, HLA-A or HLA-B in PharmCAT's report that the table does not pass on, with the same diplotype, comes from an earlier step 36 run that step 07 has not replaced: it gets no drug guidance, is listed as not called, and a warning says to rerun step 07. When the table is missing (a caller step ran again since) but PharmCAT's report has outside calls of those genes, the step passes the missing table, so none of them is confirmed. MT-RNR1's outside call comes from step 20's chrM calls through step 07, never from step 36: the table never lists it, so it is not checked against it and keeps its drug guidance. 7. With a medication list, each line is matched by its first word (a space, comma, semicolon, slash or plus ends a word, so `lopinavir/ritonavir` is matched as lopinavir; leading commas, semicolons, slashes and pluses are ignored, and a line of only those is skipped) against `assets/pgx_inhibitors.tsv`: the FDA's strong and moderate inhibitors of CYP2D6 and CYP2C19, its moderate CYP2C9 inhibitors (it lists no strong one) and its strong CYP3A inhibitors, one row each for CYP3A4 and CYP3A5, as the FDA's table of clinical CYP inhibitors classes them, each row with its source. A ritonavir-boosted combination the FDA classes as one (`lopinavir and ritonavir`) has a row under ritonavir and one under its partner. For CYP2D6 the step applies CPIC's rule from its opioid guideline: "for patients on strong CYP2D6 inhibitors, the CYP2D6 activity score is adjusted to 0 and the predicted phenotype is a poor metabolizer", and for a moderate inhibitor "the activity score is multiplied by 0.5 and then converted to the predicted phenotype" (CPIC's bands: 0 poor, under 1.25 intermediate, 1.25 to 2.25 normal, above 2.25 ultrarapid). The activity score is the one PharmCAT gives the called diplotype. With several inhibitors of one gene the strongest counts. The result is written as `Poor Metabolizer (phenoconverted by paroxetine)` beside the genotype phenotype, never in its place, with a `WARNING:` line in the section "Medications and Phenoconversion", so the summary carries it. Nothing is converted for a gene that is not called, ambiguous or unclassified, or for a moderate inhibitor when the report gives no activity score: the line says why. CPIC has no activity score for CYP2C19, and gives no rule that converts the phenotype of CYP2C9, CYP3A4 or CYP3A5, so an inhibitor of one of these genes gets a note and a `not computed: ...` value, with no converted phenotype. For a poor metabolizer of these genes (the most common CYP3A5 result) the note does not say the inhibitor can lower the activity further. A line that names no drug of the table is listed as "Not in the inhibitor table", never dropped. A list that names no drug converts nothing and leaves the phenotype table as it is without a list. An unreadable list or table is reported in that section; the rest of the report is written as usual. The FDA's table lists no strong or moderate CYP2B6 inhibitor, so CYP2B6 has no rows. -8. With step 06's ClinVar hits, each hit classed `Pathogenic` or `Likely_pathogenic` (one part of `CLNSIG`; a conflicting classification or `risk_factor` alone does not count) in a gene of `assets/pgx_contraindications.tsv` adds a `WARNING:` line in the section "Drug Contraindications From Disease Variants" for each drug the table names for the gene, with the label's note, the CPIC level and the source; a drug whose name is on your medication list is marked `ON THE MEDICATION LIST`. The table holds the CPIC gene-drug pairs at level A, B or A/B (CPIC API, queried 2026-11-03) whose gene is a disease gene PharmCAT does not call and whose drug the variant makes risky: POLG with valproic acid (the valproate label contraindicates it in POLG mitochondrial disorders), ASL, ASS1, CPS1, NAGS and OTC with valproic acid (the label contraindicates it in urea cycle disorders), HPRT1 with mycophenolic acid (the label says to avoid it in hereditary HGPRT deficiency) and SCN1A with carbamazepine and phenytoin (CPIC level B, no guideline or label names the gene). Without the hits table the section lists those genes as not checked. The line does not judge inheritance: one variant in a recessive gene usually makes a carrier, not a patient, so the section says to review it with a clinician. +8. With step 06's ClinVar hits, each hit classed `Pathogenic` or `Likely_pathogenic` (one part of `CLNSIG`; a conflicting classification or `risk_factor` alone does not count) in a gene of `assets/pgx_contraindications.tsv` adds a line in the section "Drug Contraindications From Disease Variants" for each drug the table names for the gene, with the label's note, the provisional CPIC level and the label; a drug whose name is on your medication list is marked `ON THE MEDICATION LIST`. The table's `Report_if` column says when the line is a `WARNING:`. For POLG, OTC and HPRT1 (`any`) one variant gives it. For the recessive urea cycle genes ASL, ASS1, CPS1 and NAGS (`biallelic`) it needs a genotype with both copies changed (`1/1`, `1/2`, a hemizygous `1`) or two distinct variants in the gene, and the line says which. The hits are not phased, so two variants on the same copy still give the warning. One heterozygous variant in such a gene makes a carrier: it gives a `NOTE:` line instead, which says so, and only `WARNING:` lines reach the summary. The table holds the CPIC gene-drug pairs at level A, B or A/B (CPIC API, queried 2026-11-03) whose gene is a disease gene PharmCAT does not call, whose drug the variant makes risky and whose risk a drug label names: POLG with valproic acid (the valproate label contraindicates it in POLG mitochondrial disorders), ASL, ASS1, CPS1, NAGS and OTC with valproic acid (the label contraindicates it in urea cycle disorders), and HPRT1 with mycophenolic acid (the label says to avoid it in hereditary HGPRT deficiency). Every one of these pairs is provisional in CPIC: no CPIC guideline covers it, and the line says so. SCN1A with carbamazepine and phenytoin is left out: no label or guideline names the gene, and SCN1A also has gain-of-function disease variants, for which a warning against these drugs does not follow. Without the hits table the section lists the table's genes as not checked. 9. A report that cannot be read, or that yields no gene, writes a "PARSING FAILED" report and the step exits 1. It never writes an all-clear report from a format it could not read. A field of the report with an unexpected type (a number where a name belongs, say) reads as missing. 10. The `Date:` line of the recommendations is today's date in UTC. The output files are written under a temporary name and renamed into place, so a step that stops half way leaves no half-written table. @@ -93,7 +93,7 @@ Only with a medication list. A drug can make you metabolize like a different phe ### Drug Contraindications From Disease Variants -A pathogenic variant in some disease genes makes a drug dangerous, and PharmCAT does not call those genes. This section checks step 06's ClinVar hits against `assets/pgx_contraindications.tsv`. A `WARNING:` line names the drug, the variant and its genotype, what the drug label says, and where to read it. Whether it applies to you depends on the disorder and on how many copies carry the variant, so read it with your clinician. Without step 06 the section says these genes were not checked; that is not an all-clear. +A pathogenic variant in some disease genes makes a drug dangerous, and PharmCAT does not call those genes. This section checks step 06's ClinVar hits against `assets/pgx_contraindications.tsv`. A `WARNING:` line names the drug, the variant and its genotype, what the drug label says, and where to read it. In a recessive gene one variant makes a carrier, not a patient: that gets a `NOTE:` line, not a warning. Whether a line applies to you depends on the disorder, so read it with your clinician. Without step 06 the section says these genes were not checked; that is not an all-clear. ### PharmCAT and pypgx diff --git a/modules/local/cpic_lookup/main.nf b/modules/local/cpic_lookup/main.nf index 1d9c4d6ca..962036931 100644 --- a/modules/local/cpic_lookup/main.nf +++ b/modules/local/cpic_lookup/main.nf @@ -20,8 +20,10 @@ or CYP3A5 gets a note. With CLINVAR_SCREEN's hits table and assets/pgx_contraindications.tsv, a pathogenic or likely pathogenic hit in one of its disease genes (POLG - and the urea-cycle genes with valproate, HPRT1, SCN1A) adds a WARNING - naming the drug; without the hits the report says they were not checked. + and the urea-cycle genes with valproate, HPRT1 with mycophenolate) adds + a WARNING naming the drug, or a NOTE for one heterozygous hit in a + recessive urea-cycle gene (a carrier); without the hits the report says + they were not checked. Fails when the report yields no gene. Equivalent to: scripts/27-cpic-lookup.sh diff --git a/rust/crates/og-pgx/src/cli.rs b/rust/crates/og-pgx/src/cli.rs index cd839bbef..580901eaa 100644 --- a/rust/crates/og-pgx/src/cli.rs +++ b/rust/crates/og-pgx/src/cli.rs @@ -355,9 +355,11 @@ fn contraindication_section(a: &CpicArgs, table_path: &str) -> Vec { out.extend(found); out.extend( [ - " Whether a warning applies depends on the disorder the gene causes and on how many", - " copies carry the variant: one copy in a recessive gene usually makes a carrier, not a", - " patient. Review it with your clinician before taking or stopping the drug.", + " In a recessive gene a WARNING needs both copies changed or two variants in the gene.", + " Two variants are not phased here: on the same copy they make a carrier. One variant", + " in a recessive gene gives a NOTE: a carrier, not a patient, unless a second variant", + " was missed. Whether a line applies depends on the disorder the gene causes.", + " Review it with your clinician before taking or stopping the drug.", "", ] .map(str::to_owned), diff --git a/rust/crates/og-pgx/src/sections.rs b/rust/crates/og-pgx/src/sections.rs index 4ec2173bf..f2e59a628 100644 --- a/rust/crates/og-pgx/src/sections.rs +++ b/rust/crates/og-pgx/src/sections.rs @@ -6,8 +6,8 @@ use crate::pyio::PyErr; use crate::report::{GeneCall, STEP36_GENES, position_read, text, truthy_field}; use crate::status::{Status, is_normal}; use crate::tables::{ - ClinvarHit, ConsensusRow, Contraindication, Inhibitors, Medication, PypgxCall, norm_drug, - pypgx_called, + ClinvarHit, ConsensusRow, Contraindication, Inhibitors, Medication, PypgxCall, ReportIf, + norm_drug, pypgx_called, }; use pycompat::fmt::general; use pycompat::json::Value; @@ -643,22 +643,52 @@ pub fn hit_genes(geneinfo: Option<&str>) -> BTreeSet { .collect() } -/// WARNING lines for each pathogenic or likely pathogenic hit in a gene of -/// the table, one per drug the table names for the gene. A drug whose names -/// hold a name of the medication list is marked as on it. +/// Every allele of the genotype is called and none is the reference: both +/// copies changed (`1/1`, `1|2`), or the one copy of a hemizygous call (`1`). +/// `0/1`, `1|0` and `./1` are one changed copy. +fn both_copies(genotype: &str) -> bool { + genotype + .split(['/', '|']) + .all(|a| !matches!(a.trim(), "" | "." | "0")) +} + +/// The lines for each pathogenic or likely pathogenic hit in a gene of the +/// table, one entry per drug the table names for the gene: a `WARNING:` +/// line, or a `NOTE:` line for one heterozygous variant in a gene whose +/// row needs both copies (`Report_if` biallelic), which makes a carrier. +/// A recessive gene warns when a genotype shows both copies changed or +/// when it holds two distinct variants; their phase is not known here, so +/// two variants on one copy still warn. A drug whose names hold a name of +/// the medication list is marked as on it. Only `WARNING:` lines reach the +/// summary. pub fn contraindication_lines( hits: &[ClinvarHit], table: &[Contraindication], med_names: &HashSet, ) -> Vec { - let mut lines = Vec::new(); - for h in hits { - if !is_plp(h.clnsig.as_deref()) { - continue; + let plp: Vec<(&ClinvarHit, BTreeSet)> = hits + .iter() + .filter(|h| is_plp(h.clnsig.as_deref())) + .map(|h| (h, hit_genes(h.geneinfo.as_deref()))) + .collect(); + // The distinct variants of each gene, for the two-variant rule. + let mut variants: HashMap<&str, HashSet<(&str, &str, &str, &str)>> = HashMap::new(); + for (h, genes) in &plp { + for gene in genes { + variants.entry(gene.as_str()).or_default().insert(( + h.chrom.as_str(), + h.pos.as_str(), + h.ref_.as_str(), + h.alt.as_str(), + )); } - for gene in hit_genes(h.geneinfo.as_deref()) { + } + let mut lines = Vec::new(); + for (h, genes) in &plp { + for gene in genes { let where_ = format!("{gene} {}:{} {}>{}", h.chrom, h.pos, h.ref_, h.alt); - for r in table.iter().filter(|r| r.gene == gene) { + let n = variants.get(gene.as_str()).map_or(0, HashSet::len); + for r in table.iter().filter(|r| &r.gene == gene) { let on_list = r .names .split(',') @@ -669,13 +699,31 @@ pub fn contraindication_lines( } else { "" }; - lines.push(format!( - " WARNING: {} -- {where_}, genotype {}, ClinVar {}{on}.", + let head = format!( + "{} -- {where_}, genotype {}, ClinVar {}", r.drug, h.genotype, h.clnsig.as_deref().unwrap_or("None") + ); + match r.report_if { + ReportIf::Any => lines.push(format!(" WARNING: {head}{on}.")), + ReportIf::Biallelic if both_copies(&h.genotype) => lines.push(format!( + " WARNING: {head}, both copies changed in this recessive gene{on}." + )), + ReportIf::Biallelic if n >= 2 => lines.push(format!( + " WARNING: {head}, {n} variants in {gene}, a recessive gene (phase not known){on}." + )), + ReportIf::Biallelic => { + lines.push(format!(" NOTE: {head}{on}.")); + lines.push(format!( + " One variant in {gene}, a recessive gene: a carrier, not a patient, unless a second variant was missed." + )); + } + } + lines.push(format!( + " {} (provisional CPIC level {}: no CPIC guideline covers this pair)", + r.note, r.cpic_level )); - lines.push(format!(" {} (CPIC level {})", r.note, r.cpic_level)); let source = if r.label_source == "-" { &r.pair_source } else { @@ -734,4 +782,14 @@ mod tests { let g: Vec = hit_genes(Some("POLG:5428| FOO:1|:3")).into_iter().collect(); assert_eq!(g, ["FOO", "POLG"]); } + + #[test] + fn genotypes_with_both_copies_changed() { + for gt in ["1/1", "1|1", "1/2", "2|1", "1"] { + assert!(both_copies(gt), "{gt}"); + } + for gt in ["0/1", "1|0", "./1", "1/.", "0", "", "./."] { + assert!(!both_copies(gt), "{gt}"); + } + } } diff --git a/rust/crates/og-pgx/src/tables.rs b/rust/crates/og-pgx/src/tables.rs index cc7305a4f..ef8455d7e 100644 --- a/rust/crates/og-pgx/src/tables.rs +++ b/rust/crates/og-pgx/src/tables.rs @@ -238,7 +238,7 @@ pub fn read_medications(path: &str) -> Result, PyErr> { // --- contraindications --------------------------------------------------- /// The columns of `assets/pgx_contraindications.tsv`, in order. -pub const CONTRA_COLUMNS: [&str; 7] = [ +pub const CONTRA_COLUMNS: [&str; 8] = [ "Gene", "Drug", "Names", @@ -246,8 +246,20 @@ pub const CONTRA_COLUMNS: [&str; 7] = [ "Note", "Pair_source", "Label_source", + "Report_if", ]; +/// When a pathogenic hit in a row's gene gives a WARNING (the table's +/// `Report_if` column, in the words of `assets/acmg_sf.tsv`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReportIf { + /// One variant is enough (`any`). + Any, + /// The disorder is recessive: both copies changed, or two variants in + /// the gene (`biallelic`). One heterozygous variant is a carrier. + Biallelic, +} + /// One rule of the contraindication table: a disease gene PharmCAT does not /// call, and a drug a pathogenic variant in it makes risky. #[derive(Debug, Clone, PartialEq, Eq)] @@ -260,11 +272,12 @@ pub struct Contraindication { pub note: String, pub pair_source: String, pub label_source: String, + pub report_if: ReportIf, } /// The rows of the contraindication table; lines starting with `#` are -/// comments. A missing column or an empty cell is an error, never a row -/// read as half a rule. +/// comments. A missing column, an empty cell or a `Report_if` other than +/// `any` or `biallelic` is an error, never a row read as half a rule. pub fn read_contraindications(path: &str) -> Result, PyErr> { let (header, rows) = dict_rows(&uncommented(path)?)?; if header.as_deref() != Some(&CONTRA_COLUMNS.map(str::to_owned)[..]) { @@ -288,6 +301,17 @@ pub fn read_contraindications(path: &str) -> Result, PyErr repr_row(r) ))); } + let report_if = match cells[7] { + "any" => ReportIf::Any, + "biallelic" => ReportIf::Biallelic, + other => { + return Err(PyErr::value(format!( + "{path}: Report_if is {}, not any or biallelic: {}", + repr(other), + repr_row(r) + ))); + } + }; let own = |i: usize| cells[i].to_owned(); Ok(Contraindication { gene: own(0), @@ -297,6 +321,7 @@ pub fn read_contraindications(path: &str) -> Result, PyErr note: own(4), pair_source: own(5), label_source: own(6), + report_if, }) }) .collect() @@ -391,10 +416,13 @@ mod tests { Some(0.0) ); let contra = read_contraindications(&format!("{root}pgx_contraindications.tsv")).unwrap(); + assert!(contra.iter().any(|c| c.gene == "POLG" + && c.drug == "valproic acid" + && c.report_if == ReportIf::Any)); assert!( contra .iter() - .any(|c| c.gene == "POLG" && c.drug == "valproic acid") + .any(|c| c.gene == "ASS1" && c.report_if == ReportIf::Biallelic) ); } @@ -428,17 +456,27 @@ mod tests { assert_eq!( read_contraindications(&p).unwrap_err().to_string(), format!( - "ValueError: {p}: the header is ['Gene', 'Drug'], not ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source']" + "ValueError: {p}: the header is ['Gene', 'Drug'], not ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source', 'Report_if']" ) ); let p = tmp( "contra2.tsv", - "Gene\tDrug\tNames\tCPIC_level\tNote\tPair_source\tLabel_source\nPOLG\tx\ty\n", + "Gene\tDrug\tNames\tCPIC_level\tNote\tPair_source\tLabel_source\tReport_if\nPOLG\tx\ty\n", + ); + assert_eq!( + read_contraindications(&p).unwrap_err().to_string(), + format!( + "ValueError: {p}: a row has an empty or missing cell: {{'Gene': 'POLG', 'Drug': 'x', 'Names': 'y', 'CPIC_level': None, 'Note': None, 'Pair_source': None, 'Label_source': None, 'Report_if': None}}" + ) + ); + let p = tmp( + "contra3.tsv", + "Gene\tDrug\tNames\tCPIC_level\tNote\tPair_source\tLabel_source\tReport_if\nASL\tv\tv\tB\tn\tp\tl\trecessive\n", ); assert_eq!( read_contraindications(&p).unwrap_err().to_string(), format!( - "ValueError: {p}: a row has an empty or missing cell: {{'Gene': 'POLG', 'Drug': 'x', 'Names': 'y', 'CPIC_level': None, 'Note': None, 'Pair_source': None, 'Label_source': None}}" + "ValueError: {p}: Report_if is 'recessive', not any or biallelic: {{'Gene': 'ASL', 'Drug': 'v', 'Names': 'v', 'CPIC_level': 'B', 'Note': 'n', 'Pair_source': 'p', 'Label_source': 'l', 'Report_if': 'recessive'}}" ) ); let p = tmp("hits.tsv", "nothing\there\n"); diff --git a/rust/crates/og-pgx/tests/i411_contraindications.rs b/rust/crates/og-pgx/tests/i411_contraindications.rs index 33ce246ba..3d31046d3 100644 --- a/rust/crates/og-pgx/tests/i411_contraindications.rs +++ b/rust/crates/og-pgx/tests/i411_contraindications.rs @@ -131,10 +131,13 @@ fn every_row_quotes_a_label() { fn the_cpic_level_reads_as_provisional() { let out = lines(&[hit("POLG", "89300000", "T", "0/1")]); assert!( - out.iter().any(|l| l.ends_with( - "(provisional CPIC level A/B: no CPIC guideline covers this pair)" - )), + out.iter().any( + |l| l.ends_with("(provisional CPIC level A/B: no CPIC guideline covers this pair)") + ), + "{out:#?}" + ); + assert!( + !out.iter().any(|l| l.ends_with("(CPIC level A/B)")), "{out:#?}" ); - assert!(!out.iter().any(|l| l.ends_with("(CPIC level A/B)")), "{out:#?}"); } diff --git a/scripts/27-cpic-lookup.sh b/scripts/27-cpic-lookup.sh index d880d4a53..4a7581891 100755 --- a/scripts/27-cpic-lookup.sh +++ b/scripts/27-cpic-lookup.sh @@ -23,9 +23,9 @@ # Disease-variant contraindications: with step 06's ClinVar hits # (clinvar/_clinvar_hits.tsv), a pathogenic or likely pathogenic hit # in a gene of assets/pgx_contraindications.tsv (POLG and the urea-cycle -# genes with valproate, HPRT1 with mycophenolate, SCN1A) adds a WARNING -# naming the drug. Without step 06 the report says those genes were not -# checked. +# genes with valproate, HPRT1 with mycophenolate) adds a WARNING naming the +# drug; one heterozygous hit in a recessive urea-cycle gene gives a NOTE (a +# carrier). Without step 06 the report says those genes were not checked. # # Requires: PharmCAT output from step 7. No internet connection needed. set -euo pipefail diff --git a/tests/e2e/pgx-inhibitor-rows-1-contraindication.sh b/tests/e2e/pgx-inhibitor-rows-1-contraindication.sh index 0fa7bcdd5..6ffec9a70 100755 --- a/tests/e2e/pgx-inhibitor-rows-1-contraindication.sh +++ b/tests/e2e/pgx-inhibitor-rows-1-contraindication.sh @@ -5,7 +5,9 @@ # - step 27 on a synthetic PharmCAT report and a synthetic step 06 hits # table with a pathogenic POLG hit: the report gives the valproic acid # WARNING, marked when valproate is on MEDICATIONS; fluconazole on the list -# gives CYP2C9 a note in the fifth column; +# gives CYP2C9 a note in the fifth column; one heterozygous ASS1 hit (a +# recessive urea-cycle gene) gives a carrier NOTE and no WARNING, and an +# SCN1A hit gives no drug line; # - the same sample without the hits table lists the genes as not checked # and gives no valproic acid line (the bash control); # - Nextflow with --tools pharmcat,clinvar,cpic on the HG002 VCF and a @@ -30,8 +32,12 @@ cat > "${G}/${S2}/pharmcat/${S2}.report.json" <<'JSON' "drugs": {}} JSON HITS="${G}/${S2}/clinvar/${S2}_clinvar_hits.tsv" -printf 'chrom\tpos\tref\talt\tgenotype\tclinvar_id\tgeneinfo\tclnsig\tclnrevstat\n' > "$HITS" -printf 'chr15\t89300000\tC\tT\t0/1\t9900001\tPOLG:5428\tPathogenic\tcriteria_provided,_single_submitter\n' >> "$HITS" +{ + printf 'chrom\tpos\tref\talt\tgenotype\tclinvar_id\tgeneinfo\tclnsig\tclnrevstat\n' + printf 'chr15\t89300000\tC\tT\t0/1\t9900001\tPOLG:5428\tPathogenic\tcriteria_provided,_single_submitter\n' + printf 'chr9\t130500000\tG\tA\t0/1\t9900002\tASS1:445\tLikely_pathogenic\tcriteria_provided,_single_submitter\n' + printf 'chr2\t166000000\tC\tT\t0/1\t9900003\tSCN1A:6323\tPathogenic\tcriteria_provided,_single_submitter\n' +} > "$HITS" MED="${CASE_TMP}/medications.txt" printf 'valproate 500 mg\nfluconazole\n' > "$MED" REC="${G}/${S2}/cpic/${S2}_cpic_recommendations.txt" @@ -43,8 +49,12 @@ check_step_exit 27-cpic-lookup.sh OUT=$(cat "$REC" 2>/dev/null) check "step 27: a pathogenic POLG hit gives the valproic acid WARNING, marked as on the list" \ has 'WARNING: valproic acid -- POLG chr15:89300000 C>T, genotype 0/1, ClinVar Pathogenic; ON THE MEDICATION LIST\.' "$OUT" -check "step 27: the line carries the label's note and its CPIC level" \ - has 'mitochondrial disorders caused by POLG mutations.*\(CPIC level A/B\)' "$OUT" +check "step 27: the line carries the label's note and its provisional CPIC level" \ + has 'mitochondrial disorders caused by POLG mutations.*\(provisional CPIC level A/B: no CPIC guideline covers this pair\)' "$OUT" +check "step 27: one heterozygous ASS1 hit gives the carrier NOTE" \ + has 'NOTE: valproic acid -- ASS1 chr9:130500000 G>A, genotype 0/1, ClinVar Likely_pathogenic; ON THE MEDICATION LIST\.' "$OUT" +check "step 27: one heterozygous ASS1 hit gives no WARNING" lacks 'WARNING: valproic acid -- ASS1' "$OUT" +check "step 27: an SCN1A hit gives no drug line" lacks 'SCN1A' "$OUT" check "step 27: the step's log names the hits table" grep -q 'ClinVar hits (step 06): .*_clinvar_hits\.tsv' "$STEP_LOG" check "step 27: fluconazole gives CYP2C9 a note in the fifth column" \ grep -q $'^CYP2C9\t\\*1/\\*1\tNormal Metabolizer\tnormal\tnot computed: fluconazole inhibits CYP2C9' "$PHENO" diff --git a/tests/parity/golden/pgx-parse/contra-bad-hits/out/CONTRA_BAD_HITS_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-bad-hits/out/CONTRA_BAD_HITS_cpic_recommendations.txt new file mode 100644 index 000000000..d7b252989 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-bad-hits/out/CONTRA_BAD_HITS_cpic_recommendations.txt @@ -0,0 +1,37 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_BAD_HITS +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-bad-hits.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + Could not read the ClinVar hits table (ValueError: contra-bad-hits-hits.tsv: no column alt, chrom, clnsig, geneinfo, genotype, pos, ref). + Not checked: POLG, ASL, ASS1, CPS1, NAGS, OTC, HPRT1. Run step 06 again, then this step. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-bad-hits/stderr b/tests/parity/golden/pgx-parse/contra-bad-hits/stderr new file mode 100644 index 000000000..224ee4856 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-bad-hits/stderr @@ -0,0 +1 @@ +WARNING: could not read the ClinVar hits table (ValueError: contra-bad-hits-hits.tsv: no column alt, chrom, clnsig, geneinfo, genotype, pos, ref) diff --git a/tests/parity/golden/pgx-parse/contra-bad-table/out/CONTRA_BAD_TABLE_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-bad-table/out/CONTRA_BAD_TABLE_cpic_recommendations.txt new file mode 100644 index 000000000..75e7e8c4c --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-bad-table/out/CONTRA_BAD_TABLE_cpic_recommendations.txt @@ -0,0 +1,36 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_BAD_TABLE +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-bad-table.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + Could not read the contraindication table (ValueError: contraindications-bad.tsv: the header is ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source'], not ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source', 'Report_if']). + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-bad-table/stderr b/tests/parity/golden/pgx-parse/contra-bad-table/stderr new file mode 100644 index 000000000..b6a974ac0 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-bad-table/stderr @@ -0,0 +1 @@ +WARNING: could not read the contraindication table (ValueError: contraindications-bad.tsv: the header is ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source'], not ['Gene', 'Drug', 'Names', 'CPIC_level', 'Note', 'Pair_source', 'Label_source', 'Report_if']) diff --git a/tests/parity/golden/pgx-parse/contra-no-hits/out/CONTRA_NO_HITS_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-no-hits/out/CONTRA_NO_HITS_cpic_recommendations.txt new file mode 100644 index 000000000..fb7d6d100 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-no-hits/out/CONTRA_NO_HITS_cpic_recommendations.txt @@ -0,0 +1,37 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_NO_HITS +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-no-hits.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + Not checked: no ClinVar hits table (step 06, or clinvar in --tools, did not run). + Genes it would check: POLG, ASL, ASS1, CPS1, NAGS, OTC, HPRT1. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-no-hits/stderr b/tests/parity/golden/pgx-parse/contra-no-hits/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/pgx-parse/contra-none/out/CONTRA_NONE_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-none/out/CONTRA_NONE_cpic_recommendations.txt new file mode 100644 index 000000000..c53dffc7e --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-none/out/CONTRA_NONE_cpic_recommendations.txt @@ -0,0 +1,36 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_NONE +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-none.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + No pathogenic or likely pathogenic ClinVar hit in POLG, ASL, ASS1, CPS1, NAGS, OTC, HPRT1. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-none/stderr b/tests/parity/golden/pgx-parse/contra-none/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/pgx-parse/contra-on-list/out/CONTRA_ON_LIST_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-on-list/out/CONTRA_ON_LIST_cpic_recommendations.txt new file mode 100644 index 000000000..9fdd02c8f --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-on-list/out/CONTRA_ON_LIST_cpic_recommendations.txt @@ -0,0 +1,53 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_ON_LIST +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-on-list.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Medications and Phenoconversion (CYP2D6, CYP2C19, CYP2C9, CYP3A4, CYP3A5): +------------------------------------------------------------------------ + + Medication list: contra-on-list.txt (1 line). + Not in the inhibitor table: Valproate 500 mg. + These were not checked. A drug missing from the table (a brand name, a weak inhibitor) + can still change the activity of CYP2D6, CYP2C19, CYP2C9, CYP3A4, CYP3A5. + The genotype phenotype in the table above is unchanged. A phenoconverted phenotype + holds while the inhibitor is taken. This is an aid for your clinician, not advice. + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + WARNING: valproic acid -- POLG chr15:89300000 C>T, genotype 0/1, ClinVar Likely_pathogenic; ON THE MEDICATION LIST. + The valproate label contraindicates it in people with mitochondrial disorders caused by POLG mutations, and in children under two suspected of a POLG-related disorder: valproate can cause acute liver failure in them. (provisional CPIC level A/B: no CPIC guideline covers this pair) + https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f + In a recessive gene a WARNING needs both copies changed or two variants in the gene. + Two variants are not phased here: on the same copy they make a carrier. One variant + in a recessive gene gives a NOTE: a carrier, not a patient, unless a second variant + was missed. Whether a line applies depends on the disorder the gene causes. + Review it with your clinician before taking or stopping the drug. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-on-list/stderr b/tests/parity/golden/pgx-parse/contra-on-list/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/pgx-parse/contra-plp/out/CONTRA_PLP_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-plp/out/CONTRA_PLP_cpic_recommendations.txt new file mode 100644 index 000000000..e1a4a8e39 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-plp/out/CONTRA_PLP_cpic_recommendations.txt @@ -0,0 +1,46 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_PLP +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-plp.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + WARNING: valproic acid -- POLG chr15:89300000 C>T, genotype 0/1, ClinVar Pathogenic. + The valproate label contraindicates it in people with mitochondrial disorders caused by POLG mutations, and in children under two suspected of a POLG-related disorder: valproate can cause acute liver failure in them. (provisional CPIC level A/B: no CPIC guideline covers this pair) + https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f + WARNING: valproic acid -- POLG chr15:166000000 C>T, genotype 1/1, ClinVar Pathogenic/Likely_pathogenic. + The valproate label contraindicates it in people with mitochondrial disorders caused by POLG mutations, and in children under two suspected of a POLG-related disorder: valproate can cause acute liver failure in them. (provisional CPIC level A/B: no CPIC guideline covers this pair) + https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=08a65cf4-7749-4ceb-6895-8f4805e2b01f + In a recessive gene a WARNING needs both copies changed or two variants in the gene. + Two variants are not phased here: on the same copy they make a carrier. One variant + in a recessive gene gives a NOTE: a carrier, not a patient, unless a second variant + was missed. Whether a line applies depends on the disorder the gene causes. + Review it with your clinician before taking or stopping the drug. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-plp/stderr b/tests/parity/golden/pgx-parse/contra-plp/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/pgx-parse/contra-short-row/out/CONTRA_SHORT_ROW_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/contra-short-row/out/CONTRA_SHORT_ROW_cpic_recommendations.txt new file mode 100644 index 000000000..3fd5f84d0 --- /dev/null +++ b/tests/parity/golden/pgx-parse/contra-short-row/out/CONTRA_SHORT_ROW_cpic_recommendations.txt @@ -0,0 +1,36 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: CONTRA_SHORT_ROW +Date: 2026-11-15 +Source: PharmCAT 3.4.0 report (contra-short-row.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer + +Affected Medications: +------------------------------------------------------------------------ + + No gene with a non-normal phenotype. + +Drug Contraindications From Disease Variants (ClinVar hits, step 06): +------------------------------------------------------------------------ + + No pathogenic or likely pathogenic ClinVar hit in POLG, ASL, ASS1, CPS1, NAGS, OTC, HPRT1. + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/contra-short-row/stderr b/tests/parity/golden/pgx-parse/contra-short-row/stderr new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_cpic_parser.py b/tests/test_cpic_parser.py index 2713ad6d0..9fc7d96f5 100644 --- a/tests/test_cpic_parser.py +++ b/tests/test_cpic_parser.py @@ -89,7 +89,7 @@ # The genes a medication list is checked for, and the contraindication # table's columns, as og reads them. PHENOCONVERSION_GENES = ("CYP2D6", "CYP2C19", "CYP2C9", "CYP3A4", "CYP3A5") -CONTRA_COLUMNS = ["Gene", "Drug", "Names", "CPIC_level", "Note", "Pair_source", "Label_source"] +CONTRA_COLUMNS = ["Gene", "Drug", "Names", "CPIC_level", "Note", "Pair_source", "Label_source", "Report_if"] def og(*args): @@ -374,20 +374,22 @@ def contraindication_checks(work): header = lines[0].rstrip("\n").split("\t") check("contraindications: the header", header == CONTRA_COLUMNS, header) rows = [line.rstrip("\n").split("\t") for line in lines[1:] if line.strip()] - bad = [r for r in rows if len(r) != 7 or not all(c.strip() for c in r) + bad = [r for r in rows if len(r) != 8 or not all(c.strip() for c in r) or r[3] not in ("A", "B", "A/B") or r[5] != f"https://api.cpicpgx.org/v1/pair_view?genesymbol=eq.{r[0]}" - or not (r[6] == "-" or r[6].startswith("https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=")) + or not r[6].startswith("https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=") + or r[7] not in ("any", "biallelic") or any(n != n.strip().lower() or not n for n in r[2].split(",")) or r[1].split()[0] not in r[2].split(",")] check("contraindications: every row has a gene, a drug, its names, a CPIC level, a note and its sources", not bad, bad) keys = [(r[0], r[1]) for r in rows] check("contraindications: no gene-drug pair twice", len(keys) == len(set(keys)), keys) - check("contraindications: POLG with valproic acid, the urea-cycle genes, HPRT1 and SCN1A are there", + check("contraindications: POLG with valproic acid, the urea-cycle genes and HPRT1 are there", {("POLG", "valproic acid"), ("ASL", "valproic acid"), ("ASS1", "valproic acid"), ("CPS1", "valproic acid"), - ("NAGS", "valproic acid"), ("OTC", "valproic acid"), ("HPRT1", "mycophenolic acid"), - ("SCN1A", "carbamazepine"), ("SCN1A", "phenytoin")} <= set(keys), keys) + ("NAGS", "valproic acid"), ("OTC", "valproic acid"), ("HPRT1", "mycophenolic acid")} <= set(keys), keys) + check("contraindications: SCN1A is left out (no label or guideline names it)", + not any(r[0] == "SCN1A" for r in rows), keys) _, rec, _, _ = run_report(work, "CONTRAREAD", pheno_report(), contraindications=CONTRA) check("contraindications: the parser reads the table", "Could not read" not in contra_section(rec) and "Genes it would check: " + ", ".join(dict.fromkeys(r[0] for r in rows)) + "." in contra_section(rec), @@ -402,7 +404,8 @@ def contraindication_checks(work): check("contraindications: a pathogenic POLG hit adds the valproic acid WARNING", "WARNING: valproic acid -- POLG chr15:89300000 C>T, genotype 0/1, ClinVar Pathogenic." in sec, sec) check("contraindications: the line carries the label's note, the CPIC level and the label", - "mitochondrial disorders caused by POLG mutations" in sec and "(CPIC level A/B)" in sec + "mitochondrial disorders caused by POLG mutations" in sec + and "(provisional CPIC level A/B: no CPIC guideline covers this pair)" in sec and "dailymed.nlm.nih.gov" in sec, sec) check("contraindications: the section says to review it with a clinician", "Review it with your clinician" in sec, sec) From 4e4945ff5b2b378d0d10c4eeb37912e172f0e703 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:22:47 +0100 Subject: [PATCH 109/255] ci(guard): run every test with the og-leg marker instead of a hand-kept list Guard's rust job ran the tests that need og from two hand-kept lists, so every port edited the same lines. Each of those 41 tests now carries the line "# og-leg: OG_REQUIRED", and one step runs every tests/test_* file that has it. A planted marked test with a failing assert must be picked and fail first. A second step drops one planned verb's arm and the planned-verb test must fail. Bead: D1 Bead: D14 --- .github/workflows/guard.yml | 98 ++++++++++++++++------------- tests/test_ancestry_fine_report.py | 1 + tests/test_ancient_report.py | 1 + tests/test_annotsv_benign_mane.sh | 1 + tests/test_annotsv_postfilter.sh | 1 + tests/test_annotsv_sv_inputs.py | 1 + tests/test_annotsv_sv_inputs.sh | 1 + tests/test_archaic_report.py | 1 + tests/test_callability.py | 1 + tests/test_chip_report.py | 1 + tests/test_clinvar_join_adjacent.py | 1 + tests/test_collect_summary.py | 1 + tests/test_copy_number.py | 1 + tests/test_exomiser_report.py | 1 + tests/test_family_report.py | 1 + tests/test_family_segments.py | 1 + tests/test_hba_copy_number.py | 1 + tests/test_hla_second_caller.py | 1 + tests/test_ibd_report.py | 1 + tests/test_isec_columns.sh | 1 + tests/test_locityper_report.py | 1 + tests/test_lpa_kiv2.py | 1 + tests/test_mei_report.py | 1 + tests/test_mocha_report.py | 1 + tests/test_og_setup_zip.py | 1 + tests/test_paraphase_report.py | 1 + tests/test_pgx_mito_merge.py | 1 + tests/test_pgx_outside_calls.py | 1 + tests/test_pgx_refblock_fill.py | 1 + tests/test_render_report.py | 1 + tests/test_results_index.py | 1 + tests/test_run_delta.py | 1 + tests/test_setup_fine_maps.sh | 1 + tests/test_str_catalog.py | 1 + tests/test_str_ranges.py | 1 + tests/test_traits_report.py | 1 + tests/test_traits_table.py | 1 + tests/test_viral_report.py | 1 + tests/test_vntyper_report.py | 1 + tests/test_y_private_snps.py | 1 + tests/test_y_tree_place.py | 1 + tests/test_ystr_report.py | 1 + 42 files changed, 95 insertions(+), 44 deletions(-) diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index a2aee4fac..f1a4c1821 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -374,6 +374,24 @@ jobs: - name: cargo test run: cargo test --workspace --locked --no-fail-fast + # The planned-verb test (og/tests/port_scaffold.rs) must fail when a + # later merge drops a verb's arm. + - name: The planned-verb test fails when a verb is dropped + run: | + cp og/src/main.rs "${RUNNER_TEMP}/main.rs" + trap 'cp "${RUNNER_TEMP}/main.rs" og/src/main.rs' EXIT + sed -i '/Some("survivor-prep") =>/d' og/src/main.rs + if grep -q 'survivor-prep") =>' og/src/main.rs; then echo "ERROR: the arm was not dropped"; exit 1; fi + log="${RUNNER_TEMP}/planned-verbs-red.log" + if cargo test --locked --no-fail-fast -p og --test port_scaffold > "$log" 2>&1; then + cat "$log" + echo "ERROR: the test passed with og sv survivor-prep dropped" + exit 1 + fi + cat "$log" + grep -F "og sv survivor-prep: not routed" "$log" + echo "Control: dropping og sv survivor-prep failed the planned-verb test." + # Each pinned test-only commit still contains the scanner it attacks. - name: Scanner controls (both rounds must fail against their pre-fix code) run: | @@ -571,19 +589,6 @@ jobs: file ../bin/.og/og-linux-* ../bin/og version - # Step 06 runs og clinvar in BCFTOOLS_IMAGE, so the tests that run the - # real step, or compare og clinvar with the summary, need bin/.og: they - # run here, and unit-tests skips them (no OG there). - - name: Step 06 with og (the tests that run og clinvar) - working-directory: . - env: - OG: ${{ github.workspace }}/bin/og - OG_REQUIRED: 1 - run: | - python3 tests/test_clinvar_join_adjacent.py - python3 tests/test_collect_summary.py - bash tests/test_isec_columns.sh - - name: The stale-binary guard refuses the real binaries once rust/ changes run: | echo "// edited" >> og/src/main.rs @@ -602,42 +607,47 @@ jobs: grep -q 'summary.json validates (og dev schema)' <<< "$out" \ || { echo "ERROR: reports-stale-step did not run og dev schema"; exit 1; } - # The tests of the og filters steps 04, 05 and 07 run, og chip's (step 49), og cn's (steps 35, 41 and 46), og family's (step 38), og run delta's, step 26b's og ancestry-fine, - # the opt-in step reports' (45, 47, 48, 50 to 53), step 37's Y tree tools, the - # og tools setup.sh runs (og str catalog, og setup unzip and genetic-map), and the step 25 - # tests that compare og prs format with the folders they write by hand: - # unit-tests has no og, so there they skip; here OG_REQUIRED turns a - # skip into a failure. The step 05 tests run the real step with bin/og. - - name: The og filter and setup tests, and step 05, with og + # Every test that carries the line `# og-leg: OG_REQUIRED` runs here, + # with bin/og and OG_REQUIRED=1 (the tests that run og clinvar, the + # og filters, setup and step report tools, and the step 05 tests, which + # run the real step with bin/og). unit-tests has no og, so there they + # skip with a note; here OG_REQUIRED turns a skip into a failure. A new + # test that needs og adds the marker line; there is no list to edit. + - name: The og-leg tests (every tests/test_* with the og-leg marker), with og working-directory: . env: OG: ${{ github.workspace }}/bin/og OG_REQUIRED: 1 run: | - fail=0 - for t in tests/test_annotsv_sv_inputs.py tests/test_pgx_refblock_fill.py tests/test_pgx_mito_merge.py tests/test_run_delta.py \ - tests/test_ancestry_fine_report.py tests/test_archaic_report.py tests/test_ancient_report.py \ - tests/test_annotsv_sv_inputs.sh tests/test_annotsv_postfilter.sh tests/test_annotsv_benign_mane.sh \ - tests/test_str_catalog.py tests/test_str_ranges.py tests/test_og_setup_zip.py tests/test_setup_fine_maps.sh \ - tests/test_render_report.py tests/test_collect_summary.py tests/test_chip_report.py \ - tests/test_family_report.py tests/test_family_segments.py tests/test_ibd_report.py \ - tests/test_vntyper_report.py tests/test_locityper_report.py tests/test_mocha_report.py \ - tests/test_mei_report.py tests/test_viral_report.py tests/test_ystr_report.py \ - tests/test_exomiser_report.py tests/test_y_tree_place.py tests/test_y_private_snps.py \ - tests/test_results_index.py \ - tests/test_pgx_outside_calls.py tests/test_hla_second_caller.py tests/test_paraphase_report.py \ - tests/test_copy_number.py tests/test_hba_copy_number.py \ - tests/test_lpa_kiv2.py tests/test_traits_report.py tests/test_traits_table.py tests/test_callability.py; do - rc=0 - echo "::group::${t}" - case "$t" in - *.py) python3 "$t" || rc=$? ;; - *.sh) bash "$t" || rc=$? ;; - esac - echo "::endgroup::" - if [ "$rc" -eq 0 ]; then echo "PASS ${t}"; else echo "FAIL ${t} (exit ${rc})"; fail=1; fi - done - exit "$fail" + marker='# og-leg: OG_REQUIRED' + og_leg() { + local fail=0 t rc + for t in "$@"; do + rc=0 + echo "::group::${t}" + case "$t" in + *.py) python3 "$t" || rc=$? ;; + *.sh) bash "$t" || rc=$? ;; + esac + echo "::endgroup::" + if [ "$rc" -eq 0 ]; then echo "PASS ${t}"; else echo "FAIL ${t} (exit ${rc})"; fail=1; fi + done + return "$fail" + } + # Control: a planted test with the marker and a failing assert is + # picked by the glob and turns the leg red. + probe=tests/test_og_leg_probe.py + trap 'rm -f "$probe"' EXIT + printf '%s\nassert False, "planted"\n' "$marker" > "$probe" + picked=$(grep -lxF "$marker" tests/test_*.py tests/test_*.sh) + grep -qxF "$probe" <<< "$picked" || { echo "ERROR: the marker glob missed the planted test"; exit 1; } + if og_leg "$probe"; then echo "ERROR: the planted failing test passed"; exit 1; fi + rm -f "$probe" + echo "Control: the planted marked test was picked and failed." + mapfile -t tests < <(grep -lxF "$marker" tests/test_*.py tests/test_*.sh | sort) + [ "${#tests[@]}" -gt 0 ] || { echo "ERROR: no test carries the og-leg marker"; exit 1; } + echo "${#tests[@]} tests carry the og-leg marker" + og_leg "${tests[@]}" - name: Traits independent expectations reject changed reference labels and marker metadata working-directory: . diff --git a/tests/test_ancestry_fine_report.py b/tests/test_ancestry_fine_report.py index 1f00da38c..196c713bb 100755 --- a/tests/test_ancestry_fine_report.py +++ b/tests/test_ancestry_fine_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og ancestry-fine, step 26b's commands, on made-up inputs: panel-map a group of fewer than 10 unrelated people (one Oceanian, three diff --git a/tests/test_ancient_report.py b/tests/test_ancient_report.py index b0e7efee9..aacc2535b 100644 --- a/tests/test_ancient_report.py +++ b/tests/test_ancient_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og ancient (step 44) on synthetic files, run as the pipeline runs it. Checks: diff --git a/tests/test_annotsv_benign_mane.sh b/tests/test_annotsv_benign_mane.sh index 00fadb8ce..88d1fe91f 100755 --- a/tests/test_annotsv_benign_mane.sh +++ b/tests/test_annotsv_benign_mane.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # Step 05 (AnnotSV) with the two files setup.sh adds to AnnotSV's data, on a # made-up annotation set small enough for the unit-test job, with the real # AnnotSV image (ANNOTSV_IMAGE). diff --git a/tests/test_annotsv_postfilter.sh b/tests/test_annotsv_postfilter.sh index 2df9dd45b..861c61639 100755 --- a/tests/test_annotsv_postfilter.sh +++ b/tests/test_annotsv_postfilter.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # The class 4 and 5 checks with the real tools: step 05 (AnnotSV, then # og sv postfilter in ANNOTSV_IMAGE) and step 22 (SURVIVOR_IMAGE, # BCFTOOLS_IMAGE) on the made-up data of tests/fixtures/annotsv/class45.sh, diff --git a/tests/test_annotsv_sv_inputs.py b/tests/test_annotsv_sv_inputs.py index 04fda4a7a..e79c1b481 100755 --- a/tests/test_annotsv_sv_inputs.py +++ b/tests/test_annotsv_sv_inputs.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og sv inv-filter, og sv svlen-fix and og sv add-callers on made-up VCFs. The checks run og, the binary OG names (the rust job of guard.yml sets diff --git a/tests/test_annotsv_sv_inputs.sh b/tests/test_annotsv_sv_inputs.sh index b3ef6af48..c60a4cf27 100755 --- a/tests/test_annotsv_sv_inputs.sh +++ b/tests/test_annotsv_sv_inputs.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # Step 05 (AnnotSV) on what it now reads: Manta's calls after step 04, with # the net length change of sequence-resolved records, plus the Delly and # CNVpytor calls no Manta call covers. The real AnnotSV image (ANNOTSV_IMAGE) diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index 9fbeb2ea9..86e0fbf83 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og archaic (step 43) on a synthetic archaic panel, with the real IBDmix image. Checks: diff --git a/tests/test_callability.py b/tests/test_callability.py index 2fd1aace9..f085d245f 100644 --- a/tests/test_callability.py +++ b/tests/test_callability.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """The gene callability of step 16b and the ALT contig refusal. assets/callability_genes.tsv lists the genes short reads often cannot place diff --git a/tests/test_chip_report.py b/tests/test_chip_report.py index 42340069d..f68cb8608 100644 --- a/tests/test_chip_report.py +++ b/tests/test_chip_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """Step 49 (clonal haematopoiesis): og chip on synthetic bcftools output and on assets/chip_genes.tsv, and what the reports read of it. diff --git a/tests/test_clinvar_join_adjacent.py b/tests/test_clinvar_join_adjacent.py index aac315a6b..733fa4828 100755 --- a/tests/test_clinvar_join_adjacent.py +++ b/tests/test_clinvar_join_adjacent.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og clinvar join-adjacent, the combined-allele joiner of step 06 and the CLINVAR_SCREEN module, on hand-made records at real GRCh38 chr22 positions. diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 20eb59617..2a762dabe 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """bin/collect_summary.py and bin/render_report.py on synthetic sample folders. Checks: diff --git a/tests/test_copy_number.py b/tests/test_copy_number.py index 04c53623c..c4f493616 100644 --- a/tests/test_copy_number.py +++ b/tests/test_copy_number.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og cn and bin/cn_loci.tsv: one depth-ratio engine for every copy-number locus, compared with the pinned historical catalog reader and assess() oracle. diff --git a/tests/test_exomiser_report.py b/tests/test_exomiser_report.py index c13e52e96..e1f2a3a37 100644 --- a/tests/test_exomiser_report.py +++ b/tests/test_exomiser_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og exomiser prepare and og exomiser report: the symptom-driven ranking of step 48 (Exomiser). Every input is synthetic: a few-term OBO file, one- and four-sample VCF diff --git a/tests/test_family_report.py b/tests/test_family_report.py index 54d279915..f056e76b7 100644 --- a/tests/test_family_report.py +++ b/tests/test_family_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og family (step 38 and the Nextflow FAMILY workflow) on synthetic pedigrees, somalier tables and joint VCFs, and the pedigree rules other Python tools still import from bin/family_report.py. diff --git a/tests/test_family_segments.py b/tests/test_family_segments.py index 073e47e36..5efacacc2 100755 --- a/tests/test_family_segments.py +++ b/tests/test_family_segments.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og family segments (step 38's chromosome map) on a synthetic three-generation family: two grandparents, their son (the father), the mother, and a grandchild. Genotypes are drawn from known haplotypes, so the diff --git a/tests/test_hba_copy_number.py b/tests/test_hba_copy_number.py index 4dc92a38c..e9150c012 100755 --- a/tests/test_hba_copy_number.py +++ b/tests/test_hba_copy_number.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og cn call --locus HBA: step 35's HBA1/HBA2 count from short-read depth. It needs og: OG names the binary (OG=rust/target/release/og, or bin/og); without it the test is skipped, and with OG_REQUIRED set too it fails. diff --git a/tests/test_hla_second_caller.py b/tests/test_hla_second_caller.py index b978fde46..acde4f5bc 100644 --- a/tests/test_hla_second_caller.py +++ b/tests/test_hla_second_caller.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og pgx outside-calls with OptiType, the opt-in second HLA caller. With HLA_QC=true (bash) or --hla_qc (Nextflow) step 08 also runs OptiType on diff --git a/tests/test_ibd_report.py b/tests/test_ibd_report.py index f3b39082d..25494cd6c 100644 --- a/tests/test_ibd_report.py +++ b/tests/test_ibd_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og family ibd: the DNA segments two sequenced relatives share (step 38). Each case writes synthetic Beagle VCFs, hap-ibd segment files, PLINK maps and diff --git a/tests/test_isec_columns.sh b/tests/test_isec_columns.sh index 599fd8dbd..7c23588ab 100755 --- a/tests/test_isec_columns.sh +++ b/tests/test_isec_columns.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # Step 06 + both reports on a tiny synthetic genome. # # Builds a 200-base reference, a three-record sample VCF and a three-record diff --git a/tests/test_locityper_report.py b/tests/test_locityper_report.py index 122feb26a..3460d4615 100755 --- a/tests/test_locityper_report.py +++ b/tests/test_locityper_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og locityper report: step 53's table from what Locityper genotype wrote, and its section in both reports. diff --git a/tests/test_lpa_kiv2.py b/tests/test_lpa_kiv2.py index ae634adb0..e597f1bda 100644 --- a/tests/test_lpa_kiv2.py +++ b/tests/test_lpa_kiv2.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og cn call --locus LPA_KIV2: step 46's LPA KIV-2 repeat count from short-read depth, with KIV-2 4925G>A beside it, and its Lp(a) section in both reports. Part 6 runs without og; the rest needs it: OG names the binary diff --git a/tests/test_mei_report.py b/tests/test_mei_report.py index 34873cdc9..71604d325 100644 --- a/tests/test_mei_report.py +++ b/tests/test_mei_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og mei report: the Alu, L1 and SVA insertions MEGAnE calls, and the known disease insertions (step 50). Each case writes a synthetic VCF in the format MEGAnE writes (the header and diff --git a/tests/test_mocha_report.py b/tests/test_mocha_report.py index 80aa58c86..469e2e0f5 100644 --- a/tests/test_mocha_report.py +++ b/tests/test_mocha_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og mocha report: step 51's mosaic chromosomal alterations from MoChA's tables. Each case writes MoChA's --stats and --calls tables and the per-chromosome diff --git a/tests/test_og_setup_zip.py b/tests/test_og_setup_zip.py index d15642265..557b0c653 100644 --- a/tests/test_og_setup_zip.py +++ b/tests/test_og_setup_zip.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og setup unzip and og setup genetic-map against the Python they replaced. scripts/setup.sh unpacked Exomiser's data zips with diff --git a/tests/test_paraphase_report.py b/tests/test_paraphase_report.py index 34758def9..f2f03d8cb 100755 --- a/tests/test_paraphase_report.py +++ b/tests/test_paraphase_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og paraphase: step 35b's read-group check and its two tables. Checks: diff --git a/tests/test_pgx_mito_merge.py b/tests/test_pgx_mito_merge.py index d2e64e324..6473c9e64 100644 --- a/tests/test_pgx_mito_merge.py +++ b/tests/test_pgx_mito_merge.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og pgx mito-merge: the MT-RNR1 outside call PharmCAT gets from the chrM calls of step 20 / MITO_VARIANTS (step 07 and PHARMCAT run it). diff --git a/tests/test_pgx_outside_calls.py b/tests/test_pgx_outside_calls.py index 33664a9b1..579cb6cc1 100644 --- a/tests/test_pgx_outside_calls.py +++ b/tests/test_pgx_outside_calls.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og pgx outside-calls: what reaches PharmCAT as an outside call. CYP2D6 reaches PharmCAT only when pypgx and a second caller (Cyrius) give the diff --git a/tests/test_pgx_refblock_fill.py b/tests/test_pgx_refblock_fill.py index 3f535a926..077cbedca 100644 --- a/tests/test_pgx_refblock_fill.py +++ b/tests/test_pgx_refblock_fill.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og pgx refblock-fill: PharmCAT's multi-base positions called 0/0 from the reference blocks of a gVCF (steps 07 and PHARMCAT_PREPROCESS run it). diff --git a/tests/test_render_report.py b/tests/test_render_report.py index 36add70fe..d8ce8fdf8 100644 --- a/tests/test_render_report.py +++ b/tests/test_render_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """The PRS path around pgsc_calc (og prs format and og prs table, then bin/collect_summary.py), and its card in bin/render_report.py, on synthetic files shaped like pgsc_calc's own. diff --git a/tests/test_results_index.py b/tests/test_results_index.py index db40a5348..c8c0b7839 100755 --- a/tests/test_results_index.py +++ b/tests/test_results_index.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """bin/results_index.py, its report appendix and the live search on a synthetic sample. The sample (tests/fixtures/results_index/SYNTH, invented) has ClinVar hits in diff --git a/tests/test_run_delta.py b/tests/test_run_delta.py index 97c4f9bca..173694ba8 100644 --- a/tests/test_run_delta.py +++ b/tests/test_run_delta.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og run delta on two synthetic runs of the same sample. Checks: diff --git a/tests/test_setup_fine_maps.sh b/tests/test_setup_fine_maps.sh index befc3a99a..a0d9d9bb9 100755 --- a/tests/test_setup_fine_maps.sh +++ b/tests/test_setup_fine_maps.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # setup.sh's install_fine_maps (step 26b's genetic maps) rebuilds a map an # older install left cut short. Before og, setup.sh skipped every chromosome # whose .rfmix.map was not empty, so a map a kill had cut short stayed diff --git a/tests/test_str_catalog.py b/tests/test_str_catalog.py index 4cfee57ba..051ceb690 100644 --- a/tests/test_str_catalog.py +++ b/tests/test_str_catalog.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og str catalog, and Stranger on the catalog it builds. Reads the two STRchive files versions.env pins (STRCHIVE_VERSION and the diff --git a/tests/test_str_ranges.py b/tests/test_str_ranges.py index 6482d4152..b842e9633 100644 --- a/tests/test_str_ranges.py +++ b/tests/test_str_ranges.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """The repeat ranges: the generated str_ranges.tsv, the report's cited subset and the docs. 1. str_ranges.tsv, which og str catalog writes beside the STRchive diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py index 07fb6de3b..b1ad8e57b 100644 --- a/tests/test_traits_report.py +++ b/tests/test_traits_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """Step 42 CLI goldens, rule-allele controls and og cn marker outputs. Internal validation, phase and bounded-reader assertions live in diff --git a/tests/test_traits_table.py b/tests/test_traits_table.py index 984d426f1..f9bdeb845 100644 --- a/tests/test_traits_table.py +++ b/tests/test_traits_table.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """Validate the curated traits catalog and its GRCh38 reference bases. Schema checks use og traits check. Rust unit tests plant invalid catalog diff --git a/tests/test_viral_report.py b/tests/test_viral_report.py index 7d16316a9..6ca3492e3 100644 --- a/tests/test_viral_report.py +++ b/tests/test_viral_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og viral report: the viral DNA in the reads that do not map to the human genome (step 47). Each case writes a synthetic viral FASTA, the samtools coverage and depth -aa diff --git a/tests/test_vntyper_report.py b/tests/test_vntyper_report.py index 8c1ac8384..c68b9b274 100644 --- a/tests/test_vntyper_report.py +++ b/tests/test_vntyper_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og vntyper report: step 52's reading of VNtyper's MUC1 VNTR result, and its section in both reports. diff --git a/tests/test_y_private_snps.py b/tests/test_y_private_snps.py index 5755f1617..dd93d7026 100755 --- a/tests/test_y_private_snps.py +++ b/tests/test_y_private_snps.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og y private-snps on a synthetic VCF and synthetic position tables (no real sample). diff --git a/tests/test_y_tree_place.py b/tests/test_y_tree_place.py index dbd385518..34691e828 100755 --- a/tests/test_y_tree_place.py +++ b/tests/test_y_tree_place.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og y tree-place and og y tree-positions on a synthetic tree, synthetic position tables and synthetic pileups (no real sample). diff --git a/tests/test_ystr_report.py b/tests/test_ystr_report.py index 64e4815df..d8608e342 100644 --- a/tests/test_ystr_report.py +++ b/tests/test_ystr_report.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# og-leg: OG_REQUIRED """og y str-bed and og y str-report: Y-STR repeat counts from HipSTR's calls (step 45). Each case writes a synthetic HipSTR VCF (one sample, only the records it From 2b43c3c9d506601878e7e01d865bdb29daa5078a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:23:30 +0100 Subject: [PATCH 110/255] test(roh): pin scoped summaries and prove the controls fail Record ROH-only parity goldens and scope limits. Add the planted Guard control and migrate the retained report tests and missed e2e callers to the Rust collector contract. Keep the existing imprinting keys as neutral placeholders. Bead: lin-10 Bead: lin-13 Bead: lin-15 Bead: lin-16 Bead: lin-17 Bead: lic-41 --- .github/workflows/guard.yml | 31 + CHANGELOG.md | 4 + docs/11-roh-analysis.md | 36 +- .../og-summary/src/collect/group_c/roh.rs | 2 +- .../og-summary/src/collect/group_c/tests.rs | 6 + scripts/ci/e2e-case-paths.tsv | 4 +- tests/e2e/align-read-count-gate-1.sh | 4 +- tests/e2e/vendor-vcf-intake-2-nf-report.sh | 10 +- tests/parity/gen/summary-collect-c.sh | 7 +- .../summary-collect/all-bad/all-bad.json | 759 + .../all-history/all-history.json | 15569 ++++++++++++++++ .../golden/summary-collect/all-nf/all-nf.json | 14043 ++++++++++++++ .../golden/summary-collect/all/all.json | 15495 +++++++++++++++ .../golden/summary-collect/c-bad1/bad1.json | 762 + .../golden/summary-collect/c-full/full.json | 2402 +++ .../golden/summary-collect/c-nf/nf.json | 1200 ++ tests/test_collect_summary.py | 40 +- 17 files changed, 50326 insertions(+), 48 deletions(-) create mode 100644 tests/parity/golden/summary-collect/all-bad/all-bad.json create mode 100644 tests/parity/golden/summary-collect/all-history/all-history.json create mode 100644 tests/parity/golden/summary-collect/all-nf/all-nf.json create mode 100644 tests/parity/golden/summary-collect/all/all.json create mode 100644 tests/parity/golden/summary-collect/c-bad1/bad1.json create mode 100644 tests/parity/golden/summary-collect/c-full/full.json create mode 100644 tests/parity/golden/summary-collect/c-nf/nf.json diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index a2aee4fac..a77130baa 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -371,6 +371,37 @@ jobs: cp -f "$saved" "$src" trap - EXIT + - name: Control (ROH scope and neutral hints reject planted faults) + run: | + reader=crates/og-summary/src/collect/group_c/roh.rs + table=../assets/imprinted_regions.tsv + cp "$reader" "$RUNNER_TEMP/roh-reader.rs" + cp "$table" "$RUNNER_TEMP/imprinted-regions.tsv" + restore_roh() { + cp "$RUNNER_TEMP/roh-reader.rs" "$reader" + cp "$RUNNER_TEMP/imprinted-regions.tsv" "$table" + } + trap restore_roh EXIT + sed -i 's/let denominator = covered_unmasked_bp(&covered);/let denominator = autosome_unmasked_bp();/' "$reader" + sed -i 's/s("-")/s("Example syndrome")/g' "$reader" + sed -i 's/26248010;29959430/not-pmid/' "$table" + log="$RUNNER_TEMP/roh-scope-planted.log" + rc=0 + cargo test --locked -p og-summary --no-fail-fast > "$log" 2>&1 || rc=$? + cat "$log" + test "$rc" -ne 0 + for name in \ + roh_scope_chr20_uses_its_own_unmasked_denominator \ + roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty \ + roh_scope_a_segment_only_file_never_establishes_scope \ + roh_scope_upd_is_a_region_hint_without_syndrome_names \ + imprinted_asset_is_the_only_table; do + grep -E "test .*${name} .* FAILED" "$log" + done + grep -E 'test result: FAILED\. [0-9]+ passed; 5 failed;' "$log" + restore_roh + trap - EXIT + - name: cargo test run: cargo test --workspace --locked --no-fail-fast diff --git a/CHANGELOG.md b/CHANGELOG.md index 58cac8762..78de0ffed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- ROH summaries now divide F_ROH by the GRCh38 length outside the bands of the assessed autosomes. Per-site rows identify those chromosomes, including chromosomes with no ROH. A new `roh.data.scope` records the chromosomes, denominator and scope source. Relationship labels and UPD flags require sites on all 22 autosomes and the full unmasked autosome length. Segment-only files use their observed chromosomes as an unverified lower bound and get neither label. Empty input has null F_ROH. Segment counts and the step 11 summary table keep their format. +- A one-chromosome ROH flag now names regions and citations without syndrome names. Its note includes deletion, acquired loss of heterozygosity and shared ancestry as alternatives. The maternal and paternal JSON keys remain as `"-"` for reader compatibility. The attributed `assets/imprinted_regions.tsv` is embedded at build time, replacing the duplicated Rust table. Every listed chromosome uses the same lookup, including chr11 and chr15. A region hint does not establish overlap with its imprinted locus or parental origin. Broad cytoband names remain reference labels, not diagnostic intervals. +- The ROH collector now streams per-site rows instead of keeping the whole file in memory. On 1.1 million synthetic site rows, peak RSS fell from 65.7-67.0 MB to 3.0-3.5 MB. CPU time stayed near 0.2 seconds. Under shared host load, wall time ranged from 1.40-21.46 seconds before and 7.89-38.16 seconds after; these timings do not establish a speedup. + - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/11-roh-analysis.md b/docs/11-roh-analysis.md index 20a8146bb..07fc1e338 100644 --- a/docs/11-roh-analysis.md +++ b/docs/11-roh-analysis.md @@ -42,7 +42,7 @@ For chip data (no `FORMAT/PL`), the script adds `-G30`. ### Centromere and heterochromatin bands -Few markers map in GRCh38's centromeres (cytoband stain `acen`), its large heterochromatin blocks (`gvar`, such as 1q12, 9q12 and 16q11.2) and the acrocentric stalks (`stalk`), so `bcftools roh` reads long homozygous runs there that are artefacts. The script, the Nextflow module and the reports take the part of each segment that lies in one of these bands out of its length; a segment wholly inside one counts as nothing. The bands are the `acen`, `gvar` and `stalk` rows of UCSC's `cytoBand.hg38.txt` (the file `setup.sh` installs for step 10), neighbouring rows merged into one span around each centromere (two on chrY), written into `scripts/11-roh-analysis.sh`, `modules/local/roh/main.nf` and `og summary collect` as the same line (a unit test checks they agree). On chr1 the span is 121.7-143.2 Mb, on chr9 42.2-61.5 Mb, on chr18 15.4-21.5 Mb. +Few markers map in GRCh38's centromeres (cytoband stain `acen`), its large heterochromatin blocks (`gvar`, such as 1q12, 9q12 and 16q11.2) and the acrocentric stalks (`stalk`), so `bcftools roh` reads long homozygous runs there that are artefacts. The script, the Nextflow module and the reports take the part of each segment that lies in one of these bands out of its length; a segment wholly inside one counts as nothing. The bands are the `acen`, `gvar` and `stalk` rows of UCSC's `cytoBand.hg38.txt` (the file `setup.sh` installs for step 10). Neighbouring rows are merged around each centromere, with two spans on chrY. `MASKED_BANDS` in `rust/crates/og-summary/src/collect/group_c/roh.rs` is their only copy. The step and the Nextflow module call `og roh summary`, which reads that same constant. On chr1 the span is 121.7-143.2 Mb, on chr9 42.2-61.5 Mb, on chr18 15.4-21.5 Mb. The reports count autosomes only: Segments, Total and Largest keep chr1 to chr22, so they leave out chrX and chrY (a man's single X reads as one long run) and any other contig (chrM, chrEBV, chrUn_*), and the part of each segment in a band. They say how many chrX and chrY segments and how many megabases in bands were left out. @@ -63,8 +63,15 @@ When parents are related, a child is autozygous (both copies from the same ances These are averages. Inheritance is random, so one person's total can be well above or below the value for their parents' relationship, and the ranges of neighbouring rows overlap. A total near one row is consistent with it; it does not prove it. -Both reports print this as **F_ROH**, the measure of McQuillan et al. 2008 (PMID 18760389): the length of the autosomal segments of 5 Mb or more, without their band parts, divided by the length of chr1 to chr22 without the bands (2,875,001,522 bp less 217,000,000 bp, so 2,658,001,522 bp). Beside it they name the row of the table above whose F is nearest to F_ROH on a log scale: the boundaries are the geometric means of neighbouring rows (for example 1/16 and 1/8 meet at about 0.088), and an F_ROH under 1/128 (about 21 Mb) reads "no sign of related parents". When one chromosome carries most of the runs (next section), there is no band: that chromosome fills F_ROH by itself. +`og summary collect` writes **F_ROH**, the measure of McQuillan et al. 2008 ([PMID 18760389](https://pubmed.ncbi.nlm.nih.gov/18760389/)). Its numerator is the length of autosomal runs of 5 Mb or more outside the bands. Its denominator is the GRCh38 length of the chromosomes this run assessed, with their bands removed. With all 22 autosomes, that is 2,875,001,522 bp less 217,000,000 bp, or 2,658,001,522 bp. With chr20 alone it is 64,444,167 bp less 4,700,000 bp, or 59,744,167 bp. A 21 Mb run then gives F_ROH 0.3515, rather than 0.0079. +The collector streams the per-site `ST` rows to identify assessed autosomes. A chromosome with sites counts even when it has no ROH. No marker-count floor is imposed, so sparse exome and chip input remain supported. This is a chromosome-length denominator, **not a count of callable bases**. One site does not establish coverage across a chromosome. A capture panel or a small interval on every autosome can still look complete at this resolution. + +The new `roh.data.scope` object records `autosomes` in numeric order, `unmasked_bp`, `source`, `partial` and a note. Older segment-only files have `source: segments_only`; their observed chromosomes give only a lower bound on scope. An empty file has no denominator and `f_roh: null`. The segment list and length totals remain available. + +A relationship label or a one-chromosome UPD flag needs sites on all 22 autosomes and the full 2,658,001,522 bp denominator. Below that floor, or without per-site evidence, `relationship` and `upd_flag` are null. A subset F_ROH is a descriptive fraction of the listed chromosomes and must not be mapped to the parental relationship table. The report and live-view wording for this scope is handled separately. + +For complete chromosome scope, the relationship is the row whose F is nearest on a log scale. Boundaries are the geometric means of neighbouring rows; for example 1/16 and 1/8 meet at about 0.088. An F_ROH under 1/128 reads "no sign of related parents". A one-chromosome flag suppresses that relationship label. To compute the total the table means from the summary file, which takes the band parts out before it keeps the segments of 5 Mb or more, as the reports do: ```bash @@ -78,24 +85,19 @@ In uniparental disomy (UPD) both copies of a chromosome come from one parent. Wh The reports flag a chromosome when its segments of 5 Mb or more (band parts left out) add up to at least 20 Mb **and** to more than half of the length of all such autosomal segments. Both numbers are our thresholds, not a published cut-off: 20 Mb on one chromosome is the long-standing sign of isodisomy (see the uniparental disomy note under [Important Notes](#important-notes)), and the share rule keeps out a child of related parents, whose runs spread over many chromosomes. The flag says "may": a run that long can also be chance in a child of related parents. -A flagged chromosome is looked up in `assets/imprinted_regions.tsv`, the chromosomes where UPD from one parent causes an imprinting disorder (overview: Eggermann et al. 2023, Nat Rev Dis Primers, PMID 37386011). A row with a region applies only when a run on that chromosome overlaps it; the others apply to the whole chromosome. +A flagged chromosome is looked up in [assets/imprinted_regions.tsv](../assets/imprinted_regions.tsv). That file is the only imprinting table and is embedded into `og` at build time. Its header credits [Eggermann et al. 2023](https://doi.org/10.1038/s41572-023-00443-4), with per-row PMIDs for the reference facts. -| Chromosome or region | UPD from the mother | UPD from the father | Source (PMID) | -|---|---|---|---| -| chr6 (6q24, PLAGL1, HYMAI) | none known | Transient neonatal diabetes mellitus | 20301706 | -| chr7 (GRB10 at 7p12, MEST at 7q32) | Silver-Russell syndrome | none known | 27585961 | -| 11p15, chr11:0-22,000,000 (H19/IGF2, KCNQ1OT1) | Silver-Russell syndrome | Beckwith-Wiedemann syndrome | 29377879, 27585961 | -| chr14 (14q32.2, DLK1-MEG3) | Temple syndrome | Kagami-Ogata syndrome | 24891339, 26377239 | -| 15q11-q13, chr15:20,500,000-33,400,000 (SNRPN, UBE3A) | Prader-Willi syndrome | Angelman syndrome | 22237428, 27615419 | -| chr20 (20q13.32, GNAS) | Mulchandani-Bhoj-Conlin syndrome (growth failure) | Pseudohypoparathyroidism type 1B | 26248010, 29959430 | +The flag lists the chromosome, run length, share, region names and citations. It names no syndrome. The existing `maternal_upd` and `paternal_upd` JSON keys stay as `"-"` for reader compatibility. The asset retains their reference facts. Those facts cannot be inferred from a region match in one genome. -`og summary collect` holds the same rows (`IMPRINTED_REGIONS` in `rust/crates/og-summary`; a unit test checks that the two agree). A flagged chromosome with no row still matters: a recessive variant inside the run, if there is one, would be homozygous. +All six chromosomes use the same chromosome-wide lookup, including chr11 and chr15. Mixed iso/heterodisomy can leave an imprinted locus outside the observed homozygous runs ([Robinson 2000](https://pubmed.ncbi.nlm.nih.gov/10797485/)). A region hint therefore does not claim that a run overlaps the imprinted locus. Region names are broad cytoband labels, not diagnostic locus intervals or a specialist-reviewed assay. No interval was narrowed on the basis of a ROH alone. -What this cannot tell, and the reports say so: +The flag's `note` describes long chromosome-focused ROH and suggests investigating possible isodisomy. Its limits are: -- **Which parent.** A single genome has no parent of origin; both columns are listed. The parents' genotypes (step 38, the family workflow) confirm a UPD and tell which. Where a row applies, a methylation test of its imprinted region tests for that imprinting disorder; on its own it does not prove a UPD, since an imprinting defect gives the same result. -- **Heterodisomy.** Two different copies from one parent leave no run at all, so it is not detected here. It needs the parents' genotypes. -- **Mosaic or segmental UPD** that covers only a short stretch, or a share of the cells (common in Beckwith-Wiedemann syndrome), may stay under the thresholds. +- A large deletion of one copy or acquired loss of heterozygosity can give the same pattern. Check the copy-number results and the MoChA results. The collector does not establish copy neutrality or perform that cross-check. +- Shared ancestry, including related parents, can also produce long runs. A flag is a screening hint, not a UPD diagnosis. +- A single genome cannot determine parental origin. The parents' genotypes can establish it. Methylation testing assesses an imprinting defect and alone does not establish UPD. +- Heterodisomy can leave no ROH. Mosaic or short segmental UPD can fall below the thresholds. +- Sites on all autosomes establish chromosome presence only. They do not prove assay completeness within any chromosome. ### Single segments @@ -112,5 +114,5 @@ What this cannot tell, and the reports say so: - bcftools roh reads a site's frequency from INFO/AC and INFO/AN when the record has them, and uses 0.4 only where it does not. DeepVariant writes neither, so on its calls every site uses 0.4. A VCF from another caller may carry them: in a single-sample file they are the sample's own allele counts, not a population's. - No population allele frequencies are used, and the reports say so: read the total and the short segments as rough, the long segments are the signal. A population allele-frequency file (`--AF-file`, for example from gnomAD) would replace the one default value with real frequencies per site and give better segment edges; neither the script nor the module uses one today, and its effect on the total has not been measured. - The centromere and heterochromatin bands (above) are taken out of every count; the step and the reports do it, so there is nothing to subtract by hand -- Long runs concentrated on one chromosome may indicate uniparental disomy; the reports flag a chromosome whose segments of 5 Mb or more add up to at least 20 Mb and to more than half of the length of all such autosomal segments (above). Confirm with the parents' genotypes; a methylation test of a listed imprinted region tests for its imprinting disorder +- Long runs concentrated on one chromosome may indicate uniparental disomy; the reports flag a chromosome whose segments of 5 Mb or more add up to at least 20 Mb and to more than half of the length of all such autosomal segments (above). The flag requires sites on all 22 autosomes. Check copy number and acquired LOH, and confirm with the parents' genotypes. A methylation test assesses an imprinting defect - ROH within known disease gene regions warrants checking for homozygous pathogenic variants diff --git a/rust/crates/og-summary/src/collect/group_c/roh.rs b/rust/crates/og-summary/src/collect/group_c/roh.rs index 2724c416d..11a538c28 100644 --- a/rust/crates/og-summary/src/collect/group_c/roh.rs +++ b/rust/crates/og-summary/src/collect/group_c/roh.rs @@ -289,7 +289,7 @@ pub fn section(c: &Ctx<'_>) -> Out { let has_sites = assessed.iter().any(|yes| *yes); // Old -O r files omit ST rows. Their runs give a lower bound on // chromosome scope, never evidence for a genome-wide interpretation. - let covered = std::array::from_fn(|i| assessed[i] || observed[i]); + let covered: [bool; 22] = std::array::from_fn(|i| assessed[i] || observed[i]); let denominator = covered_unmasked_bp(&covered); let complete = has_sites && assessed.iter().all(|yes| *yes) diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index 4f34ef10f..e4cad1c0e 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -259,6 +259,12 @@ fn roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty() { assert_eq!(at(&r, &["f_roh"]), &Value::Float(0.1717)); assert_eq!(at(&r, &["relationship"]), &Value::Null); assert_eq!(at(&r, &["scope", "unmasked_bp"]), &Value::Int(93_162_635)); + let mixed = roh_scope_summary(&[20, 22], "RG\tS\tchr20\t1000001\t9000000\t8000000\n"); + assert_eq!(at(&mixed, &["f_roh"]), &Value::Float(0.0859)); + assert_eq!( + at(&mixed, &["scope", "unmasked_bp"]), + &Value::Int(93_162_635) + ); let no_runs = roh_scope_summary(&[20], ""); assert_eq!(at(&no_runs, &["f_roh"]), &Value::Float(0.0)); assert_eq!(at(&no_runs, &["relationship"]), &Value::Null); diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index f4dd8e3cd..d49f98629 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -41,7 +41,7 @@ 60-nextflow - 90-report-assets - acmg-sf-rules-1-reports assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh -align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/og bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* rust/crates/og-summary/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh alignment-markdup-gvcf-sex-1-align scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-2-ploidy scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh @@ -169,7 +169,7 @@ sv-mito-telomere-steps-6-scatter scripts/03a-gatk-haplotypecaller.sh scripts/03b traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vcf-precheck-ref-spotcheck-1 assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/vcf/grch37_no_contigs.vcf vendor-vcf-intake-1-bash bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* rust/crates/og-summary/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/og modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh diff --git a/tests/e2e/align-read-count-gate-1.sh b/tests/e2e/align-read-count-gate-1.sh index 0fe8e1dcb..c9edca8dc 100755 --- a/tests/e2e/align-read-count-gate-1.sh +++ b/tests/e2e/align-read-count-gate-1.sh @@ -50,7 +50,7 @@ TT="${T}/qc/${T}_read_count.tsv" check_eq "input_reads equals fastp's total_reads" "$(val input_reads "$TT")" "$FASTP" check_eq "the BAM's primary records equal fastp's total_reads" "$(primary "${T}/aligned/${T}_sorted.bam")" "$FASTP" check_eq "read_count_check" "$(val read_count_check "$TT")" ok -python3 "${REPO}/bin/collect_summary.py" --sample "$T" --sample-dir "${G}/${T}" --out "${CASE_TMP}/T.json" >/dev/null +"${OG:-${REPO}/bin/og}" summary collect --sample "$T" --sample-dir "${G}/${T}" --out "${CASE_TMP}/T.json" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/T.json" -o "${CASE_TMP}/T.txt" >/dev/null check "the text report shows both counts" \ has "^ Alignment: ${FASTP} reads in, ${FASTP} primary records in the BAM\$" "$(cat "${CASE_TMP}/T.txt" 2>/dev/null)" @@ -141,7 +141,7 @@ check_eq "the published table: primary_records" "$(val primary_records "${R}/qc/ check_eq "the published table: missing_reads" "$(val missing_reads "${R}/qc/${X}_read_count.tsv")" 8000 check_eq "the published BAM's primary records" "$(primary "${R}/aligned/${X}_sorted.bam")" 24000 check "the table is not published beside the BAM" test ! -e "${G}/${R}/aligned/${X}_read_count.tsv" -python3 "${REPO}/bin/collect_summary.py" --sample "$X" --sample-dir "${G}/${R}" --out "${CASE_TMP}/X.json" >/dev/null +"${OG:-${REPO}/bin/og}" summary collect --sample "$X" --sample-dir "${G}/${R}" --out "${CASE_TMP}/X.json" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/X.json" -o "${CASE_TMP}/X.txt" >/dev/null python3 "${REPO}/bin/render_report.py" --summary "${CASE_TMP}/X.json" -o "${CASE_TMP}/X.html" >/dev/null check "the text report flags the missing reads" \ diff --git a/tests/e2e/vendor-vcf-intake-2-nf-report.sh b/tests/e2e/vendor-vcf-intake-2-nf-report.sh index 936730d4c..9fb6a49f6 100755 --- a/tests/e2e/vendor-vcf-intake-2-nf-report.sh +++ b/tests/e2e/vendor-vcf-intake-2-nf-report.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # A VCF-first Nextflow run (no BAM) with the tools a vendor-VCF user picks: # - the HTML report shows ROH, the mito haplogroup and CPIC, with the same -# ROH numbers and haplogroup bin/collect_summary.py (the bash report) reads +# ROH numbers and haplogroup og summary collect (the bash report) reads # from the same files, and it starts after the steps it shows; # - multiqc is selected but has nothing to read: the log says so in one line; # - the completion message prints, with no onComplete handler error; @@ -35,18 +35,18 @@ check "the report exists" test -s "$HTML" for h in 'Runs of Homozygosity' 'Mitochondrial Haplogroup' 'CPIC Drug Recommendations'; do check_eq "report card '${h}' is filled" "$(html_stat "$HTML" "$h" Status)" Complete done -python3 "${REPO}/bin/collect_summary.py" --sample "$SAMPLE" --sample-dir "$R" --out "${CASE_TMP}/summary.json" \ +"${OG:-${REPO}/bin/og}" summary collect --sample "$SAMPLE" --sample-dir "$R" --out "${CASE_TMP}/summary.json" \ > "${CASE_TMP}/summary.log" 2>&1 || cat "${CASE_TMP}/summary.log" want() { python3 -c "import json, sys; d = json.load(open('${CASE_TMP}/summary.json'))['sections']$1; print($2)" 2>/dev/null; } ROH_TOTAL=$(want "['roh']['data']['total_mb']" "'%.1f MB' % d") ROH_LARGEST=$(want "['roh']['data']['largest_mb']" "'%.1f MB' % d") HG=$(want "['haplogroup']['data']['haplogroup']" "d") -echo "collect_summary.py: ROH total ${ROH_TOTAL:-?}, largest ${ROH_LARGEST:-?}, haplogroup ${HG:-?}" -check "collect_summary.py read the ROH file" test -n "$ROH_TOTAL" +echo "og summary collect: ROH total ${ROH_TOTAL:-?}, largest ${ROH_LARGEST:-?}, haplogroup ${HG:-?}" +check "og summary collect read the ROH file" test -n "$ROH_TOTAL" check_eq "ROH total matches the bash report's" "$(html_stat "$HTML" 'Runs of Homozygosity' 'ROH total')" "$ROH_TOTAL" check_eq "ROH largest segment matches the bash report's" \ "$(html_stat "$HTML" 'Runs of Homozygosity' 'ROH largest segment')" "$ROH_LARGEST" -check "collect_summary.py read the haplogroup" test -n "$HG" +check "og summary collect read the haplogroup" test -n "$HG" check_eq "haplogroup matches the bash report's" "$(html_stat "$HTML" 'Mitochondrial Haplogroup' Haplogroup)" "$HG" # The report waits for the steps it shows. diff --git a/tests/parity/gen/summary-collect-c.sh b/tests/parity/gen/summary-collect-c.sh index 9c853a9e4..dd1380417 100644 --- a/tests/parity/gen/summary-collect-c.sh +++ b/tests/parity/gen/summary-collect-c.sh @@ -165,6 +165,11 @@ RG\tS\tchr1\t120000001\t145000000\t25000000\t500\t50 RG\tS\tchr2\t10000001\t17000000\t7000000\t900\t50 RG\tS\tchr15\t23000001\t50000000\t27000000\t900\t50 RG\tS\tchr3\t1\t2\tx\t1\t1\n" +# Sparse whole-genome scope, with chromosomes that have no RG rows too. +for chrom in {1..22}; do + printf 'ST\tS\tchr%s\t40000000\t0\t60\n' "$chrom" >> "$f/vcf/S_roh.txt" +done +at "$NEW" "$f/vcf/S_roh.txt" put $f/mito/S_haplogroup.txt "\"SampleID\"\t\"Haplogroup\"\t\"Rank\"\n\"S\"\t\"H5a1\"\t\"0.9\"\n" put $f/mito/S_haplocheck.txt "\"Sample\"\t\"Contamination Status\"\t\"Contamination Level\"\n\"S\"\t\"NO\"\t\"\"\n" put $f/y_haplogroup/S_y_haplogroup.txt "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\nS\tR1b1a1b1a1a2*(xR-L21)\tR-M269\t100\t40\t0.98\n" @@ -248,7 +253,7 @@ put $f/kir/S_kir_genotype.tsv "# too few KIR reads\n" put $f/cnvpytor/S_cnvs.txt "deletion\tchrY:2800001-2900000\t100000\t0.0\t1e-15\t1\t1\t1\t0.05\t0\t9\ndeletion\tchr2:10000001-10100000\t100000\t0.4\t1e-9\n" at "$OLD" $f/cnvpytor/S_cnvs.txt put $f/telomere/x/deep/T_summary.tsv "PID\tsample\tintratelomeric_reads\ttel_content\nS\ttumor\t61873\t334\n" -put $f/roh/S_roh.txt "RG\tS\tchr2\t10000001\t17000000\t7000000\t900\t50\n" +put $f/roh/S_roh.txt "ST\tS\tchr2\t40000000\t0\t60\nRG\tS\tchr2\t10000001\t17000000\t7000000\t900\t50\n" at "$OLD" $f/roh/S_roh.txt put $f/mito/S_haplogroup.txt "SampleID\tRank\nS\tU5a\n" at "$OLD" $f/mito/S_haplogroup.txt diff --git a/tests/parity/golden/summary-collect/all-bad/all-bad.json b/tests/parity/golden/summary-collect/all-bad/all-bad.json new file mode 100644 index 000000000..a20f67e6c --- /dev/null +++ b/tests/parity/golden/summary-collect/all-bad/all-bad.json @@ -0,0 +1,759 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:19 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "ValueError: could not convert string to float: 'abc'", + "data": {} + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_callability.bed.gz is not mosdepth's thresholds table of one depth", + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "indexcov-indexcov.ped has no sex column or no sample row", + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_sample_qc.tsv has no inferred_sex row", + "data": {} + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_sample_qc.tsv has no mosaic_call row (detected, not_detected, gain, not_estimated)", + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_read_count.tsv has no whole number for input_reads", + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S.vcf.gz: Compressed file ended before the end-of-stream marker was reached", + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "ValueError: invalid literal for int() with base 10: 'x'", + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_couple_carrier.tsv line 2 is not a row of S's couple table", + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "ValueError: invalid literal for int() with base 10: 'x'", + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S.report.json: Expecting value: line 1 column 21 (char 20)", + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "ok", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [], + "non_normal": 0, + "ambiguous": 0, + "not_called": 0, + "unclassified": 0, + "parse_failed": true, + "warnings": [] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "KeyError: 'Gene'", + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/S_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "none", + "filter": "" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_smn_copy_number.tsv lacks the chrom, start, end, agCN, agCN_filter, agCN_qual, psCN, psCN_qual or psCN_filter column", + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_paraphase_copy_number.tsv lacks the region, gene, copy_number or status column", + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01" + ], + "quality": [ + "x" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 not typed; HLA-DQB1 not typed", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": null + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A low confidence" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DQB1 not typed" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + } + ] + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_ancestry_fine.tsv has no rows or lacks a column of ['code', 'fraction', 'group', 'high', 'level', 'low', 'se']", + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_sv_class4_5_checked.tsv lacks a column of ['ACMG_class', 'Caller', 'Common', 'Exon_edge', 'Population_AF', 'SV_chrom', 'SV_end', 'SV_start', 'SV_type']", + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 2, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "CFTR_T", + "repeat_count": "7/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "7-7/9-9", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/FLANKING", + "adsp": "8/0", + "ci": "11-11/12-12", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": false, + "stranger_outdated": false, + "unsized": [ + "CFTR_TG" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "not_sized", + "genotype": ".", + "phase": ".", + "reading": "ExpansionHunter did not size both tracts (TG 11/12, flanking_only; poly-T 7/9, sized): not read." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "not_sized", + "genotype": ".", + "pharmcat": ".", + "agreement": "not_compared", + "reading": "not in step 09's output: STR_EXTRA_LOCI=yes ./scripts/setup.sh --strchive-data adds the repeat to the catalog (docs/09-str-expansions.md). PharmCAT (step 7) still reads UGT1A1 *28." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ] + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 0, + "total_mb": 0.0, + "largest_mb": 0.0, + "autosomal_over_5mb": [], + "masked_mb": 0.0, + "sex_chromosome_segments": 0, + "long_mb": -0.0, + "f_roh": null, + "relationship": null, + "upd_flag": null, + "scope": { + "source": "segments_only", + "autosomes": [], + "unmasked_bp": 0, + "partial": true, + "note": "Partial or unverified chromosome scope: no relationship band or UPD flag. F_ROH uses only the listed chromosomes, not callable bases." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_haplocheck.txt has no Contamination Status column", + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_haplogroup.txt has no Hg column or no sample row", + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_tree.txt lacks a column of Hg, Level, Tree", + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_str.tsv has a confidence other than high, low, no call or not callable", + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_consumer_export.tsv has no whole-number 'sites' row", + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_bloodtype.tsv has no system, phenotype and confidence columns, or no row", + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_lpa_kiv2.tsv has no status row (estimated, too little depth or not assessed)", + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_vntyper.tsv has no status row (positive, possible, negative, not assessed)", + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_locityper.tsv lacks a row or one of the columns locus, region, status, genotype, haplotype_1, haplotype_2, quality, total_reads, unexplained_reads, weighted_distance, warnings, filter, reason", + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "KeyError: 'length'", + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_traits.tsv line 2 does not have one field per column", + "data": {} + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_archaic.tsv has no percentage for neanderthal_percent", + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_ancient.tsv has no rows or lacks a column of ['country', 'dates', 'distance', 'group', 'locality', 'n', 'rank']", + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_viral.tsv has no rows or lacks a column of virus, accession, reads, breadth, relative_depth, copies_per_cell, call", + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_exomiser_top.tsv lacks a column of rank, gene, moi, exomiser_score, phenotype_score, variant, genotype, consequence, hgvs, acmg, disease", + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_chip.tsv lacks a column of gene, protein_change, vaf, alt_reads, depth, status, note", + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_mei.tsv is missing beside S_mei_sites.tsv", + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_mocha.tsv has 1 reported calls, S_mocha_summary.tsv says 2", + "data": {} + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S.cpsr.grch38.classification.tsv.gz: Compressed file ended before the end-of-stream marker was reached", + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "ValueError: invalid literal for int() with base 10: 'x'", + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_prioritized.vcf.gz: Compressed file ended before the end-of-stream marker was reached", + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": "none" + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Polygenic risk scores", + "Structural variants (Manta)", + "Structural variants (Delly)", + "Copy number (CNVpytor)", + "SV consensus merge" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/parity/golden/summary-collect/all-history/all-history.json b/tests/parity/golden/summary-collect/all-history/all-history.json new file mode 100644 index 000000000..d3637eeb9 --- /dev/null +++ b/tests/parity/golden/summary-collect/all-history/all-history.json @@ -0,0 +1,15569 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:19 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "male" + }, + "databases": { + "clinvar_release": "2021-03-01", + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [ + [ + "run", + "written_utc", + "2021-03-04 05:06:07 UTC" + ], + [ + "image", + "DEEPVARIANT_IMAGE", + "v1.10.0" + ], + [ + "data", + "clinvar_file_date", + "2021-03-01" + ] + ], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": 31.25 + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S_callability.bed.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "min_depth": 10, + "min_mapq": 20, + "min_share": 0.9, + "catalog_genes": 1572, + "read": 217, + "not_read": [ + { + "gene": "OR4F5", + "hgnc_id": "HGNC:14825", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR4F29", + "hgnc_id": "HGNC:31275", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F16", + "hgnc_id": "HGNC:15079", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "AGRN", + "hgnc_id": "HGNC:329", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ATAD3B", + "hgnc_id": "HGNC:24007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MMP23B", + "hgnc_id": "HGNC:7171", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CDK11B", + "hgnc_id": "HGNC:1729", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35E2B", + "hgnc_id": "HGNC:33941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDK11A", + "hgnc_id": "HGNC:1730", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TTC34", + "hgnc_id": "HGNC:34297", + "reasons": [ + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPL22", + "hgnc_id": "HGNC:10315", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MASP2", + "hgnc_id": "HGNC:6902", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPPA", + "hgnc_id": "HGNC:7939", + "reasons": [ + "CMRG" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KIAA2013", + "hgnc_id": "HGNC:28513", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF1", + "hgnc_id": "HGNC:28840", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF11", + "hgnc_id": "HGNC:14086", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL1", + "hgnc_id": "HGNC:29295", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRAMEF4", + "hgnc_id": "HGNC:31971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF10", + "hgnc_id": "HGNC:27997", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRAMEF7", + "hgnc_id": "HGNC:28415", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF6", + "hgnc_id": "HGNC:30583", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF27", + "hgnc_id": "HGNC:51234", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL3", + "hgnc_id": "HGNC:51235", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HNRNPCL2", + "hgnc_id": "HGNC:48813", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF26", + "hgnc_id": "HGNC:49178", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HNRNPCL4", + "hgnc_id": "HGNC:51333", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "PRAMEF9", + "hgnc_id": "HGNC:27996", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF13", + "hgnc_id": "HGNC:13262", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRAMEF18", + "hgnc_id": "HGNC:30693", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRAMEF8", + "hgnc_id": "HGNC:24074", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF33", + "hgnc_id": "HGNC:49193", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PRAMEF15", + "hgnc_id": "HGNC:26764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRAMEF14", + "hgnc_id": "HGNC:13576", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF19", + "hgnc_id": "HGNC:24908", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "PRAMEF17", + "hgnc_id": "HGNC:29485", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CELA2B", + "hgnc_id": "HGNC:29995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CLCNKA", + "hgnc_id": "HGNC:2026", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CLCNKB", + "hgnc_id": "HGNC:2027", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NBPF1", + "hgnc_id": "HGNC:26088", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CROCC", + "hgnc_id": "HGNC:21299", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AKR7L", + "hgnc_id": "HGNC:24056", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF3", + "hgnc_id": "HGNC:25076", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CELA3B", + "hgnc_id": "HGNC:15945", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CELA3A", + "hgnc_id": "HGNC:15944", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HMGCL", + "hgnc_id": "HGNC:5005", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNR2", + "hgnc_id": "HGNC:2160", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RSRP1", + "hgnc_id": "HGNC:25234", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGV", + "hgnc_id": "HGNC:26031", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "XKR8", + "hgnc_id": "HGNC:25508", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BMP8A", + "hgnc_id": "HGNC:21650", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BMP8B", + "hgnc_id": "HGNC:1075", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OXCT2", + "hgnc_id": "HGNC:18606", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CYP4Z1", + "hgnc_id": "HGNC:20583", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4A22", + "hgnc_id": "HGNC:20575", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GBP1", + "hgnc_id": "HGNC:4182", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SETSIP", + "hgnc_id": "HGNC:42937", + "reasons": [ + "segdup" + ], + "share": 0.4994, + "status": "not read" + }, + { + "gene": "AGL", + "hgnc_id": "HGNC:321", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "EXTL2", + "hgnc_id": "HGNC:3516", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AMY2A", + "hgnc_id": "HGNC:477", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF4", + "hgnc_id": "HGNC:26550", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NBPF6", + "hgnc_id": "HGNC:31988", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LRIG2", + "hgnc_id": "HGNC:20889", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "VANGL1", + "hgnc_id": "HGNC:15512", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IGSF3", + "hgnc_id": "HGNC:5950", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SEC22B", + "hgnc_id": "HGNC:10700", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF26", + "hgnc_id": "HGNC:49571", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2NLR", + "hgnc_id": "HGNC:53925", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4A", + "hgnc_id": "HGNC:24369", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM72B", + "hgnc_id": "HGNC:24805", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SRGAP2C", + "hgnc_id": "HGNC:30584", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3-7", + "hgnc_id": "HGNC:32060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPIAL4E", + "hgnc_id": "HGNC:33997", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF15", + "hgnc_id": "HGNC:28791", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4F", + "hgnc_id": "HGNC:33999", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "SRGAP2B", + "hgnc_id": "HGNC:35237", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM72D", + "hgnc_id": "HGNC:33593", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PPIAL4D", + "hgnc_id": "HGNC:33998", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "GPR89A", + "hgnc_id": "HGNC:31984", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBM8A", + "hgnc_id": "HGNC:9905", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "NBPF10", + "hgnc_id": "HGNC:31992", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLA", + "hgnc_id": "HGNC:31862", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPIAL4H", + "hgnc_id": "HGNC:53889", + "reasons": [ + "segdup" + ], + "share": 0.6658, + "status": "not read" + }, + { + "gene": "NBPF12", + "hgnc_id": "HGNC:24297", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF11", + "hgnc_id": "HGNC:31993", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PPIAL4G", + "hgnc_id": "HGNC:33996", + "reasons": [ + "segdup" + ], + "share": 0.1656, + "status": "not read" + }, + { + "gene": "NBPF14", + "hgnc_id": "HGNC:25232", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLB", + "hgnc_id": "HGNC:53923", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NUDT4B", + "hgnc_id": "HGNC:18012", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PDE4DIP", + "hgnc_id": "HGNC:15580", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NOTCH2NLC", + "hgnc_id": "HGNC:53924", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF19", + "hgnc_id": "HGNC:31999", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4C", + "hgnc_id": "HGNC:33995", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "H2BC18", + "hgnc_id": "HGNC:24700", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3C13", + "hgnc_id": "HGNC:25311", + "reasons": [ + "segdup" + ], + "share": 0.8316, + "status": "not read" + }, + { + "gene": "H3C14", + "hgnc_id": "HGNC:20503", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H2AC18", + "hgnc_id": "HGNC:4736", + "reasons": [ + "segdup" + ], + "share": 0.1651, + "status": "not read" + }, + { + "gene": "H2AC19", + "hgnc_id": "HGNC:29668", + "reasons": [ + "segdup" + ], + "share": 0.3328, + "status": "not read" + }, + { + "gene": "H3C15", + "hgnc_id": "HGNC:20505", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H4C15", + "hgnc_id": "HGNC:29607", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HRNR", + "hgnc_id": "HGNC:20846", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LCE3E", + "hgnc_id": "HGNC:29463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCE3D", + "hgnc_id": "HGNC:16615", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "S100A8", + "hgnc_id": "HGNC:10498", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "S100A7A", + "hgnc_id": "HGNC:21657", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "S100A7", + "hgnc_id": "HGNC:10497", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FLAD1", + "hgnc_id": "HGNC:24671", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MTX1", + "hgnc_id": "HGNC:7504", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN3", + "hgnc_id": "HGNC:19183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FCRL5", + "hgnc_id": "HGNC:18508", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA6", + "hgnc_id": "HGNC:5239", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FCGR2B", + "hgnc_id": "HGNC:3618", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD17B7", + "hgnc_id": "HGNC:5215", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD247", + "hgnc_id": "HGNC:1677", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "XCL2", + "hgnc_id": "HGNC:10646", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "PRG4", + "hgnc_id": "HGNC:9364", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PTPRC", + "hgnc_id": "HGNC:9666", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPP1R12B", + "hgnc_id": "HGNC:7619", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KISS1", + "hgnc_id": "HGNC:6341", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SRGAP2", + "hgnc_id": "HGNC:19751", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CR1", + "hgnc_id": "HGNC:2334", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CR1L", + "hgnc_id": "HGNC:2335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZC3H11B", + "hgnc_id": "HGNC:25659", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LBR", + "hgnc_id": "HGNC:6518", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "LEFTY2", + "hgnc_id": "HGNC:3122", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "OR2T33", + "hgnc_id": "HGNC:31255", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T7", + "hgnc_id": "HGNC:15019", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T2", + "hgnc_id": "HGNC:14725", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T3", + "hgnc_id": "HGNC:14727", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR2T5", + "hgnc_id": "HGNC:15017", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "OR2G6", + "hgnc_id": "HGNC:27019", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR2T34", + "hgnc_id": "HGNC:31256", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T11", + "hgnc_id": "HGNC:19574", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T35", + "hgnc_id": "HGNC:31257", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T27", + "hgnc_id": "HGNC:31252", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SNTG2", + "hgnc_id": "HGNC:13741", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TPO", + "hgnc_id": "HGNC:12015", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KLF11", + "hgnc_id": "HGNC:11811", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CCDC121", + "hgnc_id": "HGNC:25833", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "ABCG8", + "hgnc_id": "HGNC:13887", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PCBP1", + "hgnc_id": "HGNC:8647", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "NAT8", + "hgnc_id": "HGNC:18069", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "C2orf78", + "hgnc_id": "HGNC:34349", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CD8B", + "hgnc_id": "HGNC:1707", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RGPD1", + "hgnc_id": "HGNC:32414", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLGLB1", + "hgnc_id": "HGNC:9072", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLGLB2", + "hgnc_id": "HGNC:9073", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RGPD2", + "hgnc_id": "HGNC:32415", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPIA", + "hgnc_id": "HGNC:10297", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEKT4", + "hgnc_id": "HGNC:31012", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAHD2A", + "hgnc_id": "HGNC:24252", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TRIM43B", + "hgnc_id": "HGNC:37146", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM43", + "hgnc_id": "HGNC:19015", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANKRD36C", + "hgnc_id": "HGNC:32946", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPAT2", + "hgnc_id": "HGNC:27168", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD36", + "hgnc_id": "HGNC:24079", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANKRD36B", + "hgnc_id": "HGNC:29333", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "RGPD3", + "hgnc_id": "HGNC:32416", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD4", + "hgnc_id": "HGNC:32417", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RANBP2", + "hgnc_id": "HGNC:9848", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD5", + "hgnc_id": "HGNC:32418", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MALL", + "hgnc_id": "HGNC:6818", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MTLN", + "hgnc_id": "HGNC:27339", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LIMS4", + "hgnc_id": "HGNC:39941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD6", + "hgnc_id": "HGNC:32419", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANAPC1", + "hgnc_id": "HGNC:19988", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD8", + "hgnc_id": "HGNC:9849", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FOXD4L1", + "hgnc_id": "HGNC:18521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RABL2A", + "hgnc_id": "HGNC:9799", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RAB6C", + "hgnc_id": "HGNC:16525", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "POTEF", + "hgnc_id": "HGNC:33905", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CCDC74B", + "hgnc_id": "HGNC:25267", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SMPD4", + "hgnc_id": "HGNC:32949", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3E", + "hgnc_id": "HGNC:20765", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEI", + "hgnc_id": "HGNC:37093", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CFC1B", + "hgnc_id": "HGNC:33983", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CFC1", + "hgnc_id": "HGNC:18292", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "POTEJ", + "hgnc_id": "HGNC:37094", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PLEKHB2", + "hgnc_id": "HGNC:19236", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB6D", + "hgnc_id": "HGNC:30272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MZT2A", + "hgnc_id": "HGNC:33187", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TUBA3D", + "hgnc_id": "HGNC:24071", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CCDC74A", + "hgnc_id": "HGNC:25197", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "ANKRD30BL", + "hgnc_id": "HGNC:35167", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NOSTRIN", + "hgnc_id": "HGNC:20203", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IFT70B", + "hgnc_id": "HGNC:26425", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CASP10", + "hgnc_id": "HGNC:1500", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ALPP", + "hgnc_id": "HGNC:439", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ALPG", + "hgnc_id": "HGNC:441", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT1A6", + "hgnc_id": "HGNC:12538", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UGT1A5", + "hgnc_id": "HGNC:12537", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT1A3", + "hgnc_id": "HGNC:12535", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MLPH", + "hgnc_id": "HGNC:29643", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR6B2", + "hgnc_id": "HGNC:15041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AQP12B", + "hgnc_id": "HGNC:6096", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "AQP12A", + "hgnc_id": "HGNC:19941", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ANO7", + "hgnc_id": "HGNC:31677", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CHL1", + "hgnc_id": "HGNC:1939", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMC3", + "hgnc_id": "HGNC:23999", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANCD2", + "hgnc_id": "HGNC:3585", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DAZL", + "hgnc_id": "HGNC:2685", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF35", + "hgnc_id": "HGNC:13099", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LZTFL1", + "hgnc_id": "HGNC:6741", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MST1", + "hgnc_id": "HGNC:7380", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MST1R", + "hgnc_id": "HGNC:7381", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HYAL1", + "hgnc_id": "HGNC:5320", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FRG2C", + "hgnc_id": "HGNC:33626", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "ZNF717", + "hgnc_id": "HGNC:29448", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR5H6", + "hgnc_id": "HGNC:14767", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ROPN1", + "hgnc_id": "HGNC:17692", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ROPN1B", + "hgnc_id": "HGNC:31927", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RPN1", + "hgnc_id": "HGNC:10381", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ALG1L2", + "hgnc_id": "HGNC:37258", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BFSP2", + "hgnc_id": "HGNC:1041", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRR23B", + "hgnc_id": "HGNC:33764", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SOX2", + "hgnc_id": "HGNC:11195", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "EIF2B5", + "hgnc_id": "HGNC:3261", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KNG1", + "hgnc_id": "HGNC:6383", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TM4SF19", + "hgnc_id": "HGNC:25167", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF595", + "hgnc_id": "HGNC:27196", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDE6B", + "hgnc_id": "HGNC:8786", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAK", + "hgnc_id": "HGNC:4113", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FGFRL1", + "hgnc_id": "HGNC:3693", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RNF212", + "hgnc_id": "HGNC:27729", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UVSSA", + "hgnc_id": "HGNC:29304", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DOK7", + "hgnc_id": "HGNC:26594", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HMX1", + "hgnc_id": "HGNC:5017", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A26", + "hgnc_id": "HGNC:43746", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "USP17L10", + "hgnc_id": "HGNC:44438", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L11", + "hgnc_id": "HGNC:44439", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L12", + "hgnc_id": "HGNC:44440", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L13", + "hgnc_id": "HGNC:44441", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L17", + "hgnc_id": "HGNC:44445", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L18", + "hgnc_id": "HGNC:44446", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L19", + "hgnc_id": "HGNC:44447", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L20", + "hgnc_id": "HGNC:44448", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L21", + "hgnc_id": "HGNC:44449", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L22", + "hgnc_id": "HGNC:44450", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L24", + "hgnc_id": "HGNC:44453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L25", + "hgnc_id": "HGNC:44452", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L26", + "hgnc_id": "HGNC:44454", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L5", + "hgnc_id": "HGNC:37177", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L27", + "hgnc_id": "HGNC:44455", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L28", + "hgnc_id": "HGNC:44456", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L30", + "hgnc_id": "HGNC:44458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB131A", + "hgnc_id": "HGNC:18108", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRD5", + "hgnc_id": "HGNC:3026", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "UGT2B10", + "hgnc_id": "HGNC:12544", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B28", + "hgnc_id": "HGNC:13479", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UGT2A1", + "hgnc_id": "HGNC:12542", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UGT2A2", + "hgnc_id": "HGNC:28183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HTN3", + "hgnc_id": "HGNC:5284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HTN1", + "hgnc_id": "HGNC:5283", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "AFP", + "hgnc_id": "HGNC:317", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGY", + "hgnc_id": "HGNC:28213", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PYURF", + "hgnc_id": "HGNC:44317", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CISD2", + "hgnc_id": "HGNC:24212", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC9B1", + "hgnc_id": "HGNC:24244", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KLHL2", + "hgnc_id": "HGNC:6353", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GK3", + "hgnc_id": "HGNC:4292", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "HPF1", + "hgnc_id": "HGNC:26051", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDLIM3", + "hgnc_id": "HGNC:20767", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAT1", + "hgnc_id": "HGNC:3595", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG1", + "hgnc_id": "HGNC:3954", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FRG2", + "hgnc_id": "HGNC:19136", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DUX4", + "hgnc_id": "HGNC:50800", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SDHA", + "hgnc_id": "HGNC:10680", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDCD6", + "hgnc_id": "HGNC:8765", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZDHHC11B", + "hgnc_id": "HGNC:32962", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZDHHC11", + "hgnc_id": "HGNC:19158", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TERT", + "hgnc_id": "HGNC:11730", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A3", + "hgnc_id": "HGNC:11049", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAF11L2", + "hgnc_id": "HGNC:53845", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L3", + "hgnc_id": "HGNC:53846", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L4", + "hgnc_id": "HGNC:53847", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L5", + "hgnc_id": "HGNC:53848", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "TAF11L7", + "hgnc_id": "HGNC:53850", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAF11L8", + "hgnc_id": "HGNC:53851", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "TAF11L9", + "hgnc_id": "HGNC:53852", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L10", + "hgnc_id": "HGNC:53853", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L11", + "hgnc_id": "HGNC:53854", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L12", + "hgnc_id": "HGNC:53855", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "TAF11L14", + "hgnc_id": "HGNC:53857", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMB", + "hgnc_id": "HGNC:30465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MARVELD2", + "hgnc_id": "HGNC:26401", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERF1B", + "hgnc_id": "HGNC:10756", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NAIP", + "hgnc_id": "HGNC:7634", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SKIC3", + "hgnc_id": "HGNC:23639", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIX1", + "hgnc_id": "HGNC:18581", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SAR1B", + "hgnc_id": "HGNC:10535", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DND1", + "hgnc_id": "HGNC:23799", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDHA10", + "hgnc_id": "HGNC:8664", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PCDHB7", + "hgnc_id": "HGNC:8692", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PCDHB8", + "hgnc_id": "HGNC:8693", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PCDHB11", + "hgnc_id": "HGNC:8682", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PCDHB12", + "hgnc_id": "HGNC:8683", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPP1R2B", + "hgnc_id": "HGNC:16318", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPM1", + "hgnc_id": "HGNC:7910", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "THOC3", + "hgnc_id": "HGNC:19072", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SIMC1", + "hgnc_id": "HGNC:24779", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM153A", + "hgnc_id": "HGNC:29940", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FAM153CP", + "hgnc_id": "HGNC:33936", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FLT4", + "hgnc_id": "HGNC:3767", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR4F3", + "hgnc_id": "HGNC:8300", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "DUSP22", + "hgnc_id": "HGNC:16077", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TUBB2A", + "hgnc_id": "HGNC:12412", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "H2AC13", + "hgnc_id": "HGNC:4725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H2AC14", + "hgnc_id": "HGNC:4727", + "reasons": [ + "segdup" + ], + "share": 0.8319, + "status": "not read" + }, + { + "gene": "OR2J1", + "hgnc_id": "HGNC:8259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2J2", + "hgnc_id": "HGNC:8260", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HLA-G", + "hgnc_id": "HGNC:4964", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HLA-A", + "hgnc_id": "HGNC:4931", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "MICA", + "hgnc_id": "HGNC:7090", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MICB", + "hgnc_id": "HGNC:7091", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZBTB12", + "hgnc_id": "HGNC:19066", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PBX2", + "hgnc_id": "HGNC:8633", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HLA-DRB1", + "hgnc_id": "HGNC:4948", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GSTA2", + "hgnc_id": "HGNC:4627", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GSTA1", + "hgnc_id": "HGNC:4626", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTA5", + "hgnc_id": "HGNC:19662", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRIM2", + "hgnc_id": "HGNC:9370", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FKBP1C", + "hgnc_id": "HGNC:21376", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SLC17A5", + "hgnc_id": "HGNC:10933", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEC63", + "hgnc_id": "HGNC:21082", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR2A4", + "hgnc_id": "HGNC:14729", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE9", + "hgnc_id": "HGNC:37275", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "PCMT1", + "hgnc_id": "HGNC:8728", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RAET1G", + "hgnc_id": "HGNC:16795", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ULBP1", + "hgnc_id": "HGNC:14893", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RAET1L", + "hgnc_id": "HGNC:16798", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ULBP3", + "hgnc_id": "HGNC:14895", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC22A1", + "hgnc_id": "HGNC:10963", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMOC2", + "hgnc_id": "HGNC:20323", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RBAK-RBAKDN", + "hgnc_id": "HGNC:42971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBAK", + "hgnc_id": "HGNC:17680", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC29A4", + "hgnc_id": "HGNC:23097", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FSCN1", + "hgnc_id": "HGNC:11148", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM", + "hgnc_id": "HGNC:8105", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCZ1", + "hgnc_id": "HGNC:21691", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RSPH10B2", + "hgnc_id": "HGNC:34385", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CCZ1B", + "hgnc_id": "HGNC:21717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOD1", + "hgnc_id": "HGNC:16390", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FKBP9", + "hgnc_id": "HGNC:3725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DPY19L1", + "hgnc_id": "HGNC:22205", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SEPTIN7", + "hgnc_id": "HGNC:1717", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE1", + "hgnc_id": "HGNC:16408", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIA", + "hgnc_id": "HGNC:9253", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDC14C", + "hgnc_id": "HGNC:22427", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSPH", + "hgnc_id": "HGNC:9577", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCT6A", + "hgnc_id": "HGNC:1620", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF716", + "hgnc_id": "HGNC:32458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF727", + "hgnc_id": "HGNC:22785", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF735", + "hgnc_id": "HGNC:32466", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF736", + "hgnc_id": "HGNC:32467", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GUSB", + "hgnc_id": "HGNC:4696", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RABGEF1", + "hgnc_id": "HGNC:17676", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TYW1", + "hgnc_id": "HGNC:25598", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE21", + "hgnc_id": "HGNC:51517", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYW1B", + "hgnc_id": "HGNC:33908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POM121", + "hgnc_id": "HGNC:19702", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM74", + "hgnc_id": "HGNC:17453", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPDYE8", + "hgnc_id": "HGNC:33771", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE9", + "hgnc_id": "HGNC:45034", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE10", + "hgnc_id": "HGNC:51506", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NSUN5", + "hgnc_id": "HGNC:16385", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM50", + "hgnc_id": "HGNC:19017", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLIP2", + "hgnc_id": "HGNC:2586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GTF2IRD2", + "hgnc_id": "HGNC:30775", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE12", + "hgnc_id": "HGNC:51508", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CASTOR2", + "hgnc_id": "HGNC:37073", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GTF2IRD2B", + "hgnc_id": "HGNC:33125", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPDYE13", + "hgnc_id": "HGNC:51509", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE14", + "hgnc_id": "HGNC:51510", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM73", + "hgnc_id": "HGNC:18162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POM121C", + "hgnc_id": "HGNC:34005", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPDYE5", + "hgnc_id": "HGNC:35464", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DTX2", + "hgnc_id": "HGNC:15973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UPK3B", + "hgnc_id": "HGNC:21444", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE16", + "hgnc_id": "HGNC:51512", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE17", + "hgnc_id": "HGNC:51513", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE18", + "hgnc_id": "HGNC:51514", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "STEAP1", + "hgnc_id": "HGNC:11378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ASNS", + "hgnc_id": "HGNC:753", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM2", + "hgnc_id": "HGNC:34396", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP3A7-CYP3A51P", + "hgnc_id": "HGNC:51504", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP3A4", + "hgnc_id": "HGNC:2637", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "STAG3", + "hgnc_id": "HGNC:11356", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PVRIG", + "hgnc_id": "HGNC:32190", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SPDYE3", + "hgnc_id": "HGNC:35462", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SAP25", + "hgnc_id": "HGNC:41908", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZAN", + "hgnc_id": "HGNC:12857", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC12", + "hgnc_id": "HGNC:7510", + "reasons": [ + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE6", + "hgnc_id": "HGNC:35465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "POLR2J", + "hgnc_id": "HGNC:9197", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RASA4B", + "hgnc_id": "HGNC:35202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J3", + "hgnc_id": "HGNC:33853", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UPK3BL2", + "hgnc_id": "HGNC:53444", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "RASA4", + "hgnc_id": "HGNC:23181", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UPK3BL1", + "hgnc_id": "HGNC:37278", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "POLR2J3-UPK3BL2", + "hgnc_id": "HGNC:58364", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SPDYE2B", + "hgnc_id": "HGNC:48334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J2", + "hgnc_id": "HGNC:23208", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAMB1", + "hgnc_id": "HGNC:6486", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2B", + "hgnc_id": "HGNC:18272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KLF14", + "hgnc_id": "HGNC:23025", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AKR1B10", + "hgnc_id": "HGNC:382", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1B15", + "hgnc_id": "HGNC:37281", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BRAF", + "hgnc_id": "HGNC:1097", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRBV9", + "hgnc_id": "HGNC:12246", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRSS2", + "hgnc_id": "HGNC:9483", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAGE15", + "hgnc_id": "HGNC:37295", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TCAF2", + "hgnc_id": "HGNC:26878", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TCAF2C", + "hgnc_id": "HGNC:53641", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CTAGE6", + "hgnc_id": "HGNC:28644", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "OR2F2", + "hgnc_id": "HGNC:8247", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CTAGE4", + "hgnc_id": "HGNC:24772", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2A42", + "hgnc_id": "HGNC:31230", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2A7", + "hgnc_id": "HGNC:8234", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE8", + "hgnc_id": "HGNC:37294", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "OR2A1", + "hgnc_id": "HGNC:8229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARHGEF5", + "hgnc_id": "HGNC:13209", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACTR3C", + "hgnc_id": "HGNC:37282", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KMT2C", + "hgnc_id": "HGNC:13726", + "reasons": [ + "CMRG", + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ACTR3B", + "hgnc_id": "HGNC:17256", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F21", + "hgnc_id": "HGNC:19583", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "FBXO25", + "hgnc_id": "HGNC:13596", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARHGEF10", + "hgnc_id": "HGNC:14103", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A15", + "hgnc_id": "HGNC:32263", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A3", + "hgnc_id": "HGNC:32251", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A13", + "hgnc_id": "HGNC:32261", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A5", + "hgnc_id": "HGNC:32253", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM90A20", + "hgnc_id": "HGNC:32268", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L1", + "hgnc_id": "HGNC:37182", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L4", + "hgnc_id": "HGNC:37176", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNF705G", + "hgnc_id": "HGNC:37134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB108C", + "hgnc_id": "HGNC:30847", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB4B", + "hgnc_id": "HGNC:30193", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "DEFB103B", + "hgnc_id": "HGNC:31702", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "DEFB104B", + "hgnc_id": "HGNC:26165", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB106B", + "hgnc_id": "HGNC:28879", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "DEFB105B", + "hgnc_id": "HGNC:29930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "DEFB107B", + "hgnc_id": "HGNC:31918", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRR23D1", + "hgnc_id": "HGNC:49420", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A7", + "hgnc_id": "HGNC:32255", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FAM90A23", + "hgnc_id": "HGNC:32271", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A14", + "hgnc_id": "HGNC:32262", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A18", + "hgnc_id": "HGNC:32266", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A16", + "hgnc_id": "HGNC:32264", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A8", + "hgnc_id": "HGNC:32256", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A17", + "hgnc_id": "HGNC:32265", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A9", + "hgnc_id": "HGNC:32257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A10", + "hgnc_id": "HGNC:32258", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PRR23D2", + "hgnc_id": "HGNC:49396", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB107A", + "hgnc_id": "HGNC:18086", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB105A", + "hgnc_id": "HGNC:18087", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DEFB106A", + "hgnc_id": "HGNC:18088", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPAG11A", + "hgnc_id": "HGNC:33342", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB103A", + "hgnc_id": "HGNC:15967", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DEFB4A", + "hgnc_id": "HGNC:2767", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNF705B", + "hgnc_id": "HGNC:32284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "USP17L8", + "hgnc_id": "HGNC:37181", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L3", + "hgnc_id": "HGNC:37175", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "FAM90A11", + "hgnc_id": "HGNC:32259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A24", + "hgnc_id": "HGNC:32272", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A12", + "hgnc_id": "HGNC:32260", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLC35G5", + "hgnc_id": "HGNC:15546", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "DEFB130B", + "hgnc_id": "HGNC:39814", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF705D", + "hgnc_id": "HGNC:33202", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L2", + "hgnc_id": "HGNC:34434", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB109D", + "hgnc_id": "HGNC:30838", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM86B1", + "hgnc_id": "HGNC:28268", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB130A", + "hgnc_id": "HGNC:18107", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM86B2", + "hgnc_id": "HGNC:32222", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RBPMS", + "hgnc_id": "HGNC:19097", + "reasons": [ + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IMPA1", + "hgnc_id": "HGNC:6050", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC10A5", + "hgnc_id": "HGNC:22981", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH17", + "hgnc_id": "HGNC:1756", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP11B1", + "hgnc_id": "HGNC:2591", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP11B2", + "hgnc_id": "HGNC:2592", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GPIHBP1", + "hgnc_id": "HGNC:24945", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "WASHC1", + "hgnc_id": "HGNC:24361", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOXD4", + "hgnc_id": "HGNC:3805", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNG1A", + "hgnc_id": "HGNC:17134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KCNV2", + "hgnc_id": "HGNC:19698", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RLN2", + "hgnc_id": "HGNC:10027", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RLN1", + "hgnc_id": "HGNC:10026", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "IFNA21", + "hgnc_id": "HGNC:5424", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA4", + "hgnc_id": "HGNC:5425", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "IFNA7", + "hgnc_id": "HGNC:5428", + "reasons": [ + "segdup" + ], + "share": 0.3324, + "status": "not read" + }, + { + "gene": "IFNA10", + "hgnc_id": "HGNC:5418", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "IFNA16", + "hgnc_id": "HGNC:5421", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "IFNA17", + "hgnc_id": "HGNC:5422", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "IFNA13", + "hgnc_id": "HGNC:5419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA1", + "hgnc_id": "HGNC:5417", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "ANKRD18B", + "hgnc_id": "HGNC:23644", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRSS3", + "hgnc_id": "HGNC:9486", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALT", + "hgnc_id": "HGNC:4135", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31F1", + "hgnc_id": "HGNC:41911", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CNTNAP3", + "hgnc_id": "HGNC:13834", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31A1", + "hgnc_id": "HGNC:23394", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L6", + "hgnc_id": "HGNC:31986", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNG1F", + "hgnc_id": "HGNC:31978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNTNAP3B", + "hgnc_id": "HGNC:32035", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31A6", + "hgnc_id": "HGNC:32006", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPATA31A7", + "hgnc_id": "HGNC:32007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CNTNAP3C", + "hgnc_id": "HGNC:53878", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L5", + "hgnc_id": "HGNC:18522", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNG1E", + "hgnc_id": "HGNC:24584", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXD4L4", + "hgnc_id": "HGNC:23762", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF658", + "hgnc_id": "HGNC:25226", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD20A1", + "hgnc_id": "HGNC:23665", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNG1C", + "hgnc_id": "HGNC:18519", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L3", + "hgnc_id": "HGNC:18523", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PGM5", + "hgnc_id": "HGNC:8908", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FXN", + "hgnc_id": "HGNC:3951", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TJP2", + "hgnc_id": "HGNC:11828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPATA31D4", + "hgnc_id": "HGNC:38601", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31D3", + "hgnc_id": "HGNC:38603", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPATA31D1", + "hgnc_id": "HGNC:37283", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "KIF27", + "hgnc_id": "HGNC:18632", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPATA31C1", + "hgnc_id": "HGNC:27846", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPATA31C2", + "hgnc_id": "HGNC:24508", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC71A2", + "hgnc_id": "HGNC:23376", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2G", + "hgnc_id": "HGNC:23449", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CTSV", + "hgnc_id": "HGNC:2538", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR13C5", + "hgnc_id": "HGNC:15100", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR13C2", + "hgnc_id": "HGNC:14701", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR13C9", + "hgnc_id": "HGNC:15104", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "POLE3", + "hgnc_id": "HGNC:13546", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ORM1", + "hgnc_id": "HGNC:8498", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ORM2", + "hgnc_id": "HGNC:8499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SLC27A4", + "hgnc_id": "HGNC:10998", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PKN3", + "hgnc_id": "HGNC:17999", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ABO", + "hgnc_id": "HGNC:79", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCN1", + "hgnc_id": "HGNC:6525", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OBP2A", + "hgnc_id": "HGNC:23380", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SOHLH1", + "hgnc_id": "HGNC:27845", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "INPP5E", + "hgnc_id": "HGNC:21474", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AJM1", + "hgnc_id": "HGNC:37284", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRIN1", + "hgnc_id": "HGNC:4584", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8", + "hgnc_id": "HGNC:20773", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "AKR1C1", + "hgnc_id": "HGNC:384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1C2", + "hgnc_id": "HGNC:385", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DCLRE1C", + "hgnc_id": "HGNC:17642", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRC1", + "hgnc_id": "HGNC:7228", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD30A", + "hgnc_id": "HGNC:17234", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF33A", + "hgnc_id": "HGNC:13096", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF37A", + "hgnc_id": "HGNC:13102", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLD5P1", + "hgnc_id": "HGNC:55072", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF33B", + "hgnc_id": "HGNC:13097", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BMS1", + "hgnc_id": "HGNC:23505", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AGAP4", + "hgnc_id": "HGNC:23459", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23", + "hgnc_id": "HGNC:17312", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ANTXRL", + "hgnc_id": "HGNC:27277", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANXA8L1", + "hgnc_id": "HGNC:23334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPY4R", + "hgnc_id": "HGNC:9329", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPRIN2", + "hgnc_id": "HGNC:23730", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PTPN20", + "hgnc_id": "HGNC:23423", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANXA8", + "hgnc_id": "HGNC:546", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM25G", + "hgnc_id": "HGNC:23590", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP9", + "hgnc_id": "HGNC:23463", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SYT15B", + "hgnc_id": "HGNC:51487", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPY4R2", + "hgnc_id": "HGNC:52383", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PARG", + "hgnc_id": "HGNC:8605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23B", + "hgnc_id": "HGNC:23581", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TIMM23B-AGAP6", + "hgnc_id": "HGNC:45009", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP6", + "hgnc_id": "HGNC:23466", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASHC2A", + "hgnc_id": "HGNC:23416", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ASAH2B", + "hgnc_id": "HGNC:23456", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EIF5AL1", + "hgnc_id": "HGNC:17419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2B", + "hgnc_id": "HGNC:23445", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NUTM2E", + "hgnc_id": "HGNC:23448", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMEM254", + "hgnc_id": "HGNC:25804", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PLAC9", + "hgnc_id": "HGNC:19255", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ADIRF", + "hgnc_id": "HGNC:24043", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NUTM2A", + "hgnc_id": "HGNC:23438", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2D", + "hgnc_id": "HGNC:23447", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PAPSS2", + "hgnc_id": "HGNC:8604", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PTEN", + "hgnc_id": "HGNC:9588", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIPN", + "hgnc_id": "HGNC:23452", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "BTRC", + "hgnc_id": "HGNC:1144", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SFR1", + "hgnc_id": "HGNC:29574", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DHX32", + "hgnc_id": "HGNC:16717", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANK1", + "hgnc_id": "HGNC:23527", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ECHS1", + "hgnc_id": "HGNC:3151", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPRN", + "hgnc_id": "HGNC:16871", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG2B", + "hgnc_id": "HGNC:33518", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SCGB1C1", + "hgnc_id": "HGNC:18394", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFITM3", + "hgnc_id": "HGNC:5414", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DRD4", + "hgnc_id": "HGNC:3025", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEAF1", + "hgnc_id": "HGNC:14677", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP5-2", + "hgnc_id": "HGNC:23597", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP5-4", + "hgnc_id": "HGNC:23599", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MRPL23", + "hgnc_id": "HGNC:10322", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H19", + "hgnc_id": "HGNC:4713", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSU72L5", + "hgnc_id": "HGNC:43624", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "SSU72L2", + "hgnc_id": "HGNC:43621", + "reasons": [ + "segdup" + ], + "share": 0.666, + "status": "not read" + }, + { + "gene": "SSU72L4", + "hgnc_id": "HGNC:43623", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "SSU72L1", + "hgnc_id": "HGNC:43626", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR51A4", + "hgnc_id": "HGNC:14795", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR51A2", + "hgnc_id": "HGNC:14764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HBG1", + "hgnc_id": "HGNC:4831", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR10A5", + "hgnc_id": "HGNC:15131", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRGPRX4", + "hgnc_id": "HGNC:17617", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MDK", + "hgnc_id": "HGNC:6972", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPI1", + "hgnc_id": "HGNC:11241", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C3", + "hgnc_id": "HGNC:14697", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4C5", + "hgnc_id": "HGNC:14702", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4A47", + "hgnc_id": "HGNC:31266", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM51G", + "hgnc_id": "HGNC:43972", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64C", + "hgnc_id": "HGNC:37148", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOLH1", + "hgnc_id": "HGNC:3788", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C13", + "hgnc_id": "HGNC:15169", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4C12", + "hgnc_id": "HGNC:15168", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4C45", + "hgnc_id": "HGNC:31270", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4C46", + "hgnc_id": "HGNC:31271", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4A5", + "hgnc_id": "HGNC:15162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM48", + "hgnc_id": "HGNC:19021", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR4A15", + "hgnc_id": "HGNC:15152", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM51", + "hgnc_id": "HGNC:19023", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR5F1", + "hgnc_id": "HGNC:8343", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR8H2", + "hgnc_id": "HGNC:15308", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR5M8", + "hgnc_id": "HGNC:14846", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR9G1", + "hgnc_id": "HGNC:15319", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR1S2", + "hgnc_id": "HGNC:15141", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1S1", + "hgnc_id": "HGNC:8227", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GLYATL1B", + "hgnc_id": "HGNC:37865", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PGA3", + "hgnc_id": "HGNC:8885", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "B3GAT3", + "hgnc_id": "HGNC:923", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ESRRA", + "hgnc_id": "HGNC:3471", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC22A12", + "hgnc_id": "HGNC:17989", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RBM4B", + "hgnc_id": "HGNC:28842", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TPCN2", + "hgnc_id": "HGNC:20820", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FGF3", + "hgnc_id": "HGNC:3681", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KRTAP5-8", + "hgnc_id": "HGNC:23603", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP5-10", + "hgnc_id": "HGNC:23605", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "KRTAP5-11", + "hgnc_id": "HGNC:23606", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "XNDC1N-ZNF705EP-ALG1L9P", + "hgnc_id": "HGNC:55871", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DEFB108B", + "hgnc_id": "HGNC:29966", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "XNDC1N", + "hgnc_id": "HGNC:54661", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "INTS4", + "hgnc_id": "HGNC:25048", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOX4", + "hgnc_id": "HGNC:7891", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TRIM77", + "hgnc_id": "HGNC:34228", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "TRIM49", + "hgnc_id": "HGNC:13431", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM64B", + "hgnc_id": "HGNC:37147", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM49D1", + "hgnc_id": "HGNC:43973", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64", + "hgnc_id": "HGNC:14663", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM49C", + "hgnc_id": "HGNC:38877", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UBTFL1", + "hgnc_id": "HGNC:14533", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CARD16", + "hgnc_id": "HGNC:33701", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA8", + "hgnc_id": "HGNC:5241", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR8B2", + "hgnc_id": "HGNC:8471", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TMEM45B", + "hgnc_id": "HGNC:25194", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CD4", + "hgnc_id": "HGNC:1678", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PEX5", + "hgnc_id": "HGNC:9719", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC1", + "hgnc_id": "HGNC:604", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GDF3", + "hgnc_id": "HGNC:4218", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NANOG", + "hgnc_id": "HGNC:20857", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A1", + "hgnc_id": "HGNC:25526", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRH1", + "hgnc_id": "HGNC:9366", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRR4", + "hgnc_id": "HGNC:18020", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRH2", + "hgnc_id": "HGNC:9367", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAS2R14", + "hgnc_id": "HGNC:14920", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TAS2R31", + "hgnc_id": "HGNC:19113", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAS2R46", + "hgnc_id": "HGNC:18877", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAS2R43", + "hgnc_id": "HGNC:18875", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "TAS2R30", + "hgnc_id": "HGNC:19112", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "PRB4", + "hgnc_id": "HGNC:9340", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRB1", + "hgnc_id": "HGNC:9337", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "DDX11", + "hgnc_id": "HGNC:2736", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H3-5", + "hgnc_id": "HGNC:33164", + "reasons": [ + "segdup" + ], + "share": 0.1661, + "status": "not read" + }, + { + "gene": "ALG10", + "hgnc_id": "HGNC:23162", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ALG10B", + "hgnc_id": "HGNC:31088", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBA1B", + "hgnc_id": "HGNC:18809", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "COX14", + "hgnc_id": "HGNC:28216", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NACA", + "hgnc_id": "HGNC:7629", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DPY19L2", + "hgnc_id": "HGNC:19414", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNF1A", + "hgnc_id": "HGNC:11621", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HPD", + "hgnc_id": "HGNC:5147", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCAR2", + "hgnc_id": "HGNC:24827", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HCAR3", + "hgnc_id": "HGNC:16824", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "P2RX2", + "hgnc_id": "HGNC:15459", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PGAM5", + "hgnc_id": "HGNC:28763", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA3", + "hgnc_id": "HGNC:4426", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "C1QTNF9B", + "hgnc_id": "HGNC:34072", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C1QTNF9", + "hgnc_id": "HGNC:28732", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PARP4", + "hgnc_id": "HGNC:271", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PABPC3", + "hgnc_id": "HGNC:8556", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "MRPS31", + "hgnc_id": "HGNC:16632", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNRNPA1L2", + "hgnc_id": "HGNC:27067", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRR20C", + "hgnc_id": "HGNC:37221", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRR20B", + "hgnc_id": "HGNC:37220", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "PRR20D", + "hgnc_id": "HGNC:37222", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRR20E", + "hgnc_id": "HGNC:37223", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "F7", + "hgnc_id": "HGNC:3544", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PROZ", + "hgnc_id": "HGNC:9460", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEM", + "hgnc_id": "HGNC:37096", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEG", + "hgnc_id": "HGNC:33896", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR11H2", + "hgnc_id": "HGNC:14716", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4N2", + "hgnc_id": "HGNC:14742", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR4Q3", + "hgnc_id": "HGNC:15426", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR4M1", + "hgnc_id": "HGNC:14735", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RNASE3", + "hgnc_id": "HGNC:10046", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RNASE2", + "hgnc_id": "HGNC:10045", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10G2", + "hgnc_id": "HGNC:8170", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "DHRS4", + "hgnc_id": "HGNC:16985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHRS4L2", + "hgnc_id": "HGNC:19731", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARF6", + "hgnc_id": "HGNC:659", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACOT2", + "hgnc_id": "HGNC:18431", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NDUFB1", + "hgnc_id": "HGNC:7695", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "JAG2", + "hgnc_id": "HGNC:6189", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BTBD6", + "hgnc_id": "HGNC:19897", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "IGHV3-21", + "hgnc_id": "HGNC:5586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA6L6", + "hgnc_id": "HGNC:37225", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEB3", + "hgnc_id": "HGNC:51240", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LINC02203", + "hgnc_id": "HGNC:53069", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4M2B", + "hgnc_id": "HGNC:55109", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4N4C", + "hgnc_id": "HGNC:55110", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "POTEB", + "hgnc_id": "HGNC:33734", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4M2-OT1", + "hgnc_id": "HGNC:56199", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4N4", + "hgnc_id": "HGNC:15375", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L22", + "hgnc_id": "HGNC:50289", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6L1", + "hgnc_id": "HGNC:37444", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L26", + "hgnc_id": "HGNC:56306", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8S", + "hgnc_id": "HGNC:44409", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6L2", + "hgnc_id": "HGNC:26695", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HERC2", + "hgnc_id": "HGNC:4868", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L24", + "hgnc_id": "HGNC:55710", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA8F", + "hgnc_id": "HGNC:32378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8G", + "hgnc_id": "HGNC:25328", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6L25", + "hgnc_id": "HGNC:55711", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8M", + "hgnc_id": "HGNC:44404", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8J", + "hgnc_id": "HGNC:38650", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8T", + "hgnc_id": "HGNC:44410", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8R", + "hgnc_id": "HGNC:44407", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8Q", + "hgnc_id": "HGNC:44408", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8H", + "hgnc_id": "HGNC:37443", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8O", + "hgnc_id": "HGNC:44406", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8N", + "hgnc_id": "HGNC:44405", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8A", + "hgnc_id": "HGNC:31972", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8B", + "hgnc_id": "HGNC:31973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NANOGP8", + "hgnc_id": "HGNC:23106", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "NDUFAF1", + "hgnc_id": "HGNC:18828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CKMT1B", + "hgnc_id": "HGNC:1995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CATSPER2", + "hgnc_id": "HGNC:18810", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CKMT1A", + "hgnc_id": "HGNC:31736", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TERB2", + "hgnc_id": "HGNC:28520", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LDHAL6B", + "hgnc_id": "HGNC:21481", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6B", + "hgnc_id": "HGNC:32205", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6A", + "hgnc_id": "HGNC:13567", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ISLR2", + "hgnc_id": "HGNC:29286", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6C", + "hgnc_id": "HGNC:32206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6D", + "hgnc_id": "HGNC:32204", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ADAMTS7", + "hgnc_id": "HGNC:223", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EFL1", + "hgnc_id": "HGNC:25789", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SAXO2", + "hgnc_id": "HGNC:33727", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L10", + "hgnc_id": "HGNC:37228", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GOLGA6L9", + "hgnc_id": "HGNC:37229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "WHAMM", + "hgnc_id": "HGNC:30493", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4F4", + "hgnc_id": "HGNC:8301", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MPG", + "hgnc_id": "HGNC:7211", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HBM", + "hgnc_id": "HGNC:4826", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AXIN1", + "hgnc_id": "HGNC:903", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSTR5", + "hgnc_id": "HGNC:11334", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TPSD1", + "hgnc_id": "HGNC:14118", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GNPTG", + "hgnc_id": "HGNC:23026", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UNKL", + "hgnc_id": "HGNC:14184", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PKD1", + "hgnc_id": "HGNC:9008", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PDPK1", + "hgnc_id": "HGNC:8816", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "EEF2KMT", + "hgnc_id": "HGNC:32221", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NPIPA3", + "hgnc_id": "HGNC:41978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPA2", + "hgnc_id": "HGNC:41979", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NOMO1", + "hgnc_id": "HGNC:30060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDXDC1", + "hgnc_id": "HGNC:28995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RRN3", + "hgnc_id": "HGNC:30346", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPA5", + "hgnc_id": "HGNC:41980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MPV17L", + "hgnc_id": "HGNC:26827", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NOMO3", + "hgnc_id": "HGNC:25242", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPA6", + "hgnc_id": "HGNC:27893", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPA8", + "hgnc_id": "HGNC:41983", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPA9", + "hgnc_id": "HGNC:41984", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NOMO2", + "hgnc_id": "HGNC:22652", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SMG1", + "hgnc_id": "HGNC:30045", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IQCK", + "hgnc_id": "HGNC:28556", + "reasons": [ + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACSM5", + "hgnc_id": "HGNC:26060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACSM2B", + "hgnc_id": "HGNC:30931", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPB3", + "hgnc_id": "HGNC:28989", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB4", + "hgnc_id": "HGNC:41985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB5", + "hgnc_id": "HGNC:37233", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB6", + "hgnc_id": "HGNC:37454", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB7", + "hgnc_id": "HGNC:33832", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A2", + "hgnc_id": "HGNC:11454", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB8", + "hgnc_id": "HGNC:37490", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EIF3C", + "hgnc_id": "HGNC:3279", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB9", + "hgnc_id": "HGNC:41987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB11", + "hgnc_id": "HGNC:37453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BOLA2-SMG1P6", + "hgnc_id": "HGNC:53563", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BOLA2", + "hgnc_id": "HGNC:29488", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLX1B", + "hgnc_id": "HGNC:28748", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SULT1A4", + "hgnc_id": "HGNC:30004", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NPIPB12", + "hgnc_id": "HGNC:37491", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLX1A", + "hgnc_id": "HGNC:20922", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A3", + "hgnc_id": "HGNC:11455", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB13", + "hgnc_id": "HGNC:41989", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PHKG2", + "hgnc_id": "HGNC:8931", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "VKORC1", + "hgnc_id": "HGNC:23663", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRABD5", + "hgnc_id": "HGNC:26987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TP53TG3", + "hgnc_id": "HGNC:30759", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TP53TG3C", + "hgnc_id": "HGNC:42962", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "TP53TG3E", + "hgnc_id": "HGNC:51816", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TP53TG3B", + "hgnc_id": "HGNC:37202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TP53TG3F", + "hgnc_id": "HGNC:51817", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "CCNYL1B", + "hgnc_id": "HGNC:56313", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CES1", + "hgnc_id": "HGNC:1863", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MT1M", + "hgnc_id": "HGNC:14296", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MATCAP1", + "hgnc_id": "HGNC:34408", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD11B2", + "hgnc_id": "HGNC:5209", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLEC18A", + "hgnc_id": "HGNC:30388", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PDPR", + "hgnc_id": "HGNC:30264", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EXOSC6", + "hgnc_id": "HGNC:19055", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HPR", + "hgnc_id": "HGNC:5156", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB15", + "hgnc_id": "HGNC:34409", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CLEC18B", + "hgnc_id": "HGNC:33849", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CA5A", + "hgnc_id": "HGNC:1377", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF469", + "hgnc_id": "HGNC:23216", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH15", + "hgnc_id": "HGNC:1754", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANKRD11", + "hgnc_id": "HGNC:21316", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MC1R", + "hgnc_id": "HGNC:6929", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PRDM7", + "hgnc_id": "HGNC:9351", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ABR", + "hgnc_id": "HGNC:81", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SERPINF2", + "hgnc_id": "HGNC:9075", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SRR", + "hgnc_id": "HGNC:14398", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1D5", + "hgnc_id": "HGNC:8186", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "P2RX5", + "hgnc_id": "HGNC:8536", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ALOX15", + "hgnc_id": "HGNC:433", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ENO3", + "hgnc_id": "HGNC:3354", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP6", + "hgnc_id": "HGNC:12629", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35G6", + "hgnc_id": "HGNC:31351", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "ALOXE3", + "hgnc_id": "HGNC:13743", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HES7", + "hgnc_id": "HGNC:15977", + "reasons": [ + "CMRG" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CDRT15", + "hgnc_id": "HGNC:14395", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "FBXW10B", + "hgnc_id": "HGNC:14379", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM16", + "hgnc_id": "HGNC:17241", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF286A-TBC1D26", + "hgnc_id": "HGNC:55384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF286A", + "hgnc_id": "HGNC:13501", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D26", + "hgnc_id": "HGNC:28745", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UBB", + "hgnc_id": "HGNC:12463", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "ATPAF2", + "hgnc_id": "HGNC:18802", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EVPLL", + "hgnc_id": "HGNC:35236", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS9C", + "hgnc_id": "HGNC:33874", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D28", + "hgnc_id": "HGNC:26858", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXO3B", + "hgnc_id": "HGNC:3822", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FBXW10", + "hgnc_id": "HGNC:1211", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRAP", + "hgnc_id": "HGNC:4562", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GRAPL", + "hgnc_id": "HGNC:37240", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LGALS9B", + "hgnc_id": "HGNC:24842", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDRT15L2", + "hgnc_id": "HGNC:34075", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MEIS3P2", + "hgnc_id": "HGNC:17638", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAP2K3", + "hgnc_id": "HGNC:6843", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KCNJ18", + "hgnc_id": "HGNC:39080", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS9", + "hgnc_id": "HGNC:6570", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXN1", + "hgnc_id": "HGNC:12765", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SUZ12", + "hgnc_id": "HGNC:17101", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37B", + "hgnc_id": "HGNC:29070", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SLC35G3", + "hgnc_id": "HGNC:26848", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CCL4", + "hgnc_id": "HGNC:10630", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3B", + "hgnc_id": "HGNC:27011", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCL4L2", + "hgnc_id": "HGNC:24066", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "TBC1D3I", + "hgnc_id": "HGNC:32709", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TBC1D3G", + "hgnc_id": "HGNC:29860", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TBC1D3F", + "hgnc_id": "HGNC:18257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3K", + "hgnc_id": "HGNC:51245", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TBC1D3L", + "hgnc_id": "HGNC:51246", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D3D", + "hgnc_id": "HGNC:28944", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D3C", + "hgnc_id": "HGNC:24889", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TBC1D3E", + "hgnc_id": "HGNC:27071", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MRPL45", + "hgnc_id": "HGNC:16651", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP1-1", + "hgnc_id": "HGNC:16772", + "reasons": [ + "CMRG" + ], + "share": 0.1659, + "status": "not read" + }, + { + "gene": "KRTAP4-7", + "hgnc_id": "HGNC:18898", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP4-8", + "hgnc_id": "HGNC:17230", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP4-16", + "hgnc_id": "HGNC:18921", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRTAP4-9", + "hgnc_id": "HGNC:18910", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP4-12", + "hgnc_id": "HGNC:16776", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP9-7", + "hgnc_id": "HGNC:18915", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "KRT14", + "hgnc_id": "HGNC:6416", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRT16", + "hgnc_id": "HGNC:6423", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRT17", + "hgnc_id": "HGNC:6427", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HSD17B1", + "hgnc_id": "HGNC:5210", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "G6PC3", + "hgnc_id": "HGNC:24861", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ARHGAP27", + "hgnc_id": "HGNC:31813", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLEKHM1", + "hgnc_id": "HGNC:29017", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ARL17B", + "hgnc_id": "HGNC:32387", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37A", + "hgnc_id": "HGNC:29069", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARL17A", + "hgnc_id": "HGNC:24096", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NSF", + "hgnc_id": "HGNC:8016", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPEPPS", + "hgnc_id": "HGNC:7900", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HOXB8", + "hgnc_id": "HGNC:5119", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GIP", + "hgnc_id": "HGNC:4270", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHX40", + "hgnc_id": "HGNC:18018", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPS6KB1", + "hgnc_id": "HGNC:10436", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2A", + "hgnc_id": "HGNC:25755", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TLK2", + "hgnc_id": "HGNC:11842", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CSH2", + "hgnc_id": "HGNC:2441", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "GH2", + "hgnc_id": "HGNC:4262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CSH1", + "hgnc_id": "HGNC:2440", + "reasons": [ + "segdup" + ], + "share": 0.6663, + "status": "not read" + }, + { + "gene": "CSHL1", + "hgnc_id": "HGNC:2442", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "LRRC37A3", + "hgnc_id": "HGNC:32427", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C17orf58", + "hgnc_id": "HGNC:27568", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "AMZ2", + "hgnc_id": "HGNC:28041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMC6", + "hgnc_id": "HGNC:18021", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CANT1", + "hgnc_id": "HGNC:19721", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RNF213", + "hgnc_id": "HGNC:14539", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ARL16", + "hgnc_id": "HGNC:27902", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8B", + "hgnc_id": "HGNC:24983", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYMS", + "hgnc_id": "HGNC:12441", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC35G4", + "hgnc_id": "HGNC:31043", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "POTEC", + "hgnc_id": "HGNC:33894", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ANKRD30B", + "hgnc_id": "HGNC:24165", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC25A52", + "hgnc_id": "HGNC:23324", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KATNAL2", + "hgnc_id": "HGNC:25387", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SERPINB4", + "hgnc_id": "HGNC:10570", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERPINB3", + "hgnc_id": "HGNC:10569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALR1", + "hgnc_id": "HGNC:4132", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CTDP1", + "hgnc_id": "HGNC:2498", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "BSG", + "hgnc_id": "HGNC:1116", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN2", + "hgnc_id": "HGNC:4846", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KISS1R", + "hgnc_id": "HGNC:4510", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "STK11", + "hgnc_id": "HGNC:11389", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TCF3", + "hgnc_id": "HGNC:11633", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ABHD17A", + "hgnc_id": "HGNC:28756", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "GIPC3", + "hgnc_id": "HGNC:18183", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBXA2R", + "hgnc_id": "HGNC:11608", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CREB3L3", + "hgnc_id": "HGNC:18855", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RFX2", + "hgnc_id": "HGNC:9983", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "C3", + "hgnc_id": "HGNC:1318", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MBD3L2B", + "hgnc_id": "HGNC:53435", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MBD3L4", + "hgnc_id": "HGNC:37206", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBD3L2", + "hgnc_id": "HGNC:18532", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "STXBP2", + "hgnc_id": "HGNC:11445", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TYK2", + "hgnc_id": "HGNC:12440", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF799", + "hgnc_id": "HGNC:28071", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF443", + "hgnc_id": "HGNC:20878", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP4F3", + "hgnc_id": "HGNC:2646", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4F12", + "hgnc_id": "HGNC:18857", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10H2", + "hgnc_id": "HGNC:8173", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CALR3", + "hgnc_id": "HGNC:20407", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MYO9B", + "hgnc_id": "HGNC:7609", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "INSL3", + "hgnc_id": "HGNC:6086", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF738", + "hgnc_id": "HGNC:32469", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF100", + "hgnc_id": "HGNC:12880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF208", + "hgnc_id": "HGNC:12999", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF676", + "hgnc_id": "HGNC:20429", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF729", + "hgnc_id": "HGNC:32464", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF492", + "hgnc_id": "HGNC:23707", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF728", + "hgnc_id": "HGNC:32463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHPN2", + "hgnc_id": "HGNC:19974", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GPI", + "hgnc_id": "HGNC:4458", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF302", + "hgnc_id": "HGNC:13848", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF181", + "hgnc_id": "HGNC:12971", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FFAR3", + "hgnc_id": "HGNC:4499", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "COX6B1", + "hgnc_id": "HGNC:2280", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS7", + "hgnc_id": "HGNC:6568", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS7B", + "hgnc_id": "HGNC:34447", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "IFNL3", + "hgnc_id": "HGNC:18365", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IFNL4", + "hgnc_id": "HGNC:44480", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "IFNL2", + "hgnc_id": "HGNC:18364", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CYP2A7", + "hgnc_id": "HGNC:2611", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CYP2G1P", + "hgnc_id": "HGNC:2633", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A13", + "hgnc_id": "HGNC:2608", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CYP2F1", + "hgnc_id": "HGNC:2632", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG8", + "hgnc_id": "HGNC:9525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG1", + "hgnc_id": "HGNC:9514", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PSG6", + "hgnc_id": "HGNC:9523", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PSG7", + "hgnc_id": "HGNC:9524", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSG11", + "hgnc_id": "HGNC:9516", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PSG2", + "hgnc_id": "HGNC:9519", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG4", + "hgnc_id": "HGNC:9521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG9", + "hgnc_id": "HGNC:9526", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ETHE1", + "hgnc_id": "HGNC:23287", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF285", + "hgnc_id": "HGNC:13079", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "APOC1", + "hgnc_id": "HGNC:607", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOC4", + "hgnc_id": "HGNC:611", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BLOC1S3", + "hgnc_id": "HGNC:20914", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DMPK", + "hgnc_id": "HGNC:2933", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FUT1", + "hgnc_id": "HGNC:4012", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BAX", + "hgnc_id": "HGNC:959", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LHB", + "hgnc_id": "HGNC:6584", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "CGB3", + "hgnc_id": "HGNC:1886", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CGB2", + "hgnc_id": "HGNC:16722", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "CGB1", + "hgnc_id": "HGNC:16721", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "CGB5", + "hgnc_id": "HGNC:16452", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CGB8", + "hgnc_id": "HGNC:16453", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CGB7", + "hgnc_id": "HGNC:16451", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PNKP", + "hgnc_id": "HGNC:9154", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SIGLEC16", + "hgnc_id": "HGNC:24851", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KLK4", + "hgnc_id": "HGNC:6365", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ETFB", + "hgnc_id": "HGNC:3482", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF808", + "hgnc_id": "HGNC:33230", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF600", + "hgnc_id": "HGNC:30951", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF468", + "hgnc_id": "HGNC:33105", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VN1R4", + "hgnc_id": "HGNC:19871", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "FAM90A27P", + "hgnc_id": "HGNC:43617", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF765", + "hgnc_id": "HGNC:25092", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF761", + "hgnc_id": "HGNC:23179", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF813", + "hgnc_id": "HGNC:33257", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRKCG", + "hgnc_id": "HGNC:9402", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBOAT7", + "hgnc_id": "HGNC:15505", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LILRA6", + "hgnc_id": "HGNC:15495", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "LILRB2", + "hgnc_id": "HGNC:6606", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "LAIR1", + "hgnc_id": "HGNC:6477", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAIR2", + "hgnc_id": "HGNC:6478", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LILRB1", + "hgnc_id": "HGNC:6605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KIR3DL3", + "hgnc_id": "HGNC:16312", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KIR2DL3", + "hgnc_id": "HGNC:6331", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KIR2DL1", + "hgnc_id": "HGNC:6329", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KIR2DL4", + "hgnc_id": "HGNC:6332", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KIR3DL1", + "hgnc_id": "HGNC:6338", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KIR3DL2", + "hgnc_id": "HGNC:6339", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NLRP7", + "hgnc_id": "HGNC:22947", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NLRP2", + "hgnc_id": "HGNC:22948", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GP6", + "hgnc_id": "HGNC:14388", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TNNT1", + "hgnc_id": "HGNC:11948", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TNNI3", + "hgnc_id": "HGNC:11947", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL4A", + "hgnc_id": "HGNC:16449", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RFPL4AL1", + "hgnc_id": "HGNC:45147", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZNF419", + "hgnc_id": "HGNC:20648", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF773", + "hgnc_id": "HGNC:30487", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF587B", + "hgnc_id": "HGNC:37142", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF814", + "hgnc_id": "HGNC:33258", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF417", + "hgnc_id": "HGNC:20646", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC27A5", + "hgnc_id": "HGNC:10999", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SIRPB1", + "hgnc_id": "HGNC:15928", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CST4", + "hgnc_id": "HGNC:2476", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CST1", + "hgnc_id": "HGNC:2473", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CST2", + "hgnc_id": "HGNC:2474", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PLAGL2", + "hgnc_id": "HGNC:9047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEMG1", + "hgnc_id": "HGNC:10742", + "reasons": [ + "CMRG" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PLTP", + "hgnc_id": "HGNC:9093", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRNA4", + "hgnc_id": "HGNC:1958", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EEF1A2", + "hgnc_id": "HGNC:3192", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PTK6", + "hgnc_id": "HGNC:9617", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MYT1", + "hgnc_id": "HGNC:7622", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCMTD2", + "hgnc_id": "HGNC:15882", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TPTE", + "hgnc_id": "HGNC:12023", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POTED", + "hgnc_id": "HGNC:23822", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP21-1", + "hgnc_id": "HGNC:18945", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SMIM11", + "hgnc_id": "HGNC:1293", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMIM34", + "hgnc_id": "HGNC:39601", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KCNE1", + "hgnc_id": "HGNC:6240", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CBR3", + "hgnc_id": "HGNC:1549", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NDUFV3", + "hgnc_id": "HGNC:7719", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CBS", + "hgnc_id": "HGNC:1550", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "U2AF1", + "hgnc_id": "HGNC:12453", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SIK1", + "hgnc_id": "HGNC:11142", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSF2BP", + "hgnc_id": "HGNC:5226", + "reasons": [ + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BC12L", + "hgnc_id": "HGNC:4762", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3319, + "status": "not read" + }, + { + "gene": "TRAPPC10", + "hgnc_id": "HGNC:11868", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PWP2", + "hgnc_id": "HGNC:9711", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GATD3", + "hgnc_id": "HGNC:1273", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DNMT3L", + "hgnc_id": "HGNC:2980", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-1", + "hgnc_id": "HGNC:22966", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "KRTAP10-2", + "hgnc_id": "HGNC:22967", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRTAP10-4", + "hgnc_id": "HGNC:20521", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "KRTAP10-5", + "hgnc_id": "HGNC:22969", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "KRTAP10-6", + "hgnc_id": "HGNC:20523", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "KRTAP10-8", + "hgnc_id": "HGNC:20525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-9", + "hgnc_id": "HGNC:22971", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KRTAP10-10", + "hgnc_id": "HGNC:22972", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP12-2", + "hgnc_id": "HGNC:20530", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "KRTAP12-1", + "hgnc_id": "HGNC:20529", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "COL18A1", + "hgnc_id": "HGNC:2195", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR11H1", + "hgnc_id": "HGNC:15404", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEH", + "hgnc_id": "HGNC:133", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCT8L2", + "hgnc_id": "HGNC:15553", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "USP18", + "hgnc_id": "HGNC:12616", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GGTLC3", + "hgnc_id": "HGNC:33426", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TMEM191B", + "hgnc_id": "HGNC:33600", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM246B", + "hgnc_id": "HGNC:54843", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DGCR6", + "hgnc_id": "HGNC:2846", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRODH", + "hgnc_id": "HGNC:9453", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM246C", + "hgnc_id": "HGNC:54842", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "CCDC188", + "hgnc_id": "HGNC:51899", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RTN4R", + "hgnc_id": "HGNC:18601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3FP", + "hgnc_id": "HGNC:24067", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LRRC74B", + "hgnc_id": "HGNC:34301", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM246A", + "hgnc_id": "HGNC:54844", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RIMBP3B", + "hgnc_id": "HGNC:33891", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HIC2", + "hgnc_id": "HGNC:18595", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TMEM191C", + "hgnc_id": "HGNC:33601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GGTLC2", + "hgnc_id": "HGNC:18596", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IGLL1", + "hgnc_id": "HGNC:5870", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRICH1", + "hgnc_id": "HGNC:28031", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTT2B", + "hgnc_id": "HGNC:33437", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DDTL", + "hgnc_id": "HGNC:33446", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GSTT2", + "hgnc_id": "HGNC:4642", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL1", + "hgnc_id": "HGNC:9977", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C22orf42", + "hgnc_id": "HGNC:27160", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RFPL3", + "hgnc_id": "HGNC:9980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3D", + "hgnc_id": "HGNC:17354", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOBEC3H", + "hgnc_id": "HGNC:24100", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ATF4", + "hgnc_id": "HGNC:786", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NDUFA6", + "hgnc_id": "HGNC:7690", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2D7", + "hgnc_id": "HGNC:2624", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RRP7A", + "hgnc_id": "HGNC:24286", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYB5R3", + "hgnc_id": "HGNC:2873", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "A4GALT", + "hgnc_id": "HGNC:18149", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TTLL1", + "hgnc_id": "HGNC:1312", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MLC1", + "hgnc_id": "HGNC:17082", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBGCP6", + "hgnc_id": "HGNC:18127", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACR", + "hgnc_id": "HGNC:126", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "XG", + "hgnc_id": "HGNC:12806", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GYG2", + "hgnc_id": "HGNC:4700", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ARSD", + "hgnc_id": "HGNC:717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MXRA5", + "hgnc_id": "HGNC:7539", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRKX", + "hgnc_id": "HGNC:9441", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "VCX3A", + "hgnc_id": "HGNC:18159", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "VCX2", + "hgnc_id": "HGNC:18158", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VCX3B", + "hgnc_id": "HGNC:31838", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM9A", + "hgnc_id": "HGNC:18403", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM9C", + "hgnc_id": "HGNC:18405", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SUPT20HL2", + "hgnc_id": "HGNC:31797", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SUPT20HL1", + "hgnc_id": "HGNC:30773", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "MAGEB6", + "hgnc_id": "HGNC:23796", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DCAF8L1", + "hgnc_id": "HGNC:31810", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM47B", + "hgnc_id": "HGNC:26659", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "H2AL1Q", + "hgnc_id": "HGNC:53959", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "FAM47C", + "hgnc_id": "HGNC:25301", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "H2AL3", + "hgnc_id": "HGNC:53960", + "reasons": [ + "segdup" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "SPACA5B", + "hgnc_id": "HGNC:19142", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SSX5", + "hgnc_id": "HGNC:11339", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SSX1", + "hgnc_id": "HGNC:11335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSX3", + "hgnc_id": "HGNC:11337", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SSX4", + "hgnc_id": "HGNC:11338", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX4B", + "hgnc_id": "HGNC:16880", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12J", + "hgnc_id": "HGNC:17778", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE13", + "hgnc_id": "HGNC:29081", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE2E", + "hgnc_id": "HGNC:31960", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE12B", + "hgnc_id": "HGNC:26779", + "reasons": [ + "segdup" + ], + "share": 0.4957, + "status": "not read" + }, + { + "gene": "GAGE12C", + "hgnc_id": "HGNC:28402", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GAGE12D", + "hgnc_id": "HGNC:31904", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12F", + "hgnc_id": "HGNC:31906", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE12G", + "hgnc_id": "HGNC:31907", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE12H", + "hgnc_id": "HGNC:31908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE2A", + "hgnc_id": "HGNC:4099", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GAGE1", + "hgnc_id": "HGNC:4098", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CENPVL3", + "hgnc_id": "HGNC:43880", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CENPVL1", + "hgnc_id": "HGNC:31851", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MAGED4B", + "hgnc_id": "HGNC:22880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MAGED4", + "hgnc_id": "HGNC:23793", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE1A", + "hgnc_id": "HGNC:4111", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "XAGE1B", + "hgnc_id": "HGNC:25400", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX7", + "hgnc_id": "HGNC:19653", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSX2B", + "hgnc_id": "HGNC:22263", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN5", + "hgnc_id": "HGNC:33178", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "XAGE5", + "hgnc_id": "HGNC:30930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE3", + "hgnc_id": "HGNC:14618", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM156B", + "hgnc_id": "HGNC:31962", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM156A", + "hgnc_id": "HGNC:30114", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPIN2B", + "hgnc_id": "HGNC:33147", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPIN2A", + "hgnc_id": "HGNC:20694", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZXDA", + "hgnc_id": "HGNC:13198", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CXorf49", + "hgnc_id": "HGNC:30891", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CXorf49B", + "hgnc_id": "HGNC:34229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "DMRTC1B", + "hgnc_id": "HGNC:31686", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM236D", + "hgnc_id": "HGNC:52642", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DMRTC1", + "hgnc_id": "HGNC:13910", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM236C", + "hgnc_id": "HGNC:52641", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM236A", + "hgnc_id": "HGNC:44268", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PABPC1L2B", + "hgnc_id": "HGNC:31852", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PABPC1L2A", + "hgnc_id": "HGNC:27989", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TGIF2LX", + "hgnc_id": "HGNC:18570", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDH11X", + "hgnc_id": "HGNC:8656", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TCEAL6", + "hgnc_id": "HGNC:24553", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "TCP11X1", + "hgnc_id": "HGNC:48369", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NXF2", + "hgnc_id": "HGNC:8072", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NXF2B", + "hgnc_id": "HGNC:23984", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB40AL", + "hgnc_id": "HGNC:25410", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TMSB15B", + "hgnc_id": "HGNC:28612", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BW1", + "hgnc_id": "HGNC:27252", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "H2BW2", + "hgnc_id": "HGNC:27867", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "TMSB15C", + "hgnc_id": "HGNC:55173", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PSMD10", + "hgnc_id": "HGNC:9555", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CT47C1", + "hgnc_id": "HGNC:53820", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHOXF2B", + "hgnc_id": "HGNC:33519", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RHOXF2", + "hgnc_id": "HGNC:30011", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT47B1", + "hgnc_id": "HGNC:33293", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A12", + "hgnc_id": "HGNC:33292", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT47A11", + "hgnc_id": "HGNC:27397", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CT47A9", + "hgnc_id": "HGNC:33290", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A8", + "hgnc_id": "HGNC:33289", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "CT47A7", + "hgnc_id": "HGNC:33288", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT47A6", + "hgnc_id": "HGNC:33287", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A5", + "hgnc_id": "HGNC:33286", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CT47A4", + "hgnc_id": "HGNC:33285", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT47A2", + "hgnc_id": "HGNC:33283", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A1", + "hgnc_id": "HGNC:33282", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "RTL8B", + "hgnc_id": "HGNC:33156", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "RTL8C", + "hgnc_id": "HGNC:2569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RTL8A", + "hgnc_id": "HGNC:24514", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "SMIM10L2B", + "hgnc_id": "HGNC:34500", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT55", + "hgnc_id": "HGNC:26047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ETDA", + "hgnc_id": "HGNC:53449", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "SMIM10L2A", + "hgnc_id": "HGNC:34499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT45A1", + "hgnc_id": "HGNC:33267", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A3", + "hgnc_id": "HGNC:33268", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT45A5", + "hgnc_id": "HGNC:33270", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT45A2", + "hgnc_id": "HGNC:28400", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT45A7", + "hgnc_id": "HGNC:51260", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CT45A8", + "hgnc_id": "HGNC:51261", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT45A9", + "hgnc_id": "HGNC:51262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A10", + "hgnc_id": "HGNC:51263", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SAGE1", + "hgnc_id": "HGNC:30369", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPANXC", + "hgnc_id": "HGNC:14331", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXA1", + "hgnc_id": "HGNC:11218", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "SPANXA2", + "hgnc_id": "HGNC:14328", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "SPANXD", + "hgnc_id": "HGNC:14332", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "MAGEC1", + "hgnc_id": "HGNC:6812", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPANXN3", + "hgnc_id": "HGNC:33176", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UBE2NL", + "hgnc_id": "HGNC:31710", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN1", + "hgnc_id": "HGNC:33174", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CXorf51B", + "hgnc_id": "HGNC:42787", + "reasons": [ + "segdup" + ], + "share": 0.3323, + "status": "not read" + }, + { + "gene": "CXorf51A", + "hgnc_id": "HGNC:30533", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "EOLA1", + "hgnc_id": "HGNC:28089", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HSFX3", + "hgnc_id": "HGNC:52395", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HSFX2", + "hgnc_id": "HGNC:32701", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSFX1", + "hgnc_id": "HGNC:29603", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MAGEA9", + "hgnc_id": "HGNC:6807", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "EOLA2", + "hgnc_id": "HGNC:17402", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSFX4", + "hgnc_id": "HGNC:52398", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAGEA3", + "hgnc_id": "HGNC:6801", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MAGEA2B", + "hgnc_id": "HGNC:19340", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CSAG1", + "hgnc_id": "HGNC:24294", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "MAGEA12", + "hgnc_id": "HGNC:6799", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "MAGEA2", + "hgnc_id": "HGNC:6800", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CSAG3", + "hgnc_id": "HGNC:26237", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MAGEA6", + "hgnc_id": "HGNC:6804", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PWWP4", + "hgnc_id": "HGNC:55197", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A8", + "hgnc_id": "HGNC:11055", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BCAP31", + "hgnc_id": "HGNC:16695", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OPN1LW", + "hgnc_id": "HGNC:9936", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OPN1MW", + "hgnc_id": "HGNC:4206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OPN1MW2", + "hgnc_id": "HGNC:26952", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEX28", + "hgnc_id": "HGNC:2563", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAG1A", + "hgnc_id": "HGNC:24198", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "CTAG1B", + "hgnc_id": "HGNC:2491", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAG2", + "hgnc_id": "HGNC:2492", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H2AB1", + "hgnc_id": "HGNC:22516", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "F8A1", + "hgnc_id": "HGNC:3547", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "F8A2", + "hgnc_id": "HGNC:31849", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "F8A3", + "hgnc_id": "HGNC:31850", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "H2AB3", + "hgnc_id": "HGNC:14455", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IL9R", + "hgnc_id": "HGNC:6030", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASH6P", + "hgnc_id": "HGNC:31685", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "TGIF2LY", + "hgnc_id": "HGNC:18569", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "PCDH11Y", + "hgnc_id": "HGNC:15813", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY2", + "hgnc_id": "HGNC:23924", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY4", + "hgnc_id": "HGNC:37287", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY8", + "hgnc_id": "HGNC:37471", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY3", + "hgnc_id": "HGNC:33876", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY1", + "hgnc_id": "HGNC:12381", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY9", + "hgnc_id": "HGNC:37472", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY10", + "hgnc_id": "HGNC:37473", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY", + "hgnc_id": "HGNC:12668", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY1B", + "hgnc_id": "HGNC:31751", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2B", + "hgnc_id": "HGNC:23921", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2A", + "hgnc_id": "HGNC:1810", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY1", + "hgnc_id": "HGNC:18568", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY2", + "hgnc_id": "HGNC:23950", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1B", + "hgnc_id": "HGNC:23914", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1A1", + "hgnc_id": "HGNC:9912", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1D", + "hgnc_id": "HGNC:23915", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1E", + "hgnc_id": "HGNC:23916", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1F", + "hgnc_id": "HGNC:23974", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1J", + "hgnc_id": "HGNC:23917", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2", + "hgnc_id": "HGNC:13508", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ1", + "hgnc_id": "HGNC:2682", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ2", + "hgnc_id": "HGNC:15964", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1B", + "hgnc_id": "HGNC:23920", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2B", + "hgnc_id": "HGNC:25449", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ3", + "hgnc_id": "HGNC:15965", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ4", + "hgnc_id": "HGNC:15966", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2C", + "hgnc_id": "HGNC:18225", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1", + "hgnc_id": "HGNC:1809", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "C4B_2", + "hgnc_id": "HGNC:42398", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTT1", + "hgnc_id": "HGNC:4641", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTTP2", + "hgnc_id": "HGNC:33606", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GTF2H2C_2", + "hgnc_id": "HGNC:35418", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB3", + "hgnc_id": "HGNC:4951", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB4", + "hgnc_id": "HGNC:4952", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL2", + "hgnc_id": "HGNC:6330", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5A", + "hgnc_id": "HGNC:16345", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5B", + "hgnc_id": "HGNC:16346", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS1", + "hgnc_id": "HGNC:6333", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS2", + "hgnc_id": "HGNC:6334", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS3", + "hgnc_id": "HGNC:6335", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS5", + "hgnc_id": "HGNC:6337", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR3DS1", + "hgnc_id": "HGNC:6340", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "LILRA3", + "hgnc_id": "HGNC:6604", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U8", + "hgnc_id": "HGNC:27538", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U9", + "hgnc_id": "HGNC:29166", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR9G9", + "hgnc_id": "HGNC:31940", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TAS2R45", + "hgnc_id": "HGNC:18876", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + } + ], + "not_read_owned": [ + { + "gene": "RHCE", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "AMY1A", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AMY1B", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AMY1C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GSTM1", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FCGR1A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GBA1", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FCGR2A", + "owner": "parascopy", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FCGR2C", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "FCGR3B", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR3", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR1", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PROS1", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B17", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT2B15", + "owner": "pypgx", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPE", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPB", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GTF2H2C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMN2", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMN1", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2H2", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSPA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "C4A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "C4B", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP21A2", + "owner": "cn_loci:CYP21A2", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LPA", + "owner": "cn_loci:LPA_KIV2", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PMS2", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2I", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DEFA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CEL", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC5B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KLRC2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KRT6B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRT6C", + "owner": "parascopy", + "share": 0.1665, + "status": "not read" + }, + { + "gene": "KRT6A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRFAM7A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "STRC", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SORD", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HBA2", + "owner": "cn_loci:HBA", + "share": 0.5, + "status": "not read" + }, + { + "gene": "HBA1", + "owner": "cn_loci:HBA", + "share": 0.6659, + "status": "not read" + }, + { + "gene": "TPSB2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TPSAB1", + "owner": "parascopy", + "share": 0.3331, + "status": "not read" + }, + { + "gene": "OTOA", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SULT1A1", + "owner": "pypgx", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HYDIN", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HP", + "owner": "cn_loci:HP", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CCL3L3", + "owner": "cn_loci:CCL3L1", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A6", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP2B6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD177", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "APOBEC3B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP2D6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + } + ], + "not_on_reference": [ + "TGIF2LY", + "PCDH11Y", + "TSPY2", + "TSPY4", + "TSPY8", + "TSPY3", + "TSPY1", + "TSPY9", + "TSPY10", + "VCY", + "VCY1B", + "CDY2B", + "CDY2A", + "HSFY1", + "HSFY2", + "RBMY1B", + "RBMY1A1", + "RBMY1D", + "RBMY1E", + "RBMY1F", + "RBMY1J", + "BPY2", + "DAZ1", + "DAZ2", + "CDY1B", + "BPY2B", + "DAZ3", + "DAZ4", + "BPY2C", + "CDY1", + "C4B_2", + "GSTT1", + "GSTTP2", + "GTF2H2C_2", + "HLA-DRB3", + "HLA-DRB4", + "KIR2DL2", + "KIR2DL5A", + "KIR2DL5B", + "KIR2DS1", + "KIR2DS2", + "KIR2DS3", + "KIR2DS5", + "KIR3DS1", + "LILRA3", + "OR8U8", + "OR8U9", + "OR9G9", + "TAS2R45" + ] + } + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "ok", + "source": "indexcov/indexcov-indexcov.ped", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "inferred_sex": "male", + "cn_chrX": "1.01", + "cn_chrY": "0.98", + "karyotype": null + } + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "declared_sex": "male", + "mosaic_call": "not_detected", + "mosaic_depth_ratio": "0.98", + "mosaic_depth_ratio_expected": "1", + "mosaic_loss_depth": "0.02", + "mosaic_loss_depth_low": "-0.01", + "mosaic_loss_depth_high": "0.05", + "mosaic_loss_af": "NA", + "mosaic_floor": "0.1", + "mosaic_x_het_sites": "0", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "call": "not_detected", + "depth_ratio": 0.98, + "depth_ratio_expected": 1.0, + "loss_depth": 0.02, + "loss_depth_low": -0.01, + "loss_depth_high": 0.05, + "loss_af": null, + "floor": 0.1, + "x_het_sites": 0, + "loss_af_low": null, + "loss_af_high": null + } + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "ok", + "source": "qc/S_read_count.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "input_reads": 1000, + "primary_records": 1000, + "missing_reads": 0, + "read_count_check": "ok" + } + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "ok", + "source": "vcf/S.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1, + "snps": 0, + "indels": 1 + } + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 8, + "by_stars": { + "4": 0, + "3": 2, + "2": 3, + "1": 1, + "0": 2 + }, + "hits": [ + { + "stars": 4, + "chrom": "chr7", + "pos": 117559590, + "ref": "ATCT", + "alt": "A", + "genotype": "het", + "gene": "CFTR", + "significance": "Pathogenic", + "review_status": "practice guideline", + "combined_allele": ".", + "dp": 30, + "gq": 15, + "low_confidence": true, + "gnomad_exome_af": 0.008, + "gnomad_genome_af": 0.007, + "rsids": [ + "rs113993960" + ], + "af": 0.008, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr17", + "pos": 43057100, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "BRCA1,NBR2", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": "part of a combined allele: chr17:43057100 C>T, chr17:43057102 A>G; unphased, so the parts may sit on different haplotypes", + "dp": 35, + "gq": 60, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr6", + "pos": 26092913, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HFE", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": ".", + "dp": 40, + "gq": 99, + "low_confidence": false, + "gnomad_exome_af": 0.06, + "gnomad_genome_af": 0.05, + "rsids": [ + "rs99", + "rs1800562" + ], + "af": 0.06, + "common": true, + "cdna": { + "check": "clean", + "note": "", + "alt_reads": 20, + "alt_clipped": 0, + "ref_reads": 20, + "ref_clipped": 0, + "construct": null + } + }, + { + "stars": 2, + "chrom": "chr1", + "pos": 45331556, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "MUTYH", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 30, + "gq": 50, + "low_confidence": false, + "gnomad_exome_af": 0.002, + "gnomad_genome_af": 0.0005, + "rsids": [ + "rs36053993" + ], + "af": 0.002, + "common": false, + "cdna": { + "check": "flagged", + "note": "possible cDNA/transgene contamination: check the reads", + "alt_reads": 12, + "alt_clipped": 6, + "ref_reads": 10, + "ref_clipped": 0, + "construct": "yes" + } + }, + { + "stars": 2, + "chrom": "chr16", + "pos": 176700, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HBA1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 2, + "chrom": "chr5", + "pos": 70951946, + "ref": "C", + "alt": "T", + "genotype": "hom", + "gene": "SMN1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr13", + "pos": 20189473, + "ref": "C", + "alt": "T", + "genotype": "one copy, other copy not called ref (1/.)", + "gene": "GJB2", + "significance": "Likely pathogenic", + "review_status": "criteria provided, single submitter", + "combined_allele": "combined allele of chr13:20189473 C>T + chr13:20189475 G>A", + "dp": 5, + "gq": 40, + "low_confidence": true, + "gnomad_exome_af": 0.03, + "gnomad_genome_af": null, + "rsids": [ + "rs80338939" + ], + "af": 0.03, + "common": true, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr3", + "pos": 15645186, + "ref": "G", + "alt": "C", + "genotype": "het", + "gene": "BTD", + "significance": "Pathogenic/Likely pathogenic", + "review_status": "criteria provided, conflicting classifications", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chr2", + "pos": 100, + "ref": "A", + "alt": "G", + "genotype": "het", + "gene": ".", + "significance": ".", + "review_status": ".", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chrX", + "pos": 101398000, + "ref": "C", + "alt": "T", + "genotype": "1", + "gene": "GLA", + "significance": "Pathogenic", + "review_status": "no assertion criteria provided", + "combined_allele": "possibly one allele with chrX:101398004 G>A: check ClinVar for a combined record", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "low_confidence": 2, + "call_floor": { + "GQ": 20, + "DP": 8 + }, + "frequency": { + "source": "vep/S_vep.vcf.gz", + "note": null, + "common_af": 0.01 + }, + "cdna_flagged": 1, + "construct_sites": 1, + "cdna_scan": { + "note": "2 genes with spliced reads", + "loci": [ + { + "chrom": "chr1", + "start": 100, + "end": 200, + "junctions": 2, + "reads": 5, + "junction_list": "A:1-2,A:3-4" + } + ] + }, + "deep_intronic": [ + { + "chrom": "chr7", + "pos": 117639961, + "ref": "C", + "alt": "T", + "gene": "CFTR", + "hgvs": "c.3718-2477C>T", + "clinvar_id": "7129", + "call": "no call", + "detail": "" + } + ] + } + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_on_list": 3292, + "list": "assets/carrier_genes.tsv", + "carriers": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "MUTYH-related polyposis" + }, + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Symptomatic form of HFE-related hemochromatosis" + }, + { + "gene": "CFTR", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 4, + "inheritance": "AR", + "condition": "Cystic fibrosis" + }, + { + "gene": "GJB2", + "variant": "chr13:20189473 C>T", + "genotype": "one copy, other copy not called ref (1/.)", + "significance": "Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Nonsyndromic hearing loss, recessive (DFNB1A)" + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Fanconi anemia" + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 0, + "inheritance": "XL", + "condition": "Fabry disease" + }, + { + "gene": "HBA1", + "variant": "chr16:176700 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Alpha-thalassemia" + }, + { + "gene": "SMN1", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Spinal muscular atrophy", + "hemizygous": "called homozygous inside one SMN1 copy (Parascopy, step 35)" + } + ], + "set_aside": [ + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "significance": "Pathogenic/Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Biotinidase deficiency", + "kind": "hypomorph", + "name": "p.Asp444His (p.Asp424His on NM_001370658.1), partial biotinidase deficiency" + } + ], + "two_hits": { + "genes": [ + { + "gene": "SMN1", + "verdict": "biallelic", + "why": "the variant is called homozygous inside the deletion: it is hemizygous, on the one copy the deletion leaves", + "deletion": "one SMN1 copy (Parascopy, step 35)", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "allele": null, + "condition": "Spinal muscular atrophy", + "inheritance": "AR" + } + ], + "deletions_read": [ + "AnnotSV class 4 and 5 deletions (step 05)", + "step 35's interval table (read depth)", + "Parascopy's SMN1 count (step 35)" + ], + "deletions_not_read": [ + "step 35's HBA1/HBA2 count: not called" + ] + } + } + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "ok", + "source": "pharmcat/S.report.json", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "version": "3.4.0", + "html_report": true + } + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "ok", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "gene": "UGT1A1", + "diplotype": "*1/*28", + "phenotype": "Intermediate", + "status": "non-normal" + } + ], + "non_normal": 1, + "ambiguous": 0, + "not_called": 0, + "unclassified": 0, + "parse_failed": false, + "warnings": [ + "CYP2C19 has two possible phenotypes", + "last" + ] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "ok", + "source": "pypgx/S_pypgx_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_total": 6, + "genes_called": 3, + "cyp2d6": "*1/*2", + "cnv_call": { + "CYP2D6": "Normal", + "CYP2A6": "not called: pypgx failed", + "CYP2B6": "not called: the CYP2D6 depth check failed", + "SLCO1B1": "not called: pypgx gave no copy-number call" + }, + "not_callable": [ + "CYP2E1" + ], + "low_gq": [ + { + "gene": "UGT1A1", + "diplotype": "Indeterminate", + "min_gq": 12, + "weakest": "chr2:1:A:G" + }, + { + "gene": "SLCO1B1", + "diplotype": "*1/*5", + "min_gq": null, + "weakest": "" + } + ], + "cyp2d6_evidence": { + "phenotype": "Normal Metabolizer", + "defining_filter": "PASS", + "quality": "ok" + }, + "depth_check": { + "status": "ok", + "gene_mapq1_share": "0.99", + "body_mapq20_share": "0.95" + }, + "comparison": { + "conflicts": 1, + "one_tool_only": 2, + "concordant": 1 + } + } + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/S_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "*2/*1", + "filter": "PASS" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": 1, + "text": "1 (quality 31, filter PASS)", + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": 3, + "text": "3 (quality 25, filter PASS)" + }, + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "one SMN1 copy: the pattern of an SMA carrier", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "called", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality 50, filter PASS); chr7:5980001-5990000 3 (quality 12, filter LowQual)" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": 1, + "text": "1 (quality 31, filter PASS)" + }, + "agree": "disagree" + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01:01", + "A*01:01:01" + ], + "quality": [ + "60", + "50" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-B", + "alleles": [ + "B*57:01:01:02", + "B*08:01:01,B*08:01:02,B*08:01:03" + ], + "quality": [ + "0", + "0" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1, + 5 + ], + "one_allele_type": false + }, + { + "gene": "HLA-C", + "alleles": [ + "C*07:01:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*05:01:01", + "DQA1*02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01:01", + "DQB1*02:02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*03:01:01", + "DRB1*07:01:01" + ], + "quality": [ + "40", + "nan" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HFE", + "alleles": [ + "HFE*001" + ], + "quality": [ + "20" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true, + "label": "T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the variant calls (ClinVar section)" + } + ], + "derived": { + "coeliac": { + "status": "derived", + "reason": "", + "dq2_5": "present", + "dq2_5_phase": "likely cis (DRB1*03 present)", + "dq2_2": "present", + "dq8": "absent", + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": 0.95, + "low": 0.93, + "high": 0.97 + }, + { + "code": "AFR", + "group": "Africa", + "fraction": 0.05, + "low": null, + "high": null + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": 0.6, + "low": 0.5, + "high": 0.7 + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": null, + "low": null, + "high": null + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": null, + "low": null, + "high": null + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 8, + "deletions": 5, + "duplications": 3, + "significant": 7, + "significant_filtered": 4, + "filtered_chry": 1, + "dropped": { + "q0": 1, + "gap": 1, + "band": 1 + }, + "declared_sex": "male" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "depth": 30.0, + "read_floor": 2000, + "low_reads": true + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 3, + "total_mb": 37.5, + "largest_mb": 27.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + }, + { + "region": "chr15:23000001-50000000", + "mb": 27.0 + } + ], + "masked_mb": 46.5, + "sex_chromosome_segments": 1, + "long_mb": 34.0, + "f_roh": 0.0128, + "relationship": null, + "upd_flag": { + "chrom": "chr15", + "mb": 27.0, + "share": 0.79, + "imprinted": [ + { + "region": "15q11-q13 (SNRPN, UBE3A)", + "maternal_upd": "-", + "paternal_upd": "-", + "pmid": "22237428;27615419" + } + ], + "note": "Long chromosome-focused ROH; investigate possible isodisomy. A deletion, acquired loss of heterozygosity or shared ancestry can give this pattern. Copy neutrality and parental origin are not established; check copy-number results and parents' genotypes. Heterodisomy can leave no ROH." + }, + "scope": { + "source": "sites", + "autosomes": [ + "chr1", + "chr2", + "chr3", + "chr4", + "chr5", + "chr6", + "chr7", + "chr8", + "chr9", + "chr10", + "chr11", + "chr12", + "chr13", + "chr14", + "chr15", + "chr16", + "chr17", + "chr18", + "chr19", + "chr20", + "chr21", + "chr22" + ], + "unmasked_bp": 2658001522, + "partial": false, + "note": "Sites on all 22 autosomes; marker presence does not establish callable breadth or copy neutrality." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*03:01:01/DRB1*07:01:01" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32): *1/*2; Cyrius (step 21): *2/*1" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + }, + { + "hgvs": "m.8344A>G", + "gene": "MT-TK", + "disease": "MERRF (myoclonic epilepsy with ragged-red fibres)", + "clinvar_id": "9579", + "pmids": "2112427", + "note": "Usually heteroplasmic; the level differs between tissues.", + "pos": 8344, + "filter": "weak_evidence", + "af": "0.30" + }, + { + "hgvs": "m.11778G>A", + "gene": "MT-ND4", + "disease": "Leber hereditary optic neuropathy (LHON)", + "clinvar_id": "9708", + "pmids": "3201231", + "note": "Usually homoplasmic; not everyone who carries it loses vision.", + "pos": 11778, + "filter": "PASS", + "af": "0.99" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": 0.5, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + }, + { + "bkp5": 100, + "bkp3": 200, + "length": null, + "level_pct": 2.5 + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 7, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [ + { + "variant": "chrM:302 A>AC,ACC", + "af": 0.3 + }, + { + "variant": "chrM:16189 T>C", + "af": 0.5 + } + ], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "30.5", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TET2", + "mean_depth": "40", + "detection_limit_vaf": "0.08", + "status": "covered" + }, + { + "gene": "ASXL1", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + }, + { + "gene": "JAK2", + "mean_depth": "35", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TP53", + "mean_depth": "25", + "detection_limit_vaf": "0.1", + "status": "covered" + } + ], + "median_depth": 32.75, + "median_limit": 0.12901665405107993, + "not_covered": [ + "ASXL1" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "ok", + "source": "cpsr/S.cpsr.grch38.classification.tsv.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "html_report": true, + "classification_column": "CLASSIFICATION", + "classification": { + "Pathogenic": 1, + "Likely_Pathogenic": 1, + "VUS": 1, + "Benign": 1, + ".": 1 + }, + "plp": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "13:32338000 G>A", + "gene": "BRCA2", + "final": "Likely_Pathogenic", + "cpsr": "VUS", + "clinvar": null, + "authority": "CPSR", + "acmg_codes": null, + "dp": 40, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "5:112839000 A>T", + "gene": "APC", + "final": ".", + "cpsr": "Pathogenic", + "clinvar": null, + "authority": null, + "acmg_codes": null, + "dp": 30, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "disagreements": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "final_plp_own_benign": 1, + "own_class_column": "CPSR_CLASSIFICATION", + "low_confidence": 1, + "call_floor_dp": 8, + "cdna_flagged": 1 + } + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 5, + "by_impact": { + "HIGH": 3, + "MODERATE": 1, + ".": 1 + }, + "genes": 3, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "low_confidence": false, + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "low_confidence": false, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "1", + "significance": "Pathogenic", + "stars": 0, + "low_confidence": false, + "rule": "All hemi, het, homozygous P and LP", + "rule_note": null, + "severity": null + } + ], + "clinvar_checked": true, + "high_impact": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "consequence": "stop_gained", + "transcript": "MANE", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": 40, + "dp": 30, + "ad": "15,15", + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 2, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": true, + "structural_note": null, + "not_reported": [ + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "source": "ClinVar", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + }, + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "source": "ClinVar", + "reason": "one heterozygous variant; BTD is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + }, + "frequency_unknown": 1, + "weak_calls": 2 + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_slivar_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 2, + "frequency_unknown": 1, + "compound_het_variants": 4, + "compound_het_genes": 2, + "compound_het_pairs": 26, + "compound_het_phase": { + "trans": 23, + "cis": 1, + "unknown": 1 + }, + "compound_het_trans": [ + { + "gene": "G1", + "variants": "chr1:10 A>G + chr1:15 C>T", + "reads": "1 of 2" + }, + { + "gene": "G2", + "variants": "chr1:20 A>G + chr1:25 C>T", + "reads": "2 of 3" + }, + { + "gene": "G3", + "variants": "chr1:30 A>G + chr1:35 C>T", + "reads": "3 of 4" + }, + { + "gene": "G4", + "variants": "chr1:40 A>G + chr1:45 C>T", + "reads": "4 of 5" + }, + { + "gene": "G5", + "variants": "chr1:50 A>G + chr1:55 C>T", + "reads": "5 of 6" + }, + { + "gene": "G6", + "variants": "chr1:60 A>G + chr1:65 C>T", + "reads": "6 of 7" + }, + { + "gene": "G7", + "variants": "chr1:70 A>G + chr1:75 C>T", + "reads": "7 of 8" + }, + { + "gene": "G8", + "variants": "chr1:80 A>G + chr1:85 C>T", + "reads": "8 of 9" + }, + { + "gene": "G9", + "variants": "chr1:90 A>G + chr1:95 C>T", + "reads": "9 of 10" + }, + { + "gene": "G10", + "variants": "chr1:100 A>G + chr1:105 C>T", + "reads": "10 of 11" + }, + { + "gene": "G11", + "variants": "chr1:110 A>G + chr1:115 C>T", + "reads": "11 of 12" + }, + { + "gene": "G12", + "variants": "chr1:120 A>G + chr1:125 C>T", + "reads": "12 of 13" + }, + { + "gene": "G13", + "variants": "chr1:130 A>G + chr1:135 C>T", + "reads": "13 of 14" + }, + { + "gene": "G14", + "variants": "chr1:140 A>G + chr1:145 C>T", + "reads": "14 of 15" + }, + { + "gene": "G15", + "variants": "chr1:150 A>G + chr1:155 C>T", + "reads": "15 of 16" + }, + { + "gene": "G16", + "variants": "chr1:160 A>G + chr1:165 C>T", + "reads": "16 of 17" + }, + { + "gene": "G17", + "variants": "chr1:170 A>G + chr1:175 C>T", + "reads": "17 of 18" + }, + { + "gene": "G18", + "variants": "chr1:180 A>G + chr1:185 C>T", + "reads": "18 of 19" + }, + { + "gene": "G19", + "variants": "chr1:190 A>G + chr1:195 C>T", + "reads": "19 of 20" + }, + { + "gene": "G20", + "variants": "chr1:200 A>G + chr1:205 C>T", + "reads": "20 of 21" + } + ] + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": "*1/*2", + "Cyrius": "*2/*1" + }, + "agree": true, + "consensus": { + "result": "*1/*2", + "passed_to_pharmcat": true, + "reason": "pypgx and Cyrius agree", + "source": "pgx_consensus/S_pgx_consensus.tsv" + }, + "probable": null + }, + "not_run": [], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)" + ], + "recheck": { + "checked_utc": "2021-03-05 00:00:00 UTC", + "items": [ + { + "item": "clinvar", + "label": "ClinVar", + "class": "newer-release", + "ran_with": "2021-03-01", + "upstream": "2021-03-04", + "action": "rerun step 06" + }, + { + "item": "vep", + "label": "VEP", + "class": "no-newer-release", + "ran_with": null, + "upstream": null, + "action": null + } + ], + "affected_steps": [ + "06" + ], + "rerun": { + "command": "run-all.sh" + } + }, + "new_since": { + "previous_utc": "2021-01-01 00:00 UTC", + "sections": [ + "ancestry_fine", + "ancient", + "annotsv", + "archaic", + "bloodtype", + "callability", + "carrier", + "chip", + "clinical", + "cnvpytor", + "consumer_export", + "couple", + "coverage", + "cpsr", + "cyrius", + "delly", + "exomiser", + "expansions", + "haplogroup", + "hla", + "locityper", + "lpa", + "manta", + "mito", + "mosaic", + "mosaic_sex_loss", + "paralogs", + "parascopy", + "pharmcat", + "read_count", + "roh", + "sample_qc", + "sex_check", + "slivar", + "sv_consensus", + "telomere", + "traits", + "variants", + "viral", + "vntyper", + "y_haplogroup", + "y_str", + "y_tree", + "clinvar", + "cpic", + "pypgx", + "prs" + ] + } +} diff --git a/tests/parity/golden/summary-collect/all-nf/all-nf.json b/tests/parity/golden/summary-collect/all-nf/all-nf.json new file mode 100644 index 000000000..ec0c1438c --- /dev/null +++ b/tests/parity/golden/summary-collect/all-nf/all-nf.json @@ -0,0 +1,14043 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:19 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "female" + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "coverage/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": NaN + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "ok", + "source": "coverage/S_callability.bed.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "min_depth": 10, + "min_mapq": 20, + "min_share": 0.9, + "catalog_genes": 1572, + "read": 217, + "not_read": [ + { + "gene": "OR4F5", + "hgnc_id": "HGNC:14825", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR4F29", + "hgnc_id": "HGNC:31275", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F16", + "hgnc_id": "HGNC:15079", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "AGRN", + "hgnc_id": "HGNC:329", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ATAD3B", + "hgnc_id": "HGNC:24007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MMP23B", + "hgnc_id": "HGNC:7171", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CDK11B", + "hgnc_id": "HGNC:1729", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35E2B", + "hgnc_id": "HGNC:33941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDK11A", + "hgnc_id": "HGNC:1730", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TTC34", + "hgnc_id": "HGNC:34297", + "reasons": [ + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPL22", + "hgnc_id": "HGNC:10315", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MASP2", + "hgnc_id": "HGNC:6902", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPPA", + "hgnc_id": "HGNC:7939", + "reasons": [ + "CMRG" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KIAA2013", + "hgnc_id": "HGNC:28513", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF1", + "hgnc_id": "HGNC:28840", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF11", + "hgnc_id": "HGNC:14086", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL1", + "hgnc_id": "HGNC:29295", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRAMEF4", + "hgnc_id": "HGNC:31971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF10", + "hgnc_id": "HGNC:27997", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRAMEF7", + "hgnc_id": "HGNC:28415", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF6", + "hgnc_id": "HGNC:30583", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF27", + "hgnc_id": "HGNC:51234", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL3", + "hgnc_id": "HGNC:51235", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HNRNPCL2", + "hgnc_id": "HGNC:48813", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF26", + "hgnc_id": "HGNC:49178", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HNRNPCL4", + "hgnc_id": "HGNC:51333", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "PRAMEF9", + "hgnc_id": "HGNC:27996", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF13", + "hgnc_id": "HGNC:13262", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRAMEF18", + "hgnc_id": "HGNC:30693", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRAMEF8", + "hgnc_id": "HGNC:24074", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF33", + "hgnc_id": "HGNC:49193", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PRAMEF15", + "hgnc_id": "HGNC:26764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRAMEF14", + "hgnc_id": "HGNC:13576", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF19", + "hgnc_id": "HGNC:24908", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "PRAMEF17", + "hgnc_id": "HGNC:29485", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CELA2B", + "hgnc_id": "HGNC:29995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CLCNKA", + "hgnc_id": "HGNC:2026", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CLCNKB", + "hgnc_id": "HGNC:2027", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NBPF1", + "hgnc_id": "HGNC:26088", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CROCC", + "hgnc_id": "HGNC:21299", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AKR7L", + "hgnc_id": "HGNC:24056", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF3", + "hgnc_id": "HGNC:25076", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CELA3B", + "hgnc_id": "HGNC:15945", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CELA3A", + "hgnc_id": "HGNC:15944", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HMGCL", + "hgnc_id": "HGNC:5005", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNR2", + "hgnc_id": "HGNC:2160", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RSRP1", + "hgnc_id": "HGNC:25234", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGV", + "hgnc_id": "HGNC:26031", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "XKR8", + "hgnc_id": "HGNC:25508", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BMP8A", + "hgnc_id": "HGNC:21650", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BMP8B", + "hgnc_id": "HGNC:1075", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OXCT2", + "hgnc_id": "HGNC:18606", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CYP4Z1", + "hgnc_id": "HGNC:20583", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4A22", + "hgnc_id": "HGNC:20575", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GBP1", + "hgnc_id": "HGNC:4182", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SETSIP", + "hgnc_id": "HGNC:42937", + "reasons": [ + "segdup" + ], + "share": 0.4994, + "status": "not read" + }, + { + "gene": "AGL", + "hgnc_id": "HGNC:321", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "EXTL2", + "hgnc_id": "HGNC:3516", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AMY2A", + "hgnc_id": "HGNC:477", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF4", + "hgnc_id": "HGNC:26550", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NBPF6", + "hgnc_id": "HGNC:31988", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LRIG2", + "hgnc_id": "HGNC:20889", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "VANGL1", + "hgnc_id": "HGNC:15512", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IGSF3", + "hgnc_id": "HGNC:5950", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SEC22B", + "hgnc_id": "HGNC:10700", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF26", + "hgnc_id": "HGNC:49571", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2NLR", + "hgnc_id": "HGNC:53925", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4A", + "hgnc_id": "HGNC:24369", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM72B", + "hgnc_id": "HGNC:24805", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SRGAP2C", + "hgnc_id": "HGNC:30584", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3-7", + "hgnc_id": "HGNC:32060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPIAL4E", + "hgnc_id": "HGNC:33997", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF15", + "hgnc_id": "HGNC:28791", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4F", + "hgnc_id": "HGNC:33999", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "SRGAP2B", + "hgnc_id": "HGNC:35237", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM72D", + "hgnc_id": "HGNC:33593", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PPIAL4D", + "hgnc_id": "HGNC:33998", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "GPR89A", + "hgnc_id": "HGNC:31984", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBM8A", + "hgnc_id": "HGNC:9905", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "NBPF10", + "hgnc_id": "HGNC:31992", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLA", + "hgnc_id": "HGNC:31862", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPIAL4H", + "hgnc_id": "HGNC:53889", + "reasons": [ + "segdup" + ], + "share": 0.6658, + "status": "not read" + }, + { + "gene": "NBPF12", + "hgnc_id": "HGNC:24297", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF11", + "hgnc_id": "HGNC:31993", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PPIAL4G", + "hgnc_id": "HGNC:33996", + "reasons": [ + "segdup" + ], + "share": 0.1656, + "status": "not read" + }, + { + "gene": "NBPF14", + "hgnc_id": "HGNC:25232", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLB", + "hgnc_id": "HGNC:53923", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NUDT4B", + "hgnc_id": "HGNC:18012", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PDE4DIP", + "hgnc_id": "HGNC:15580", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NOTCH2NLC", + "hgnc_id": "HGNC:53924", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF19", + "hgnc_id": "HGNC:31999", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4C", + "hgnc_id": "HGNC:33995", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "H2BC18", + "hgnc_id": "HGNC:24700", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3C13", + "hgnc_id": "HGNC:25311", + "reasons": [ + "segdup" + ], + "share": 0.8316, + "status": "not read" + }, + { + "gene": "H3C14", + "hgnc_id": "HGNC:20503", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H2AC18", + "hgnc_id": "HGNC:4736", + "reasons": [ + "segdup" + ], + "share": 0.1651, + "status": "not read" + }, + { + "gene": "H2AC19", + "hgnc_id": "HGNC:29668", + "reasons": [ + "segdup" + ], + "share": 0.3328, + "status": "not read" + }, + { + "gene": "H3C15", + "hgnc_id": "HGNC:20505", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H4C15", + "hgnc_id": "HGNC:29607", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HRNR", + "hgnc_id": "HGNC:20846", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LCE3E", + "hgnc_id": "HGNC:29463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCE3D", + "hgnc_id": "HGNC:16615", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "S100A8", + "hgnc_id": "HGNC:10498", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "S100A7A", + "hgnc_id": "HGNC:21657", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "S100A7", + "hgnc_id": "HGNC:10497", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FLAD1", + "hgnc_id": "HGNC:24671", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MTX1", + "hgnc_id": "HGNC:7504", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN3", + "hgnc_id": "HGNC:19183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FCRL5", + "hgnc_id": "HGNC:18508", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA6", + "hgnc_id": "HGNC:5239", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FCGR2B", + "hgnc_id": "HGNC:3618", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD17B7", + "hgnc_id": "HGNC:5215", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD247", + "hgnc_id": "HGNC:1677", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "XCL2", + "hgnc_id": "HGNC:10646", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "PRG4", + "hgnc_id": "HGNC:9364", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PTPRC", + "hgnc_id": "HGNC:9666", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPP1R12B", + "hgnc_id": "HGNC:7619", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KISS1", + "hgnc_id": "HGNC:6341", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SRGAP2", + "hgnc_id": "HGNC:19751", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CR1", + "hgnc_id": "HGNC:2334", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CR1L", + "hgnc_id": "HGNC:2335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZC3H11B", + "hgnc_id": "HGNC:25659", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LBR", + "hgnc_id": "HGNC:6518", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "LEFTY2", + "hgnc_id": "HGNC:3122", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "OR2T33", + "hgnc_id": "HGNC:31255", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T7", + "hgnc_id": "HGNC:15019", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T2", + "hgnc_id": "HGNC:14725", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T3", + "hgnc_id": "HGNC:14727", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR2T5", + "hgnc_id": "HGNC:15017", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "OR2G6", + "hgnc_id": "HGNC:27019", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR2T34", + "hgnc_id": "HGNC:31256", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T11", + "hgnc_id": "HGNC:19574", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T35", + "hgnc_id": "HGNC:31257", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T27", + "hgnc_id": "HGNC:31252", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SNTG2", + "hgnc_id": "HGNC:13741", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TPO", + "hgnc_id": "HGNC:12015", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KLF11", + "hgnc_id": "HGNC:11811", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CCDC121", + "hgnc_id": "HGNC:25833", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "ABCG8", + "hgnc_id": "HGNC:13887", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PCBP1", + "hgnc_id": "HGNC:8647", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "NAT8", + "hgnc_id": "HGNC:18069", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "C2orf78", + "hgnc_id": "HGNC:34349", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CD8B", + "hgnc_id": "HGNC:1707", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RGPD1", + "hgnc_id": "HGNC:32414", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLGLB1", + "hgnc_id": "HGNC:9072", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLGLB2", + "hgnc_id": "HGNC:9073", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RGPD2", + "hgnc_id": "HGNC:32415", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPIA", + "hgnc_id": "HGNC:10297", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEKT4", + "hgnc_id": "HGNC:31012", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAHD2A", + "hgnc_id": "HGNC:24252", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TRIM43B", + "hgnc_id": "HGNC:37146", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM43", + "hgnc_id": "HGNC:19015", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANKRD36C", + "hgnc_id": "HGNC:32946", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPAT2", + "hgnc_id": "HGNC:27168", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD36", + "hgnc_id": "HGNC:24079", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANKRD36B", + "hgnc_id": "HGNC:29333", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "RGPD3", + "hgnc_id": "HGNC:32416", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD4", + "hgnc_id": "HGNC:32417", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RANBP2", + "hgnc_id": "HGNC:9848", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD5", + "hgnc_id": "HGNC:32418", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MALL", + "hgnc_id": "HGNC:6818", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MTLN", + "hgnc_id": "HGNC:27339", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LIMS4", + "hgnc_id": "HGNC:39941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD6", + "hgnc_id": "HGNC:32419", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANAPC1", + "hgnc_id": "HGNC:19988", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD8", + "hgnc_id": "HGNC:9849", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FOXD4L1", + "hgnc_id": "HGNC:18521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RABL2A", + "hgnc_id": "HGNC:9799", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RAB6C", + "hgnc_id": "HGNC:16525", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "POTEF", + "hgnc_id": "HGNC:33905", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CCDC74B", + "hgnc_id": "HGNC:25267", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SMPD4", + "hgnc_id": "HGNC:32949", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3E", + "hgnc_id": "HGNC:20765", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEI", + "hgnc_id": "HGNC:37093", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CFC1B", + "hgnc_id": "HGNC:33983", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CFC1", + "hgnc_id": "HGNC:18292", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "POTEJ", + "hgnc_id": "HGNC:37094", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PLEKHB2", + "hgnc_id": "HGNC:19236", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB6D", + "hgnc_id": "HGNC:30272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MZT2A", + "hgnc_id": "HGNC:33187", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TUBA3D", + "hgnc_id": "HGNC:24071", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CCDC74A", + "hgnc_id": "HGNC:25197", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "ANKRD30BL", + "hgnc_id": "HGNC:35167", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NOSTRIN", + "hgnc_id": "HGNC:20203", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IFT70B", + "hgnc_id": "HGNC:26425", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CASP10", + "hgnc_id": "HGNC:1500", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ALPP", + "hgnc_id": "HGNC:439", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ALPG", + "hgnc_id": "HGNC:441", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT1A6", + "hgnc_id": "HGNC:12538", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UGT1A5", + "hgnc_id": "HGNC:12537", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT1A3", + "hgnc_id": "HGNC:12535", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MLPH", + "hgnc_id": "HGNC:29643", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR6B2", + "hgnc_id": "HGNC:15041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AQP12B", + "hgnc_id": "HGNC:6096", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "AQP12A", + "hgnc_id": "HGNC:19941", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ANO7", + "hgnc_id": "HGNC:31677", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CHL1", + "hgnc_id": "HGNC:1939", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMC3", + "hgnc_id": "HGNC:23999", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANCD2", + "hgnc_id": "HGNC:3585", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DAZL", + "hgnc_id": "HGNC:2685", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF35", + "hgnc_id": "HGNC:13099", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LZTFL1", + "hgnc_id": "HGNC:6741", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MST1", + "hgnc_id": "HGNC:7380", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MST1R", + "hgnc_id": "HGNC:7381", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HYAL1", + "hgnc_id": "HGNC:5320", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FRG2C", + "hgnc_id": "HGNC:33626", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "ZNF717", + "hgnc_id": "HGNC:29448", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR5H6", + "hgnc_id": "HGNC:14767", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ROPN1", + "hgnc_id": "HGNC:17692", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ROPN1B", + "hgnc_id": "HGNC:31927", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RPN1", + "hgnc_id": "HGNC:10381", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ALG1L2", + "hgnc_id": "HGNC:37258", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BFSP2", + "hgnc_id": "HGNC:1041", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRR23B", + "hgnc_id": "HGNC:33764", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SOX2", + "hgnc_id": "HGNC:11195", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "EIF2B5", + "hgnc_id": "HGNC:3261", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KNG1", + "hgnc_id": "HGNC:6383", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TM4SF19", + "hgnc_id": "HGNC:25167", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF595", + "hgnc_id": "HGNC:27196", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDE6B", + "hgnc_id": "HGNC:8786", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAK", + "hgnc_id": "HGNC:4113", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FGFRL1", + "hgnc_id": "HGNC:3693", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RNF212", + "hgnc_id": "HGNC:27729", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UVSSA", + "hgnc_id": "HGNC:29304", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DOK7", + "hgnc_id": "HGNC:26594", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HMX1", + "hgnc_id": "HGNC:5017", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A26", + "hgnc_id": "HGNC:43746", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "USP17L10", + "hgnc_id": "HGNC:44438", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L11", + "hgnc_id": "HGNC:44439", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L12", + "hgnc_id": "HGNC:44440", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L13", + "hgnc_id": "HGNC:44441", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L17", + "hgnc_id": "HGNC:44445", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L18", + "hgnc_id": "HGNC:44446", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L19", + "hgnc_id": "HGNC:44447", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L20", + "hgnc_id": "HGNC:44448", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L21", + "hgnc_id": "HGNC:44449", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L22", + "hgnc_id": "HGNC:44450", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L24", + "hgnc_id": "HGNC:44453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L25", + "hgnc_id": "HGNC:44452", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L26", + "hgnc_id": "HGNC:44454", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L5", + "hgnc_id": "HGNC:37177", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L27", + "hgnc_id": "HGNC:44455", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L28", + "hgnc_id": "HGNC:44456", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L30", + "hgnc_id": "HGNC:44458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB131A", + "hgnc_id": "HGNC:18108", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRD5", + "hgnc_id": "HGNC:3026", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "UGT2B10", + "hgnc_id": "HGNC:12544", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B28", + "hgnc_id": "HGNC:13479", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UGT2A1", + "hgnc_id": "HGNC:12542", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UGT2A2", + "hgnc_id": "HGNC:28183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HTN3", + "hgnc_id": "HGNC:5284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HTN1", + "hgnc_id": "HGNC:5283", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "AFP", + "hgnc_id": "HGNC:317", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGY", + "hgnc_id": "HGNC:28213", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PYURF", + "hgnc_id": "HGNC:44317", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CISD2", + "hgnc_id": "HGNC:24212", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC9B1", + "hgnc_id": "HGNC:24244", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KLHL2", + "hgnc_id": "HGNC:6353", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GK3", + "hgnc_id": "HGNC:4292", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "HPF1", + "hgnc_id": "HGNC:26051", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDLIM3", + "hgnc_id": "HGNC:20767", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAT1", + "hgnc_id": "HGNC:3595", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG1", + "hgnc_id": "HGNC:3954", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FRG2", + "hgnc_id": "HGNC:19136", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DUX4", + "hgnc_id": "HGNC:50800", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SDHA", + "hgnc_id": "HGNC:10680", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDCD6", + "hgnc_id": "HGNC:8765", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZDHHC11B", + "hgnc_id": "HGNC:32962", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZDHHC11", + "hgnc_id": "HGNC:19158", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TERT", + "hgnc_id": "HGNC:11730", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A3", + "hgnc_id": "HGNC:11049", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAF11L2", + "hgnc_id": "HGNC:53845", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L3", + "hgnc_id": "HGNC:53846", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L4", + "hgnc_id": "HGNC:53847", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L5", + "hgnc_id": "HGNC:53848", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "TAF11L7", + "hgnc_id": "HGNC:53850", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAF11L8", + "hgnc_id": "HGNC:53851", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "TAF11L9", + "hgnc_id": "HGNC:53852", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L10", + "hgnc_id": "HGNC:53853", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L11", + "hgnc_id": "HGNC:53854", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L12", + "hgnc_id": "HGNC:53855", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "TAF11L14", + "hgnc_id": "HGNC:53857", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMB", + "hgnc_id": "HGNC:30465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MARVELD2", + "hgnc_id": "HGNC:26401", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERF1B", + "hgnc_id": "HGNC:10756", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NAIP", + "hgnc_id": "HGNC:7634", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SKIC3", + "hgnc_id": "HGNC:23639", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIX1", + "hgnc_id": "HGNC:18581", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SAR1B", + "hgnc_id": "HGNC:10535", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DND1", + "hgnc_id": "HGNC:23799", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDHA10", + "hgnc_id": "HGNC:8664", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PCDHB7", + "hgnc_id": "HGNC:8692", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PCDHB8", + "hgnc_id": "HGNC:8693", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PCDHB11", + "hgnc_id": "HGNC:8682", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PCDHB12", + "hgnc_id": "HGNC:8683", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPP1R2B", + "hgnc_id": "HGNC:16318", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPM1", + "hgnc_id": "HGNC:7910", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "THOC3", + "hgnc_id": "HGNC:19072", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SIMC1", + "hgnc_id": "HGNC:24779", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM153A", + "hgnc_id": "HGNC:29940", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FAM153CP", + "hgnc_id": "HGNC:33936", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FLT4", + "hgnc_id": "HGNC:3767", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR4F3", + "hgnc_id": "HGNC:8300", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "DUSP22", + "hgnc_id": "HGNC:16077", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TUBB2A", + "hgnc_id": "HGNC:12412", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "H2AC13", + "hgnc_id": "HGNC:4725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H2AC14", + "hgnc_id": "HGNC:4727", + "reasons": [ + "segdup" + ], + "share": 0.8319, + "status": "not read" + }, + { + "gene": "OR2J1", + "hgnc_id": "HGNC:8259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2J2", + "hgnc_id": "HGNC:8260", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HLA-G", + "hgnc_id": "HGNC:4964", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HLA-A", + "hgnc_id": "HGNC:4931", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "MICA", + "hgnc_id": "HGNC:7090", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MICB", + "hgnc_id": "HGNC:7091", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZBTB12", + "hgnc_id": "HGNC:19066", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PBX2", + "hgnc_id": "HGNC:8633", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HLA-DRB1", + "hgnc_id": "HGNC:4948", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GSTA2", + "hgnc_id": "HGNC:4627", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GSTA1", + "hgnc_id": "HGNC:4626", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTA5", + "hgnc_id": "HGNC:19662", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRIM2", + "hgnc_id": "HGNC:9370", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FKBP1C", + "hgnc_id": "HGNC:21376", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SLC17A5", + "hgnc_id": "HGNC:10933", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEC63", + "hgnc_id": "HGNC:21082", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR2A4", + "hgnc_id": "HGNC:14729", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE9", + "hgnc_id": "HGNC:37275", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "PCMT1", + "hgnc_id": "HGNC:8728", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RAET1G", + "hgnc_id": "HGNC:16795", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ULBP1", + "hgnc_id": "HGNC:14893", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RAET1L", + "hgnc_id": "HGNC:16798", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ULBP3", + "hgnc_id": "HGNC:14895", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC22A1", + "hgnc_id": "HGNC:10963", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMOC2", + "hgnc_id": "HGNC:20323", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RBAK-RBAKDN", + "hgnc_id": "HGNC:42971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBAK", + "hgnc_id": "HGNC:17680", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC29A4", + "hgnc_id": "HGNC:23097", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FSCN1", + "hgnc_id": "HGNC:11148", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM", + "hgnc_id": "HGNC:8105", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCZ1", + "hgnc_id": "HGNC:21691", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RSPH10B2", + "hgnc_id": "HGNC:34385", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CCZ1B", + "hgnc_id": "HGNC:21717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOD1", + "hgnc_id": "HGNC:16390", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FKBP9", + "hgnc_id": "HGNC:3725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DPY19L1", + "hgnc_id": "HGNC:22205", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SEPTIN7", + "hgnc_id": "HGNC:1717", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE1", + "hgnc_id": "HGNC:16408", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIA", + "hgnc_id": "HGNC:9253", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDC14C", + "hgnc_id": "HGNC:22427", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSPH", + "hgnc_id": "HGNC:9577", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCT6A", + "hgnc_id": "HGNC:1620", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF716", + "hgnc_id": "HGNC:32458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF727", + "hgnc_id": "HGNC:22785", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF735", + "hgnc_id": "HGNC:32466", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF736", + "hgnc_id": "HGNC:32467", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GUSB", + "hgnc_id": "HGNC:4696", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RABGEF1", + "hgnc_id": "HGNC:17676", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TYW1", + "hgnc_id": "HGNC:25598", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE21", + "hgnc_id": "HGNC:51517", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYW1B", + "hgnc_id": "HGNC:33908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POM121", + "hgnc_id": "HGNC:19702", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM74", + "hgnc_id": "HGNC:17453", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPDYE8", + "hgnc_id": "HGNC:33771", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE9", + "hgnc_id": "HGNC:45034", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE10", + "hgnc_id": "HGNC:51506", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NSUN5", + "hgnc_id": "HGNC:16385", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM50", + "hgnc_id": "HGNC:19017", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLIP2", + "hgnc_id": "HGNC:2586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GTF2IRD2", + "hgnc_id": "HGNC:30775", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE12", + "hgnc_id": "HGNC:51508", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CASTOR2", + "hgnc_id": "HGNC:37073", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GTF2IRD2B", + "hgnc_id": "HGNC:33125", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPDYE13", + "hgnc_id": "HGNC:51509", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE14", + "hgnc_id": "HGNC:51510", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM73", + "hgnc_id": "HGNC:18162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POM121C", + "hgnc_id": "HGNC:34005", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPDYE5", + "hgnc_id": "HGNC:35464", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DTX2", + "hgnc_id": "HGNC:15973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UPK3B", + "hgnc_id": "HGNC:21444", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE16", + "hgnc_id": "HGNC:51512", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE17", + "hgnc_id": "HGNC:51513", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE18", + "hgnc_id": "HGNC:51514", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "STEAP1", + "hgnc_id": "HGNC:11378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ASNS", + "hgnc_id": "HGNC:753", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM2", + "hgnc_id": "HGNC:34396", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP3A7-CYP3A51P", + "hgnc_id": "HGNC:51504", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP3A4", + "hgnc_id": "HGNC:2637", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "STAG3", + "hgnc_id": "HGNC:11356", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PVRIG", + "hgnc_id": "HGNC:32190", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SPDYE3", + "hgnc_id": "HGNC:35462", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SAP25", + "hgnc_id": "HGNC:41908", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZAN", + "hgnc_id": "HGNC:12857", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC12", + "hgnc_id": "HGNC:7510", + "reasons": [ + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE6", + "hgnc_id": "HGNC:35465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "POLR2J", + "hgnc_id": "HGNC:9197", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RASA4B", + "hgnc_id": "HGNC:35202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J3", + "hgnc_id": "HGNC:33853", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UPK3BL2", + "hgnc_id": "HGNC:53444", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "RASA4", + "hgnc_id": "HGNC:23181", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UPK3BL1", + "hgnc_id": "HGNC:37278", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "POLR2J3-UPK3BL2", + "hgnc_id": "HGNC:58364", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SPDYE2B", + "hgnc_id": "HGNC:48334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J2", + "hgnc_id": "HGNC:23208", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAMB1", + "hgnc_id": "HGNC:6486", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2B", + "hgnc_id": "HGNC:18272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KLF14", + "hgnc_id": "HGNC:23025", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AKR1B10", + "hgnc_id": "HGNC:382", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1B15", + "hgnc_id": "HGNC:37281", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BRAF", + "hgnc_id": "HGNC:1097", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRBV9", + "hgnc_id": "HGNC:12246", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRSS2", + "hgnc_id": "HGNC:9483", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAGE15", + "hgnc_id": "HGNC:37295", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TCAF2", + "hgnc_id": "HGNC:26878", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TCAF2C", + "hgnc_id": "HGNC:53641", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CTAGE6", + "hgnc_id": "HGNC:28644", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "OR2F2", + "hgnc_id": "HGNC:8247", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CTAGE4", + "hgnc_id": "HGNC:24772", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2A42", + "hgnc_id": "HGNC:31230", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2A7", + "hgnc_id": "HGNC:8234", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE8", + "hgnc_id": "HGNC:37294", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "OR2A1", + "hgnc_id": "HGNC:8229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARHGEF5", + "hgnc_id": "HGNC:13209", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACTR3C", + "hgnc_id": "HGNC:37282", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KMT2C", + "hgnc_id": "HGNC:13726", + "reasons": [ + "CMRG", + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ACTR3B", + "hgnc_id": "HGNC:17256", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F21", + "hgnc_id": "HGNC:19583", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "FBXO25", + "hgnc_id": "HGNC:13596", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARHGEF10", + "hgnc_id": "HGNC:14103", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A15", + "hgnc_id": "HGNC:32263", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A3", + "hgnc_id": "HGNC:32251", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A13", + "hgnc_id": "HGNC:32261", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A5", + "hgnc_id": "HGNC:32253", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM90A20", + "hgnc_id": "HGNC:32268", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L1", + "hgnc_id": "HGNC:37182", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L4", + "hgnc_id": "HGNC:37176", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNF705G", + "hgnc_id": "HGNC:37134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB108C", + "hgnc_id": "HGNC:30847", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB4B", + "hgnc_id": "HGNC:30193", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "DEFB103B", + "hgnc_id": "HGNC:31702", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "DEFB104B", + "hgnc_id": "HGNC:26165", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB106B", + "hgnc_id": "HGNC:28879", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "DEFB105B", + "hgnc_id": "HGNC:29930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "DEFB107B", + "hgnc_id": "HGNC:31918", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRR23D1", + "hgnc_id": "HGNC:49420", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A7", + "hgnc_id": "HGNC:32255", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FAM90A23", + "hgnc_id": "HGNC:32271", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A14", + "hgnc_id": "HGNC:32262", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A18", + "hgnc_id": "HGNC:32266", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A16", + "hgnc_id": "HGNC:32264", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A8", + "hgnc_id": "HGNC:32256", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A17", + "hgnc_id": "HGNC:32265", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A9", + "hgnc_id": "HGNC:32257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A10", + "hgnc_id": "HGNC:32258", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PRR23D2", + "hgnc_id": "HGNC:49396", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB107A", + "hgnc_id": "HGNC:18086", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB105A", + "hgnc_id": "HGNC:18087", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DEFB106A", + "hgnc_id": "HGNC:18088", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPAG11A", + "hgnc_id": "HGNC:33342", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB103A", + "hgnc_id": "HGNC:15967", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DEFB4A", + "hgnc_id": "HGNC:2767", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNF705B", + "hgnc_id": "HGNC:32284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "USP17L8", + "hgnc_id": "HGNC:37181", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L3", + "hgnc_id": "HGNC:37175", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "FAM90A11", + "hgnc_id": "HGNC:32259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A24", + "hgnc_id": "HGNC:32272", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A12", + "hgnc_id": "HGNC:32260", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLC35G5", + "hgnc_id": "HGNC:15546", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "DEFB130B", + "hgnc_id": "HGNC:39814", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF705D", + "hgnc_id": "HGNC:33202", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L2", + "hgnc_id": "HGNC:34434", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB109D", + "hgnc_id": "HGNC:30838", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM86B1", + "hgnc_id": "HGNC:28268", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB130A", + "hgnc_id": "HGNC:18107", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM86B2", + "hgnc_id": "HGNC:32222", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RBPMS", + "hgnc_id": "HGNC:19097", + "reasons": [ + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IMPA1", + "hgnc_id": "HGNC:6050", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC10A5", + "hgnc_id": "HGNC:22981", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH17", + "hgnc_id": "HGNC:1756", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP11B1", + "hgnc_id": "HGNC:2591", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP11B2", + "hgnc_id": "HGNC:2592", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GPIHBP1", + "hgnc_id": "HGNC:24945", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "WASHC1", + "hgnc_id": "HGNC:24361", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOXD4", + "hgnc_id": "HGNC:3805", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNG1A", + "hgnc_id": "HGNC:17134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KCNV2", + "hgnc_id": "HGNC:19698", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RLN2", + "hgnc_id": "HGNC:10027", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RLN1", + "hgnc_id": "HGNC:10026", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "IFNA21", + "hgnc_id": "HGNC:5424", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA4", + "hgnc_id": "HGNC:5425", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "IFNA7", + "hgnc_id": "HGNC:5428", + "reasons": [ + "segdup" + ], + "share": 0.3324, + "status": "not read" + }, + { + "gene": "IFNA10", + "hgnc_id": "HGNC:5418", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "IFNA16", + "hgnc_id": "HGNC:5421", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "IFNA17", + "hgnc_id": "HGNC:5422", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "IFNA13", + "hgnc_id": "HGNC:5419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA1", + "hgnc_id": "HGNC:5417", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "ANKRD18B", + "hgnc_id": "HGNC:23644", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRSS3", + "hgnc_id": "HGNC:9486", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALT", + "hgnc_id": "HGNC:4135", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31F1", + "hgnc_id": "HGNC:41911", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CNTNAP3", + "hgnc_id": "HGNC:13834", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31A1", + "hgnc_id": "HGNC:23394", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L6", + "hgnc_id": "HGNC:31986", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNG1F", + "hgnc_id": "HGNC:31978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNTNAP3B", + "hgnc_id": "HGNC:32035", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31A6", + "hgnc_id": "HGNC:32006", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPATA31A7", + "hgnc_id": "HGNC:32007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CNTNAP3C", + "hgnc_id": "HGNC:53878", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L5", + "hgnc_id": "HGNC:18522", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNG1E", + "hgnc_id": "HGNC:24584", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXD4L4", + "hgnc_id": "HGNC:23762", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF658", + "hgnc_id": "HGNC:25226", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD20A1", + "hgnc_id": "HGNC:23665", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNG1C", + "hgnc_id": "HGNC:18519", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L3", + "hgnc_id": "HGNC:18523", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PGM5", + "hgnc_id": "HGNC:8908", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FXN", + "hgnc_id": "HGNC:3951", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TJP2", + "hgnc_id": "HGNC:11828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPATA31D4", + "hgnc_id": "HGNC:38601", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31D3", + "hgnc_id": "HGNC:38603", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPATA31D1", + "hgnc_id": "HGNC:37283", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "KIF27", + "hgnc_id": "HGNC:18632", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPATA31C1", + "hgnc_id": "HGNC:27846", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPATA31C2", + "hgnc_id": "HGNC:24508", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC71A2", + "hgnc_id": "HGNC:23376", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2G", + "hgnc_id": "HGNC:23449", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CTSV", + "hgnc_id": "HGNC:2538", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR13C5", + "hgnc_id": "HGNC:15100", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR13C2", + "hgnc_id": "HGNC:14701", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR13C9", + "hgnc_id": "HGNC:15104", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "POLE3", + "hgnc_id": "HGNC:13546", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ORM1", + "hgnc_id": "HGNC:8498", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ORM2", + "hgnc_id": "HGNC:8499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SLC27A4", + "hgnc_id": "HGNC:10998", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PKN3", + "hgnc_id": "HGNC:17999", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ABO", + "hgnc_id": "HGNC:79", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCN1", + "hgnc_id": "HGNC:6525", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OBP2A", + "hgnc_id": "HGNC:23380", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SOHLH1", + "hgnc_id": "HGNC:27845", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "INPP5E", + "hgnc_id": "HGNC:21474", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AJM1", + "hgnc_id": "HGNC:37284", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRIN1", + "hgnc_id": "HGNC:4584", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8", + "hgnc_id": "HGNC:20773", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "AKR1C1", + "hgnc_id": "HGNC:384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1C2", + "hgnc_id": "HGNC:385", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DCLRE1C", + "hgnc_id": "HGNC:17642", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRC1", + "hgnc_id": "HGNC:7228", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD30A", + "hgnc_id": "HGNC:17234", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF33A", + "hgnc_id": "HGNC:13096", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF37A", + "hgnc_id": "HGNC:13102", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLD5P1", + "hgnc_id": "HGNC:55072", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF33B", + "hgnc_id": "HGNC:13097", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BMS1", + "hgnc_id": "HGNC:23505", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AGAP4", + "hgnc_id": "HGNC:23459", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23", + "hgnc_id": "HGNC:17312", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ANTXRL", + "hgnc_id": "HGNC:27277", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANXA8L1", + "hgnc_id": "HGNC:23334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPY4R", + "hgnc_id": "HGNC:9329", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPRIN2", + "hgnc_id": "HGNC:23730", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PTPN20", + "hgnc_id": "HGNC:23423", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANXA8", + "hgnc_id": "HGNC:546", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM25G", + "hgnc_id": "HGNC:23590", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP9", + "hgnc_id": "HGNC:23463", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SYT15B", + "hgnc_id": "HGNC:51487", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPY4R2", + "hgnc_id": "HGNC:52383", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PARG", + "hgnc_id": "HGNC:8605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23B", + "hgnc_id": "HGNC:23581", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TIMM23B-AGAP6", + "hgnc_id": "HGNC:45009", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP6", + "hgnc_id": "HGNC:23466", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASHC2A", + "hgnc_id": "HGNC:23416", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ASAH2B", + "hgnc_id": "HGNC:23456", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EIF5AL1", + "hgnc_id": "HGNC:17419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2B", + "hgnc_id": "HGNC:23445", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NUTM2E", + "hgnc_id": "HGNC:23448", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMEM254", + "hgnc_id": "HGNC:25804", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PLAC9", + "hgnc_id": "HGNC:19255", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ADIRF", + "hgnc_id": "HGNC:24043", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NUTM2A", + "hgnc_id": "HGNC:23438", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2D", + "hgnc_id": "HGNC:23447", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PAPSS2", + "hgnc_id": "HGNC:8604", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PTEN", + "hgnc_id": "HGNC:9588", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIPN", + "hgnc_id": "HGNC:23452", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "BTRC", + "hgnc_id": "HGNC:1144", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SFR1", + "hgnc_id": "HGNC:29574", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DHX32", + "hgnc_id": "HGNC:16717", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANK1", + "hgnc_id": "HGNC:23527", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ECHS1", + "hgnc_id": "HGNC:3151", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPRN", + "hgnc_id": "HGNC:16871", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG2B", + "hgnc_id": "HGNC:33518", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SCGB1C1", + "hgnc_id": "HGNC:18394", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFITM3", + "hgnc_id": "HGNC:5414", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DRD4", + "hgnc_id": "HGNC:3025", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEAF1", + "hgnc_id": "HGNC:14677", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP5-2", + "hgnc_id": "HGNC:23597", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP5-4", + "hgnc_id": "HGNC:23599", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MRPL23", + "hgnc_id": "HGNC:10322", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H19", + "hgnc_id": "HGNC:4713", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSU72L5", + "hgnc_id": "HGNC:43624", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "SSU72L2", + "hgnc_id": "HGNC:43621", + "reasons": [ + "segdup" + ], + "share": 0.666, + "status": "not read" + }, + { + "gene": "SSU72L4", + "hgnc_id": "HGNC:43623", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "SSU72L1", + "hgnc_id": "HGNC:43626", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR51A4", + "hgnc_id": "HGNC:14795", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR51A2", + "hgnc_id": "HGNC:14764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HBG1", + "hgnc_id": "HGNC:4831", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR10A5", + "hgnc_id": "HGNC:15131", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRGPRX4", + "hgnc_id": "HGNC:17617", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MDK", + "hgnc_id": "HGNC:6972", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPI1", + "hgnc_id": "HGNC:11241", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C3", + "hgnc_id": "HGNC:14697", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4C5", + "hgnc_id": "HGNC:14702", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4A47", + "hgnc_id": "HGNC:31266", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM51G", + "hgnc_id": "HGNC:43972", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64C", + "hgnc_id": "HGNC:37148", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOLH1", + "hgnc_id": "HGNC:3788", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C13", + "hgnc_id": "HGNC:15169", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4C12", + "hgnc_id": "HGNC:15168", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4C45", + "hgnc_id": "HGNC:31270", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4C46", + "hgnc_id": "HGNC:31271", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4A5", + "hgnc_id": "HGNC:15162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM48", + "hgnc_id": "HGNC:19021", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR4A15", + "hgnc_id": "HGNC:15152", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM51", + "hgnc_id": "HGNC:19023", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR5F1", + "hgnc_id": "HGNC:8343", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR8H2", + "hgnc_id": "HGNC:15308", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR5M8", + "hgnc_id": "HGNC:14846", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR9G1", + "hgnc_id": "HGNC:15319", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR1S2", + "hgnc_id": "HGNC:15141", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1S1", + "hgnc_id": "HGNC:8227", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GLYATL1B", + "hgnc_id": "HGNC:37865", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PGA3", + "hgnc_id": "HGNC:8885", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "B3GAT3", + "hgnc_id": "HGNC:923", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ESRRA", + "hgnc_id": "HGNC:3471", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC22A12", + "hgnc_id": "HGNC:17989", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RBM4B", + "hgnc_id": "HGNC:28842", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TPCN2", + "hgnc_id": "HGNC:20820", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FGF3", + "hgnc_id": "HGNC:3681", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KRTAP5-8", + "hgnc_id": "HGNC:23603", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP5-10", + "hgnc_id": "HGNC:23605", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "KRTAP5-11", + "hgnc_id": "HGNC:23606", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "XNDC1N-ZNF705EP-ALG1L9P", + "hgnc_id": "HGNC:55871", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DEFB108B", + "hgnc_id": "HGNC:29966", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "XNDC1N", + "hgnc_id": "HGNC:54661", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "INTS4", + "hgnc_id": "HGNC:25048", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOX4", + "hgnc_id": "HGNC:7891", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TRIM77", + "hgnc_id": "HGNC:34228", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "TRIM49", + "hgnc_id": "HGNC:13431", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM64B", + "hgnc_id": "HGNC:37147", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM49D1", + "hgnc_id": "HGNC:43973", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64", + "hgnc_id": "HGNC:14663", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM49C", + "hgnc_id": "HGNC:38877", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UBTFL1", + "hgnc_id": "HGNC:14533", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CARD16", + "hgnc_id": "HGNC:33701", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA8", + "hgnc_id": "HGNC:5241", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR8B2", + "hgnc_id": "HGNC:8471", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TMEM45B", + "hgnc_id": "HGNC:25194", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CD4", + "hgnc_id": "HGNC:1678", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PEX5", + "hgnc_id": "HGNC:9719", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC1", + "hgnc_id": "HGNC:604", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GDF3", + "hgnc_id": "HGNC:4218", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NANOG", + "hgnc_id": "HGNC:20857", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A1", + "hgnc_id": "HGNC:25526", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRH1", + "hgnc_id": "HGNC:9366", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRR4", + "hgnc_id": "HGNC:18020", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRH2", + "hgnc_id": "HGNC:9367", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAS2R14", + "hgnc_id": "HGNC:14920", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TAS2R31", + "hgnc_id": "HGNC:19113", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAS2R46", + "hgnc_id": "HGNC:18877", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAS2R43", + "hgnc_id": "HGNC:18875", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "TAS2R30", + "hgnc_id": "HGNC:19112", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "PRB4", + "hgnc_id": "HGNC:9340", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRB1", + "hgnc_id": "HGNC:9337", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "DDX11", + "hgnc_id": "HGNC:2736", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H3-5", + "hgnc_id": "HGNC:33164", + "reasons": [ + "segdup" + ], + "share": 0.1661, + "status": "not read" + }, + { + "gene": "ALG10", + "hgnc_id": "HGNC:23162", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ALG10B", + "hgnc_id": "HGNC:31088", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBA1B", + "hgnc_id": "HGNC:18809", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "COX14", + "hgnc_id": "HGNC:28216", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NACA", + "hgnc_id": "HGNC:7629", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DPY19L2", + "hgnc_id": "HGNC:19414", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNF1A", + "hgnc_id": "HGNC:11621", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HPD", + "hgnc_id": "HGNC:5147", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCAR2", + "hgnc_id": "HGNC:24827", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HCAR3", + "hgnc_id": "HGNC:16824", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "P2RX2", + "hgnc_id": "HGNC:15459", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PGAM5", + "hgnc_id": "HGNC:28763", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA3", + "hgnc_id": "HGNC:4426", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "C1QTNF9B", + "hgnc_id": "HGNC:34072", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C1QTNF9", + "hgnc_id": "HGNC:28732", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PARP4", + "hgnc_id": "HGNC:271", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PABPC3", + "hgnc_id": "HGNC:8556", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "MRPS31", + "hgnc_id": "HGNC:16632", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNRNPA1L2", + "hgnc_id": "HGNC:27067", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRR20C", + "hgnc_id": "HGNC:37221", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRR20B", + "hgnc_id": "HGNC:37220", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "PRR20D", + "hgnc_id": "HGNC:37222", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRR20E", + "hgnc_id": "HGNC:37223", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "F7", + "hgnc_id": "HGNC:3544", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PROZ", + "hgnc_id": "HGNC:9460", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEM", + "hgnc_id": "HGNC:37096", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEG", + "hgnc_id": "HGNC:33896", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR11H2", + "hgnc_id": "HGNC:14716", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4N2", + "hgnc_id": "HGNC:14742", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR4Q3", + "hgnc_id": "HGNC:15426", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR4M1", + "hgnc_id": "HGNC:14735", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RNASE3", + "hgnc_id": "HGNC:10046", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RNASE2", + "hgnc_id": "HGNC:10045", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10G2", + "hgnc_id": "HGNC:8170", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "DHRS4", + "hgnc_id": "HGNC:16985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHRS4L2", + "hgnc_id": "HGNC:19731", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARF6", + "hgnc_id": "HGNC:659", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACOT2", + "hgnc_id": "HGNC:18431", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NDUFB1", + "hgnc_id": "HGNC:7695", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "JAG2", + "hgnc_id": "HGNC:6189", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BTBD6", + "hgnc_id": "HGNC:19897", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "IGHV3-21", + "hgnc_id": "HGNC:5586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA6L6", + "hgnc_id": "HGNC:37225", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEB3", + "hgnc_id": "HGNC:51240", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LINC02203", + "hgnc_id": "HGNC:53069", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4M2B", + "hgnc_id": "HGNC:55109", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4N4C", + "hgnc_id": "HGNC:55110", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "POTEB", + "hgnc_id": "HGNC:33734", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4M2-OT1", + "hgnc_id": "HGNC:56199", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4N4", + "hgnc_id": "HGNC:15375", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L22", + "hgnc_id": "HGNC:50289", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6L1", + "hgnc_id": "HGNC:37444", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L26", + "hgnc_id": "HGNC:56306", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8S", + "hgnc_id": "HGNC:44409", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6L2", + "hgnc_id": "HGNC:26695", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HERC2", + "hgnc_id": "HGNC:4868", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L24", + "hgnc_id": "HGNC:55710", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA8F", + "hgnc_id": "HGNC:32378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8G", + "hgnc_id": "HGNC:25328", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6L25", + "hgnc_id": "HGNC:55711", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8M", + "hgnc_id": "HGNC:44404", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8J", + "hgnc_id": "HGNC:38650", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8T", + "hgnc_id": "HGNC:44410", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8R", + "hgnc_id": "HGNC:44407", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8Q", + "hgnc_id": "HGNC:44408", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8H", + "hgnc_id": "HGNC:37443", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8O", + "hgnc_id": "HGNC:44406", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8N", + "hgnc_id": "HGNC:44405", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8A", + "hgnc_id": "HGNC:31972", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8B", + "hgnc_id": "HGNC:31973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NANOGP8", + "hgnc_id": "HGNC:23106", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "NDUFAF1", + "hgnc_id": "HGNC:18828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CKMT1B", + "hgnc_id": "HGNC:1995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CATSPER2", + "hgnc_id": "HGNC:18810", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CKMT1A", + "hgnc_id": "HGNC:31736", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TERB2", + "hgnc_id": "HGNC:28520", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LDHAL6B", + "hgnc_id": "HGNC:21481", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6B", + "hgnc_id": "HGNC:32205", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6A", + "hgnc_id": "HGNC:13567", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ISLR2", + "hgnc_id": "HGNC:29286", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6C", + "hgnc_id": "HGNC:32206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6D", + "hgnc_id": "HGNC:32204", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ADAMTS7", + "hgnc_id": "HGNC:223", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EFL1", + "hgnc_id": "HGNC:25789", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SAXO2", + "hgnc_id": "HGNC:33727", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L10", + "hgnc_id": "HGNC:37228", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GOLGA6L9", + "hgnc_id": "HGNC:37229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "WHAMM", + "hgnc_id": "HGNC:30493", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4F4", + "hgnc_id": "HGNC:8301", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MPG", + "hgnc_id": "HGNC:7211", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HBM", + "hgnc_id": "HGNC:4826", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AXIN1", + "hgnc_id": "HGNC:903", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSTR5", + "hgnc_id": "HGNC:11334", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TPSD1", + "hgnc_id": "HGNC:14118", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GNPTG", + "hgnc_id": "HGNC:23026", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UNKL", + "hgnc_id": "HGNC:14184", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PKD1", + "hgnc_id": "HGNC:9008", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PDPK1", + "hgnc_id": "HGNC:8816", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "EEF2KMT", + "hgnc_id": "HGNC:32221", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NPIPA3", + "hgnc_id": "HGNC:41978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPA2", + "hgnc_id": "HGNC:41979", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NOMO1", + "hgnc_id": "HGNC:30060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDXDC1", + "hgnc_id": "HGNC:28995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RRN3", + "hgnc_id": "HGNC:30346", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPA5", + "hgnc_id": "HGNC:41980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MPV17L", + "hgnc_id": "HGNC:26827", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NOMO3", + "hgnc_id": "HGNC:25242", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPA6", + "hgnc_id": "HGNC:27893", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPA8", + "hgnc_id": "HGNC:41983", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPA9", + "hgnc_id": "HGNC:41984", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NOMO2", + "hgnc_id": "HGNC:22652", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SMG1", + "hgnc_id": "HGNC:30045", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IQCK", + "hgnc_id": "HGNC:28556", + "reasons": [ + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACSM5", + "hgnc_id": "HGNC:26060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACSM2B", + "hgnc_id": "HGNC:30931", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPB3", + "hgnc_id": "HGNC:28989", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB4", + "hgnc_id": "HGNC:41985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB5", + "hgnc_id": "HGNC:37233", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB6", + "hgnc_id": "HGNC:37454", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB7", + "hgnc_id": "HGNC:33832", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A2", + "hgnc_id": "HGNC:11454", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB8", + "hgnc_id": "HGNC:37490", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EIF3C", + "hgnc_id": "HGNC:3279", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB9", + "hgnc_id": "HGNC:41987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB11", + "hgnc_id": "HGNC:37453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BOLA2-SMG1P6", + "hgnc_id": "HGNC:53563", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BOLA2", + "hgnc_id": "HGNC:29488", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLX1B", + "hgnc_id": "HGNC:28748", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SULT1A4", + "hgnc_id": "HGNC:30004", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NPIPB12", + "hgnc_id": "HGNC:37491", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLX1A", + "hgnc_id": "HGNC:20922", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A3", + "hgnc_id": "HGNC:11455", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB13", + "hgnc_id": "HGNC:41989", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PHKG2", + "hgnc_id": "HGNC:8931", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "VKORC1", + "hgnc_id": "HGNC:23663", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRABD5", + "hgnc_id": "HGNC:26987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TP53TG3", + "hgnc_id": "HGNC:30759", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TP53TG3C", + "hgnc_id": "HGNC:42962", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "TP53TG3E", + "hgnc_id": "HGNC:51816", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TP53TG3B", + "hgnc_id": "HGNC:37202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TP53TG3F", + "hgnc_id": "HGNC:51817", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "CCNYL1B", + "hgnc_id": "HGNC:56313", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CES1", + "hgnc_id": "HGNC:1863", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MT1M", + "hgnc_id": "HGNC:14296", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MATCAP1", + "hgnc_id": "HGNC:34408", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD11B2", + "hgnc_id": "HGNC:5209", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLEC18A", + "hgnc_id": "HGNC:30388", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PDPR", + "hgnc_id": "HGNC:30264", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EXOSC6", + "hgnc_id": "HGNC:19055", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HPR", + "hgnc_id": "HGNC:5156", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB15", + "hgnc_id": "HGNC:34409", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CLEC18B", + "hgnc_id": "HGNC:33849", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CA5A", + "hgnc_id": "HGNC:1377", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF469", + "hgnc_id": "HGNC:23216", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH15", + "hgnc_id": "HGNC:1754", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANKRD11", + "hgnc_id": "HGNC:21316", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MC1R", + "hgnc_id": "HGNC:6929", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PRDM7", + "hgnc_id": "HGNC:9351", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ABR", + "hgnc_id": "HGNC:81", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SERPINF2", + "hgnc_id": "HGNC:9075", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SRR", + "hgnc_id": "HGNC:14398", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1D5", + "hgnc_id": "HGNC:8186", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "P2RX5", + "hgnc_id": "HGNC:8536", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ALOX15", + "hgnc_id": "HGNC:433", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ENO3", + "hgnc_id": "HGNC:3354", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP6", + "hgnc_id": "HGNC:12629", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35G6", + "hgnc_id": "HGNC:31351", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "ALOXE3", + "hgnc_id": "HGNC:13743", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HES7", + "hgnc_id": "HGNC:15977", + "reasons": [ + "CMRG" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CDRT15", + "hgnc_id": "HGNC:14395", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "FBXW10B", + "hgnc_id": "HGNC:14379", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM16", + "hgnc_id": "HGNC:17241", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF286A-TBC1D26", + "hgnc_id": "HGNC:55384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF286A", + "hgnc_id": "HGNC:13501", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D26", + "hgnc_id": "HGNC:28745", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UBB", + "hgnc_id": "HGNC:12463", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "ATPAF2", + "hgnc_id": "HGNC:18802", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EVPLL", + "hgnc_id": "HGNC:35236", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS9C", + "hgnc_id": "HGNC:33874", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D28", + "hgnc_id": "HGNC:26858", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXO3B", + "hgnc_id": "HGNC:3822", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FBXW10", + "hgnc_id": "HGNC:1211", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRAP", + "hgnc_id": "HGNC:4562", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GRAPL", + "hgnc_id": "HGNC:37240", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LGALS9B", + "hgnc_id": "HGNC:24842", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDRT15L2", + "hgnc_id": "HGNC:34075", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MEIS3P2", + "hgnc_id": "HGNC:17638", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAP2K3", + "hgnc_id": "HGNC:6843", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KCNJ18", + "hgnc_id": "HGNC:39080", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS9", + "hgnc_id": "HGNC:6570", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXN1", + "hgnc_id": "HGNC:12765", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SUZ12", + "hgnc_id": "HGNC:17101", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37B", + "hgnc_id": "HGNC:29070", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SLC35G3", + "hgnc_id": "HGNC:26848", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CCL4", + "hgnc_id": "HGNC:10630", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3B", + "hgnc_id": "HGNC:27011", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCL4L2", + "hgnc_id": "HGNC:24066", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "TBC1D3I", + "hgnc_id": "HGNC:32709", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TBC1D3G", + "hgnc_id": "HGNC:29860", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TBC1D3F", + "hgnc_id": "HGNC:18257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3K", + "hgnc_id": "HGNC:51245", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TBC1D3L", + "hgnc_id": "HGNC:51246", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D3D", + "hgnc_id": "HGNC:28944", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D3C", + "hgnc_id": "HGNC:24889", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TBC1D3E", + "hgnc_id": "HGNC:27071", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MRPL45", + "hgnc_id": "HGNC:16651", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP1-1", + "hgnc_id": "HGNC:16772", + "reasons": [ + "CMRG" + ], + "share": 0.1659, + "status": "not read" + }, + { + "gene": "KRTAP4-7", + "hgnc_id": "HGNC:18898", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP4-8", + "hgnc_id": "HGNC:17230", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP4-16", + "hgnc_id": "HGNC:18921", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRTAP4-9", + "hgnc_id": "HGNC:18910", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP4-12", + "hgnc_id": "HGNC:16776", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP9-7", + "hgnc_id": "HGNC:18915", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "KRT14", + "hgnc_id": "HGNC:6416", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRT16", + "hgnc_id": "HGNC:6423", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRT17", + "hgnc_id": "HGNC:6427", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HSD17B1", + "hgnc_id": "HGNC:5210", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "G6PC3", + "hgnc_id": "HGNC:24861", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ARHGAP27", + "hgnc_id": "HGNC:31813", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLEKHM1", + "hgnc_id": "HGNC:29017", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ARL17B", + "hgnc_id": "HGNC:32387", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37A", + "hgnc_id": "HGNC:29069", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARL17A", + "hgnc_id": "HGNC:24096", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NSF", + "hgnc_id": "HGNC:8016", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPEPPS", + "hgnc_id": "HGNC:7900", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HOXB8", + "hgnc_id": "HGNC:5119", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GIP", + "hgnc_id": "HGNC:4270", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHX40", + "hgnc_id": "HGNC:18018", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPS6KB1", + "hgnc_id": "HGNC:10436", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2A", + "hgnc_id": "HGNC:25755", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TLK2", + "hgnc_id": "HGNC:11842", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CSH2", + "hgnc_id": "HGNC:2441", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "GH2", + "hgnc_id": "HGNC:4262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CSH1", + "hgnc_id": "HGNC:2440", + "reasons": [ + "segdup" + ], + "share": 0.6663, + "status": "not read" + }, + { + "gene": "CSHL1", + "hgnc_id": "HGNC:2442", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "LRRC37A3", + "hgnc_id": "HGNC:32427", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C17orf58", + "hgnc_id": "HGNC:27568", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "AMZ2", + "hgnc_id": "HGNC:28041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMC6", + "hgnc_id": "HGNC:18021", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CANT1", + "hgnc_id": "HGNC:19721", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RNF213", + "hgnc_id": "HGNC:14539", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ARL16", + "hgnc_id": "HGNC:27902", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8B", + "hgnc_id": "HGNC:24983", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYMS", + "hgnc_id": "HGNC:12441", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC35G4", + "hgnc_id": "HGNC:31043", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "POTEC", + "hgnc_id": "HGNC:33894", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ANKRD30B", + "hgnc_id": "HGNC:24165", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC25A52", + "hgnc_id": "HGNC:23324", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KATNAL2", + "hgnc_id": "HGNC:25387", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SERPINB4", + "hgnc_id": "HGNC:10570", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERPINB3", + "hgnc_id": "HGNC:10569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALR1", + "hgnc_id": "HGNC:4132", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CTDP1", + "hgnc_id": "HGNC:2498", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "BSG", + "hgnc_id": "HGNC:1116", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN2", + "hgnc_id": "HGNC:4846", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KISS1R", + "hgnc_id": "HGNC:4510", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "STK11", + "hgnc_id": "HGNC:11389", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TCF3", + "hgnc_id": "HGNC:11633", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ABHD17A", + "hgnc_id": "HGNC:28756", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "GIPC3", + "hgnc_id": "HGNC:18183", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBXA2R", + "hgnc_id": "HGNC:11608", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CREB3L3", + "hgnc_id": "HGNC:18855", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RFX2", + "hgnc_id": "HGNC:9983", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "C3", + "hgnc_id": "HGNC:1318", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MBD3L2B", + "hgnc_id": "HGNC:53435", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MBD3L4", + "hgnc_id": "HGNC:37206", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBD3L2", + "hgnc_id": "HGNC:18532", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "STXBP2", + "hgnc_id": "HGNC:11445", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TYK2", + "hgnc_id": "HGNC:12440", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF799", + "hgnc_id": "HGNC:28071", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF443", + "hgnc_id": "HGNC:20878", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP4F3", + "hgnc_id": "HGNC:2646", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4F12", + "hgnc_id": "HGNC:18857", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10H2", + "hgnc_id": "HGNC:8173", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CALR3", + "hgnc_id": "HGNC:20407", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MYO9B", + "hgnc_id": "HGNC:7609", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "INSL3", + "hgnc_id": "HGNC:6086", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF738", + "hgnc_id": "HGNC:32469", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF100", + "hgnc_id": "HGNC:12880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF208", + "hgnc_id": "HGNC:12999", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF676", + "hgnc_id": "HGNC:20429", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF729", + "hgnc_id": "HGNC:32464", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF492", + "hgnc_id": "HGNC:23707", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF728", + "hgnc_id": "HGNC:32463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHPN2", + "hgnc_id": "HGNC:19974", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GPI", + "hgnc_id": "HGNC:4458", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF302", + "hgnc_id": "HGNC:13848", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF181", + "hgnc_id": "HGNC:12971", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FFAR3", + "hgnc_id": "HGNC:4499", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "COX6B1", + "hgnc_id": "HGNC:2280", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS7", + "hgnc_id": "HGNC:6568", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS7B", + "hgnc_id": "HGNC:34447", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "IFNL3", + "hgnc_id": "HGNC:18365", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IFNL4", + "hgnc_id": "HGNC:44480", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "IFNL2", + "hgnc_id": "HGNC:18364", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CYP2A7", + "hgnc_id": "HGNC:2611", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CYP2G1P", + "hgnc_id": "HGNC:2633", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A13", + "hgnc_id": "HGNC:2608", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CYP2F1", + "hgnc_id": "HGNC:2632", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG8", + "hgnc_id": "HGNC:9525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG1", + "hgnc_id": "HGNC:9514", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PSG6", + "hgnc_id": "HGNC:9523", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PSG7", + "hgnc_id": "HGNC:9524", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSG11", + "hgnc_id": "HGNC:9516", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PSG2", + "hgnc_id": "HGNC:9519", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG4", + "hgnc_id": "HGNC:9521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG9", + "hgnc_id": "HGNC:9526", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ETHE1", + "hgnc_id": "HGNC:23287", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF285", + "hgnc_id": "HGNC:13079", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "APOC1", + "hgnc_id": "HGNC:607", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOC4", + "hgnc_id": "HGNC:611", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BLOC1S3", + "hgnc_id": "HGNC:20914", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DMPK", + "hgnc_id": "HGNC:2933", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FUT1", + "hgnc_id": "HGNC:4012", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BAX", + "hgnc_id": "HGNC:959", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LHB", + "hgnc_id": "HGNC:6584", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "CGB3", + "hgnc_id": "HGNC:1886", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CGB2", + "hgnc_id": "HGNC:16722", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "CGB1", + "hgnc_id": "HGNC:16721", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "CGB5", + "hgnc_id": "HGNC:16452", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CGB8", + "hgnc_id": "HGNC:16453", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CGB7", + "hgnc_id": "HGNC:16451", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PNKP", + "hgnc_id": "HGNC:9154", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SIGLEC16", + "hgnc_id": "HGNC:24851", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KLK4", + "hgnc_id": "HGNC:6365", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ETFB", + "hgnc_id": "HGNC:3482", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF808", + "hgnc_id": "HGNC:33230", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF600", + "hgnc_id": "HGNC:30951", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF468", + "hgnc_id": "HGNC:33105", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VN1R4", + "hgnc_id": "HGNC:19871", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "FAM90A27P", + "hgnc_id": "HGNC:43617", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF765", + "hgnc_id": "HGNC:25092", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF761", + "hgnc_id": "HGNC:23179", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF813", + "hgnc_id": "HGNC:33257", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRKCG", + "hgnc_id": "HGNC:9402", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBOAT7", + "hgnc_id": "HGNC:15505", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LILRA6", + "hgnc_id": "HGNC:15495", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "LILRB2", + "hgnc_id": "HGNC:6606", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "LAIR1", + "hgnc_id": "HGNC:6477", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAIR2", + "hgnc_id": "HGNC:6478", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LILRB1", + "hgnc_id": "HGNC:6605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KIR3DL3", + "hgnc_id": "HGNC:16312", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KIR2DL3", + "hgnc_id": "HGNC:6331", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KIR2DL1", + "hgnc_id": "HGNC:6329", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KIR2DL4", + "hgnc_id": "HGNC:6332", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KIR3DL1", + "hgnc_id": "HGNC:6338", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KIR3DL2", + "hgnc_id": "HGNC:6339", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NLRP7", + "hgnc_id": "HGNC:22947", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NLRP2", + "hgnc_id": "HGNC:22948", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GP6", + "hgnc_id": "HGNC:14388", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TNNT1", + "hgnc_id": "HGNC:11948", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TNNI3", + "hgnc_id": "HGNC:11947", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL4A", + "hgnc_id": "HGNC:16449", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RFPL4AL1", + "hgnc_id": "HGNC:45147", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZNF419", + "hgnc_id": "HGNC:20648", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF773", + "hgnc_id": "HGNC:30487", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF587B", + "hgnc_id": "HGNC:37142", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF814", + "hgnc_id": "HGNC:33258", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF417", + "hgnc_id": "HGNC:20646", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC27A5", + "hgnc_id": "HGNC:10999", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SIRPB1", + "hgnc_id": "HGNC:15928", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CST4", + "hgnc_id": "HGNC:2476", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CST1", + "hgnc_id": "HGNC:2473", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CST2", + "hgnc_id": "HGNC:2474", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PLAGL2", + "hgnc_id": "HGNC:9047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEMG1", + "hgnc_id": "HGNC:10742", + "reasons": [ + "CMRG" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PLTP", + "hgnc_id": "HGNC:9093", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRNA4", + "hgnc_id": "HGNC:1958", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EEF1A2", + "hgnc_id": "HGNC:3192", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PTK6", + "hgnc_id": "HGNC:9617", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MYT1", + "hgnc_id": "HGNC:7622", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCMTD2", + "hgnc_id": "HGNC:15882", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TPTE", + "hgnc_id": "HGNC:12023", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POTED", + "hgnc_id": "HGNC:23822", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP21-1", + "hgnc_id": "HGNC:18945", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SMIM11", + "hgnc_id": "HGNC:1293", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMIM34", + "hgnc_id": "HGNC:39601", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KCNE1", + "hgnc_id": "HGNC:6240", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CBR3", + "hgnc_id": "HGNC:1549", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NDUFV3", + "hgnc_id": "HGNC:7719", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CBS", + "hgnc_id": "HGNC:1550", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "U2AF1", + "hgnc_id": "HGNC:12453", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SIK1", + "hgnc_id": "HGNC:11142", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSF2BP", + "hgnc_id": "HGNC:5226", + "reasons": [ + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BC12L", + "hgnc_id": "HGNC:4762", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3319, + "status": "not read" + }, + { + "gene": "TRAPPC10", + "hgnc_id": "HGNC:11868", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PWP2", + "hgnc_id": "HGNC:9711", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GATD3", + "hgnc_id": "HGNC:1273", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DNMT3L", + "hgnc_id": "HGNC:2980", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-1", + "hgnc_id": "HGNC:22966", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "KRTAP10-2", + "hgnc_id": "HGNC:22967", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRTAP10-4", + "hgnc_id": "HGNC:20521", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "KRTAP10-5", + "hgnc_id": "HGNC:22969", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "KRTAP10-6", + "hgnc_id": "HGNC:20523", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "KRTAP10-8", + "hgnc_id": "HGNC:20525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-9", + "hgnc_id": "HGNC:22971", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KRTAP10-10", + "hgnc_id": "HGNC:22972", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP12-2", + "hgnc_id": "HGNC:20530", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "KRTAP12-1", + "hgnc_id": "HGNC:20529", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "COL18A1", + "hgnc_id": "HGNC:2195", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR11H1", + "hgnc_id": "HGNC:15404", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEH", + "hgnc_id": "HGNC:133", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCT8L2", + "hgnc_id": "HGNC:15553", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "USP18", + "hgnc_id": "HGNC:12616", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GGTLC3", + "hgnc_id": "HGNC:33426", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TMEM191B", + "hgnc_id": "HGNC:33600", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM246B", + "hgnc_id": "HGNC:54843", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DGCR6", + "hgnc_id": "HGNC:2846", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRODH", + "hgnc_id": "HGNC:9453", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM246C", + "hgnc_id": "HGNC:54842", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "CCDC188", + "hgnc_id": "HGNC:51899", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RTN4R", + "hgnc_id": "HGNC:18601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3FP", + "hgnc_id": "HGNC:24067", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LRRC74B", + "hgnc_id": "HGNC:34301", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM246A", + "hgnc_id": "HGNC:54844", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RIMBP3B", + "hgnc_id": "HGNC:33891", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HIC2", + "hgnc_id": "HGNC:18595", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TMEM191C", + "hgnc_id": "HGNC:33601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GGTLC2", + "hgnc_id": "HGNC:18596", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IGLL1", + "hgnc_id": "HGNC:5870", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRICH1", + "hgnc_id": "HGNC:28031", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTT2B", + "hgnc_id": "HGNC:33437", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DDTL", + "hgnc_id": "HGNC:33446", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GSTT2", + "hgnc_id": "HGNC:4642", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL1", + "hgnc_id": "HGNC:9977", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C22orf42", + "hgnc_id": "HGNC:27160", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RFPL3", + "hgnc_id": "HGNC:9980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3D", + "hgnc_id": "HGNC:17354", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOBEC3H", + "hgnc_id": "HGNC:24100", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ATF4", + "hgnc_id": "HGNC:786", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NDUFA6", + "hgnc_id": "HGNC:7690", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2D7", + "hgnc_id": "HGNC:2624", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RRP7A", + "hgnc_id": "HGNC:24286", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYB5R3", + "hgnc_id": "HGNC:2873", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "A4GALT", + "hgnc_id": "HGNC:18149", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TTLL1", + "hgnc_id": "HGNC:1312", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MLC1", + "hgnc_id": "HGNC:17082", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBGCP6", + "hgnc_id": "HGNC:18127", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACR", + "hgnc_id": "HGNC:126", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "XG", + "hgnc_id": "HGNC:12806", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GYG2", + "hgnc_id": "HGNC:4700", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ARSD", + "hgnc_id": "HGNC:717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MXRA5", + "hgnc_id": "HGNC:7539", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRKX", + "hgnc_id": "HGNC:9441", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "VCX3A", + "hgnc_id": "HGNC:18159", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "VCX2", + "hgnc_id": "HGNC:18158", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VCX3B", + "hgnc_id": "HGNC:31838", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM9A", + "hgnc_id": "HGNC:18403", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM9C", + "hgnc_id": "HGNC:18405", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SUPT20HL2", + "hgnc_id": "HGNC:31797", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SUPT20HL1", + "hgnc_id": "HGNC:30773", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "MAGEB6", + "hgnc_id": "HGNC:23796", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DCAF8L1", + "hgnc_id": "HGNC:31810", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM47B", + "hgnc_id": "HGNC:26659", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "H2AL1Q", + "hgnc_id": "HGNC:53959", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "FAM47C", + "hgnc_id": "HGNC:25301", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "H2AL3", + "hgnc_id": "HGNC:53960", + "reasons": [ + "segdup" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "SPACA5B", + "hgnc_id": "HGNC:19142", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SSX5", + "hgnc_id": "HGNC:11339", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SSX1", + "hgnc_id": "HGNC:11335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSX3", + "hgnc_id": "HGNC:11337", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SSX4", + "hgnc_id": "HGNC:11338", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX4B", + "hgnc_id": "HGNC:16880", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12J", + "hgnc_id": "HGNC:17778", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE13", + "hgnc_id": "HGNC:29081", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE2E", + "hgnc_id": "HGNC:31960", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE12B", + "hgnc_id": "HGNC:26779", + "reasons": [ + "segdup" + ], + "share": 0.4957, + "status": "not read" + }, + { + "gene": "GAGE12C", + "hgnc_id": "HGNC:28402", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GAGE12D", + "hgnc_id": "HGNC:31904", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12F", + "hgnc_id": "HGNC:31906", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE12G", + "hgnc_id": "HGNC:31907", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE12H", + "hgnc_id": "HGNC:31908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE2A", + "hgnc_id": "HGNC:4099", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GAGE1", + "hgnc_id": "HGNC:4098", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CENPVL3", + "hgnc_id": "HGNC:43880", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CENPVL1", + "hgnc_id": "HGNC:31851", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MAGED4B", + "hgnc_id": "HGNC:22880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MAGED4", + "hgnc_id": "HGNC:23793", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE1A", + "hgnc_id": "HGNC:4111", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "XAGE1B", + "hgnc_id": "HGNC:25400", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX7", + "hgnc_id": "HGNC:19653", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSX2B", + "hgnc_id": "HGNC:22263", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN5", + "hgnc_id": "HGNC:33178", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "XAGE5", + "hgnc_id": "HGNC:30930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE3", + "hgnc_id": "HGNC:14618", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM156B", + "hgnc_id": "HGNC:31962", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM156A", + "hgnc_id": "HGNC:30114", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPIN2B", + "hgnc_id": "HGNC:33147", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPIN2A", + "hgnc_id": "HGNC:20694", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZXDA", + "hgnc_id": "HGNC:13198", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CXorf49", + "hgnc_id": "HGNC:30891", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CXorf49B", + "hgnc_id": "HGNC:34229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "DMRTC1B", + "hgnc_id": "HGNC:31686", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM236D", + "hgnc_id": "HGNC:52642", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DMRTC1", + "hgnc_id": "HGNC:13910", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM236C", + "hgnc_id": "HGNC:52641", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM236A", + "hgnc_id": "HGNC:44268", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PABPC1L2B", + "hgnc_id": "HGNC:31852", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PABPC1L2A", + "hgnc_id": "HGNC:27989", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TGIF2LX", + "hgnc_id": "HGNC:18570", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDH11X", + "hgnc_id": "HGNC:8656", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TCEAL6", + "hgnc_id": "HGNC:24553", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "TCP11X1", + "hgnc_id": "HGNC:48369", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NXF2", + "hgnc_id": "HGNC:8072", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NXF2B", + "hgnc_id": "HGNC:23984", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB40AL", + "hgnc_id": "HGNC:25410", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TMSB15B", + "hgnc_id": "HGNC:28612", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BW1", + "hgnc_id": "HGNC:27252", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "H2BW2", + "hgnc_id": "HGNC:27867", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "TMSB15C", + "hgnc_id": "HGNC:55173", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PSMD10", + "hgnc_id": "HGNC:9555", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CT47C1", + "hgnc_id": "HGNC:53820", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHOXF2B", + "hgnc_id": "HGNC:33519", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RHOXF2", + "hgnc_id": "HGNC:30011", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT47B1", + "hgnc_id": "HGNC:33293", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A12", + "hgnc_id": "HGNC:33292", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT47A11", + "hgnc_id": "HGNC:27397", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CT47A9", + "hgnc_id": "HGNC:33290", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A8", + "hgnc_id": "HGNC:33289", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "CT47A7", + "hgnc_id": "HGNC:33288", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT47A6", + "hgnc_id": "HGNC:33287", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A5", + "hgnc_id": "HGNC:33286", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CT47A4", + "hgnc_id": "HGNC:33285", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT47A2", + "hgnc_id": "HGNC:33283", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A1", + "hgnc_id": "HGNC:33282", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "RTL8B", + "hgnc_id": "HGNC:33156", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "RTL8C", + "hgnc_id": "HGNC:2569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RTL8A", + "hgnc_id": "HGNC:24514", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "SMIM10L2B", + "hgnc_id": "HGNC:34500", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT55", + "hgnc_id": "HGNC:26047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ETDA", + "hgnc_id": "HGNC:53449", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "SMIM10L2A", + "hgnc_id": "HGNC:34499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT45A1", + "hgnc_id": "HGNC:33267", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A3", + "hgnc_id": "HGNC:33268", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT45A5", + "hgnc_id": "HGNC:33270", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT45A2", + "hgnc_id": "HGNC:28400", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT45A7", + "hgnc_id": "HGNC:51260", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CT45A8", + "hgnc_id": "HGNC:51261", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT45A9", + "hgnc_id": "HGNC:51262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A10", + "hgnc_id": "HGNC:51263", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SAGE1", + "hgnc_id": "HGNC:30369", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPANXC", + "hgnc_id": "HGNC:14331", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXA1", + "hgnc_id": "HGNC:11218", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "SPANXA2", + "hgnc_id": "HGNC:14328", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "SPANXD", + "hgnc_id": "HGNC:14332", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "MAGEC1", + "hgnc_id": "HGNC:6812", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPANXN3", + "hgnc_id": "HGNC:33176", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UBE2NL", + "hgnc_id": "HGNC:31710", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN1", + "hgnc_id": "HGNC:33174", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CXorf51B", + "hgnc_id": "HGNC:42787", + "reasons": [ + "segdup" + ], + "share": 0.3323, + "status": "not read" + }, + { + "gene": "CXorf51A", + "hgnc_id": "HGNC:30533", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "EOLA1", + "hgnc_id": "HGNC:28089", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HSFX3", + "hgnc_id": "HGNC:52395", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HSFX2", + "hgnc_id": "HGNC:32701", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSFX1", + "hgnc_id": "HGNC:29603", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MAGEA9", + "hgnc_id": "HGNC:6807", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "EOLA2", + "hgnc_id": "HGNC:17402", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSFX4", + "hgnc_id": "HGNC:52398", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAGEA3", + "hgnc_id": "HGNC:6801", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MAGEA2B", + "hgnc_id": "HGNC:19340", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CSAG1", + "hgnc_id": "HGNC:24294", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "MAGEA12", + "hgnc_id": "HGNC:6799", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "MAGEA2", + "hgnc_id": "HGNC:6800", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CSAG3", + "hgnc_id": "HGNC:26237", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MAGEA6", + "hgnc_id": "HGNC:6804", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PWWP4", + "hgnc_id": "HGNC:55197", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A8", + "hgnc_id": "HGNC:11055", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BCAP31", + "hgnc_id": "HGNC:16695", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OPN1LW", + "hgnc_id": "HGNC:9936", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OPN1MW", + "hgnc_id": "HGNC:4206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OPN1MW2", + "hgnc_id": "HGNC:26952", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEX28", + "hgnc_id": "HGNC:2563", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAG1A", + "hgnc_id": "HGNC:24198", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "CTAG1B", + "hgnc_id": "HGNC:2491", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAG2", + "hgnc_id": "HGNC:2492", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H2AB1", + "hgnc_id": "HGNC:22516", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "F8A1", + "hgnc_id": "HGNC:3547", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "F8A2", + "hgnc_id": "HGNC:31849", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "F8A3", + "hgnc_id": "HGNC:31850", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "H2AB3", + "hgnc_id": "HGNC:14455", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IL9R", + "hgnc_id": "HGNC:6030", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASH6P", + "hgnc_id": "HGNC:31685", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "TGIF2LY", + "hgnc_id": "HGNC:18569", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "PCDH11Y", + "hgnc_id": "HGNC:15813", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY2", + "hgnc_id": "HGNC:23924", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY4", + "hgnc_id": "HGNC:37287", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY8", + "hgnc_id": "HGNC:37471", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY3", + "hgnc_id": "HGNC:33876", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY1", + "hgnc_id": "HGNC:12381", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY9", + "hgnc_id": "HGNC:37472", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY10", + "hgnc_id": "HGNC:37473", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY", + "hgnc_id": "HGNC:12668", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY1B", + "hgnc_id": "HGNC:31751", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2B", + "hgnc_id": "HGNC:23921", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2A", + "hgnc_id": "HGNC:1810", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY1", + "hgnc_id": "HGNC:18568", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY2", + "hgnc_id": "HGNC:23950", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1B", + "hgnc_id": "HGNC:23914", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1A1", + "hgnc_id": "HGNC:9912", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1D", + "hgnc_id": "HGNC:23915", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1E", + "hgnc_id": "HGNC:23916", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1F", + "hgnc_id": "HGNC:23974", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1J", + "hgnc_id": "HGNC:23917", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2", + "hgnc_id": "HGNC:13508", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ1", + "hgnc_id": "HGNC:2682", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ2", + "hgnc_id": "HGNC:15964", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1B", + "hgnc_id": "HGNC:23920", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2B", + "hgnc_id": "HGNC:25449", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ3", + "hgnc_id": "HGNC:15965", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ4", + "hgnc_id": "HGNC:15966", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2C", + "hgnc_id": "HGNC:18225", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1", + "hgnc_id": "HGNC:1809", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "C4B_2", + "hgnc_id": "HGNC:42398", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTT1", + "hgnc_id": "HGNC:4641", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTTP2", + "hgnc_id": "HGNC:33606", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GTF2H2C_2", + "hgnc_id": "HGNC:35418", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB3", + "hgnc_id": "HGNC:4951", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB4", + "hgnc_id": "HGNC:4952", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL2", + "hgnc_id": "HGNC:6330", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5A", + "hgnc_id": "HGNC:16345", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5B", + "hgnc_id": "HGNC:16346", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS1", + "hgnc_id": "HGNC:6333", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS2", + "hgnc_id": "HGNC:6334", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS3", + "hgnc_id": "HGNC:6335", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS5", + "hgnc_id": "HGNC:6337", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR3DS1", + "hgnc_id": "HGNC:6340", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "LILRA3", + "hgnc_id": "HGNC:6604", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U8", + "hgnc_id": "HGNC:27538", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U9", + "hgnc_id": "HGNC:29166", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR9G9", + "hgnc_id": "HGNC:31940", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TAS2R45", + "hgnc_id": "HGNC:18876", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + } + ], + "not_read_owned": [ + { + "gene": "RHCE", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "AMY1A", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AMY1B", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AMY1C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GSTM1", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FCGR1A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GBA1", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FCGR2A", + "owner": "parascopy", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FCGR2C", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "FCGR3B", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR3", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR1", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PROS1", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B17", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT2B15", + "owner": "pypgx", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPE", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPB", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GTF2H2C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMN2", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMN1", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2H2", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSPA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "C4A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "C4B", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP21A2", + "owner": "cn_loci:CYP21A2", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LPA", + "owner": "cn_loci:LPA_KIV2", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PMS2", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2I", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DEFA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CEL", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC5B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KLRC2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KRT6B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRT6C", + "owner": "parascopy", + "share": 0.1665, + "status": "not read" + }, + { + "gene": "KRT6A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRFAM7A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "STRC", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SORD", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HBA2", + "owner": "cn_loci:HBA", + "share": 0.5, + "status": "not read" + }, + { + "gene": "HBA1", + "owner": "cn_loci:HBA", + "share": 0.6659, + "status": "not read" + }, + { + "gene": "TPSB2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TPSAB1", + "owner": "parascopy", + "share": 0.3331, + "status": "not read" + }, + { + "gene": "OTOA", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SULT1A1", + "owner": "pypgx", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HYDIN", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HP", + "owner": "cn_loci:HP", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CCL3L3", + "owner": "cn_loci:CCL3L1", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A6", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP2B6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD177", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "APOBEC3B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP2D6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + } + ], + "not_on_reference": [ + "TGIF2LY", + "PCDH11Y", + "TSPY2", + "TSPY4", + "TSPY8", + "TSPY3", + "TSPY1", + "TSPY9", + "TSPY10", + "VCY", + "VCY1B", + "CDY2B", + "CDY2A", + "HSFY1", + "HSFY2", + "RBMY1B", + "RBMY1A1", + "RBMY1D", + "RBMY1E", + "RBMY1F", + "RBMY1J", + "BPY2", + "DAZ1", + "DAZ2", + "CDY1B", + "BPY2B", + "DAZ3", + "DAZ4", + "BPY2C", + "CDY1", + "C4B_2", + "GSTT1", + "GSTTP2", + "GTF2H2C_2", + "HLA-DRB3", + "HLA-DRB4", + "KIR2DL2", + "KIR2DL5A", + "KIR2DL5B", + "KIR2DS1", + "KIR2DS2", + "KIR2DS3", + "KIR2DS5", + "KIR3DS1", + "LILRA3", + "OR8U8", + "OR8U9", + "OR9G9", + "TAS2R45" + ] + } + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "ok", + "source": "indexcov/A-indexcov.ped", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "inferred_sex": "female", + "cn_chrX": "2.1", + "cn_chrY": "0.02", + "karyotype": null + } + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 8, + "by_stars": { + "4": 0, + "3": 2, + "2": 3, + "1": 1, + "0": 2 + }, + "hits": [ + { + "stars": 4, + "chrom": "chr7", + "pos": 117559590, + "ref": "ATCT", + "alt": "A", + "genotype": "het", + "gene": "CFTR", + "significance": "Pathogenic", + "review_status": "practice guideline", + "combined_allele": ".", + "dp": 30, + "gq": 15, + "low_confidence": true, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 3, + "chrom": "chr17", + "pos": 43057100, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "BRCA1,NBR2", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": "part of a combined allele: chr17:43057100 C>T, chr17:43057102 A>G; unphased, so the parts may sit on different haplotypes", + "dp": 35, + "gq": 60, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 3, + "chrom": "chr6", + "pos": 26092913, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HFE", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": ".", + "dp": 40, + "gq": 99, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 2, + "chrom": "chr1", + "pos": 45331556, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "MUTYH", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 30, + "gq": 50, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 2, + "chrom": "chr16", + "pos": 176700, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HBA1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 2, + "chrom": "chr5", + "pos": 70951946, + "ref": "C", + "alt": "T", + "genotype": "hom", + "gene": "SMN1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 1, + "chrom": "chr13", + "pos": 20189473, + "ref": "C", + "alt": "T", + "genotype": "one copy, other copy not called ref (1/.)", + "gene": "GJB2", + "significance": "Likely pathogenic", + "review_status": "criteria provided, single submitter", + "combined_allele": "combined allele of chr13:20189473 C>T + chr13:20189475 G>A", + "dp": 5, + "gq": 40, + "low_confidence": true, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 1, + "chrom": "chr3", + "pos": 15645186, + "ref": "G", + "alt": "C", + "genotype": "het", + "gene": "BTD", + "significance": "Pathogenic/Likely pathogenic", + "review_status": "criteria provided, conflicting classifications", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 0, + "chrom": "chr2", + "pos": 100, + "ref": "A", + "alt": "G", + "genotype": "het", + "gene": ".", + "significance": ".", + "review_status": ".", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 0, + "chrom": "chrX", + "pos": 101398000, + "ref": "C", + "alt": "T", + "genotype": "1", + "gene": "GLA", + "significance": "Pathogenic", + "review_status": "no assertion criteria provided", + "combined_allele": "possibly one allele with chrX:101398004 G>A: check ClinVar for a combined record", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + } + ], + "low_confidence": 2, + "call_floor": { + "GQ": 20, + "DP": 8 + }, + "frequency": { + "source": null, + "note": "no VEP output (step 13) in this folder", + "common_af": 0.01 + } + } + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_on_list": 3292, + "list": "assets/carrier_genes.tsv", + "carriers": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "MUTYH-related polyposis" + }, + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Symptomatic form of HFE-related hemochromatosis" + }, + { + "gene": "CFTR", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 4, + "inheritance": "AR", + "condition": "Cystic fibrosis" + }, + { + "gene": "GJB2", + "variant": "chr13:20189473 C>T", + "genotype": "one copy, other copy not called ref (1/.)", + "significance": "Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Nonsyndromic hearing loss, recessive (DFNB1A)" + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Fanconi anemia" + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 0, + "inheritance": "XL", + "condition": "Fabry disease" + }, + { + "gene": "HBA1", + "variant": "chr16:176700 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Alpha-thalassemia" + }, + { + "gene": "SMN1", + "variant": "chr5:70951946 C>T", + "genotype": "hom", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Spinal muscular atrophy" + } + ], + "set_aside": [ + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "significance": "Pathogenic/Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Biotinidase deficiency", + "kind": "hypomorph", + "name": "p.Asp444His (p.Asp424His on NM_001370658.1), partial biotinidase deficiency" + } + ], + "two_hits": { + "genes": [ + { + "gene": "CFTR", + "verdict": "possibly biallelic", + "why": "a heterozygous call inside a called deletion: one of the two calls may be wrong; check the reads", + "deletion": "CFTR_del deletion, 1 of 2 copies (read depth, step 35)", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "allele": null, + "condition": "Cystic fibrosis", + "inheritance": "AR" + } + ], + "deletions_read": [ + "AnnotSV class 4 and 5 deletions (step 05)", + "step 35's interval table (read depth)" + ], + "deletions_not_read": [ + "step 35's HBA1/HBA2 count: not assessed: S_hba_copy_number.tsv has no status row", + "Parascopy's SMN1 count: not resolved" + ] + } + } + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "ok", + "source": "vcf/B.report.json", + "file_date": "2021-03-04 05:07 UTC", + "note": null, + "data": { + "version": "", + "html_report": false + } + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "stale", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 27 was not run in the run of 2020-01-01 00:00 UTC", + "data": { + "genes": [ + { + "gene": "CYP2D6", + "diplotype": "*4/*4", + "phenotype": "Poor", + "status": "not called" + } + ], + "non_normal": 0, + "ambiguous": 0, + "not_called": 1, + "unclassified": 0, + "parse_failed": false, + "warnings": [] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "ok", + "source": "pypgx/S_pypgx_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_total": 6, + "genes_called": 3, + "cyp2d6": "*1/*2", + "cnv_call": { + "CYP2D6": "Normal", + "CYP2A6": "not called: pypgx failed", + "CYP2B6": "not called: the CYP2D6 depth check failed", + "SLCO1B1": "not called: pypgx gave no copy-number call" + }, + "not_callable": [ + "CYP2E1" + ], + "low_gq": [ + { + "gene": "UGT1A1", + "diplotype": "Indeterminate", + "min_gq": 12, + "weakest": "chr2:1:A:G" + }, + { + "gene": "SLCO1B1", + "diplotype": "*1/*5", + "min_gq": null, + "weakest": "" + } + ], + "cyp2d6_evidence": { + "phenotype": "Normal Metabolizer", + "defining_filter": "PASS", + "quality": "ok" + } + } + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/T_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "*1/*2", + "filter": "LowQ" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": null, + "text": "no row covers exon 7", + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": "." + }, + "smn2": { + "cn": null, + "text": "no row covers exon 7" + }, + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": ".", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "SMN1 g.27134T>G: not assessed, step 35 wrote no S_smn1_marker.tsv (a run from before this reader, or the check did not run)." + }, + "rows": 1, + "hba": { + "status": "not assessed", + "message": "not assessed: S_hba_copy_number.tsv has no status row" + }, + "cn_intervals": { + "rows": 3, + "by_status": { + "called": 3 + }, + "changes": [ + { + "locus": "CFTR_del", + "name": "CFTR region", + "chrom": "chr7", + "start": "117500000", + "end": "117600000", + "copies": "1", + "expected": "2", + "change": "deletion" + }, + { + "locus": "DUP1", + "name": "x", + "chrom": "chr1", + "start": "1", + "end": "100", + "copies": "3", + "expected": "2", + "change": "duplication" + }, + { + "locus": "GJB6", + "name": "x", + "chrom": "chr13", + "start": "x", + "end": "2", + "copies": "1", + "expected": "2", + "change": "deletion" + } + ] + }, + "cyp21a2": null, + "dosage": null, + "reading": "SMN1 and SMN2 not assessed: no model: no carrier reading", + "loci": [] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "x", + "status": "called", + "detail": "." + } + ], + "other_regions": 0, + "other_regions_called": 0, + "variants": {}, + "smn1": { + "paraphase": null, + "parascopy": { + "cn": null, + "text": "no row covers SMN1 exon 7" + }, + "agree": null + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*03:01", + "DQA1*01:01" + ], + "quality": [ + "0", + "30" + ], + "low_confidence": true, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*04:01", + "DRB1*07:01" + ], + "quality": [ + "40", + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 low confidence", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A not typed" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "present", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [], + "reason": "OptiType did not run: no class I reads" + }, + "kir": { + "typed": [], + "not_called": [], + "reason": "too few KIR reads" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "stale", + "source": "prs/S_prs_summary.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 25 was failed (exit 2) in the run of 2020-01-01 00:00 UTC", + "data": { + "scores": [ + { + "condition": "X", + "pgs_id": "PGS1", + "score": "", + "matched": "", + "total": "", + "chrx": "", + "ambiguous": "", + "applies_to": "", + "sex_check": "", + "percentile": "", + "group": "", + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": false, + "ancestry": { + "population": "", + "panel": "", + "low_confidence": "" + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/x/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 0, + "pass": 0 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "stale", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 18 was skipped (opt-in) in the run of 2020-01-01 00:00 UTC", + "data": { + "total": 2, + "deletions": 2, + "duplications": 0, + "significant": 2, + "significant_filtered": 1, + "filtered_chry": 1, + "dropped": { + "q0": 0, + "gap": 0, + "band": 0, + "chry_female": 1 + }, + "declared_sex": "female" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "stale", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 22 was failed (exit 1) in the run of 2020-01-01 00:00 UTC", + "data": { + "consensus": 1 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 0, + "rows": [], + "downgraded": [], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 0, + "exon_edge": 0, + "exon_edge_checked": true, + "common": 0 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 3, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "CFTR_T", + "repeat_count": "5/5", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/5-5", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "12/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "12-12/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "UGT1A1", + "repeat_count": "8/10", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "8-8/10-10", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": false, + "stranger_outdated": true, + "unsized": [], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG12-5T/TG12-5T", + "pairings": [ + "TG12-5T/TG12-5T" + ], + "tg": [ + 12, + 12 + ], + "poly_t": [ + 5, + 5 + ], + "five_t": true, + "phase": "not_read", + "r117h": ".", + "reading": "5T: 2 copys with the 5T poly-T tract, TG12 on the 5T copies. 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. R117H was not read (no step 42 traits row for rs78655421 was called), so whether it sits on the 5T copy is not known." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*28/TA9", + "ta_units": [ + 8, + 10 + ], + "pharmcat": ".", + "agreement": "not_compared", + "reading": "*28/TA9 (TA7/TA9, 8/10 TA units as ExpansionHunter counts them): a TA length outside TA5 to TA8 is not a named allele here, so its function is not read and it is not compared with PharmCAT (step 7)." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ] + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/x/deep/T_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "334", + "intratelomeric_reads": 61873, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "roh/S_roh.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": null, + "data": { + "segments": 1, + "total_mb": 7.0, + "largest_mb": 7.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + } + ], + "masked_mb": 0.0, + "sex_chromosome_segments": 0, + "long_mb": 7.0, + "f_roh": 0.0294, + "relationship": null, + "upd_flag": null, + "scope": { + "source": "sites", + "autosomes": [ + "chr2" + ], + "unmasked_bp": 237993529, + "partial": true, + "note": "Partial or unverified chromosome scope: no relationship band or UPD flag. F_ROH uses only the listed chromosomes, not callable bases." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "stale", + "source": "mito/S_haplogroup.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 12 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "haplogroup": "U5a", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/U5a" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/U5a/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "insufficient markers", + "valid_markers": ".", + "qc_score": ".", + "explore": [] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "no branch has a derived call", + "level": ".", + "tree": ".", + "path_derived": ".", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": ".", + "min_depth": ".", + "explore": [], + "path": [] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 10, + "called": 10, + "no_call": 0, + "from_reference_blocks": 0, + "template_sites": ".", + "file_23andme": ".", + "file_ancestrydna": ".", + "no_call_reasons": {} + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "too little depth", + "kiv2_copies": ".", + "ci95_low": ".", + "ci95_high": ".", + "class": ".", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": ".", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "LPA KIV-2 4925G>A: not assessed, step 46 wrote no S_lpa_kiv2_4925.tsv (a run from before this reader, or the check did not run)." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": ".", + "status": "not_covered", + "risk_alleles": null + } + ], + "risk_alleles": 0, + "flag": ".", + "flag_text": "Not determined: neither the KIV-2 count nor the two risk variants could be read. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "confidence": ".", + "variant": ".", + "flag": ".", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "." + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "stale", + "source": "locityper/S_locityper.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 53 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*04:01/DRB1*07:01" + }, + { + "locus": "HLA-DQB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DQB1*02:01" + }, + { + "locus": "CYP2D6", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "pypgx (step 32): *1/*2; Cyrius (step 21): *1/*2 (not usable: Filter LowQ)" + } + ], + "called": 3, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + } + ], + "deletions_note": "no deletion table (mito/_chrM_deletions.tsv): step 20 ran before it looked for large deletions, eKLIPse wrote no table (see the step's log), or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "copies_note": "no copy-number table (mito/_mtdna_copies.tsv): step 20 ran before it counted mtDNA copies, or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "pass": 1, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "note": "mito/S_chrM_annotated.tsv is older than the chrM calls: rerun step 20" + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [], + "called": 0, + "not_covered": 1, + "other_allele": 0, + "rules": null + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "not assessed (no human depth)", + "human_depth": ".", + "found": [], + "genomes": 1 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [], + "genes": [] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [], + "other": 0, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + } + ], + "median_depth": null, + "median_limit": null, + "not_covered": [ + "DNMT3A" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 0, + "by_element": {}, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "F", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "seen" + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "message": "too shallow", + "depth": ".", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": ".", + "calls_mocha": ".", + "calls_reported": ".", + "phasing": ".", + "calls": [], + "not_reported": 0 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "ok", + "source": "cpsr/T.cpsr.grch37.classification.tsv.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "html_report": true, + "classification_column": "FINAL_CLASSIFICATION", + "classification": { + "Pathogenic": 1, + ".": 1 + }, + "plp": [ + { + "variant": ".", + "gene": "A", + "final": "Pathogenic", + "cpsr": "Benign", + "clinvar": null, + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false + } + ], + "disagreements": [], + "final_plp_own_benign": 1, + "own_class_column": "CPSR_CLASSIFICATION", + "low_confidence": 0, + "call_floor_dp": 8 + } + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 6, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "low_confidence": false, + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "low_confidence": false, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "1", + "significance": "Pathogenic", + "stars": 0, + "low_confidence": false, + "rule": "All hemi, het, homozygous P and LP", + "rule_note": null, + "severity": null + } + ], + "clinvar_checked": true, + "high_impact": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "consequence": "stop_gained", + "transcript": "MANE", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": 40, + "dp": 30, + "ad": "15,15", + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 2, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": true, + "structural_note": null, + "not_reported": [ + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "source": "ClinVar", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + }, + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "source": "ClinVar", + "reason": "one heterozygous variant; BTD is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + } + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_prioritized.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 2, + "compound_het_pairs": 26, + "compound_het_phase": { + "trans": 23, + "cis": 1, + "unknown": 1 + }, + "compound_het_trans": [ + { + "gene": "G1", + "variants": "chr1:10 A>G + chr1:15 C>T", + "reads": "1 of 2" + }, + { + "gene": "G2", + "variants": "chr1:20 A>G + chr1:25 C>T", + "reads": "2 of 3" + }, + { + "gene": "G3", + "variants": "chr1:30 A>G + chr1:35 C>T", + "reads": "3 of 4" + }, + { + "gene": "G4", + "variants": "chr1:40 A>G + chr1:45 C>T", + "reads": "4 of 5" + }, + { + "gene": "G5", + "variants": "chr1:50 A>G + chr1:55 C>T", + "reads": "5 of 6" + }, + { + "gene": "G6", + "variants": "chr1:60 A>G + chr1:65 C>T", + "reads": "6 of 7" + }, + { + "gene": "G7", + "variants": "chr1:70 A>G + chr1:75 C>T", + "reads": "7 of 8" + }, + { + "gene": "G8", + "variants": "chr1:80 A>G + chr1:85 C>T", + "reads": "8 of 9" + }, + { + "gene": "G9", + "variants": "chr1:90 A>G + chr1:95 C>T", + "reads": "9 of 10" + }, + { + "gene": "G10", + "variants": "chr1:100 A>G + chr1:105 C>T", + "reads": "10 of 11" + }, + { + "gene": "G11", + "variants": "chr1:110 A>G + chr1:115 C>T", + "reads": "11 of 12" + }, + { + "gene": "G12", + "variants": "chr1:120 A>G + chr1:125 C>T", + "reads": "12 of 13" + }, + { + "gene": "G13", + "variants": "chr1:130 A>G + chr1:135 C>T", + "reads": "13 of 14" + }, + { + "gene": "G14", + "variants": "chr1:140 A>G + chr1:145 C>T", + "reads": "14 of 15" + }, + { + "gene": "G15", + "variants": "chr1:150 A>G + chr1:155 C>T", + "reads": "15 of 16" + }, + { + "gene": "G16", + "variants": "chr1:160 A>G + chr1:165 C>T", + "reads": "16 of 17" + }, + { + "gene": "G17", + "variants": "chr1:170 A>G + chr1:175 C>T", + "reads": "17 of 18" + }, + { + "gene": "G18", + "variants": "chr1:180 A>G + chr1:185 C>T", + "reads": "18 of 19" + }, + { + "gene": "G19", + "variants": "chr1:190 A>G + chr1:195 C>T", + "reads": "19 of 20" + }, + { + "gene": "G20", + "variants": "chr1:200 A>G + chr1:205 C>T", + "reads": "20 of 21" + } + ] + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": "not called", + "pypgx": "*1/*2", + "Cyrius": "*1/*2 (not usable: Filter LowQ)" + }, + "agree": null, + "consensus": null, + "probable": { + "diplotype": "*1/*2", + "phenotype": "Normal Metabolizer", + "label": "probable (one caller: pypgx)" + } + }, + "not_run": [ + "Mosaic loss of chromosome Y or X", + "Variant calling", + "Fine-grained ancestry (ADMIXTURE, RFMix)", + "Structural variants (Delly)", + "Y-STR markers (HipSTR)", + "Blood groups (predicted)", + "Archaic ancestry (IBDmix)", + "Ancient populations (AADR PCA)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/parity/golden/summary-collect/all/all.json b/tests/parity/golden/summary-collect/all/all.json new file mode 100644 index 000000000..0ccd9b68a --- /dev/null +++ b/tests/parity/golden/summary-collect/all/all.json @@ -0,0 +1,15495 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:19 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "male" + }, + "databases": { + "clinvar_release": "2026-10-01", + "cpsr_clinvar_release": "20250901", + "hla_database": "IPD-IMGT/HLA 3.60\textra" + }, + "manifest": [ + [ + "run", + "declared_sex", + "male" + ], + [ + "data", + "clinvar_file_date", + "2026-10-01" + ], + [ + "data", + "cpsr_clinvar_release", + "20250901" + ], + [ + "data", + "hla_database", + "IPD-IMGT/HLA 3.60\textra" + ] + ], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": 31.25 + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S_callability.bed.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "min_depth": 10, + "min_mapq": 20, + "min_share": 0.9, + "catalog_genes": 1572, + "read": 217, + "not_read": [ + { + "gene": "OR4F5", + "hgnc_id": "HGNC:14825", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR4F29", + "hgnc_id": "HGNC:31275", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F16", + "hgnc_id": "HGNC:15079", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "AGRN", + "hgnc_id": "HGNC:329", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ATAD3B", + "hgnc_id": "HGNC:24007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MMP23B", + "hgnc_id": "HGNC:7171", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CDK11B", + "hgnc_id": "HGNC:1729", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35E2B", + "hgnc_id": "HGNC:33941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDK11A", + "hgnc_id": "HGNC:1730", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TTC34", + "hgnc_id": "HGNC:34297", + "reasons": [ + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPL22", + "hgnc_id": "HGNC:10315", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MASP2", + "hgnc_id": "HGNC:6902", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPPA", + "hgnc_id": "HGNC:7939", + "reasons": [ + "CMRG" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KIAA2013", + "hgnc_id": "HGNC:28513", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF1", + "hgnc_id": "HGNC:28840", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF11", + "hgnc_id": "HGNC:14086", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL1", + "hgnc_id": "HGNC:29295", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRAMEF4", + "hgnc_id": "HGNC:31971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF10", + "hgnc_id": "HGNC:27997", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRAMEF7", + "hgnc_id": "HGNC:28415", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF6", + "hgnc_id": "HGNC:30583", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF27", + "hgnc_id": "HGNC:51234", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL3", + "hgnc_id": "HGNC:51235", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HNRNPCL2", + "hgnc_id": "HGNC:48813", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF26", + "hgnc_id": "HGNC:49178", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HNRNPCL4", + "hgnc_id": "HGNC:51333", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "PRAMEF9", + "hgnc_id": "HGNC:27996", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF13", + "hgnc_id": "HGNC:13262", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRAMEF18", + "hgnc_id": "HGNC:30693", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRAMEF8", + "hgnc_id": "HGNC:24074", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF33", + "hgnc_id": "HGNC:49193", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PRAMEF15", + "hgnc_id": "HGNC:26764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRAMEF14", + "hgnc_id": "HGNC:13576", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF19", + "hgnc_id": "HGNC:24908", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "PRAMEF17", + "hgnc_id": "HGNC:29485", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CELA2B", + "hgnc_id": "HGNC:29995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CLCNKA", + "hgnc_id": "HGNC:2026", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CLCNKB", + "hgnc_id": "HGNC:2027", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NBPF1", + "hgnc_id": "HGNC:26088", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CROCC", + "hgnc_id": "HGNC:21299", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AKR7L", + "hgnc_id": "HGNC:24056", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF3", + "hgnc_id": "HGNC:25076", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CELA3B", + "hgnc_id": "HGNC:15945", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CELA3A", + "hgnc_id": "HGNC:15944", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HMGCL", + "hgnc_id": "HGNC:5005", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNR2", + "hgnc_id": "HGNC:2160", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RSRP1", + "hgnc_id": "HGNC:25234", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGV", + "hgnc_id": "HGNC:26031", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "XKR8", + "hgnc_id": "HGNC:25508", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BMP8A", + "hgnc_id": "HGNC:21650", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BMP8B", + "hgnc_id": "HGNC:1075", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OXCT2", + "hgnc_id": "HGNC:18606", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CYP4Z1", + "hgnc_id": "HGNC:20583", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4A22", + "hgnc_id": "HGNC:20575", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GBP1", + "hgnc_id": "HGNC:4182", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SETSIP", + "hgnc_id": "HGNC:42937", + "reasons": [ + "segdup" + ], + "share": 0.4994, + "status": "not read" + }, + { + "gene": "AGL", + "hgnc_id": "HGNC:321", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "EXTL2", + "hgnc_id": "HGNC:3516", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AMY2A", + "hgnc_id": "HGNC:477", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF4", + "hgnc_id": "HGNC:26550", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NBPF6", + "hgnc_id": "HGNC:31988", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LRIG2", + "hgnc_id": "HGNC:20889", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "VANGL1", + "hgnc_id": "HGNC:15512", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IGSF3", + "hgnc_id": "HGNC:5950", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SEC22B", + "hgnc_id": "HGNC:10700", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF26", + "hgnc_id": "HGNC:49571", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2NLR", + "hgnc_id": "HGNC:53925", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4A", + "hgnc_id": "HGNC:24369", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM72B", + "hgnc_id": "HGNC:24805", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SRGAP2C", + "hgnc_id": "HGNC:30584", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3-7", + "hgnc_id": "HGNC:32060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPIAL4E", + "hgnc_id": "HGNC:33997", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF15", + "hgnc_id": "HGNC:28791", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4F", + "hgnc_id": "HGNC:33999", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "SRGAP2B", + "hgnc_id": "HGNC:35237", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM72D", + "hgnc_id": "HGNC:33593", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PPIAL4D", + "hgnc_id": "HGNC:33998", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "GPR89A", + "hgnc_id": "HGNC:31984", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBM8A", + "hgnc_id": "HGNC:9905", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "NBPF10", + "hgnc_id": "HGNC:31992", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLA", + "hgnc_id": "HGNC:31862", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPIAL4H", + "hgnc_id": "HGNC:53889", + "reasons": [ + "segdup" + ], + "share": 0.6658, + "status": "not read" + }, + { + "gene": "NBPF12", + "hgnc_id": "HGNC:24297", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF11", + "hgnc_id": "HGNC:31993", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PPIAL4G", + "hgnc_id": "HGNC:33996", + "reasons": [ + "segdup" + ], + "share": 0.1656, + "status": "not read" + }, + { + "gene": "NBPF14", + "hgnc_id": "HGNC:25232", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLB", + "hgnc_id": "HGNC:53923", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NUDT4B", + "hgnc_id": "HGNC:18012", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PDE4DIP", + "hgnc_id": "HGNC:15580", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NOTCH2NLC", + "hgnc_id": "HGNC:53924", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF19", + "hgnc_id": "HGNC:31999", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4C", + "hgnc_id": "HGNC:33995", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "H2BC18", + "hgnc_id": "HGNC:24700", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3C13", + "hgnc_id": "HGNC:25311", + "reasons": [ + "segdup" + ], + "share": 0.8316, + "status": "not read" + }, + { + "gene": "H3C14", + "hgnc_id": "HGNC:20503", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H2AC18", + "hgnc_id": "HGNC:4736", + "reasons": [ + "segdup" + ], + "share": 0.1651, + "status": "not read" + }, + { + "gene": "H2AC19", + "hgnc_id": "HGNC:29668", + "reasons": [ + "segdup" + ], + "share": 0.3328, + "status": "not read" + }, + { + "gene": "H3C15", + "hgnc_id": "HGNC:20505", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H4C15", + "hgnc_id": "HGNC:29607", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HRNR", + "hgnc_id": "HGNC:20846", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LCE3E", + "hgnc_id": "HGNC:29463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCE3D", + "hgnc_id": "HGNC:16615", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "S100A8", + "hgnc_id": "HGNC:10498", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "S100A7A", + "hgnc_id": "HGNC:21657", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "S100A7", + "hgnc_id": "HGNC:10497", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FLAD1", + "hgnc_id": "HGNC:24671", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MTX1", + "hgnc_id": "HGNC:7504", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN3", + "hgnc_id": "HGNC:19183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FCRL5", + "hgnc_id": "HGNC:18508", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA6", + "hgnc_id": "HGNC:5239", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FCGR2B", + "hgnc_id": "HGNC:3618", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD17B7", + "hgnc_id": "HGNC:5215", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD247", + "hgnc_id": "HGNC:1677", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "XCL2", + "hgnc_id": "HGNC:10646", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "PRG4", + "hgnc_id": "HGNC:9364", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PTPRC", + "hgnc_id": "HGNC:9666", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPP1R12B", + "hgnc_id": "HGNC:7619", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KISS1", + "hgnc_id": "HGNC:6341", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SRGAP2", + "hgnc_id": "HGNC:19751", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CR1", + "hgnc_id": "HGNC:2334", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CR1L", + "hgnc_id": "HGNC:2335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZC3H11B", + "hgnc_id": "HGNC:25659", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LBR", + "hgnc_id": "HGNC:6518", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "LEFTY2", + "hgnc_id": "HGNC:3122", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "OR2T33", + "hgnc_id": "HGNC:31255", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T7", + "hgnc_id": "HGNC:15019", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T2", + "hgnc_id": "HGNC:14725", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T3", + "hgnc_id": "HGNC:14727", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR2T5", + "hgnc_id": "HGNC:15017", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "OR2G6", + "hgnc_id": "HGNC:27019", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR2T34", + "hgnc_id": "HGNC:31256", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T11", + "hgnc_id": "HGNC:19574", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T35", + "hgnc_id": "HGNC:31257", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T27", + "hgnc_id": "HGNC:31252", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SNTG2", + "hgnc_id": "HGNC:13741", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TPO", + "hgnc_id": "HGNC:12015", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KLF11", + "hgnc_id": "HGNC:11811", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CCDC121", + "hgnc_id": "HGNC:25833", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "ABCG8", + "hgnc_id": "HGNC:13887", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PCBP1", + "hgnc_id": "HGNC:8647", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "NAT8", + "hgnc_id": "HGNC:18069", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "C2orf78", + "hgnc_id": "HGNC:34349", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CD8B", + "hgnc_id": "HGNC:1707", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RGPD1", + "hgnc_id": "HGNC:32414", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLGLB1", + "hgnc_id": "HGNC:9072", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLGLB2", + "hgnc_id": "HGNC:9073", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RGPD2", + "hgnc_id": "HGNC:32415", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPIA", + "hgnc_id": "HGNC:10297", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEKT4", + "hgnc_id": "HGNC:31012", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAHD2A", + "hgnc_id": "HGNC:24252", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TRIM43B", + "hgnc_id": "HGNC:37146", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM43", + "hgnc_id": "HGNC:19015", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANKRD36C", + "hgnc_id": "HGNC:32946", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPAT2", + "hgnc_id": "HGNC:27168", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD36", + "hgnc_id": "HGNC:24079", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANKRD36B", + "hgnc_id": "HGNC:29333", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "RGPD3", + "hgnc_id": "HGNC:32416", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD4", + "hgnc_id": "HGNC:32417", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RANBP2", + "hgnc_id": "HGNC:9848", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD5", + "hgnc_id": "HGNC:32418", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MALL", + "hgnc_id": "HGNC:6818", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MTLN", + "hgnc_id": "HGNC:27339", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LIMS4", + "hgnc_id": "HGNC:39941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD6", + "hgnc_id": "HGNC:32419", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANAPC1", + "hgnc_id": "HGNC:19988", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD8", + "hgnc_id": "HGNC:9849", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FOXD4L1", + "hgnc_id": "HGNC:18521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RABL2A", + "hgnc_id": "HGNC:9799", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RAB6C", + "hgnc_id": "HGNC:16525", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "POTEF", + "hgnc_id": "HGNC:33905", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CCDC74B", + "hgnc_id": "HGNC:25267", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SMPD4", + "hgnc_id": "HGNC:32949", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3E", + "hgnc_id": "HGNC:20765", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEI", + "hgnc_id": "HGNC:37093", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CFC1B", + "hgnc_id": "HGNC:33983", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CFC1", + "hgnc_id": "HGNC:18292", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "POTEJ", + "hgnc_id": "HGNC:37094", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PLEKHB2", + "hgnc_id": "HGNC:19236", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB6D", + "hgnc_id": "HGNC:30272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MZT2A", + "hgnc_id": "HGNC:33187", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TUBA3D", + "hgnc_id": "HGNC:24071", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CCDC74A", + "hgnc_id": "HGNC:25197", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "ANKRD30BL", + "hgnc_id": "HGNC:35167", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NOSTRIN", + "hgnc_id": "HGNC:20203", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IFT70B", + "hgnc_id": "HGNC:26425", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CASP10", + "hgnc_id": "HGNC:1500", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ALPP", + "hgnc_id": "HGNC:439", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ALPG", + "hgnc_id": "HGNC:441", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT1A6", + "hgnc_id": "HGNC:12538", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UGT1A5", + "hgnc_id": "HGNC:12537", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT1A3", + "hgnc_id": "HGNC:12535", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MLPH", + "hgnc_id": "HGNC:29643", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR6B2", + "hgnc_id": "HGNC:15041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AQP12B", + "hgnc_id": "HGNC:6096", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "AQP12A", + "hgnc_id": "HGNC:19941", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ANO7", + "hgnc_id": "HGNC:31677", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CHL1", + "hgnc_id": "HGNC:1939", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMC3", + "hgnc_id": "HGNC:23999", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANCD2", + "hgnc_id": "HGNC:3585", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DAZL", + "hgnc_id": "HGNC:2685", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF35", + "hgnc_id": "HGNC:13099", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LZTFL1", + "hgnc_id": "HGNC:6741", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MST1", + "hgnc_id": "HGNC:7380", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MST1R", + "hgnc_id": "HGNC:7381", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HYAL1", + "hgnc_id": "HGNC:5320", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FRG2C", + "hgnc_id": "HGNC:33626", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "ZNF717", + "hgnc_id": "HGNC:29448", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR5H6", + "hgnc_id": "HGNC:14767", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ROPN1", + "hgnc_id": "HGNC:17692", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ROPN1B", + "hgnc_id": "HGNC:31927", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RPN1", + "hgnc_id": "HGNC:10381", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ALG1L2", + "hgnc_id": "HGNC:37258", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BFSP2", + "hgnc_id": "HGNC:1041", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRR23B", + "hgnc_id": "HGNC:33764", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SOX2", + "hgnc_id": "HGNC:11195", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "EIF2B5", + "hgnc_id": "HGNC:3261", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KNG1", + "hgnc_id": "HGNC:6383", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TM4SF19", + "hgnc_id": "HGNC:25167", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF595", + "hgnc_id": "HGNC:27196", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDE6B", + "hgnc_id": "HGNC:8786", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAK", + "hgnc_id": "HGNC:4113", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FGFRL1", + "hgnc_id": "HGNC:3693", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RNF212", + "hgnc_id": "HGNC:27729", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UVSSA", + "hgnc_id": "HGNC:29304", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DOK7", + "hgnc_id": "HGNC:26594", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HMX1", + "hgnc_id": "HGNC:5017", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A26", + "hgnc_id": "HGNC:43746", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "USP17L10", + "hgnc_id": "HGNC:44438", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L11", + "hgnc_id": "HGNC:44439", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L12", + "hgnc_id": "HGNC:44440", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L13", + "hgnc_id": "HGNC:44441", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L17", + "hgnc_id": "HGNC:44445", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L18", + "hgnc_id": "HGNC:44446", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L19", + "hgnc_id": "HGNC:44447", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L20", + "hgnc_id": "HGNC:44448", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L21", + "hgnc_id": "HGNC:44449", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L22", + "hgnc_id": "HGNC:44450", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L24", + "hgnc_id": "HGNC:44453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L25", + "hgnc_id": "HGNC:44452", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L26", + "hgnc_id": "HGNC:44454", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L5", + "hgnc_id": "HGNC:37177", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L27", + "hgnc_id": "HGNC:44455", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L28", + "hgnc_id": "HGNC:44456", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L30", + "hgnc_id": "HGNC:44458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB131A", + "hgnc_id": "HGNC:18108", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRD5", + "hgnc_id": "HGNC:3026", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "UGT2B10", + "hgnc_id": "HGNC:12544", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B28", + "hgnc_id": "HGNC:13479", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UGT2A1", + "hgnc_id": "HGNC:12542", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UGT2A2", + "hgnc_id": "HGNC:28183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HTN3", + "hgnc_id": "HGNC:5284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HTN1", + "hgnc_id": "HGNC:5283", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "AFP", + "hgnc_id": "HGNC:317", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGY", + "hgnc_id": "HGNC:28213", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PYURF", + "hgnc_id": "HGNC:44317", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CISD2", + "hgnc_id": "HGNC:24212", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC9B1", + "hgnc_id": "HGNC:24244", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KLHL2", + "hgnc_id": "HGNC:6353", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GK3", + "hgnc_id": "HGNC:4292", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "HPF1", + "hgnc_id": "HGNC:26051", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDLIM3", + "hgnc_id": "HGNC:20767", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAT1", + "hgnc_id": "HGNC:3595", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG1", + "hgnc_id": "HGNC:3954", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FRG2", + "hgnc_id": "HGNC:19136", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DUX4", + "hgnc_id": "HGNC:50800", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SDHA", + "hgnc_id": "HGNC:10680", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDCD6", + "hgnc_id": "HGNC:8765", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZDHHC11B", + "hgnc_id": "HGNC:32962", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZDHHC11", + "hgnc_id": "HGNC:19158", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TERT", + "hgnc_id": "HGNC:11730", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A3", + "hgnc_id": "HGNC:11049", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAF11L2", + "hgnc_id": "HGNC:53845", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L3", + "hgnc_id": "HGNC:53846", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L4", + "hgnc_id": "HGNC:53847", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L5", + "hgnc_id": "HGNC:53848", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "TAF11L7", + "hgnc_id": "HGNC:53850", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAF11L8", + "hgnc_id": "HGNC:53851", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "TAF11L9", + "hgnc_id": "HGNC:53852", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L10", + "hgnc_id": "HGNC:53853", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L11", + "hgnc_id": "HGNC:53854", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L12", + "hgnc_id": "HGNC:53855", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "TAF11L14", + "hgnc_id": "HGNC:53857", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMB", + "hgnc_id": "HGNC:30465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MARVELD2", + "hgnc_id": "HGNC:26401", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERF1B", + "hgnc_id": "HGNC:10756", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NAIP", + "hgnc_id": "HGNC:7634", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SKIC3", + "hgnc_id": "HGNC:23639", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIX1", + "hgnc_id": "HGNC:18581", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SAR1B", + "hgnc_id": "HGNC:10535", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DND1", + "hgnc_id": "HGNC:23799", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDHA10", + "hgnc_id": "HGNC:8664", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PCDHB7", + "hgnc_id": "HGNC:8692", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PCDHB8", + "hgnc_id": "HGNC:8693", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PCDHB11", + "hgnc_id": "HGNC:8682", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PCDHB12", + "hgnc_id": "HGNC:8683", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPP1R2B", + "hgnc_id": "HGNC:16318", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPM1", + "hgnc_id": "HGNC:7910", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "THOC3", + "hgnc_id": "HGNC:19072", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SIMC1", + "hgnc_id": "HGNC:24779", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM153A", + "hgnc_id": "HGNC:29940", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FAM153CP", + "hgnc_id": "HGNC:33936", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FLT4", + "hgnc_id": "HGNC:3767", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR4F3", + "hgnc_id": "HGNC:8300", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "DUSP22", + "hgnc_id": "HGNC:16077", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TUBB2A", + "hgnc_id": "HGNC:12412", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "H2AC13", + "hgnc_id": "HGNC:4725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H2AC14", + "hgnc_id": "HGNC:4727", + "reasons": [ + "segdup" + ], + "share": 0.8319, + "status": "not read" + }, + { + "gene": "OR2J1", + "hgnc_id": "HGNC:8259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2J2", + "hgnc_id": "HGNC:8260", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HLA-G", + "hgnc_id": "HGNC:4964", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HLA-A", + "hgnc_id": "HGNC:4931", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "MICA", + "hgnc_id": "HGNC:7090", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MICB", + "hgnc_id": "HGNC:7091", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZBTB12", + "hgnc_id": "HGNC:19066", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PBX2", + "hgnc_id": "HGNC:8633", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HLA-DRB1", + "hgnc_id": "HGNC:4948", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GSTA2", + "hgnc_id": "HGNC:4627", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GSTA1", + "hgnc_id": "HGNC:4626", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTA5", + "hgnc_id": "HGNC:19662", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRIM2", + "hgnc_id": "HGNC:9370", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FKBP1C", + "hgnc_id": "HGNC:21376", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SLC17A5", + "hgnc_id": "HGNC:10933", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEC63", + "hgnc_id": "HGNC:21082", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR2A4", + "hgnc_id": "HGNC:14729", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE9", + "hgnc_id": "HGNC:37275", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "PCMT1", + "hgnc_id": "HGNC:8728", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RAET1G", + "hgnc_id": "HGNC:16795", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ULBP1", + "hgnc_id": "HGNC:14893", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RAET1L", + "hgnc_id": "HGNC:16798", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ULBP3", + "hgnc_id": "HGNC:14895", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC22A1", + "hgnc_id": "HGNC:10963", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMOC2", + "hgnc_id": "HGNC:20323", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RBAK-RBAKDN", + "hgnc_id": "HGNC:42971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBAK", + "hgnc_id": "HGNC:17680", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC29A4", + "hgnc_id": "HGNC:23097", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FSCN1", + "hgnc_id": "HGNC:11148", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM", + "hgnc_id": "HGNC:8105", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCZ1", + "hgnc_id": "HGNC:21691", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RSPH10B2", + "hgnc_id": "HGNC:34385", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CCZ1B", + "hgnc_id": "HGNC:21717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOD1", + "hgnc_id": "HGNC:16390", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FKBP9", + "hgnc_id": "HGNC:3725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DPY19L1", + "hgnc_id": "HGNC:22205", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SEPTIN7", + "hgnc_id": "HGNC:1717", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE1", + "hgnc_id": "HGNC:16408", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIA", + "hgnc_id": "HGNC:9253", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDC14C", + "hgnc_id": "HGNC:22427", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSPH", + "hgnc_id": "HGNC:9577", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCT6A", + "hgnc_id": "HGNC:1620", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF716", + "hgnc_id": "HGNC:32458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF727", + "hgnc_id": "HGNC:22785", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF735", + "hgnc_id": "HGNC:32466", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF736", + "hgnc_id": "HGNC:32467", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GUSB", + "hgnc_id": "HGNC:4696", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RABGEF1", + "hgnc_id": "HGNC:17676", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TYW1", + "hgnc_id": "HGNC:25598", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE21", + "hgnc_id": "HGNC:51517", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYW1B", + "hgnc_id": "HGNC:33908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POM121", + "hgnc_id": "HGNC:19702", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM74", + "hgnc_id": "HGNC:17453", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPDYE8", + "hgnc_id": "HGNC:33771", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE9", + "hgnc_id": "HGNC:45034", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE10", + "hgnc_id": "HGNC:51506", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NSUN5", + "hgnc_id": "HGNC:16385", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM50", + "hgnc_id": "HGNC:19017", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLIP2", + "hgnc_id": "HGNC:2586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GTF2IRD2", + "hgnc_id": "HGNC:30775", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE12", + "hgnc_id": "HGNC:51508", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CASTOR2", + "hgnc_id": "HGNC:37073", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GTF2IRD2B", + "hgnc_id": "HGNC:33125", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPDYE13", + "hgnc_id": "HGNC:51509", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE14", + "hgnc_id": "HGNC:51510", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM73", + "hgnc_id": "HGNC:18162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POM121C", + "hgnc_id": "HGNC:34005", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPDYE5", + "hgnc_id": "HGNC:35464", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DTX2", + "hgnc_id": "HGNC:15973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UPK3B", + "hgnc_id": "HGNC:21444", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE16", + "hgnc_id": "HGNC:51512", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE17", + "hgnc_id": "HGNC:51513", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE18", + "hgnc_id": "HGNC:51514", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "STEAP1", + "hgnc_id": "HGNC:11378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ASNS", + "hgnc_id": "HGNC:753", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM2", + "hgnc_id": "HGNC:34396", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP3A7-CYP3A51P", + "hgnc_id": "HGNC:51504", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP3A4", + "hgnc_id": "HGNC:2637", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "STAG3", + "hgnc_id": "HGNC:11356", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PVRIG", + "hgnc_id": "HGNC:32190", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SPDYE3", + "hgnc_id": "HGNC:35462", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SAP25", + "hgnc_id": "HGNC:41908", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZAN", + "hgnc_id": "HGNC:12857", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC12", + "hgnc_id": "HGNC:7510", + "reasons": [ + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE6", + "hgnc_id": "HGNC:35465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "POLR2J", + "hgnc_id": "HGNC:9197", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RASA4B", + "hgnc_id": "HGNC:35202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J3", + "hgnc_id": "HGNC:33853", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UPK3BL2", + "hgnc_id": "HGNC:53444", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "RASA4", + "hgnc_id": "HGNC:23181", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UPK3BL1", + "hgnc_id": "HGNC:37278", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "POLR2J3-UPK3BL2", + "hgnc_id": "HGNC:58364", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SPDYE2B", + "hgnc_id": "HGNC:48334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J2", + "hgnc_id": "HGNC:23208", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAMB1", + "hgnc_id": "HGNC:6486", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2B", + "hgnc_id": "HGNC:18272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KLF14", + "hgnc_id": "HGNC:23025", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AKR1B10", + "hgnc_id": "HGNC:382", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1B15", + "hgnc_id": "HGNC:37281", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BRAF", + "hgnc_id": "HGNC:1097", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRBV9", + "hgnc_id": "HGNC:12246", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRSS2", + "hgnc_id": "HGNC:9483", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAGE15", + "hgnc_id": "HGNC:37295", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TCAF2", + "hgnc_id": "HGNC:26878", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TCAF2C", + "hgnc_id": "HGNC:53641", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CTAGE6", + "hgnc_id": "HGNC:28644", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "OR2F2", + "hgnc_id": "HGNC:8247", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CTAGE4", + "hgnc_id": "HGNC:24772", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2A42", + "hgnc_id": "HGNC:31230", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2A7", + "hgnc_id": "HGNC:8234", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE8", + "hgnc_id": "HGNC:37294", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "OR2A1", + "hgnc_id": "HGNC:8229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARHGEF5", + "hgnc_id": "HGNC:13209", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACTR3C", + "hgnc_id": "HGNC:37282", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KMT2C", + "hgnc_id": "HGNC:13726", + "reasons": [ + "CMRG", + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ACTR3B", + "hgnc_id": "HGNC:17256", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F21", + "hgnc_id": "HGNC:19583", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "FBXO25", + "hgnc_id": "HGNC:13596", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARHGEF10", + "hgnc_id": "HGNC:14103", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A15", + "hgnc_id": "HGNC:32263", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A3", + "hgnc_id": "HGNC:32251", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A13", + "hgnc_id": "HGNC:32261", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A5", + "hgnc_id": "HGNC:32253", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM90A20", + "hgnc_id": "HGNC:32268", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L1", + "hgnc_id": "HGNC:37182", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L4", + "hgnc_id": "HGNC:37176", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNF705G", + "hgnc_id": "HGNC:37134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB108C", + "hgnc_id": "HGNC:30847", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB4B", + "hgnc_id": "HGNC:30193", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "DEFB103B", + "hgnc_id": "HGNC:31702", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "DEFB104B", + "hgnc_id": "HGNC:26165", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB106B", + "hgnc_id": "HGNC:28879", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "DEFB105B", + "hgnc_id": "HGNC:29930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "DEFB107B", + "hgnc_id": "HGNC:31918", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRR23D1", + "hgnc_id": "HGNC:49420", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A7", + "hgnc_id": "HGNC:32255", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FAM90A23", + "hgnc_id": "HGNC:32271", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A14", + "hgnc_id": "HGNC:32262", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A18", + "hgnc_id": "HGNC:32266", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A16", + "hgnc_id": "HGNC:32264", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A8", + "hgnc_id": "HGNC:32256", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A17", + "hgnc_id": "HGNC:32265", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A9", + "hgnc_id": "HGNC:32257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A10", + "hgnc_id": "HGNC:32258", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PRR23D2", + "hgnc_id": "HGNC:49396", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB107A", + "hgnc_id": "HGNC:18086", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB105A", + "hgnc_id": "HGNC:18087", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DEFB106A", + "hgnc_id": "HGNC:18088", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPAG11A", + "hgnc_id": "HGNC:33342", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB103A", + "hgnc_id": "HGNC:15967", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DEFB4A", + "hgnc_id": "HGNC:2767", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNF705B", + "hgnc_id": "HGNC:32284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "USP17L8", + "hgnc_id": "HGNC:37181", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L3", + "hgnc_id": "HGNC:37175", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "FAM90A11", + "hgnc_id": "HGNC:32259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A24", + "hgnc_id": "HGNC:32272", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A12", + "hgnc_id": "HGNC:32260", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLC35G5", + "hgnc_id": "HGNC:15546", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "DEFB130B", + "hgnc_id": "HGNC:39814", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF705D", + "hgnc_id": "HGNC:33202", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L2", + "hgnc_id": "HGNC:34434", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB109D", + "hgnc_id": "HGNC:30838", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM86B1", + "hgnc_id": "HGNC:28268", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB130A", + "hgnc_id": "HGNC:18107", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM86B2", + "hgnc_id": "HGNC:32222", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RBPMS", + "hgnc_id": "HGNC:19097", + "reasons": [ + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IMPA1", + "hgnc_id": "HGNC:6050", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC10A5", + "hgnc_id": "HGNC:22981", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH17", + "hgnc_id": "HGNC:1756", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP11B1", + "hgnc_id": "HGNC:2591", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP11B2", + "hgnc_id": "HGNC:2592", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GPIHBP1", + "hgnc_id": "HGNC:24945", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "WASHC1", + "hgnc_id": "HGNC:24361", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOXD4", + "hgnc_id": "HGNC:3805", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNG1A", + "hgnc_id": "HGNC:17134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KCNV2", + "hgnc_id": "HGNC:19698", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RLN2", + "hgnc_id": "HGNC:10027", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RLN1", + "hgnc_id": "HGNC:10026", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "IFNA21", + "hgnc_id": "HGNC:5424", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA4", + "hgnc_id": "HGNC:5425", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "IFNA7", + "hgnc_id": "HGNC:5428", + "reasons": [ + "segdup" + ], + "share": 0.3324, + "status": "not read" + }, + { + "gene": "IFNA10", + "hgnc_id": "HGNC:5418", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "IFNA16", + "hgnc_id": "HGNC:5421", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "IFNA17", + "hgnc_id": "HGNC:5422", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "IFNA13", + "hgnc_id": "HGNC:5419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA1", + "hgnc_id": "HGNC:5417", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "ANKRD18B", + "hgnc_id": "HGNC:23644", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRSS3", + "hgnc_id": "HGNC:9486", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALT", + "hgnc_id": "HGNC:4135", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31F1", + "hgnc_id": "HGNC:41911", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CNTNAP3", + "hgnc_id": "HGNC:13834", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31A1", + "hgnc_id": "HGNC:23394", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L6", + "hgnc_id": "HGNC:31986", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNG1F", + "hgnc_id": "HGNC:31978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNTNAP3B", + "hgnc_id": "HGNC:32035", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31A6", + "hgnc_id": "HGNC:32006", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPATA31A7", + "hgnc_id": "HGNC:32007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CNTNAP3C", + "hgnc_id": "HGNC:53878", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L5", + "hgnc_id": "HGNC:18522", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNG1E", + "hgnc_id": "HGNC:24584", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXD4L4", + "hgnc_id": "HGNC:23762", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF658", + "hgnc_id": "HGNC:25226", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD20A1", + "hgnc_id": "HGNC:23665", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNG1C", + "hgnc_id": "HGNC:18519", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L3", + "hgnc_id": "HGNC:18523", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PGM5", + "hgnc_id": "HGNC:8908", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FXN", + "hgnc_id": "HGNC:3951", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TJP2", + "hgnc_id": "HGNC:11828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPATA31D4", + "hgnc_id": "HGNC:38601", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31D3", + "hgnc_id": "HGNC:38603", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPATA31D1", + "hgnc_id": "HGNC:37283", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "KIF27", + "hgnc_id": "HGNC:18632", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPATA31C1", + "hgnc_id": "HGNC:27846", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPATA31C2", + "hgnc_id": "HGNC:24508", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC71A2", + "hgnc_id": "HGNC:23376", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2G", + "hgnc_id": "HGNC:23449", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CTSV", + "hgnc_id": "HGNC:2538", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR13C5", + "hgnc_id": "HGNC:15100", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR13C2", + "hgnc_id": "HGNC:14701", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR13C9", + "hgnc_id": "HGNC:15104", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "POLE3", + "hgnc_id": "HGNC:13546", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ORM1", + "hgnc_id": "HGNC:8498", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ORM2", + "hgnc_id": "HGNC:8499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SLC27A4", + "hgnc_id": "HGNC:10998", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PKN3", + "hgnc_id": "HGNC:17999", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ABO", + "hgnc_id": "HGNC:79", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCN1", + "hgnc_id": "HGNC:6525", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OBP2A", + "hgnc_id": "HGNC:23380", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SOHLH1", + "hgnc_id": "HGNC:27845", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "INPP5E", + "hgnc_id": "HGNC:21474", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AJM1", + "hgnc_id": "HGNC:37284", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRIN1", + "hgnc_id": "HGNC:4584", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8", + "hgnc_id": "HGNC:20773", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "AKR1C1", + "hgnc_id": "HGNC:384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1C2", + "hgnc_id": "HGNC:385", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DCLRE1C", + "hgnc_id": "HGNC:17642", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRC1", + "hgnc_id": "HGNC:7228", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD30A", + "hgnc_id": "HGNC:17234", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF33A", + "hgnc_id": "HGNC:13096", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF37A", + "hgnc_id": "HGNC:13102", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLD5P1", + "hgnc_id": "HGNC:55072", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF33B", + "hgnc_id": "HGNC:13097", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BMS1", + "hgnc_id": "HGNC:23505", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AGAP4", + "hgnc_id": "HGNC:23459", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23", + "hgnc_id": "HGNC:17312", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ANTXRL", + "hgnc_id": "HGNC:27277", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANXA8L1", + "hgnc_id": "HGNC:23334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPY4R", + "hgnc_id": "HGNC:9329", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPRIN2", + "hgnc_id": "HGNC:23730", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PTPN20", + "hgnc_id": "HGNC:23423", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANXA8", + "hgnc_id": "HGNC:546", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM25G", + "hgnc_id": "HGNC:23590", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP9", + "hgnc_id": "HGNC:23463", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SYT15B", + "hgnc_id": "HGNC:51487", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPY4R2", + "hgnc_id": "HGNC:52383", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PARG", + "hgnc_id": "HGNC:8605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23B", + "hgnc_id": "HGNC:23581", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TIMM23B-AGAP6", + "hgnc_id": "HGNC:45009", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP6", + "hgnc_id": "HGNC:23466", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASHC2A", + "hgnc_id": "HGNC:23416", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ASAH2B", + "hgnc_id": "HGNC:23456", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EIF5AL1", + "hgnc_id": "HGNC:17419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2B", + "hgnc_id": "HGNC:23445", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NUTM2E", + "hgnc_id": "HGNC:23448", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMEM254", + "hgnc_id": "HGNC:25804", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PLAC9", + "hgnc_id": "HGNC:19255", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ADIRF", + "hgnc_id": "HGNC:24043", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NUTM2A", + "hgnc_id": "HGNC:23438", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2D", + "hgnc_id": "HGNC:23447", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PAPSS2", + "hgnc_id": "HGNC:8604", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PTEN", + "hgnc_id": "HGNC:9588", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIPN", + "hgnc_id": "HGNC:23452", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "BTRC", + "hgnc_id": "HGNC:1144", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SFR1", + "hgnc_id": "HGNC:29574", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DHX32", + "hgnc_id": "HGNC:16717", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANK1", + "hgnc_id": "HGNC:23527", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ECHS1", + "hgnc_id": "HGNC:3151", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPRN", + "hgnc_id": "HGNC:16871", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG2B", + "hgnc_id": "HGNC:33518", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SCGB1C1", + "hgnc_id": "HGNC:18394", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFITM3", + "hgnc_id": "HGNC:5414", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DRD4", + "hgnc_id": "HGNC:3025", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEAF1", + "hgnc_id": "HGNC:14677", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP5-2", + "hgnc_id": "HGNC:23597", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP5-4", + "hgnc_id": "HGNC:23599", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MRPL23", + "hgnc_id": "HGNC:10322", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H19", + "hgnc_id": "HGNC:4713", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSU72L5", + "hgnc_id": "HGNC:43624", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "SSU72L2", + "hgnc_id": "HGNC:43621", + "reasons": [ + "segdup" + ], + "share": 0.666, + "status": "not read" + }, + { + "gene": "SSU72L4", + "hgnc_id": "HGNC:43623", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "SSU72L1", + "hgnc_id": "HGNC:43626", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR51A4", + "hgnc_id": "HGNC:14795", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR51A2", + "hgnc_id": "HGNC:14764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HBG1", + "hgnc_id": "HGNC:4831", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR10A5", + "hgnc_id": "HGNC:15131", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRGPRX4", + "hgnc_id": "HGNC:17617", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MDK", + "hgnc_id": "HGNC:6972", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPI1", + "hgnc_id": "HGNC:11241", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C3", + "hgnc_id": "HGNC:14697", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4C5", + "hgnc_id": "HGNC:14702", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4A47", + "hgnc_id": "HGNC:31266", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM51G", + "hgnc_id": "HGNC:43972", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64C", + "hgnc_id": "HGNC:37148", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOLH1", + "hgnc_id": "HGNC:3788", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C13", + "hgnc_id": "HGNC:15169", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4C12", + "hgnc_id": "HGNC:15168", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4C45", + "hgnc_id": "HGNC:31270", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4C46", + "hgnc_id": "HGNC:31271", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4A5", + "hgnc_id": "HGNC:15162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM48", + "hgnc_id": "HGNC:19021", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR4A15", + "hgnc_id": "HGNC:15152", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM51", + "hgnc_id": "HGNC:19023", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR5F1", + "hgnc_id": "HGNC:8343", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR8H2", + "hgnc_id": "HGNC:15308", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR5M8", + "hgnc_id": "HGNC:14846", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR9G1", + "hgnc_id": "HGNC:15319", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR1S2", + "hgnc_id": "HGNC:15141", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1S1", + "hgnc_id": "HGNC:8227", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GLYATL1B", + "hgnc_id": "HGNC:37865", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PGA3", + "hgnc_id": "HGNC:8885", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "B3GAT3", + "hgnc_id": "HGNC:923", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ESRRA", + "hgnc_id": "HGNC:3471", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC22A12", + "hgnc_id": "HGNC:17989", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RBM4B", + "hgnc_id": "HGNC:28842", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TPCN2", + "hgnc_id": "HGNC:20820", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FGF3", + "hgnc_id": "HGNC:3681", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KRTAP5-8", + "hgnc_id": "HGNC:23603", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP5-10", + "hgnc_id": "HGNC:23605", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "KRTAP5-11", + "hgnc_id": "HGNC:23606", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "XNDC1N-ZNF705EP-ALG1L9P", + "hgnc_id": "HGNC:55871", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DEFB108B", + "hgnc_id": "HGNC:29966", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "XNDC1N", + "hgnc_id": "HGNC:54661", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "INTS4", + "hgnc_id": "HGNC:25048", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOX4", + "hgnc_id": "HGNC:7891", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TRIM77", + "hgnc_id": "HGNC:34228", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "TRIM49", + "hgnc_id": "HGNC:13431", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM64B", + "hgnc_id": "HGNC:37147", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM49D1", + "hgnc_id": "HGNC:43973", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64", + "hgnc_id": "HGNC:14663", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM49C", + "hgnc_id": "HGNC:38877", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UBTFL1", + "hgnc_id": "HGNC:14533", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CARD16", + "hgnc_id": "HGNC:33701", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA8", + "hgnc_id": "HGNC:5241", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR8B2", + "hgnc_id": "HGNC:8471", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TMEM45B", + "hgnc_id": "HGNC:25194", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CD4", + "hgnc_id": "HGNC:1678", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PEX5", + "hgnc_id": "HGNC:9719", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC1", + "hgnc_id": "HGNC:604", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GDF3", + "hgnc_id": "HGNC:4218", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NANOG", + "hgnc_id": "HGNC:20857", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A1", + "hgnc_id": "HGNC:25526", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRH1", + "hgnc_id": "HGNC:9366", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRR4", + "hgnc_id": "HGNC:18020", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRH2", + "hgnc_id": "HGNC:9367", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAS2R14", + "hgnc_id": "HGNC:14920", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TAS2R31", + "hgnc_id": "HGNC:19113", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAS2R46", + "hgnc_id": "HGNC:18877", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAS2R43", + "hgnc_id": "HGNC:18875", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "TAS2R30", + "hgnc_id": "HGNC:19112", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "PRB4", + "hgnc_id": "HGNC:9340", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRB1", + "hgnc_id": "HGNC:9337", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "DDX11", + "hgnc_id": "HGNC:2736", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H3-5", + "hgnc_id": "HGNC:33164", + "reasons": [ + "segdup" + ], + "share": 0.1661, + "status": "not read" + }, + { + "gene": "ALG10", + "hgnc_id": "HGNC:23162", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ALG10B", + "hgnc_id": "HGNC:31088", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBA1B", + "hgnc_id": "HGNC:18809", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "COX14", + "hgnc_id": "HGNC:28216", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NACA", + "hgnc_id": "HGNC:7629", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DPY19L2", + "hgnc_id": "HGNC:19414", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNF1A", + "hgnc_id": "HGNC:11621", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HPD", + "hgnc_id": "HGNC:5147", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCAR2", + "hgnc_id": "HGNC:24827", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HCAR3", + "hgnc_id": "HGNC:16824", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "P2RX2", + "hgnc_id": "HGNC:15459", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PGAM5", + "hgnc_id": "HGNC:28763", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA3", + "hgnc_id": "HGNC:4426", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "C1QTNF9B", + "hgnc_id": "HGNC:34072", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C1QTNF9", + "hgnc_id": "HGNC:28732", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PARP4", + "hgnc_id": "HGNC:271", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PABPC3", + "hgnc_id": "HGNC:8556", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "MRPS31", + "hgnc_id": "HGNC:16632", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNRNPA1L2", + "hgnc_id": "HGNC:27067", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRR20C", + "hgnc_id": "HGNC:37221", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRR20B", + "hgnc_id": "HGNC:37220", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "PRR20D", + "hgnc_id": "HGNC:37222", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRR20E", + "hgnc_id": "HGNC:37223", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "F7", + "hgnc_id": "HGNC:3544", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PROZ", + "hgnc_id": "HGNC:9460", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEM", + "hgnc_id": "HGNC:37096", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEG", + "hgnc_id": "HGNC:33896", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR11H2", + "hgnc_id": "HGNC:14716", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4N2", + "hgnc_id": "HGNC:14742", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR4Q3", + "hgnc_id": "HGNC:15426", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR4M1", + "hgnc_id": "HGNC:14735", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RNASE3", + "hgnc_id": "HGNC:10046", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RNASE2", + "hgnc_id": "HGNC:10045", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10G2", + "hgnc_id": "HGNC:8170", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "DHRS4", + "hgnc_id": "HGNC:16985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHRS4L2", + "hgnc_id": "HGNC:19731", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARF6", + "hgnc_id": "HGNC:659", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACOT2", + "hgnc_id": "HGNC:18431", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NDUFB1", + "hgnc_id": "HGNC:7695", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "JAG2", + "hgnc_id": "HGNC:6189", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BTBD6", + "hgnc_id": "HGNC:19897", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "IGHV3-21", + "hgnc_id": "HGNC:5586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA6L6", + "hgnc_id": "HGNC:37225", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEB3", + "hgnc_id": "HGNC:51240", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LINC02203", + "hgnc_id": "HGNC:53069", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4M2B", + "hgnc_id": "HGNC:55109", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4N4C", + "hgnc_id": "HGNC:55110", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "POTEB", + "hgnc_id": "HGNC:33734", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4M2-OT1", + "hgnc_id": "HGNC:56199", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4N4", + "hgnc_id": "HGNC:15375", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L22", + "hgnc_id": "HGNC:50289", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6L1", + "hgnc_id": "HGNC:37444", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L26", + "hgnc_id": "HGNC:56306", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8S", + "hgnc_id": "HGNC:44409", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6L2", + "hgnc_id": "HGNC:26695", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HERC2", + "hgnc_id": "HGNC:4868", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L24", + "hgnc_id": "HGNC:55710", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA8F", + "hgnc_id": "HGNC:32378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8G", + "hgnc_id": "HGNC:25328", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6L25", + "hgnc_id": "HGNC:55711", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8M", + "hgnc_id": "HGNC:44404", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8J", + "hgnc_id": "HGNC:38650", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8T", + "hgnc_id": "HGNC:44410", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8R", + "hgnc_id": "HGNC:44407", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8Q", + "hgnc_id": "HGNC:44408", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8H", + "hgnc_id": "HGNC:37443", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8O", + "hgnc_id": "HGNC:44406", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8N", + "hgnc_id": "HGNC:44405", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8A", + "hgnc_id": "HGNC:31972", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8B", + "hgnc_id": "HGNC:31973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NANOGP8", + "hgnc_id": "HGNC:23106", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "NDUFAF1", + "hgnc_id": "HGNC:18828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CKMT1B", + "hgnc_id": "HGNC:1995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CATSPER2", + "hgnc_id": "HGNC:18810", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CKMT1A", + "hgnc_id": "HGNC:31736", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TERB2", + "hgnc_id": "HGNC:28520", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LDHAL6B", + "hgnc_id": "HGNC:21481", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6B", + "hgnc_id": "HGNC:32205", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6A", + "hgnc_id": "HGNC:13567", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ISLR2", + "hgnc_id": "HGNC:29286", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6C", + "hgnc_id": "HGNC:32206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6D", + "hgnc_id": "HGNC:32204", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ADAMTS7", + "hgnc_id": "HGNC:223", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EFL1", + "hgnc_id": "HGNC:25789", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SAXO2", + "hgnc_id": "HGNC:33727", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L10", + "hgnc_id": "HGNC:37228", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GOLGA6L9", + "hgnc_id": "HGNC:37229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "WHAMM", + "hgnc_id": "HGNC:30493", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4F4", + "hgnc_id": "HGNC:8301", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MPG", + "hgnc_id": "HGNC:7211", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HBM", + "hgnc_id": "HGNC:4826", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AXIN1", + "hgnc_id": "HGNC:903", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSTR5", + "hgnc_id": "HGNC:11334", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TPSD1", + "hgnc_id": "HGNC:14118", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GNPTG", + "hgnc_id": "HGNC:23026", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UNKL", + "hgnc_id": "HGNC:14184", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PKD1", + "hgnc_id": "HGNC:9008", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PDPK1", + "hgnc_id": "HGNC:8816", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "EEF2KMT", + "hgnc_id": "HGNC:32221", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NPIPA3", + "hgnc_id": "HGNC:41978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPA2", + "hgnc_id": "HGNC:41979", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NOMO1", + "hgnc_id": "HGNC:30060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDXDC1", + "hgnc_id": "HGNC:28995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RRN3", + "hgnc_id": "HGNC:30346", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPA5", + "hgnc_id": "HGNC:41980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MPV17L", + "hgnc_id": "HGNC:26827", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NOMO3", + "hgnc_id": "HGNC:25242", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPA6", + "hgnc_id": "HGNC:27893", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPA8", + "hgnc_id": "HGNC:41983", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPA9", + "hgnc_id": "HGNC:41984", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NOMO2", + "hgnc_id": "HGNC:22652", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SMG1", + "hgnc_id": "HGNC:30045", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IQCK", + "hgnc_id": "HGNC:28556", + "reasons": [ + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACSM5", + "hgnc_id": "HGNC:26060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACSM2B", + "hgnc_id": "HGNC:30931", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPB3", + "hgnc_id": "HGNC:28989", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB4", + "hgnc_id": "HGNC:41985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB5", + "hgnc_id": "HGNC:37233", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB6", + "hgnc_id": "HGNC:37454", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB7", + "hgnc_id": "HGNC:33832", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A2", + "hgnc_id": "HGNC:11454", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB8", + "hgnc_id": "HGNC:37490", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EIF3C", + "hgnc_id": "HGNC:3279", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB9", + "hgnc_id": "HGNC:41987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB11", + "hgnc_id": "HGNC:37453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BOLA2-SMG1P6", + "hgnc_id": "HGNC:53563", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BOLA2", + "hgnc_id": "HGNC:29488", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLX1B", + "hgnc_id": "HGNC:28748", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SULT1A4", + "hgnc_id": "HGNC:30004", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NPIPB12", + "hgnc_id": "HGNC:37491", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLX1A", + "hgnc_id": "HGNC:20922", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A3", + "hgnc_id": "HGNC:11455", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB13", + "hgnc_id": "HGNC:41989", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PHKG2", + "hgnc_id": "HGNC:8931", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "VKORC1", + "hgnc_id": "HGNC:23663", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRABD5", + "hgnc_id": "HGNC:26987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TP53TG3", + "hgnc_id": "HGNC:30759", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TP53TG3C", + "hgnc_id": "HGNC:42962", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "TP53TG3E", + "hgnc_id": "HGNC:51816", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TP53TG3B", + "hgnc_id": "HGNC:37202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TP53TG3F", + "hgnc_id": "HGNC:51817", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "CCNYL1B", + "hgnc_id": "HGNC:56313", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CES1", + "hgnc_id": "HGNC:1863", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MT1M", + "hgnc_id": "HGNC:14296", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MATCAP1", + "hgnc_id": "HGNC:34408", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD11B2", + "hgnc_id": "HGNC:5209", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLEC18A", + "hgnc_id": "HGNC:30388", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PDPR", + "hgnc_id": "HGNC:30264", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EXOSC6", + "hgnc_id": "HGNC:19055", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HPR", + "hgnc_id": "HGNC:5156", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB15", + "hgnc_id": "HGNC:34409", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CLEC18B", + "hgnc_id": "HGNC:33849", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CA5A", + "hgnc_id": "HGNC:1377", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF469", + "hgnc_id": "HGNC:23216", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH15", + "hgnc_id": "HGNC:1754", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANKRD11", + "hgnc_id": "HGNC:21316", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MC1R", + "hgnc_id": "HGNC:6929", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PRDM7", + "hgnc_id": "HGNC:9351", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ABR", + "hgnc_id": "HGNC:81", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SERPINF2", + "hgnc_id": "HGNC:9075", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SRR", + "hgnc_id": "HGNC:14398", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1D5", + "hgnc_id": "HGNC:8186", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "P2RX5", + "hgnc_id": "HGNC:8536", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ALOX15", + "hgnc_id": "HGNC:433", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ENO3", + "hgnc_id": "HGNC:3354", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP6", + "hgnc_id": "HGNC:12629", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35G6", + "hgnc_id": "HGNC:31351", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "ALOXE3", + "hgnc_id": "HGNC:13743", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HES7", + "hgnc_id": "HGNC:15977", + "reasons": [ + "CMRG" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CDRT15", + "hgnc_id": "HGNC:14395", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "FBXW10B", + "hgnc_id": "HGNC:14379", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM16", + "hgnc_id": "HGNC:17241", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF286A-TBC1D26", + "hgnc_id": "HGNC:55384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF286A", + "hgnc_id": "HGNC:13501", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D26", + "hgnc_id": "HGNC:28745", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UBB", + "hgnc_id": "HGNC:12463", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "ATPAF2", + "hgnc_id": "HGNC:18802", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EVPLL", + "hgnc_id": "HGNC:35236", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS9C", + "hgnc_id": "HGNC:33874", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D28", + "hgnc_id": "HGNC:26858", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXO3B", + "hgnc_id": "HGNC:3822", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FBXW10", + "hgnc_id": "HGNC:1211", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRAP", + "hgnc_id": "HGNC:4562", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GRAPL", + "hgnc_id": "HGNC:37240", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LGALS9B", + "hgnc_id": "HGNC:24842", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDRT15L2", + "hgnc_id": "HGNC:34075", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MEIS3P2", + "hgnc_id": "HGNC:17638", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAP2K3", + "hgnc_id": "HGNC:6843", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KCNJ18", + "hgnc_id": "HGNC:39080", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS9", + "hgnc_id": "HGNC:6570", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXN1", + "hgnc_id": "HGNC:12765", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SUZ12", + "hgnc_id": "HGNC:17101", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37B", + "hgnc_id": "HGNC:29070", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SLC35G3", + "hgnc_id": "HGNC:26848", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CCL4", + "hgnc_id": "HGNC:10630", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3B", + "hgnc_id": "HGNC:27011", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCL4L2", + "hgnc_id": "HGNC:24066", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "TBC1D3I", + "hgnc_id": "HGNC:32709", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TBC1D3G", + "hgnc_id": "HGNC:29860", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TBC1D3F", + "hgnc_id": "HGNC:18257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3K", + "hgnc_id": "HGNC:51245", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TBC1D3L", + "hgnc_id": "HGNC:51246", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D3D", + "hgnc_id": "HGNC:28944", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D3C", + "hgnc_id": "HGNC:24889", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TBC1D3E", + "hgnc_id": "HGNC:27071", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MRPL45", + "hgnc_id": "HGNC:16651", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP1-1", + "hgnc_id": "HGNC:16772", + "reasons": [ + "CMRG" + ], + "share": 0.1659, + "status": "not read" + }, + { + "gene": "KRTAP4-7", + "hgnc_id": "HGNC:18898", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP4-8", + "hgnc_id": "HGNC:17230", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP4-16", + "hgnc_id": "HGNC:18921", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRTAP4-9", + "hgnc_id": "HGNC:18910", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP4-12", + "hgnc_id": "HGNC:16776", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP9-7", + "hgnc_id": "HGNC:18915", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "KRT14", + "hgnc_id": "HGNC:6416", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRT16", + "hgnc_id": "HGNC:6423", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRT17", + "hgnc_id": "HGNC:6427", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HSD17B1", + "hgnc_id": "HGNC:5210", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "G6PC3", + "hgnc_id": "HGNC:24861", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ARHGAP27", + "hgnc_id": "HGNC:31813", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLEKHM1", + "hgnc_id": "HGNC:29017", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ARL17B", + "hgnc_id": "HGNC:32387", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37A", + "hgnc_id": "HGNC:29069", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARL17A", + "hgnc_id": "HGNC:24096", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NSF", + "hgnc_id": "HGNC:8016", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPEPPS", + "hgnc_id": "HGNC:7900", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HOXB8", + "hgnc_id": "HGNC:5119", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GIP", + "hgnc_id": "HGNC:4270", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHX40", + "hgnc_id": "HGNC:18018", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPS6KB1", + "hgnc_id": "HGNC:10436", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2A", + "hgnc_id": "HGNC:25755", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TLK2", + "hgnc_id": "HGNC:11842", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CSH2", + "hgnc_id": "HGNC:2441", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "GH2", + "hgnc_id": "HGNC:4262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CSH1", + "hgnc_id": "HGNC:2440", + "reasons": [ + "segdup" + ], + "share": 0.6663, + "status": "not read" + }, + { + "gene": "CSHL1", + "hgnc_id": "HGNC:2442", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "LRRC37A3", + "hgnc_id": "HGNC:32427", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C17orf58", + "hgnc_id": "HGNC:27568", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "AMZ2", + "hgnc_id": "HGNC:28041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMC6", + "hgnc_id": "HGNC:18021", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CANT1", + "hgnc_id": "HGNC:19721", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RNF213", + "hgnc_id": "HGNC:14539", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ARL16", + "hgnc_id": "HGNC:27902", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8B", + "hgnc_id": "HGNC:24983", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYMS", + "hgnc_id": "HGNC:12441", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC35G4", + "hgnc_id": "HGNC:31043", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "POTEC", + "hgnc_id": "HGNC:33894", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ANKRD30B", + "hgnc_id": "HGNC:24165", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC25A52", + "hgnc_id": "HGNC:23324", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KATNAL2", + "hgnc_id": "HGNC:25387", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SERPINB4", + "hgnc_id": "HGNC:10570", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERPINB3", + "hgnc_id": "HGNC:10569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALR1", + "hgnc_id": "HGNC:4132", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CTDP1", + "hgnc_id": "HGNC:2498", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "BSG", + "hgnc_id": "HGNC:1116", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN2", + "hgnc_id": "HGNC:4846", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KISS1R", + "hgnc_id": "HGNC:4510", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "STK11", + "hgnc_id": "HGNC:11389", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TCF3", + "hgnc_id": "HGNC:11633", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ABHD17A", + "hgnc_id": "HGNC:28756", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "GIPC3", + "hgnc_id": "HGNC:18183", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBXA2R", + "hgnc_id": "HGNC:11608", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CREB3L3", + "hgnc_id": "HGNC:18855", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RFX2", + "hgnc_id": "HGNC:9983", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "C3", + "hgnc_id": "HGNC:1318", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MBD3L2B", + "hgnc_id": "HGNC:53435", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MBD3L4", + "hgnc_id": "HGNC:37206", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBD3L2", + "hgnc_id": "HGNC:18532", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "STXBP2", + "hgnc_id": "HGNC:11445", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TYK2", + "hgnc_id": "HGNC:12440", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF799", + "hgnc_id": "HGNC:28071", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF443", + "hgnc_id": "HGNC:20878", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP4F3", + "hgnc_id": "HGNC:2646", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4F12", + "hgnc_id": "HGNC:18857", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10H2", + "hgnc_id": "HGNC:8173", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CALR3", + "hgnc_id": "HGNC:20407", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MYO9B", + "hgnc_id": "HGNC:7609", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "INSL3", + "hgnc_id": "HGNC:6086", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF738", + "hgnc_id": "HGNC:32469", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF100", + "hgnc_id": "HGNC:12880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF208", + "hgnc_id": "HGNC:12999", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF676", + "hgnc_id": "HGNC:20429", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF729", + "hgnc_id": "HGNC:32464", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF492", + "hgnc_id": "HGNC:23707", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF728", + "hgnc_id": "HGNC:32463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHPN2", + "hgnc_id": "HGNC:19974", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GPI", + "hgnc_id": "HGNC:4458", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF302", + "hgnc_id": "HGNC:13848", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF181", + "hgnc_id": "HGNC:12971", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FFAR3", + "hgnc_id": "HGNC:4499", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "COX6B1", + "hgnc_id": "HGNC:2280", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS7", + "hgnc_id": "HGNC:6568", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS7B", + "hgnc_id": "HGNC:34447", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "IFNL3", + "hgnc_id": "HGNC:18365", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IFNL4", + "hgnc_id": "HGNC:44480", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "IFNL2", + "hgnc_id": "HGNC:18364", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CYP2A7", + "hgnc_id": "HGNC:2611", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CYP2G1P", + "hgnc_id": "HGNC:2633", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A13", + "hgnc_id": "HGNC:2608", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CYP2F1", + "hgnc_id": "HGNC:2632", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG8", + "hgnc_id": "HGNC:9525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG1", + "hgnc_id": "HGNC:9514", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PSG6", + "hgnc_id": "HGNC:9523", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PSG7", + "hgnc_id": "HGNC:9524", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSG11", + "hgnc_id": "HGNC:9516", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PSG2", + "hgnc_id": "HGNC:9519", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG4", + "hgnc_id": "HGNC:9521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG9", + "hgnc_id": "HGNC:9526", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ETHE1", + "hgnc_id": "HGNC:23287", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF285", + "hgnc_id": "HGNC:13079", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "APOC1", + "hgnc_id": "HGNC:607", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOC4", + "hgnc_id": "HGNC:611", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BLOC1S3", + "hgnc_id": "HGNC:20914", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DMPK", + "hgnc_id": "HGNC:2933", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FUT1", + "hgnc_id": "HGNC:4012", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BAX", + "hgnc_id": "HGNC:959", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LHB", + "hgnc_id": "HGNC:6584", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "CGB3", + "hgnc_id": "HGNC:1886", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CGB2", + "hgnc_id": "HGNC:16722", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "CGB1", + "hgnc_id": "HGNC:16721", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "CGB5", + "hgnc_id": "HGNC:16452", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CGB8", + "hgnc_id": "HGNC:16453", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CGB7", + "hgnc_id": "HGNC:16451", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PNKP", + "hgnc_id": "HGNC:9154", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SIGLEC16", + "hgnc_id": "HGNC:24851", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KLK4", + "hgnc_id": "HGNC:6365", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ETFB", + "hgnc_id": "HGNC:3482", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF808", + "hgnc_id": "HGNC:33230", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF600", + "hgnc_id": "HGNC:30951", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF468", + "hgnc_id": "HGNC:33105", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VN1R4", + "hgnc_id": "HGNC:19871", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "FAM90A27P", + "hgnc_id": "HGNC:43617", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF765", + "hgnc_id": "HGNC:25092", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF761", + "hgnc_id": "HGNC:23179", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF813", + "hgnc_id": "HGNC:33257", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRKCG", + "hgnc_id": "HGNC:9402", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBOAT7", + "hgnc_id": "HGNC:15505", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LILRA6", + "hgnc_id": "HGNC:15495", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "LILRB2", + "hgnc_id": "HGNC:6606", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "LAIR1", + "hgnc_id": "HGNC:6477", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAIR2", + "hgnc_id": "HGNC:6478", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LILRB1", + "hgnc_id": "HGNC:6605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KIR3DL3", + "hgnc_id": "HGNC:16312", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KIR2DL3", + "hgnc_id": "HGNC:6331", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KIR2DL1", + "hgnc_id": "HGNC:6329", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KIR2DL4", + "hgnc_id": "HGNC:6332", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KIR3DL1", + "hgnc_id": "HGNC:6338", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KIR3DL2", + "hgnc_id": "HGNC:6339", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NLRP7", + "hgnc_id": "HGNC:22947", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NLRP2", + "hgnc_id": "HGNC:22948", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GP6", + "hgnc_id": "HGNC:14388", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TNNT1", + "hgnc_id": "HGNC:11948", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TNNI3", + "hgnc_id": "HGNC:11947", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL4A", + "hgnc_id": "HGNC:16449", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RFPL4AL1", + "hgnc_id": "HGNC:45147", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZNF419", + "hgnc_id": "HGNC:20648", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF773", + "hgnc_id": "HGNC:30487", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF587B", + "hgnc_id": "HGNC:37142", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF814", + "hgnc_id": "HGNC:33258", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF417", + "hgnc_id": "HGNC:20646", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC27A5", + "hgnc_id": "HGNC:10999", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SIRPB1", + "hgnc_id": "HGNC:15928", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CST4", + "hgnc_id": "HGNC:2476", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CST1", + "hgnc_id": "HGNC:2473", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CST2", + "hgnc_id": "HGNC:2474", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PLAGL2", + "hgnc_id": "HGNC:9047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEMG1", + "hgnc_id": "HGNC:10742", + "reasons": [ + "CMRG" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PLTP", + "hgnc_id": "HGNC:9093", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRNA4", + "hgnc_id": "HGNC:1958", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EEF1A2", + "hgnc_id": "HGNC:3192", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PTK6", + "hgnc_id": "HGNC:9617", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MYT1", + "hgnc_id": "HGNC:7622", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCMTD2", + "hgnc_id": "HGNC:15882", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TPTE", + "hgnc_id": "HGNC:12023", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POTED", + "hgnc_id": "HGNC:23822", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP21-1", + "hgnc_id": "HGNC:18945", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SMIM11", + "hgnc_id": "HGNC:1293", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMIM34", + "hgnc_id": "HGNC:39601", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KCNE1", + "hgnc_id": "HGNC:6240", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CBR3", + "hgnc_id": "HGNC:1549", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NDUFV3", + "hgnc_id": "HGNC:7719", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CBS", + "hgnc_id": "HGNC:1550", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "U2AF1", + "hgnc_id": "HGNC:12453", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SIK1", + "hgnc_id": "HGNC:11142", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSF2BP", + "hgnc_id": "HGNC:5226", + "reasons": [ + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BC12L", + "hgnc_id": "HGNC:4762", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3319, + "status": "not read" + }, + { + "gene": "TRAPPC10", + "hgnc_id": "HGNC:11868", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PWP2", + "hgnc_id": "HGNC:9711", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GATD3", + "hgnc_id": "HGNC:1273", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DNMT3L", + "hgnc_id": "HGNC:2980", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-1", + "hgnc_id": "HGNC:22966", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "KRTAP10-2", + "hgnc_id": "HGNC:22967", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRTAP10-4", + "hgnc_id": "HGNC:20521", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "KRTAP10-5", + "hgnc_id": "HGNC:22969", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "KRTAP10-6", + "hgnc_id": "HGNC:20523", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "KRTAP10-8", + "hgnc_id": "HGNC:20525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-9", + "hgnc_id": "HGNC:22971", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KRTAP10-10", + "hgnc_id": "HGNC:22972", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP12-2", + "hgnc_id": "HGNC:20530", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "KRTAP12-1", + "hgnc_id": "HGNC:20529", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "COL18A1", + "hgnc_id": "HGNC:2195", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR11H1", + "hgnc_id": "HGNC:15404", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEH", + "hgnc_id": "HGNC:133", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCT8L2", + "hgnc_id": "HGNC:15553", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "USP18", + "hgnc_id": "HGNC:12616", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GGTLC3", + "hgnc_id": "HGNC:33426", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TMEM191B", + "hgnc_id": "HGNC:33600", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM246B", + "hgnc_id": "HGNC:54843", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DGCR6", + "hgnc_id": "HGNC:2846", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRODH", + "hgnc_id": "HGNC:9453", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM246C", + "hgnc_id": "HGNC:54842", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "CCDC188", + "hgnc_id": "HGNC:51899", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RTN4R", + "hgnc_id": "HGNC:18601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3FP", + "hgnc_id": "HGNC:24067", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LRRC74B", + "hgnc_id": "HGNC:34301", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM246A", + "hgnc_id": "HGNC:54844", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RIMBP3B", + "hgnc_id": "HGNC:33891", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HIC2", + "hgnc_id": "HGNC:18595", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TMEM191C", + "hgnc_id": "HGNC:33601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GGTLC2", + "hgnc_id": "HGNC:18596", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IGLL1", + "hgnc_id": "HGNC:5870", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRICH1", + "hgnc_id": "HGNC:28031", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTT2B", + "hgnc_id": "HGNC:33437", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DDTL", + "hgnc_id": "HGNC:33446", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GSTT2", + "hgnc_id": "HGNC:4642", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL1", + "hgnc_id": "HGNC:9977", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C22orf42", + "hgnc_id": "HGNC:27160", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RFPL3", + "hgnc_id": "HGNC:9980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3D", + "hgnc_id": "HGNC:17354", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOBEC3H", + "hgnc_id": "HGNC:24100", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ATF4", + "hgnc_id": "HGNC:786", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NDUFA6", + "hgnc_id": "HGNC:7690", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2D7", + "hgnc_id": "HGNC:2624", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RRP7A", + "hgnc_id": "HGNC:24286", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYB5R3", + "hgnc_id": "HGNC:2873", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "A4GALT", + "hgnc_id": "HGNC:18149", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TTLL1", + "hgnc_id": "HGNC:1312", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MLC1", + "hgnc_id": "HGNC:17082", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBGCP6", + "hgnc_id": "HGNC:18127", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACR", + "hgnc_id": "HGNC:126", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "XG", + "hgnc_id": "HGNC:12806", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GYG2", + "hgnc_id": "HGNC:4700", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ARSD", + "hgnc_id": "HGNC:717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MXRA5", + "hgnc_id": "HGNC:7539", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRKX", + "hgnc_id": "HGNC:9441", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "VCX3A", + "hgnc_id": "HGNC:18159", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "VCX2", + "hgnc_id": "HGNC:18158", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VCX3B", + "hgnc_id": "HGNC:31838", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM9A", + "hgnc_id": "HGNC:18403", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM9C", + "hgnc_id": "HGNC:18405", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SUPT20HL2", + "hgnc_id": "HGNC:31797", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SUPT20HL1", + "hgnc_id": "HGNC:30773", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "MAGEB6", + "hgnc_id": "HGNC:23796", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DCAF8L1", + "hgnc_id": "HGNC:31810", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM47B", + "hgnc_id": "HGNC:26659", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "H2AL1Q", + "hgnc_id": "HGNC:53959", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "FAM47C", + "hgnc_id": "HGNC:25301", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "H2AL3", + "hgnc_id": "HGNC:53960", + "reasons": [ + "segdup" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "SPACA5B", + "hgnc_id": "HGNC:19142", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SSX5", + "hgnc_id": "HGNC:11339", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SSX1", + "hgnc_id": "HGNC:11335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSX3", + "hgnc_id": "HGNC:11337", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SSX4", + "hgnc_id": "HGNC:11338", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX4B", + "hgnc_id": "HGNC:16880", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12J", + "hgnc_id": "HGNC:17778", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE13", + "hgnc_id": "HGNC:29081", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE2E", + "hgnc_id": "HGNC:31960", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE12B", + "hgnc_id": "HGNC:26779", + "reasons": [ + "segdup" + ], + "share": 0.4957, + "status": "not read" + }, + { + "gene": "GAGE12C", + "hgnc_id": "HGNC:28402", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GAGE12D", + "hgnc_id": "HGNC:31904", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12F", + "hgnc_id": "HGNC:31906", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE12G", + "hgnc_id": "HGNC:31907", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE12H", + "hgnc_id": "HGNC:31908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE2A", + "hgnc_id": "HGNC:4099", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GAGE1", + "hgnc_id": "HGNC:4098", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CENPVL3", + "hgnc_id": "HGNC:43880", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CENPVL1", + "hgnc_id": "HGNC:31851", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MAGED4B", + "hgnc_id": "HGNC:22880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MAGED4", + "hgnc_id": "HGNC:23793", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE1A", + "hgnc_id": "HGNC:4111", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "XAGE1B", + "hgnc_id": "HGNC:25400", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX7", + "hgnc_id": "HGNC:19653", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSX2B", + "hgnc_id": "HGNC:22263", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN5", + "hgnc_id": "HGNC:33178", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "XAGE5", + "hgnc_id": "HGNC:30930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE3", + "hgnc_id": "HGNC:14618", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM156B", + "hgnc_id": "HGNC:31962", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM156A", + "hgnc_id": "HGNC:30114", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPIN2B", + "hgnc_id": "HGNC:33147", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPIN2A", + "hgnc_id": "HGNC:20694", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZXDA", + "hgnc_id": "HGNC:13198", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CXorf49", + "hgnc_id": "HGNC:30891", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CXorf49B", + "hgnc_id": "HGNC:34229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "DMRTC1B", + "hgnc_id": "HGNC:31686", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM236D", + "hgnc_id": "HGNC:52642", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DMRTC1", + "hgnc_id": "HGNC:13910", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM236C", + "hgnc_id": "HGNC:52641", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM236A", + "hgnc_id": "HGNC:44268", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PABPC1L2B", + "hgnc_id": "HGNC:31852", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PABPC1L2A", + "hgnc_id": "HGNC:27989", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TGIF2LX", + "hgnc_id": "HGNC:18570", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDH11X", + "hgnc_id": "HGNC:8656", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TCEAL6", + "hgnc_id": "HGNC:24553", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "TCP11X1", + "hgnc_id": "HGNC:48369", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NXF2", + "hgnc_id": "HGNC:8072", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NXF2B", + "hgnc_id": "HGNC:23984", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB40AL", + "hgnc_id": "HGNC:25410", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TMSB15B", + "hgnc_id": "HGNC:28612", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BW1", + "hgnc_id": "HGNC:27252", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "H2BW2", + "hgnc_id": "HGNC:27867", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "TMSB15C", + "hgnc_id": "HGNC:55173", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PSMD10", + "hgnc_id": "HGNC:9555", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CT47C1", + "hgnc_id": "HGNC:53820", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHOXF2B", + "hgnc_id": "HGNC:33519", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RHOXF2", + "hgnc_id": "HGNC:30011", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT47B1", + "hgnc_id": "HGNC:33293", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A12", + "hgnc_id": "HGNC:33292", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT47A11", + "hgnc_id": "HGNC:27397", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CT47A9", + "hgnc_id": "HGNC:33290", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A8", + "hgnc_id": "HGNC:33289", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "CT47A7", + "hgnc_id": "HGNC:33288", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT47A6", + "hgnc_id": "HGNC:33287", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A5", + "hgnc_id": "HGNC:33286", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CT47A4", + "hgnc_id": "HGNC:33285", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT47A2", + "hgnc_id": "HGNC:33283", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A1", + "hgnc_id": "HGNC:33282", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "RTL8B", + "hgnc_id": "HGNC:33156", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "RTL8C", + "hgnc_id": "HGNC:2569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RTL8A", + "hgnc_id": "HGNC:24514", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "SMIM10L2B", + "hgnc_id": "HGNC:34500", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT55", + "hgnc_id": "HGNC:26047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ETDA", + "hgnc_id": "HGNC:53449", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "SMIM10L2A", + "hgnc_id": "HGNC:34499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT45A1", + "hgnc_id": "HGNC:33267", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A3", + "hgnc_id": "HGNC:33268", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT45A5", + "hgnc_id": "HGNC:33270", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT45A2", + "hgnc_id": "HGNC:28400", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT45A7", + "hgnc_id": "HGNC:51260", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CT45A8", + "hgnc_id": "HGNC:51261", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT45A9", + "hgnc_id": "HGNC:51262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A10", + "hgnc_id": "HGNC:51263", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SAGE1", + "hgnc_id": "HGNC:30369", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPANXC", + "hgnc_id": "HGNC:14331", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXA1", + "hgnc_id": "HGNC:11218", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "SPANXA2", + "hgnc_id": "HGNC:14328", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "SPANXD", + "hgnc_id": "HGNC:14332", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "MAGEC1", + "hgnc_id": "HGNC:6812", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPANXN3", + "hgnc_id": "HGNC:33176", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UBE2NL", + "hgnc_id": "HGNC:31710", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN1", + "hgnc_id": "HGNC:33174", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CXorf51B", + "hgnc_id": "HGNC:42787", + "reasons": [ + "segdup" + ], + "share": 0.3323, + "status": "not read" + }, + { + "gene": "CXorf51A", + "hgnc_id": "HGNC:30533", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "EOLA1", + "hgnc_id": "HGNC:28089", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HSFX3", + "hgnc_id": "HGNC:52395", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HSFX2", + "hgnc_id": "HGNC:32701", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSFX1", + "hgnc_id": "HGNC:29603", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MAGEA9", + "hgnc_id": "HGNC:6807", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "EOLA2", + "hgnc_id": "HGNC:17402", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSFX4", + "hgnc_id": "HGNC:52398", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAGEA3", + "hgnc_id": "HGNC:6801", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MAGEA2B", + "hgnc_id": "HGNC:19340", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CSAG1", + "hgnc_id": "HGNC:24294", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "MAGEA12", + "hgnc_id": "HGNC:6799", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "MAGEA2", + "hgnc_id": "HGNC:6800", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CSAG3", + "hgnc_id": "HGNC:26237", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MAGEA6", + "hgnc_id": "HGNC:6804", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PWWP4", + "hgnc_id": "HGNC:55197", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A8", + "hgnc_id": "HGNC:11055", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BCAP31", + "hgnc_id": "HGNC:16695", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OPN1LW", + "hgnc_id": "HGNC:9936", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OPN1MW", + "hgnc_id": "HGNC:4206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OPN1MW2", + "hgnc_id": "HGNC:26952", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEX28", + "hgnc_id": "HGNC:2563", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAG1A", + "hgnc_id": "HGNC:24198", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "CTAG1B", + "hgnc_id": "HGNC:2491", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAG2", + "hgnc_id": "HGNC:2492", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H2AB1", + "hgnc_id": "HGNC:22516", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "F8A1", + "hgnc_id": "HGNC:3547", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "F8A2", + "hgnc_id": "HGNC:31849", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "F8A3", + "hgnc_id": "HGNC:31850", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "H2AB3", + "hgnc_id": "HGNC:14455", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IL9R", + "hgnc_id": "HGNC:6030", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASH6P", + "hgnc_id": "HGNC:31685", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "TGIF2LY", + "hgnc_id": "HGNC:18569", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "PCDH11Y", + "hgnc_id": "HGNC:15813", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY2", + "hgnc_id": "HGNC:23924", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY4", + "hgnc_id": "HGNC:37287", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY8", + "hgnc_id": "HGNC:37471", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY3", + "hgnc_id": "HGNC:33876", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY1", + "hgnc_id": "HGNC:12381", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY9", + "hgnc_id": "HGNC:37472", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY10", + "hgnc_id": "HGNC:37473", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY", + "hgnc_id": "HGNC:12668", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY1B", + "hgnc_id": "HGNC:31751", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2B", + "hgnc_id": "HGNC:23921", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2A", + "hgnc_id": "HGNC:1810", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY1", + "hgnc_id": "HGNC:18568", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY2", + "hgnc_id": "HGNC:23950", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1B", + "hgnc_id": "HGNC:23914", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1A1", + "hgnc_id": "HGNC:9912", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1D", + "hgnc_id": "HGNC:23915", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1E", + "hgnc_id": "HGNC:23916", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1F", + "hgnc_id": "HGNC:23974", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1J", + "hgnc_id": "HGNC:23917", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2", + "hgnc_id": "HGNC:13508", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ1", + "hgnc_id": "HGNC:2682", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ2", + "hgnc_id": "HGNC:15964", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1B", + "hgnc_id": "HGNC:23920", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2B", + "hgnc_id": "HGNC:25449", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ3", + "hgnc_id": "HGNC:15965", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ4", + "hgnc_id": "HGNC:15966", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2C", + "hgnc_id": "HGNC:18225", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1", + "hgnc_id": "HGNC:1809", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "C4B_2", + "hgnc_id": "HGNC:42398", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTT1", + "hgnc_id": "HGNC:4641", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTTP2", + "hgnc_id": "HGNC:33606", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GTF2H2C_2", + "hgnc_id": "HGNC:35418", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB3", + "hgnc_id": "HGNC:4951", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB4", + "hgnc_id": "HGNC:4952", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL2", + "hgnc_id": "HGNC:6330", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5A", + "hgnc_id": "HGNC:16345", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5B", + "hgnc_id": "HGNC:16346", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS1", + "hgnc_id": "HGNC:6333", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS2", + "hgnc_id": "HGNC:6334", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS3", + "hgnc_id": "HGNC:6335", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS5", + "hgnc_id": "HGNC:6337", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR3DS1", + "hgnc_id": "HGNC:6340", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "LILRA3", + "hgnc_id": "HGNC:6604", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U8", + "hgnc_id": "HGNC:27538", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U9", + "hgnc_id": "HGNC:29166", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR9G9", + "hgnc_id": "HGNC:31940", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TAS2R45", + "hgnc_id": "HGNC:18876", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + } + ], + "not_read_owned": [ + { + "gene": "RHCE", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "AMY1A", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AMY1B", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AMY1C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GSTM1", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FCGR1A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GBA1", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FCGR2A", + "owner": "parascopy", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FCGR2C", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "FCGR3B", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR3", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR1", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PROS1", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B17", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT2B15", + "owner": "pypgx", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPE", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPB", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GTF2H2C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMN2", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMN1", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2H2", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSPA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "C4A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "C4B", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP21A2", + "owner": "cn_loci:CYP21A2", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LPA", + "owner": "cn_loci:LPA_KIV2", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PMS2", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2I", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DEFA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CEL", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC5B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KLRC2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KRT6B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRT6C", + "owner": "parascopy", + "share": 0.1665, + "status": "not read" + }, + { + "gene": "KRT6A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRFAM7A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "STRC", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SORD", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HBA2", + "owner": "cn_loci:HBA", + "share": 0.5, + "status": "not read" + }, + { + "gene": "HBA1", + "owner": "cn_loci:HBA", + "share": 0.6659, + "status": "not read" + }, + { + "gene": "TPSB2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TPSAB1", + "owner": "parascopy", + "share": 0.3331, + "status": "not read" + }, + { + "gene": "OTOA", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SULT1A1", + "owner": "pypgx", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HYDIN", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HP", + "owner": "cn_loci:HP", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CCL3L3", + "owner": "cn_loci:CCL3L1", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A6", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP2B6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD177", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "APOBEC3B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP2D6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + } + ], + "not_on_reference": [ + "TGIF2LY", + "PCDH11Y", + "TSPY2", + "TSPY4", + "TSPY8", + "TSPY3", + "TSPY1", + "TSPY9", + "TSPY10", + "VCY", + "VCY1B", + "CDY2B", + "CDY2A", + "HSFY1", + "HSFY2", + "RBMY1B", + "RBMY1A1", + "RBMY1D", + "RBMY1E", + "RBMY1F", + "RBMY1J", + "BPY2", + "DAZ1", + "DAZ2", + "CDY1B", + "BPY2B", + "DAZ3", + "DAZ4", + "BPY2C", + "CDY1", + "C4B_2", + "GSTT1", + "GSTTP2", + "GTF2H2C_2", + "HLA-DRB3", + "HLA-DRB4", + "KIR2DL2", + "KIR2DL5A", + "KIR2DL5B", + "KIR2DS1", + "KIR2DS2", + "KIR2DS3", + "KIR2DS5", + "KIR3DS1", + "LILRA3", + "OR8U8", + "OR8U9", + "OR9G9", + "TAS2R45" + ] + } + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "ok", + "source": "indexcov/indexcov-indexcov.ped", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "inferred_sex": "male", + "cn_chrX": "1.01", + "cn_chrY": "0.98", + "karyotype": null + } + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "declared_sex": "male", + "mosaic_call": "not_detected", + "mosaic_depth_ratio": "0.98", + "mosaic_depth_ratio_expected": "1", + "mosaic_loss_depth": "0.02", + "mosaic_loss_depth_low": "-0.01", + "mosaic_loss_depth_high": "0.05", + "mosaic_loss_af": "NA", + "mosaic_floor": "0.1", + "mosaic_x_het_sites": "0", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "call": "not_detected", + "depth_ratio": 0.98, + "depth_ratio_expected": 1.0, + "loss_depth": 0.02, + "loss_depth_low": -0.01, + "loss_depth_high": 0.05, + "loss_af": null, + "floor": 0.1, + "x_het_sites": 0, + "loss_af_low": null, + "loss_af_high": null + } + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "ok", + "source": "qc/S_read_count.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "input_reads": 1000, + "primary_records": 1000, + "missing_reads": 0, + "read_count_check": "ok" + } + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "ok", + "source": "vcf/S.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1, + "snps": 0, + "indels": 1 + } + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 8, + "by_stars": { + "4": 0, + "3": 2, + "2": 3, + "1": 1, + "0": 2 + }, + "hits": [ + { + "stars": 4, + "chrom": "chr7", + "pos": 117559590, + "ref": "ATCT", + "alt": "A", + "genotype": "het", + "gene": "CFTR", + "significance": "Pathogenic", + "review_status": "practice guideline", + "combined_allele": ".", + "dp": 30, + "gq": 15, + "low_confidence": true, + "gnomad_exome_af": 0.008, + "gnomad_genome_af": 0.007, + "rsids": [ + "rs113993960" + ], + "af": 0.008, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr17", + "pos": 43057100, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "BRCA1,NBR2", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": "part of a combined allele: chr17:43057100 C>T, chr17:43057102 A>G; unphased, so the parts may sit on different haplotypes", + "dp": 35, + "gq": 60, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr6", + "pos": 26092913, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HFE", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": ".", + "dp": 40, + "gq": 99, + "low_confidence": false, + "gnomad_exome_af": 0.06, + "gnomad_genome_af": 0.05, + "rsids": [ + "rs99", + "rs1800562" + ], + "af": 0.06, + "common": true, + "cdna": { + "check": "clean", + "note": "", + "alt_reads": 20, + "alt_clipped": 0, + "ref_reads": 20, + "ref_clipped": 0, + "construct": null + } + }, + { + "stars": 2, + "chrom": "chr1", + "pos": 45331556, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "MUTYH", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 30, + "gq": 50, + "low_confidence": false, + "gnomad_exome_af": 0.002, + "gnomad_genome_af": 0.0005, + "rsids": [ + "rs36053993" + ], + "af": 0.002, + "common": false, + "cdna": { + "check": "flagged", + "note": "possible cDNA/transgene contamination: check the reads", + "alt_reads": 12, + "alt_clipped": 6, + "ref_reads": 10, + "ref_clipped": 0, + "construct": "yes" + } + }, + { + "stars": 2, + "chrom": "chr16", + "pos": 176700, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HBA1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 2, + "chrom": "chr5", + "pos": 70951946, + "ref": "C", + "alt": "T", + "genotype": "hom", + "gene": "SMN1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr13", + "pos": 20189473, + "ref": "C", + "alt": "T", + "genotype": "one copy, other copy not called ref (1/.)", + "gene": "GJB2", + "significance": "Likely pathogenic", + "review_status": "criteria provided, single submitter", + "combined_allele": "combined allele of chr13:20189473 C>T + chr13:20189475 G>A", + "dp": 5, + "gq": 40, + "low_confidence": true, + "gnomad_exome_af": 0.03, + "gnomad_genome_af": null, + "rsids": [ + "rs80338939" + ], + "af": 0.03, + "common": true, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr3", + "pos": 15645186, + "ref": "G", + "alt": "C", + "genotype": "het", + "gene": "BTD", + "significance": "Pathogenic/Likely pathogenic", + "review_status": "criteria provided, conflicting classifications", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chr2", + "pos": 100, + "ref": "A", + "alt": "G", + "genotype": "het", + "gene": ".", + "significance": ".", + "review_status": ".", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chrX", + "pos": 101398000, + "ref": "C", + "alt": "T", + "genotype": "1", + "gene": "GLA", + "significance": "Pathogenic", + "review_status": "no assertion criteria provided", + "combined_allele": "possibly one allele with chrX:101398004 G>A: check ClinVar for a combined record", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "low_confidence": 2, + "call_floor": { + "GQ": 20, + "DP": 8 + }, + "frequency": { + "source": "vep/S_vep.vcf.gz", + "note": null, + "common_af": 0.01 + }, + "cdna_flagged": 1, + "construct_sites": 1, + "cdna_scan": { + "note": "2 genes with spliced reads", + "loci": [ + { + "chrom": "chr1", + "start": 100, + "end": 200, + "junctions": 2, + "reads": 5, + "junction_list": "A:1-2,A:3-4" + } + ] + }, + "deep_intronic": [ + { + "chrom": "chr7", + "pos": 117639961, + "ref": "C", + "alt": "T", + "gene": "CFTR", + "hgvs": "c.3718-2477C>T", + "clinvar_id": "7129", + "call": "no call", + "detail": "" + } + ] + } + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_on_list": 3292, + "list": "assets/carrier_genes.tsv", + "carriers": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "MUTYH-related polyposis" + }, + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Symptomatic form of HFE-related hemochromatosis" + }, + { + "gene": "CFTR", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 4, + "inheritance": "AR", + "condition": "Cystic fibrosis" + }, + { + "gene": "GJB2", + "variant": "chr13:20189473 C>T", + "genotype": "one copy, other copy not called ref (1/.)", + "significance": "Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Nonsyndromic hearing loss, recessive (DFNB1A)" + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Fanconi anemia" + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 0, + "inheritance": "XL", + "condition": "Fabry disease" + }, + { + "gene": "HBA1", + "variant": "chr16:176700 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Alpha-thalassemia" + }, + { + "gene": "SMN1", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Spinal muscular atrophy", + "hemizygous": "called homozygous inside one SMN1 copy (Parascopy, step 35)" + } + ], + "set_aside": [ + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "significance": "Pathogenic/Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Biotinidase deficiency", + "kind": "hypomorph", + "name": "p.Asp444His (p.Asp424His on NM_001370658.1), partial biotinidase deficiency" + } + ], + "two_hits": { + "genes": [ + { + "gene": "SMN1", + "verdict": "biallelic", + "why": "the variant is called homozygous inside the deletion: it is hemizygous, on the one copy the deletion leaves", + "deletion": "one SMN1 copy (Parascopy, step 35)", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "allele": null, + "condition": "Spinal muscular atrophy", + "inheritance": "AR" + } + ], + "deletions_read": [ + "AnnotSV class 4 and 5 deletions (step 05)", + "step 35's interval table (read depth)", + "Parascopy's SMN1 count (step 35)" + ], + "deletions_not_read": [ + "step 35's HBA1/HBA2 count: not called" + ] + } + } + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "ok", + "source": "pharmcat/S.report.json", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "version": "3.4.0", + "html_report": true + } + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "ok", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "gene": "UGT1A1", + "diplotype": "*1/*28", + "phenotype": "Intermediate", + "status": "non-normal" + } + ], + "non_normal": 1, + "ambiguous": 0, + "not_called": 0, + "unclassified": 0, + "parse_failed": false, + "warnings": [ + "CYP2C19 has two possible phenotypes", + "last" + ] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "ok", + "source": "pypgx/S_pypgx_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_total": 6, + "genes_called": 3, + "cyp2d6": "*1/*2", + "cnv_call": { + "CYP2D6": "Normal", + "CYP2A6": "not called: pypgx failed", + "CYP2B6": "not called: the CYP2D6 depth check failed", + "SLCO1B1": "not called: pypgx gave no copy-number call" + }, + "not_callable": [ + "CYP2E1" + ], + "low_gq": [ + { + "gene": "UGT1A1", + "diplotype": "Indeterminate", + "min_gq": 12, + "weakest": "chr2:1:A:G" + }, + { + "gene": "SLCO1B1", + "diplotype": "*1/*5", + "min_gq": null, + "weakest": "" + } + ], + "cyp2d6_evidence": { + "phenotype": "Normal Metabolizer", + "defining_filter": "PASS", + "quality": "ok" + }, + "depth_check": { + "status": "ok", + "gene_mapq1_share": "0.99", + "body_mapq20_share": "0.95" + }, + "comparison": { + "conflicts": 1, + "one_tool_only": 2, + "concordant": 1 + } + } + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/S_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "*2/*1", + "filter": "PASS" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": 1, + "text": "1 (quality 31, filter PASS)", + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": 3, + "text": "3 (quality 25, filter PASS)" + }, + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "one SMN1 copy: the pattern of an SMA carrier", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "called", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality 50, filter PASS); chr7:5980001-5990000 3 (quality 12, filter LowQual)" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": 1, + "text": "1 (quality 31, filter PASS)" + }, + "agree": "disagree" + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01:01", + "A*01:01:01" + ], + "quality": [ + "60", + "50" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-B", + "alleles": [ + "B*57:01:01:02", + "B*08:01:01,B*08:01:02,B*08:01:03" + ], + "quality": [ + "0", + "0" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1, + 5 + ], + "one_allele_type": false + }, + { + "gene": "HLA-C", + "alleles": [ + "C*07:01:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*05:01:01", + "DQA1*02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01:01", + "DQB1*02:02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*03:01:01", + "DRB1*07:01:01" + ], + "quality": [ + "40", + "nan" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HFE", + "alleles": [ + "HFE*001" + ], + "quality": [ + "20" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true, + "label": "T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the variant calls (ClinVar section)" + } + ], + "derived": { + "coeliac": { + "status": "derived", + "reason": "", + "dq2_5": "present", + "dq2_5_phase": "likely cis (DRB1*03 present)", + "dq2_2": "present", + "dq8": "absent", + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": 0.95, + "low": 0.93, + "high": 0.97 + }, + { + "code": "AFR", + "group": "Africa", + "fraction": 0.05, + "low": null, + "high": null + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": 0.6, + "low": 0.5, + "high": 0.7 + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": null, + "low": null, + "high": null + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": null, + "low": null, + "high": null + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 8, + "deletions": 5, + "duplications": 3, + "significant": 7, + "significant_filtered": 4, + "filtered_chry": 1, + "dropped": { + "q0": 1, + "gap": 1, + "band": 1 + }, + "declared_sex": "male" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "depth": 30.0, + "read_floor": 2000, + "low_reads": true + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 3, + "total_mb": 37.5, + "largest_mb": 27.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + }, + { + "region": "chr15:23000001-50000000", + "mb": 27.0 + } + ], + "masked_mb": 46.5, + "sex_chromosome_segments": 1, + "long_mb": 34.0, + "f_roh": 0.0128, + "relationship": null, + "upd_flag": { + "chrom": "chr15", + "mb": 27.0, + "share": 0.79, + "imprinted": [ + { + "region": "15q11-q13 (SNRPN, UBE3A)", + "maternal_upd": "-", + "paternal_upd": "-", + "pmid": "22237428;27615419" + } + ], + "note": "Long chromosome-focused ROH; investigate possible isodisomy. A deletion, acquired loss of heterozygosity or shared ancestry can give this pattern. Copy neutrality and parental origin are not established; check copy-number results and parents' genotypes. Heterodisomy can leave no ROH." + }, + "scope": { + "source": "sites", + "autosomes": [ + "chr1", + "chr2", + "chr3", + "chr4", + "chr5", + "chr6", + "chr7", + "chr8", + "chr9", + "chr10", + "chr11", + "chr12", + "chr13", + "chr14", + "chr15", + "chr16", + "chr17", + "chr18", + "chr19", + "chr20", + "chr21", + "chr22" + ], + "unmasked_bp": 2658001522, + "partial": false, + "note": "Sites on all 22 autosomes; marker presence does not establish callable breadth or copy neutrality." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*03:01:01/DRB1*07:01:01" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32): *1/*2; Cyrius (step 21): *2/*1" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + }, + { + "hgvs": "m.8344A>G", + "gene": "MT-TK", + "disease": "MERRF (myoclonic epilepsy with ragged-red fibres)", + "clinvar_id": "9579", + "pmids": "2112427", + "note": "Usually heteroplasmic; the level differs between tissues.", + "pos": 8344, + "filter": "weak_evidence", + "af": "0.30" + }, + { + "hgvs": "m.11778G>A", + "gene": "MT-ND4", + "disease": "Leber hereditary optic neuropathy (LHON)", + "clinvar_id": "9708", + "pmids": "3201231", + "note": "Usually homoplasmic; not everyone who carries it loses vision.", + "pos": 11778, + "filter": "PASS", + "af": "0.99" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": 0.5, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + }, + { + "bkp5": 100, + "bkp3": 200, + "length": null, + "level_pct": 2.5 + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 7, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [ + { + "variant": "chrM:302 A>AC,ACC", + "af": 0.3 + }, + { + "variant": "chrM:16189 T>C", + "af": 0.5 + } + ], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "30.5", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TET2", + "mean_depth": "40", + "detection_limit_vaf": "0.08", + "status": "covered" + }, + { + "gene": "ASXL1", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + }, + { + "gene": "JAK2", + "mean_depth": "35", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TP53", + "mean_depth": "25", + "detection_limit_vaf": "0.1", + "status": "covered" + } + ], + "median_depth": 32.75, + "median_limit": 0.12901665405107993, + "not_covered": [ + "ASXL1" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "ok", + "source": "cpsr/S.cpsr.grch38.classification.tsv.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "html_report": true, + "classification_column": "CLASSIFICATION", + "classification": { + "Pathogenic": 1, + "Likely_Pathogenic": 1, + "VUS": 1, + "Benign": 1, + ".": 1 + }, + "plp": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "13:32338000 G>A", + "gene": "BRCA2", + "final": "Likely_Pathogenic", + "cpsr": "VUS", + "clinvar": null, + "authority": "CPSR", + "acmg_codes": null, + "dp": 40, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "5:112839000 A>T", + "gene": "APC", + "final": ".", + "cpsr": "Pathogenic", + "clinvar": null, + "authority": null, + "acmg_codes": null, + "dp": 30, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "disagreements": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "final_plp_own_benign": 1, + "own_class_column": "CPSR_CLASSIFICATION", + "low_confidence": 1, + "call_floor_dp": 8, + "cdna_flagged": 1 + } + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 5, + "by_impact": { + "HIGH": 3, + "MODERATE": 1, + ".": 1 + }, + "genes": 3, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "low_confidence": false, + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "low_confidence": false, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "1", + "significance": "Pathogenic", + "stars": 0, + "low_confidence": false, + "rule": "All hemi, het, homozygous P and LP", + "rule_note": null, + "severity": null + } + ], + "clinvar_checked": true, + "high_impact": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "consequence": "stop_gained", + "transcript": "MANE", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": 40, + "dp": 30, + "ad": "15,15", + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 2, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": true, + "structural_note": null, + "not_reported": [ + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "source": "ClinVar", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + }, + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "source": "ClinVar", + "reason": "one heterozygous variant; BTD is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + }, + "frequency_unknown": 1, + "weak_calls": 2 + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_slivar_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 2, + "frequency_unknown": 1, + "compound_het_variants": 4, + "compound_het_genes": 2, + "compound_het_pairs": 26, + "compound_het_phase": { + "trans": 23, + "cis": 1, + "unknown": 1 + }, + "compound_het_trans": [ + { + "gene": "G1", + "variants": "chr1:10 A>G + chr1:15 C>T", + "reads": "1 of 2" + }, + { + "gene": "G2", + "variants": "chr1:20 A>G + chr1:25 C>T", + "reads": "2 of 3" + }, + { + "gene": "G3", + "variants": "chr1:30 A>G + chr1:35 C>T", + "reads": "3 of 4" + }, + { + "gene": "G4", + "variants": "chr1:40 A>G + chr1:45 C>T", + "reads": "4 of 5" + }, + { + "gene": "G5", + "variants": "chr1:50 A>G + chr1:55 C>T", + "reads": "5 of 6" + }, + { + "gene": "G6", + "variants": "chr1:60 A>G + chr1:65 C>T", + "reads": "6 of 7" + }, + { + "gene": "G7", + "variants": "chr1:70 A>G + chr1:75 C>T", + "reads": "7 of 8" + }, + { + "gene": "G8", + "variants": "chr1:80 A>G + chr1:85 C>T", + "reads": "8 of 9" + }, + { + "gene": "G9", + "variants": "chr1:90 A>G + chr1:95 C>T", + "reads": "9 of 10" + }, + { + "gene": "G10", + "variants": "chr1:100 A>G + chr1:105 C>T", + "reads": "10 of 11" + }, + { + "gene": "G11", + "variants": "chr1:110 A>G + chr1:115 C>T", + "reads": "11 of 12" + }, + { + "gene": "G12", + "variants": "chr1:120 A>G + chr1:125 C>T", + "reads": "12 of 13" + }, + { + "gene": "G13", + "variants": "chr1:130 A>G + chr1:135 C>T", + "reads": "13 of 14" + }, + { + "gene": "G14", + "variants": "chr1:140 A>G + chr1:145 C>T", + "reads": "14 of 15" + }, + { + "gene": "G15", + "variants": "chr1:150 A>G + chr1:155 C>T", + "reads": "15 of 16" + }, + { + "gene": "G16", + "variants": "chr1:160 A>G + chr1:165 C>T", + "reads": "16 of 17" + }, + { + "gene": "G17", + "variants": "chr1:170 A>G + chr1:175 C>T", + "reads": "17 of 18" + }, + { + "gene": "G18", + "variants": "chr1:180 A>G + chr1:185 C>T", + "reads": "18 of 19" + }, + { + "gene": "G19", + "variants": "chr1:190 A>G + chr1:195 C>T", + "reads": "19 of 20" + }, + { + "gene": "G20", + "variants": "chr1:200 A>G + chr1:205 C>T", + "reads": "20 of 21" + } + ] + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": "*1/*2", + "Cyrius": "*2/*1" + }, + "agree": true, + "consensus": { + "result": "*1/*2", + "passed_to_pharmcat": true, + "reason": "pypgx and Cyrius agree", + "source": "pgx_consensus/S_pgx_consensus.tsv" + }, + "probable": null + }, + "not_run": [], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)" + ] +} diff --git a/tests/parity/golden/summary-collect/c-bad1/bad1.json b/tests/parity/golden/summary-collect/c-bad1/bad1.json new file mode 100644 index 000000000..ad14386d1 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-bad1/bad1.json @@ -0,0 +1,762 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:18 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_couple_carrier.tsv line 2 is not a row of S's couple table", + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_smn_copy_number.tsv lacks the chrom, start, end, agCN, agCN_filter, agCN_qual, psCN, psCN_qual or psCN_filter column", + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_paraphase_copy_number.tsv lacks the region, gene, copy_number or status column", + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01" + ], + "quality": [ + "x" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 not typed; HLA-DQB1 not typed", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": null + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A low confidence" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DQB1 not typed" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + } + ] + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_ancestry_fine.tsv has no rows or lacks a column of ['code', 'fraction', 'group', 'high', 'level', 'low', 'se']", + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_sv_class4_5_checked.tsv lacks a column of ['ACMG_class', 'Caller', 'Common', 'Exon_edge', 'Population_AF', 'SV_chrom', 'SV_end', 'SV_start', 'SV_type']", + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 2, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "CFTR_T", + "repeat_count": "7/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "7-7/9-9", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/FLANKING", + "adsp": "8/0", + "ci": "11-11/12-12", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": false, + "stranger_outdated": false, + "unsized": [ + "CFTR_TG" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "not_sized", + "genotype": ".", + "phase": ".", + "reading": "ExpansionHunter did not size both tracts (TG 11/12, flanking_only; poly-T 7/9, sized): not read." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "not_sized", + "genotype": ".", + "pharmcat": ".", + "agreement": "not_compared", + "reading": "not in step 09's output: STR_EXTRA_LOCI=yes ./scripts/setup.sh --strchive-data adds the repeat to the catalog (docs/09-str-expansions.md). PharmCAT (step 7) still reads UGT1A1 *28." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ] + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 0, + "total_mb": 0.0, + "largest_mb": 0.0, + "autosomal_over_5mb": [], + "masked_mb": 0.0, + "sex_chromosome_segments": 0, + "long_mb": -0.0, + "f_roh": null, + "relationship": null, + "upd_flag": null, + "scope": { + "source": "segments_only", + "autosomes": [], + "unmasked_bp": 0, + "partial": true, + "note": "Partial or unverified chromosome scope: no relationship band or UPD flag. F_ROH uses only the listed chromosomes, not callable bases." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_haplocheck.txt has no Contamination Status column", + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_haplogroup.txt has no Hg column or no sample row", + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_tree.txt lacks a column of Hg, Level, Tree", + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_y_str.tsv has a confidence other than high, low, no call or not callable", + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_consumer_export.tsv has no whole-number 'sites' row", + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_bloodtype.tsv has no system, phenotype and confidence columns, or no row", + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_lpa_kiv2.tsv has no status row (estimated, too little depth or not assessed)", + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_vntyper.tsv has no status row (positive, possible, negative, not assessed)", + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_locityper.tsv lacks a row or one of the columns locus, region, status, genotype, haplotype_1, haplotype_2, quality, total_reads, unexplained_reads, weighted_distance, warnings, filter, reason", + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "KeyError: 'length'", + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_traits.tsv line 2 does not have one field per column", + "data": {} + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_archaic.tsv has no percentage for neanderthal_percent", + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_ancient.tsv has no rows or lacks a column of ['country', 'dates', 'distance', 'group', 'locality', 'n', 'rank']", + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_viral.tsv has no rows or lacks a column of virus, accession, reads, breadth, relative_depth, copies_per_cell, call", + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_exomiser_top.tsv lacks a column of rank, gene, moi, exomiser_score, phenotype_score, variant, genotype, consequence, hgvs, acmg, disease", + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_chip.tsv lacks a column of gene, protein_change, vaf, alt_reads, depth, status, note", + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_mei.tsv is missing beside S_mei_sites.tsv", + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_mocha.tsv has 1 reported calls, S_mocha_summary.tsv says 2", + "data": {} + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Coverage (mosdepth)", + "Gene callability (mosdepth)", + "Sex check (indexcov)", + "Sample identity and contamination (somalier, VerifyBamID2)", + "Mosaic loss of chromosome Y or X", + "Variant calling", + "ClinVar screen", + "PharmCAT", + "CPIC drug-gene recommendations", + "pypgx", + "Cyrius (CYP2D6)", + "Polygenic risk scores", + "Structural variants (Manta)", + "Structural variants (Delly)", + "Copy number (CNVpytor)", + "SV consensus merge", + "Cancer predisposition (CPSR)", + "Clinical variant filter", + "Variant prioritization (slivar)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/parity/golden/summary-collect/c-full/full.json b/tests/parity/golden/summary-collect/c-full/full.json new file mode 100644 index 000000000..3e4f956fc --- /dev/null +++ b/tests/parity/golden/summary-collect/c-full/full.json @@ -0,0 +1,2402 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:18 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "ok", + "source": "vcf/S.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1, + "snps": 0, + "indels": 1 + } + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "ok", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "gene": "UGT1A1", + "diplotype": "*1/*28", + "phenotype": "Intermediate", + "status": "non-normal" + } + ], + "non_normal": 1, + "ambiguous": 0, + "not_called": 0, + "unclassified": 0, + "parse_failed": false, + "warnings": [] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": 1, + "text": "1 (quality 31, filter PASS)", + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": 3, + "text": "3 (quality 25, filter PASS)" + }, + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "one SMN1 copy: the pattern of an SMA carrier", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "called", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality 50, filter PASS); chr7:5980001-5990000 3 (quality 12, filter LowQual)" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": 1, + "text": "1 (quality 31, filter PASS)" + }, + "agree": "disagree" + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01:01", + "A*01:01:01" + ], + "quality": [ + "60", + "50" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-B", + "alleles": [ + "B*57:01:01:02", + "B*08:01:01,B*08:01:02,B*08:01:03" + ], + "quality": [ + "0", + "0" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1, + 5 + ], + "one_allele_type": false + }, + { + "gene": "HLA-C", + "alleles": [ + "C*07:01:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*05:01:01", + "DQA1*02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01:01", + "DQB1*02:02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*03:01:01", + "DRB1*07:01:01" + ], + "quality": [ + "40", + "nan" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HFE", + "alleles": [ + "HFE*001" + ], + "quality": [ + "20" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true, + "label": "T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the variant calls (ClinVar section)" + } + ], + "derived": { + "coeliac": { + "status": "derived", + "reason": "", + "dq2_5": "present", + "dq2_5_phase": "likely cis (DRB1*03 present)", + "dq2_2": "present", + "dq8": "absent", + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": 0.95, + "low": 0.93, + "high": 0.97 + }, + { + "code": "AFR", + "group": "Africa", + "fraction": 0.05, + "low": null, + "high": null + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": 0.6, + "low": 0.5, + "high": 0.7 + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": null, + "low": null, + "high": null + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": null, + "low": null, + "high": null + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 8, + "deletions": 5, + "duplications": 3, + "significant": 7, + "significant_filtered": 4, + "filtered_chry": 1, + "dropped": { + "q0": 1, + "gap": 1, + "band": 1 + }, + "declared_sex": null + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "depth": 30.0, + "read_floor": 2000, + "low_reads": true + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 3, + "total_mb": 37.5, + "largest_mb": 27.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + }, + { + "region": "chr15:23000001-50000000", + "mb": 27.0 + } + ], + "masked_mb": 46.5, + "sex_chromosome_segments": 1, + "long_mb": 34.0, + "f_roh": 0.0128, + "relationship": null, + "upd_flag": { + "chrom": "chr15", + "mb": 27.0, + "share": 0.79, + "imprinted": [ + { + "region": "15q11-q13 (SNRPN, UBE3A)", + "maternal_upd": "-", + "paternal_upd": "-", + "pmid": "22237428;27615419" + } + ], + "note": "Long chromosome-focused ROH; investigate possible isodisomy. A deletion, acquired loss of heterozygosity or shared ancestry can give this pattern. Copy neutrality and parental origin are not established; check copy-number results and parents' genotypes. Heterodisomy can leave no ROH." + }, + "scope": { + "source": "sites", + "autosomes": [ + "chr1", + "chr2", + "chr3", + "chr4", + "chr5", + "chr6", + "chr7", + "chr8", + "chr9", + "chr10", + "chr11", + "chr12", + "chr13", + "chr14", + "chr15", + "chr16", + "chr17", + "chr18", + "chr19", + "chr20", + "chr21", + "chr22" + ], + "unmasked_bp": 2658001522, + "partial": false, + "note": "Sites on all 22 autosomes; marker presence does not establish callable breadth or copy neutrality." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*03:01:01/DRB1*07:01:01" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32) and Cyrius (step 21) did not run" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + }, + { + "hgvs": "m.8344A>G", + "gene": "MT-TK", + "disease": "MERRF (myoclonic epilepsy with ragged-red fibres)", + "clinvar_id": "9579", + "pmids": "2112427", + "note": "Usually heteroplasmic; the level differs between tissues.", + "pos": 8344, + "filter": "weak_evidence", + "af": "0.30" + }, + { + "hgvs": "m.11778G>A", + "gene": "MT-ND4", + "disease": "Leber hereditary optic neuropathy (LHON)", + "clinvar_id": "9708", + "pmids": "3201231", + "note": "Usually homoplasmic; not everyone who carries it loses vision.", + "pos": 11778, + "filter": "PASS", + "af": "0.99" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": 0.5, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + }, + { + "bkp5": 100, + "bkp3": 200, + "length": null, + "level_pct": 2.5 + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 7, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [ + { + "variant": "chrM:302 A>AC,ACC", + "af": 0.3 + }, + { + "variant": "chrM:16189 T>C", + "af": 0.5 + } + ], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "30.5", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TET2", + "mean_depth": "40", + "detection_limit_vaf": "0.08", + "status": "covered" + }, + { + "gene": "ASXL1", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + }, + { + "gene": "JAK2", + "mean_depth": "35", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TP53", + "mean_depth": "25", + "detection_limit_vaf": "0.1", + "status": "covered" + } + ], + "median_depth": 32.75, + "median_limit": 0.12901665405107993, + "not_covered": [ + "ASXL1" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Coverage (mosdepth)", + "Gene callability (mosdepth)", + "Sex check (indexcov)", + "Sample identity and contamination (somalier, VerifyBamID2)", + "Mosaic loss of chromosome Y or X", + "ClinVar screen", + "PharmCAT", + "pypgx", + "Cyrius (CYP2D6)", + "Cancer predisposition (CPSR)", + "Clinical variant filter", + "Variant prioritization (slivar)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)" + ] +} diff --git a/tests/parity/golden/summary-collect/c-nf/nf.json b/tests/parity/golden/summary-collect/c-nf/nf.json new file mode 100644 index 000000000..e836d9048 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-nf/nf.json @@ -0,0 +1,1200 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 00:18 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "female" + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": null, + "text": "no row covers exon 7", + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": "." + }, + "smn2": { + "cn": null, + "text": "no row covers exon 7" + }, + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": ".", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "SMN1 g.27134T>G: not assessed, step 35 wrote no S_smn1_marker.tsv (a run from before this reader, or the check did not run)." + }, + "rows": 1, + "hba": { + "status": "not assessed", + "message": "not assessed: S_hba_copy_number.tsv has no status row" + }, + "cn_intervals": null, + "cyp21a2": null, + "dosage": null, + "reading": "SMN1 and SMN2 not assessed: no model: no carrier reading", + "loci": [] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "x", + "status": "called", + "detail": "." + } + ], + "other_regions": 0, + "other_regions_called": 0, + "variants": {}, + "smn1": { + "paraphase": null, + "parascopy": { + "cn": null, + "text": "no row covers SMN1 exon 7" + }, + "agree": null + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*03:01", + "DQA1*01:01" + ], + "quality": [ + "0", + "30" + ], + "low_confidence": true, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*04:01", + "DRB1*07:01" + ], + "quality": [ + "40", + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 low confidence", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A not typed" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "present", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [], + "reason": "OptiType did not run: no class I reads" + }, + "kir": { + "typed": [], + "not_called": [], + "reason": "too few KIR reads" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "stale", + "source": "prs/S_prs_summary.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 25 was failed (exit 2) in the run of 2020-01-01 00:00 UTC", + "data": { + "scores": [ + { + "condition": "X", + "pgs_id": "PGS1", + "score": "", + "matched": "", + "total": "", + "chrx": "", + "ambiguous": "", + "applies_to": "", + "sex_check": "", + "percentile": "", + "group": "", + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": false, + "ancestry": { + "population": "", + "panel": "", + "low_confidence": "" + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/x/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 0, + "pass": 0 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "stale", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 18 was skipped (opt-in) in the run of 2020-01-01 00:00 UTC", + "data": { + "total": 2, + "deletions": 2, + "duplications": 0, + "significant": 2, + "significant_filtered": 1, + "filtered_chry": 1, + "dropped": { + "q0": 0, + "gap": 0, + "band": 0, + "chry_female": 1 + }, + "declared_sex": "female" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "stale", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 22 was failed (exit 1) in the run of 2020-01-01 00:00 UTC", + "data": { + "consensus": 1 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 0, + "rows": [], + "downgraded": [], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 0, + "exon_edge": 0, + "exon_edge_checked": true, + "common": 0 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 3, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "CFTR_T", + "repeat_count": "5/5", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/5-5", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "12/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "12-12/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "UGT1A1", + "repeat_count": "8/10", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "8-8/10-10", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": false, + "stranger_outdated": true, + "unsized": [], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG12-5T/TG12-5T", + "pairings": [ + "TG12-5T/TG12-5T" + ], + "tg": [ + 12, + 12 + ], + "poly_t": [ + 5, + 5 + ], + "five_t": true, + "phase": "not_read", + "r117h": ".", + "reading": "5T: 2 copys with the 5T poly-T tract, TG12 on the 5T copies. 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. R117H was not read (no step 42 traits row for rs78655421 was called), so whether it sits on the 5T copy is not known." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*28/TA9", + "ta_units": [ + 8, + 10 + ], + "pharmcat": ".", + "agreement": "not_compared", + "reading": "*28/TA9 (TA7/TA9, 8/10 TA units as ExpansionHunter counts them): a TA length outside TA5 to TA8 is not a named allele here, so its function is not read and it is not compared with PharmCAT (step 7)." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ] + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/x/deep/T_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "334", + "intratelomeric_reads": 61873, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "roh/S_roh.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": null, + "data": { + "segments": 1, + "total_mb": 7.0, + "largest_mb": 7.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + } + ], + "masked_mb": 0.0, + "sex_chromosome_segments": 0, + "long_mb": 7.0, + "f_roh": 0.0294, + "relationship": null, + "upd_flag": null, + "scope": { + "source": "sites", + "autosomes": [ + "chr2" + ], + "unmasked_bp": 237993529, + "partial": true, + "note": "Partial or unverified chromosome scope: no relationship band or UPD flag. F_ROH uses only the listed chromosomes, not callable bases." + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "stale", + "source": "mito/S_haplogroup.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 12 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "haplogroup": "U5a", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/U5a" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/U5a/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "insufficient markers", + "valid_markers": ".", + "qc_score": ".", + "explore": [] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "no branch has a derived call", + "level": ".", + "tree": ".", + "path_derived": ".", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": ".", + "min_depth": ".", + "explore": [], + "path": [] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 10, + "called": 10, + "no_call": 0, + "from_reference_blocks": 0, + "template_sites": ".", + "file_23andme": ".", + "file_ancestrydna": ".", + "no_call_reasons": {} + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "too little depth", + "kiv2_copies": ".", + "ci95_low": ".", + "ci95_high": ".", + "class": ".", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": ".", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "LPA KIV-2 4925G>A: not assessed, step 46 wrote no S_lpa_kiv2_4925.tsv (a run from before this reader, or the check did not run)." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": ".", + "status": "not_covered", + "risk_alleles": null + } + ], + "risk_alleles": 0, + "flag": ".", + "flag_text": "Not determined: neither the KIV-2 count nor the two risk variants could be read. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "confidence": ".", + "variant": ".", + "flag": ".", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "." + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "stale", + "source": "locityper/S_locityper.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 53 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*04:01/DRB1*07:01" + }, + { + "locus": "HLA-DQB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DQB1*02:01" + }, + { + "locus": "CYP2D6", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "pypgx (step 32) and Cyrius (step 21) did not run" + } + ], + "called": 3, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + } + ], + "deletions_note": "no deletion table (mito/_chrM_deletions.tsv): step 20 ran before it looked for large deletions, eKLIPse wrote no table (see the step's log), or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "copies_note": "no copy-number table (mito/_mtdna_copies.tsv): step 20 ran before it counted mtDNA copies, or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "pass": 1, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "note": "mito/S_chrM_annotated.tsv is older than the chrM calls: rerun step 20" + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [], + "called": 0, + "not_covered": 1, + "other_allele": 0, + "rules": null + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "not assessed (no human depth)", + "human_depth": ".", + "found": [], + "genomes": 1 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [], + "genes": [] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [], + "other": 0, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + } + ], + "median_depth": null, + "median_limit": null, + "not_covered": [ + "DNMT3A" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 0, + "by_element": {}, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "F", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "seen" + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "message": "too shallow", + "depth": ".", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": ".", + "calls_mocha": ".", + "calls_reported": ".", + "phasing": ".", + "calls": [], + "not_reported": 0 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": null, + "Cyrius": null + }, + "agree": null, + "consensus": null, + "probable": null + }, + "not_run": [ + "Coverage (mosdepth)", + "Gene callability (mosdepth)", + "Sex check (indexcov)", + "Sample identity and contamination (somalier, VerifyBamID2)", + "Mosaic loss of chromosome Y or X", + "Variant calling", + "ClinVar screen", + "PharmCAT", + "CPIC drug-gene recommendations", + "pypgx", + "Cyrius (CYP2D6)", + "Fine-grained ancestry (ADMIXTURE, RFMix)", + "Structural variants (Delly)", + "Y-STR markers (HipSTR)", + "Blood groups (predicted)", + "Archaic ancestry (IBDmix)", + "Ancient populations (AADR PCA)", + "Cancer predisposition (CPSR)", + "Clinical variant filter", + "Variant prioritization (slivar)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 20eb59617..b2902119d 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -2113,7 +2113,7 @@ def report_sections_mito_roh_telomere_cnv(work, schema): def roh_upd_karyotype(work, schema): """F_ROH, the one-chromosome isodisomy flag joined to assets/imprinted_regions.tsv, and the sex-chromosome karyotype label.""" - # The catalog and the rows the summary holds (the Nextflow report task sees bin/, not assets/). + # The one table is embedded into og; the flag emits neutral region hints. with open(os.path.join(REPO, "assets/imprinted_regions.tsv")) as f: rows = list(csv.DictReader((l for l in f if not l.startswith("#")), delimiter="\t")) asset = [(r["chrom"], int(r["start"]) if r["start"] else None, int(r["end"]) if r["end"] else None, @@ -2126,55 +2126,55 @@ def roh_upd_karyotype(work, schema): # hg38.chrom.sizes) less the 217,000,000 bp of autosomal bands, written out # here rather than recomputed from the code under test. denom = 2875001522 - 217000000 + sites = "".join(f"ST\tS\tchr{c}\t40000000\t0\t60\n" for c in range(1, 23)) d = os.path.join(work, "roh_upd", "S") # Isodisomy: the whole of chr15 in one run (chr15 is 102 Mb; its first # 20.5 Mb are the acrocentric band, so 81.5 Mb count) and 8 Mb on chr2. upd = "RG\tS\tchr15\t1\t101991189\t101991189\t9000\t50\nRG\tS\tchr2\t10000001\t18000000\t8000000\t900\t50\n" - put(f"{d}/vcf/S_roh.txt", upd) + put(f"{d}/vcf/S_roh.txt", sites + upd) summ, txt, html = render(d) r = summ["sections"]["roh"]["data"] u = r.get("upd_flag") or {} long_bp = (101991189 - 20500000) + 8000000 check("ROH UPD: chr15 holding most of the long runs raises the flag", u.get("chrom") == "chr15" and u.get("mb") == 81.5 and r.get("relationship") is None, r) - check("ROH UPD: the flag joins the 15q11-q13 row, Prader-Willi from the mother and Angelman from the father", + check("ROH UPD: the flag names the 15q11-q13 region without a syndrome", [(x["region"], x["maternal_upd"], x["paternal_upd"]) for x in u.get("imprinted", [])] - == [("15q11-q13 (SNRPN, UBE3A)", "Prader-Willi syndrome", "Angelman syndrome")], u) + == [("15q11-q13 (SNRPN, UBE3A)", "-", "-")], u) check("ROH UPD: F_ROH of the two runs", abs((r.get("f_roh") or -1) - long_bp / denom) < 0.001, r.get("f_roh")) - check("ROH UPD: both reports carry the flag and the syndromes", - all("chr15 carries 81.5 MB" in t and "Prader-Willi syndrome" in t and "Angelman syndrome" in t + check("ROH UPD: both reports carry the flag and region without syndrome names", + all("chr15 carries 81.5 MB" in t and "15q11-q13 (SNRPN, UBE3A)" in t and "Prader-Willi" not in t and "Angelman" not in t for t in (txt, html)), txt) check("ROH UPD summary validates", not validate.validate(schema, json.loads(json.dumps(summ))), validate.validate(schema, json.loads(json.dumps(summ)))) - # Red control: the run on chr15 cut to 15q21 onwards misses 15q11-q13, so no row joins. - put(f"{d}/vcf/S_roh.txt", upd.replace("chr15\t1\t101991189\t101991189", "chr15\t40000001\t101991189\t61991189")) + # A distal isodisomic run can coexist with heterodisomy at the imprinted locus. + put(f"{d}/vcf/S_roh.txt", sites + upd.replace("chr15\t1\t101991189\t101991189", "chr15\t40000001\t101991189\t61991189")) u0 = section(d, "roh")["data"].get("upd_flag") or {} - check("ROH UPD red control: a chr15 run outside 15q11-q13 is flagged but joins no row", - u0.get("chrom") == "chr15" and u0.get("imprinted") == [], u0) + check("ROH UPD red control: a distal chr15 run still names the chromosome-wide region hint", + u0.get("chrom") == "chr15" and [x["region"] for x in u0.get("imprinted", [])] == ["15q11-q13 (SNRPN, UBE3A)"], u0) _, txt0, html0 = render(d) - check("ROH UPD: a flagged chr15 run outside 15q11-q13 says no listed region overlaps it, in both reports", - all("No listed imprinting-disorder region overlaps the flagged runs on chr15" in t - and "No imprinting disorder is listed" not in t for t in (txt0, html0)), txt0) + check("ROH UPD: a distal chr15 run lists its region without a syndrome in both reports", + all("15q11-q13 (SNRPN, UBE3A)" in t and "Prader-Willi" not in t and "Angelman" not in t for t in (txt0, html0)), txt0) check("ROH UPD: with a flag there is no band, so the band sentence of the F_ROH note is left out", all("F_ROH is the share of the autosomes" in t and "The band is the row" not in t for t in (txt0, html0)), txt0) # Just over half: 25.1 of 50 Mb is 50.2%, which a whole percent rounds to 50%. - put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t35100000\t25100000\t900\t50\n" + put(f"{d}/vcf/S_roh.txt", sites + "RG\tS\tchr2\t10000001\t35100000\t25100000\t900\t50\n" "RG\tS\tchr3\t10000001\t34900000\t24900000\t900\t50\n") _, txt5, html5 = render(d) check("ROH UPD: a share just over half reads more than 50%, never 50%, in both reports", all("chr2 carries 25.1 MB, more than 50% of the length" in t and "MB, 50%" not in t for t in (txt5, html5)), txt5) # chr7 joins as a whole chromosome. - put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr7\t70000001\t110000000\t40000000\t900\t50\n") + put(f"{d}/vcf/S_roh.txt", sites + "RG\tS\tchr7\t70000001\t110000000\t40000000\t900\t50\n") u7 = section(d, "roh")["data"].get("upd_flag") or {} - check("ROH UPD: 40 Mb on chr7 joins the whole-chromosome Silver-Russell row", - [x["maternal_upd"] for x in u7.get("imprinted", [])] == ["Silver-Russell syndrome"], u7) + check("ROH UPD: 40 Mb on chr7 lists a neutral region hint", + [(x["region"], x["maternal_upd"]) for x in u7.get("imprinted", [])] == [("7p12 (GRB10), 7q32 (MEST)", "-")], u7) # The same long total over 12 chromosomes: no flag, F_ROH the same, a relationship band. each = long_bp // 12 spread = "".join(f"RG\tS\tchr{c}\t10000001\t{10000000 + each}\t{each}\t900\t50\n" for c in range(1, 13)) - put(f"{d}/vcf/S_roh.txt", spread) + put(f"{d}/vcf/S_roh.txt", sites + spread) summ, txt, html = render(d) r = summ["sections"]["roh"]["data"] check("ROH spread: 12 chromosomes raise no flag", "upd_flag" in r and r["upd_flag"] is None, r) @@ -2187,12 +2187,12 @@ def roh_upd_karyotype(work, schema): and "may come from one parent" not in t for t in (txt, html)), txt) check("ROH spread: with a band the F_ROH note says what the band is", all("The band is the row" in t for t in (txt, html)), txt) # Short runs only: F_ROH 0, no sign of related parents. - put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t13000000\t3000000\t900\t50\n") + put(f"{d}/vcf/S_roh.txt", sites + "RG\tS\tchr2\t10000001\t13000000\t3000000\t900\t50\n") r = section(d, "roh")["data"] check("ROH: runs under 5 Mb only give F_ROH 0 and no sign of related parents", r.get("f_roh") == 0 and r.get("relationship") == "no sign of related parents", r) # A run of exactly 5 Mb counts, as in step 11's summary (5 Mb or more). - put(f"{d}/vcf/S_roh.txt", "RG\tS\tchr2\t10000001\t15000000\t5000000\t900\t50\n") + put(f"{d}/vcf/S_roh.txt", sites + "RG\tS\tchr2\t10000001\t15000000\t5000000\t900\t50\n") r = section(d, "roh")["data"] check("ROH: a run of exactly 5 Mb counts in F_ROH and the long-run list, as in step 11", r.get("long_mb") == 5.0 and len(r.get("autosomal_over_5mb") or []) == 1, r) From c8bd5963ecf90ddcfa672aa94834ed5ce3f1dc53 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:23:43 +0100 Subject: [PATCH 111/255] docs(rust): name the four new crates, the planned verbs and the og-leg marker Bead: D1 Bead: D14 --- CHANGELOG.md | 3 +++ docs/rust.md | 8 +++++++- rust/README.md | 4 ++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 58cac8762..9342e5038 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Every verb the 1.0.0 release still has to port now exists in the `og` binary, each in its own module, as a stub that prints `og : not ported yet` and exits 2. The new crates are og-render, og-live, og-couple and og-admix. Each port then fills one module and never edits `rust/og/src/main.rs` or a crate's module list, so the ports can land in parallel. No step calls these verbs yet, and no output changes. + - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. + - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/rust.md b/docs/rust.md index 0e0de2374..bfe8ead4b 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -31,6 +31,10 @@ rust/ crates/og-lookup/ the genotype lookup of scripts/owngenome-lookup.sh: og lookup query, resolve, report crates/og-fresh/ og dev freshness: the monthly freshness issue's upstream lookups, typed answers crates/og-recheck/ og recheck: newer databases since a run, the rerun and the ClinVar lock + crates/og-render/ og report render|index|charts: the reports from summary.json (not ported yet) + crates/og-live/ og live serve|results|family|demo, og view serve: the live site (not ported yet) + crates/og-couple/ og couple screen|genes: step 39's couple screen (not ported yet) + crates/og-admix/ what the fine-ancestry engines share: painting and projection (not ported yet) og/ the binary: og [args] bin/og the shim tasks call; runs bin/.og/og-linux- (og-darwin- on a Mac) tests/rust/ the differential test against CPython, the image test @@ -38,6 +42,8 @@ tests/rust/ the differential test against CPython, the image test Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (were the commands of `chip_report.py`). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py`. Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Every verb the 1.0.0 release still has to port is already registered, each in its own module, as a stub that prints `og : not ported yet` on stderr and exits 2: `og report render|index|charts`, `og live serve|results|family|demo`, `og view serve`, `og couple screen|genes`, `og vcf index|shift-back`, `og setup chrm-shift|revel|roh-af`, `og sv survivor-prep|consensus-check`, `og archaic compare|genomes`, `og ancestry-fine freqs|project|paint|simulate`, `og longread clair3-model`, `og map svg`, `og dev demo-sample|demo-events|params-doc|python-gate|everything-check|licences|docs-check|stub-http|steps-check` and `og dev build roh-af|acmg-lof`. A port fills its module's body and does not edit `og/src/main.rs`, the workspace member list or a crate's module list. `rust/og/tests/port_scaffold.rs` lists these verbs and fails when one has no line in `og --help` or reaches a router's "unknown" error. + A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, `cargo fmt`, and few, well-known crates (`indexmap`, `flate2`, `zlib-rs`, `memchr`, `smallvec`, `rustc-hash` and `rust-htslib` so far, declared in `rust/Cargo.toml` and locked in `Cargo.lock`). The musl builds also link `mimalloc` as their allocator: musl's own lets a long multi-threaded read hold several times its live memory. @@ -104,7 +110,7 @@ To add a primitive: write it in `pycompat` with unit tests, add an operation to ## CI -- **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; the four builds through `scripts/setup.sh --og` (both musl ones and the macOS pair); and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. Then the fake-docker case `reports-stale-step` runs with `bin/og` and must check its `summary.json` with `og dev schema`, and the tests of the `og sv` and `og pgx` filters, the step 05 tests and `tests/test_ancestry_fine_report.py` run with `bin/og` and `OG_REQUIRED=1`; in `unit-tests`, which has no `og`, they skip with a note. `cargo test` includes `og dev contract --static` and `og dev report-only` on the checkout (`rust/crates/og-dev/tests/repo.rs`). It also runs `og dev helpers --static` and `--list` on the checkout (`rust/crates/og-dev/tests/helpers_repo.rs`); the `helper-binaries` job runs the same verb through `scripts/ci/check-container-helpers.sh`, with the og the `og` job builds. +- **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; the four builds through `scripts/setup.sh --og` (both musl ones and the macOS pair); and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. Then the fake-docker case `reports-stale-step` runs with `bin/og` and must check its `summary.json` with `og dev schema`, and every test that carries the line `# og-leg: OG_REQUIRED` (the tests that run `og`, such as the `og sv` and `og pgx` filter tests, the step 05 tests and `tests/test_ancestry_fine_report.py`) runs with `bin/og` and `OG_REQUIRED=1`; in `unit-tests`, which has no `og`, they skip with a note. A new test that needs `og` adds that line; the job has no list to edit. Before the real run the step plants a marked test with a failing assert, which must be picked and fail. The planned-verb test must fail once one verb's arm is dropped from `og/src/main.rs`. `cargo test` includes `og dev contract --static` and `og dev report-only` on the checkout (`rust/crates/og-dev/tests/repo.rs`). It also runs `og dev helpers --static` and `--list` on the checkout (`rust/crates/og-dev/tests/helpers_repo.rs`); the `helper-binaries` job runs the same verb through `scripts/ci/check-container-helpers.sh`, with the og the `og` job builds. - **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a fresh and a stale macOS pair, a tree hash that cannot be computed, a release download with a bad checksum or attestation, without the macOS pair, with `gh` not logged in, with a network error, without binaries, or whose copy into `bin/.og` fails), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. - **Rust** (`rust.yml`, on a pull request that touches the Rust code, the shim or `versions.env`): times a cold and a cached build, runs `og` by bare name in the four images on amd64 and arm64 (QEMU), runs that check once more with a wrong-arch binary planted, which must fail, runs the fetch smoke with both binaries (arm64 under QEMU) and with pysam in `PYPGX_IMAGE`, which must all match `fetch.pysam.tsv`, and keeps the binaries as the artifact `og-binaries`. Every night it runs the differential test at 10,000,000 cases per operation with a new seed. - **Release** (`release.yml`): a tag builds the four binaries, writes `SHA256SUMS`, attests them and attaches all five files to the release. A pull request that changes `rust/` or the workflow builds them as a dry run. diff --git a/rust/README.md b/rust/README.md index ce5fae19c..5a834a2e8 100644 --- a/rust/README.md +++ b/rust/README.md @@ -22,6 +22,10 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-steps` | step helpers, among them `og pgx outside-calls`, `og pgx hla-concordance` (steps 08, 36), `og pgx pypgx-summary` (step 32) and `og paraphase check-bam`, `og paraphase table` (step 35b) | | `crates/og-fresh` | `og dev freshness`: the upstream lookups of the monthly freshness issue, each answer typed, HTTP through the host's `curl` | | `crates/og-recheck` | `og recheck`: which of a finished sample's results a newer ClinVar or catalog would change, `--rerun`, `--affected`, `--launch-dir` | +| `crates/og-render` | `og report render`, `og report index`, `og report charts`: the reports from summary.json (not ported yet) | +| `crates/og-live` | `og live serve`, `og live results`, `og live family`, `og live demo`, `og view serve`: the live site (not ported yet) | +| `crates/og-couple` | `og couple screen`, `og couple genes`: step 39's couple screen (not ported yet) | +| `crates/og-admix` | what the fine-ancestry engines share: our own painting and projection (not ported yet) | | `og` | the binary: `og [args]`, `og version`, `og --tree-hash` | | `og.sh` | build or download `bin/.og/og-linux-{x86_64,aarch64}`, and check they match this `rust/` | | `rust-toolchain.toml` | the Rust release, the one `OG_BUILDER` (`versions.env`) ships | From a4fb1c573962ef11b20292e84a85aeb8695371b3 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:24:16 +0100 Subject: [PATCH 112/255] fix(ci): keep ROH caller paths within the supported e2e scope Bead: lin-17 --- scripts/ci/e2e-case-paths.tsv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index d49f98629..6dd7e7359 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -41,7 +41,7 @@ 60-nextflow - 90-report-assets - acmg-sf-rules-1-reports assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh -align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/og bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* rust/crates/og-summary/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/og bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh alignment-markdup-gvcf-sex-1-align scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-2-ploidy scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh @@ -169,7 +169,7 @@ sv-mito-telomere-steps-6-scatter scripts/03a-gatk-haplotypecaller.sh scripts/03b traits-panel-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/genotype_rules.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/traits.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/traits/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/42-traits.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vcf-precheck-ref-spotcheck-1 assets/par_grch38.bed modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/11-roh-analysis.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/vcf/grch37_no_contigs.vcf vendor-vcf-intake-1-bash bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh -vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* rust/crates/og-summary/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/og modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh From c78f72afa1ba095f843211bc89df902bac803ec0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:24:30 +0100 Subject: [PATCH 113/255] fix(recheck): say the check failed when every lookup failed, and rerun with typed arguments A failed lookup no longer ends in 'Nothing here is newer than what the run used.': the last line says how many lookups failed, or 'Could not check' when none answered. The local comparison of what is installed here with what the run used runs before the lookup's error is read, so a newer ClinVar file on disk stays rerun-ready with its steps; the item keeps its error and the command still exits 1. --rerun reads 'meta command' back as the typed call run-all.sh records (cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS) and starts that run-all.sh with bash and its arguments one by one. Any other shape, or a cd folder that is gone, is refused with exit 2 and nothing runs. Bead: plat-25 Bead: plat-26 --- CHANGELOG.md | 2 ++ docs/recheck.md | 6 +++- rust/crates/og-recheck/src/cli.rs | 6 ++-- rust/crates/og-recheck/src/lib.rs | 53 +++++++++++++++++++++++------ rust/crates/og-recheck/src/rerun.rs | 24 ++++++++++++- scripts/run-all.sh | 9 ++--- 6 files changed, 81 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..574c7d7a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- `og recheck` no longer says "Nothing here is newer than what the run used." when a lookup failed. With no network it said that line, and a cron log then read as up to date. Now the last line says how many lookups failed and that the check is incomplete, or "Could not check: all N lookups failed" when none answered. The exit code was already 1 and stays 1. A failed lookup also no longer hides a newer file or pin that is already installed here: a newer ClinVar file on disk stays `rerun-ready`, with its steps, in `state.json`, the reports and the live page, and the item keeps its `error`. `og recheck --rerun` no longer replays the stored command through `bash -c`. It reads `meta command` of `logs/run_status.tsv` back word by word as the call `run-all.sh` records (`cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS`) and starts that `run-all.sh` with its arguments one by one. A row of any other shape, such as one edited to add `; rm ...`, or a `cd` folder that is gone, is refused with exit 2 and nothing runs. A row that `run-all.sh` wrote replays as before. No option, output file or format changes. [docs/recheck.md](docs/recheck.md) + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/docs/recheck.md b/docs/recheck.md index c12066669..0d149ec73 100644 --- a/docs/recheck.md +++ b/docs/recheck.md @@ -49,6 +49,10 @@ Each item gets one class: | `unknown` | A pin of steps 50 to 53 that the run's manifest does not name (a run of an older version): nothing was compared | Nothing; the next run records it. It does not make the command exit 1 | | `error` | The lookup failed | Check again later. A failed lookup is never read as "up to date", and the command exits 1 | +A newer file or pin installed here stays `rerun-ready` when its lookup fails, because that comparison needs no network. The item then keeps its `error` too, and the command still exits 1. + +The last line of the output sums up the check. When something is `rerun-ready` or `refresh-available`, it names the steps a rerun would redo. When a lookup failed, it says how many of how many failed and that the check is incomplete, or "Could not check: all N lookups failed" when none answered. "Nothing here is newer than what the run used." comes only when every lookup answered and nothing is newer. + The 2-day margin for ClinVar is there because NCBI dates the file a day or two after the release in its `##fileDate`. A newer PGS file date means the catalog wrote the harmonised file again; it does not prove the weights changed. ## Which steps a change redoes @@ -64,7 +68,7 @@ bin/og recheck --sample your_sample --rerun bin/og recheck --sample your_sample --rerun --refresh-clinvar # refresh ClinVar first ``` -`run-all.sh` writes its own command into `logs/run_status.tsv` (`meta command`: a `cd` to the directory it was run from, the switches it reads that were set, then its arguments, each shell-quoted; `meta switches`: the names of those switches). `--rerun` runs that command again with those switches only, so a `TOOLS` set in your shell today does not change the run. `NXF_OPTS` and `NXF_JVM_ARGS` are not recorded, because they can hold a proxy password; the replay takes them from your shell. Nextflow's `-resume` then redoes only the tasks whose inputs changed. +`run-all.sh` writes its own command into `logs/run_status.tsv` (`meta command`: a `cd` to the directory it was run from, the switches it reads that were set, then its arguments, each shell-quoted; `meta switches`: the names of those switches). `--rerun` reads that command back word by word: the `cd` folder, the switch assignments, the `run-all.sh` path and its arguments. It starts that `run-all.sh` with `bash`, in that folder, with the arguments one by one and those switches only, so a `TOOLS` set in your shell today does not change the run. No shell reads the command. A row of any other shape, for example one edited to add `; rm ...`, is refused with exit 2 and nothing runs; a `cd` folder that no longer exists is refused the same way. Check such a row, then run the command by hand. `NXF_OPTS` and `NXF_JVM_ARGS` are not recorded, because they can hold a proxy password; the replay takes them from your shell. Nextflow's `-resume` then redoes only the tasks whose inputs changed. It refuses while a run of the sample is going on (`run_status.tsv` has no `finished_utc`), and while another `--rerun` of the sample holds `recheck/rerun.lock`. A run that was killed without its exit trap leaves no `finished_utc` either: run `run-all.sh` by hand once, and the next check can rerun again. diff --git a/rust/crates/og-recheck/src/cli.rs b/rust/crates/og-recheck/src/cli.rs index f30f1d66f..b3e509870 100644 --- a/rust/crates/og-recheck/src/cli.rs +++ b/rust/crates/og-recheck/src/cli.rs @@ -6,9 +6,9 @@ //! og recheck --launch-dir DIR the last bare `nextflow run` there, with -resume //! ``` //! -//! Exit codes: 0, 1 when a lookup failed, 2 on a usage error or a sample -//! with no manifest, 3 when a lock is held or a run is going on, else the -//! replay's own exit code. +//! Exit codes: 0, 1 when a lookup failed, 2 on a usage error, a sample +//! with no manifest or a recorded command `--rerun` will not run, 3 when a +//! lock is held or a run is going on, else the replay's own exit code. use crate::answers::{self, Answers, Memo, Offline}; use crate::files::{last_rerun, log_check, write_check}; diff --git a/rust/crates/og-recheck/src/lib.rs b/rust/crates/og-recheck/src/lib.rs index 1c5d8e4fb..b966f3159 100644 --- a/rust/crates/og-recheck/src/lib.rs +++ b/rust/crates/og-recheck/src/lib.rs @@ -352,11 +352,11 @@ fn truthy(v: &Option) -> Option<&str> { } /// The class of a looked-up item; may set its error or the release that -/// pins a newer one. +/// pins a newer one. What is installed here is compared with what the run +/// used before the lookup's answer is read: that needs no network, so a +/// failed lookup leaves a newer file or pin here `rerun-ready`, with its +/// error kept. pub fn classify(item: &mut Item, release: &Result, clinvar: bool) -> Class { - if item.error.is_some() { - return Class::Error; - } let (ran, inst) = (truthy(&item.ran_with), truthy(&item.installed)); if let (Some(i), Some(r)) = (inst, ran) { if ver(i) > ver(r) { @@ -373,6 +373,9 @@ pub fn classify(item: &mut Item, release: &Result, clinvar: boo return Class::RerunReady; } } + if item.error.is_some() { + return Class::Error; + } // ClinVar's date is in the manifest whenever step 06 had a file, so a run // without one cannot be compared with the file here. A pin with no run // value (HLA data not installed then) compares this checkout's. @@ -572,10 +575,12 @@ impl Check { Value::Object(m) } + /// The items whose lookup failed or that could not be compared: the + /// `error` items, and a `rerun-ready` item whose lookup failed. pub fn errors(&self) -> usize { self.items .iter() - .filter(|i| i.class == Class::Error) + .filter(|i| i.class == Class::Error || i.error.is_some()) .count() } } @@ -747,7 +752,12 @@ pub fn print_check(c: &Check, out: &mut dyn std::io::Write) -> std::io::Result<( let or_unknown = |v: &Option| truthy(v).unwrap_or("unknown").to_owned(); for i in &c.items { let mut detail = match &i.error { - Some(e) if !e.is_empty() => e.clone(), + Some(e) if !e.is_empty() && i.class == Class::Error => e.clone(), + Some(e) if !e.is_empty() => format!( + "run used {}, here {}; the lookup failed: {e}", + or_unknown(&i.ran_with), + or_unknown(&i.installed), + ), _ => format!( "run used {}, here {}, upstream {}", or_unknown(&i.ran_with), @@ -774,15 +784,38 @@ pub fn print_check(c: &Check, out: &mut dyn std::io::Write) -> std::io::Result<( )?; } } - if c.affected_steps.is_empty() { - writeln!(out, "Nothing here is newer than what the run used.") - } else { + if !c.affected_steps.is_empty() { writeln!( out, "A rerun would redo steps {} and the text report.", c.affected_steps.join(", ") - ) + )?; + } + // A failed lookup is never read as "nothing newer". An item that was + // looked up has an answer or an error; the others were never asked. + let asked = c + .items + .iter() + .filter(|i| i.upstream.is_some() || i.error.is_some()) + .count(); + let failed = c.errors(); + if failed == 0 { + if c.affected_steps.is_empty() { + writeln!(out, "Nothing here is newer than what the run used.")?; + } + } else if failed >= asked { + writeln!( + out, + "Could not check: all {asked} lookups failed (each line above says why). Check again later." + )?; + } else { + writeln!( + out, + "{failed} of {asked} lookups failed: this check is incomplete, so it cannot say \ + nothing else is newer. Check again later." + )?; } + Ok(()) } #[cfg(test)] diff --git a/rust/crates/og-recheck/src/rerun.rs b/rust/crates/og-recheck/src/rerun.rs index 78c9f242a..af72f1474 100644 --- a/rust/crates/og-recheck/src/rerun.rs +++ b/rust/crates/og-recheck/src/rerun.rs @@ -355,6 +355,24 @@ pub fn rerun( )); }; let switches = meta.get("switches").cloned().unwrap_or_default(); + let Some(call) = parse_call(&cmd, &switches) else { + return Ok(Rerun::Refused( + 2, + "the command in logs/run_status.tsv is not a run-all.sh call as run-all.sh records it \ + (cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS): not running it. Check it, \ + then run it by hand." + .into(), + )); + }; + if !Path::new(&call.dir).is_dir() { + return Ok(Rerun::Refused( + 2, + format!( + "the run's directory {} is gone: run the command by hand from where it should run.", + call.dir + ), + )); + } let started = now_utc(); // After --refresh-clinvar the new file is installed, so it is rerun-ready. let predicted = state_.rerun_ready_steps.clone(); @@ -362,10 +380,14 @@ pub fn rerun( let _ = std::io::stdout().flush(); append_log(log, &[[&started, "rerun-start", "", "", &cmd]])?; let mut c = Command::new("bash"); - c.arg("-c").arg(&cmd); + c.arg(&call.script).args(&call.args).current_dir(&call.dir); for name in switches.split_whitespace() { c.env_remove(name); } + for (name, value) in &call.env { + c.env(name, value); + } + c.env("PWD", &call.dir); let rc = c.status().map(|s| s.code().unwrap_or(1)).unwrap_or(127); let after = files(sample_dir)?; let cmp = compare( diff --git a/scripts/run-all.sh b/scripts/run-all.sh index acec2fb02..8a7dc17cf 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -37,7 +37,7 @@ # failed (logs/run_status.tsv). Results are published by hard link when work/ and GENOME_DIR share a filesystem, # you passed no -w, -c or --publish_dir_mode, and you run as root or Linux's fs.protected_hardlinks is 0 (then a user # may link the tasks' root-owned files). -# logs/run_status.tsv keeps this command with its switches: bin/owngenome_recheck.py --rerun replays it when a newer +# logs/run_status.tsv keeps this command with its switches: bin/og recheck --rerun replays it when a newer # ClinVar or catalog changes a result (docs/recheck.md). set -euo pipefail case "${1:-}" in -h|--help) sed -n '2,/^set -euo/p' "$0" | sed -e '$d' -e 's/^# \{0,1\}//'; exit 0 ;; esac @@ -46,9 +46,10 @@ SAMPLE=${1:-} SEX=${2:-} [ -z "$SEX" ] || echo "ERROR: sex must be 'male' or 'female', got '${SEX}'."; } >&2; exit 2; } ((BASH_VERSINFO[0] * 100 + BASH_VERSINFO[1] >= 404)) || { echo "ERROR: run-all.sh needs bash 4.4 or later (this is ${BASH_VERSION}); on macOS: brew install bash" >&2; exit 2; } shift 2; SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) USER_THREADS=${THREADS:-} -# This command as given, for bin/owngenome_recheck.py --rerun (logs/run_status.tsv, 'meta command'): the switches +# This command as given, for bin/og recheck --rerun (logs/run_status.tsv, 'meta command'): the switches # below that are set, then the arguments, each shell-quoted, run from this directory so relative paths still hold. -# 'meta switches' names them all, so a replay drops one set now that the run did not have. +# 'meta switches' names them all, so a replay drops one set now that the run did not have. The replay reads this +# shape back word by word and starts run-all.sh with those arguments, never through a shell; it refuses any other. SWITCHES=(GENOME_DIR SKIP_VALIDATION THREADS SKIP_TRIM INTERVALS ALIGN_DIR TOOLS KIR HLA_QC HPO REF_FASTA EH_CATALOG MANTA_CALL_REGIONS PARASCOPY_POPULATION PARAPHASE_REGIONS ANCESTRY_PANEL ANCESTRY_FINE_DIR ARCHAIC_DIR ARCHAIC_POP PGSC_CALC_DIR ANCIENT_PANEL_DIR VIRAL_REF MEGANE_KMERS MOCHA_DATA GRIDSS IMPUTATION SOMATIC EXTRA_CALLERS BENCHMARK @@ -63,7 +64,7 @@ export GENOME_DIR=${GENOME_DIR:?Set GENOME_DIR to your data directory} validate_sample "$SAMPLE" G=$GENOME_DIR S="${GENOME_DIR}/${SAMPLE}" LOG_DIR="${GENOME_DIR}/${SAMPLE}/logs" T=${TOOLS:-} C=${EXTRA_CALLERS:-} on() { [[ "${!1:-}" =~ ^(true|1)$ ]]; } -# The ClinVar files are shared by every sample of GENOME_DIR. bin/owngenome_recheck.py --refresh-clinvar holds this +# The ClinVar files are shared by every sample of GENOME_DIR. bin/og recheck --refresh-clinvar holds this # lock while it replaces them; the run holds it shared on fd 7 until it ends, so a run waits for a refresh going on # and a refresh refuses while a run goes on. Without flock (macOS: brew install flock) nothing waits here. CVLOCK="${G}/clinvar/.refresh.lock" From c406f622f0205a70880a74419f511907d37c54a5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:26:40 +0100 Subject: [PATCH 114/255] fix(archive): reject empty Nextflow checksum reports Exercise the actual module script blocks on the synthetic reads and reject successful checksum commands that produce no report. Record the local case time. Bead: plat-62 --- modules/local/cram_archive/main.nf | 4 ++-- scripts/ci/e2e-case-times.tsv | 2 +- tests/e2e/qc-3-cram-checksum.sh | 33 ++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/modules/local/cram_archive/main.nf b/modules/local/cram_archive/main.nf index c9b96c270..069e98eaa 100644 --- a/modules/local/cram_archive/main.nf +++ b/modules/local/cram_archive/main.nf @@ -76,7 +76,7 @@ process CRAM_ARCHIVE { } checksum_rows ${bam} > bam.checksum checksum_rows --input-fmt-option reference=${reference} --input-fmt-option decode_md=0 ${meta.id}_sorted.cram > ${meta.id}_sorted.cram.checksum - if ! diff bam.checksum ${meta.id}_sorted.cram.checksum >&2; then + if [ ! -s bam.checksum ] || [ ! -s ${meta.id}_sorted.cram.checksum ] || ! diff bam.checksum ${meta.id}_sorted.cram.checksum >&2; then echo "ERROR: the CRAM content of ${meta.id} differs from its BAM" >&2 exit 1 fi @@ -136,7 +136,7 @@ process CRAM_TO_BAM { } checksum_rows --input-fmt-option reference=${reference} --input-fmt-option decode_md=0 ${cram} > cram.checksum checksum_rows ${meta.id}_from_cram.bam > bam.checksum - if ! diff cram.checksum bam.checksum >&2; then + if [ ! -s cram.checksum ] || [ ! -s bam.checksum ] || ! diff cram.checksum bam.checksum >&2; then echo "ERROR: the restored BAM content differs from ${cram}" >&2 exit 1 fi diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv index 066ae31fb..0206946d3 100644 --- a/scripts/ci/e2e-case-times.tsv +++ b/scripts/ci/e2e-case-times.tsv @@ -179,4 +179,4 @@ y-str-1 shard 42 - y-tree-current-1-placement shard 21 nextflow-hardening-2-nf rust-z2e-readers shard 1 - rust-c2b-traits shard 2 - -qc-3-cram-checksum shard 40 - +qc-3-cram-checksum shard 34 - diff --git a/tests/e2e/qc-3-cram-checksum.sh b/tests/e2e/qc-3-cram-checksum.sh index e1df0c207..acb4ce323 100755 --- a/tests/e2e/qc-3-cram-checksum.sh +++ b/tests/e2e/qc-3-cram-checksum.sh @@ -112,4 +112,37 @@ check "faithful restore writes BAM and index" test -s "${A}/${C}_sorted.bam" -a sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/original.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/original.checksum" sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/${C}_sorted.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/restored.checksum" check "round trip preserves read content and explicit tags" cmp -s "${CASE_TMP}/original.checksum" "${CASE_TMP}/restored.checksum" +# Run the module's actual script blocks on the same synthetic records. Only +# Nextflow interpolation and version metadata are omitted by this harness. +# Literal Nextflow placeholders are substituted by sed, not by this shell. +# shellcheck disable=SC2016 +module_script() { + awk -v process="$1" ' + $0 == "process " process " {" {found=1} + found && /^ script:/ {script=1; next} + script && /^ """/ {body=1; next} + body && /cat <<-END_VERSIONS/ {exit} + body {print} + ' "${REPO}/modules/local/cram_archive/main.nf" | + sed -e 's/${meta.id}/module/g' -e 's/${task.cpus}/1/g' \ + -e 's|${reference}|reference/GRCh38_no_alt_analysis_set.fasta|g' \ + -e 's|${bam}|checksum/aligned/original.bam|g' \ + -e 's|${cram}|module_sorted.cram|g' \ + -e 's/\\\\/\\/g' -e 's/\\\$/\$/g' +} +# A successful command with no report must never certify a conversion. The +# other samtools verbs still run for real, so flagstat and quickcheck pass. +# The function expands its arguments in the container shell. +# shellcheck disable=SC2016 +EMPTY_CHECKSUM='samtools() { if [ "$1" = checksum ]; then return 0; else command samtools "$@"; fi; }' +for process in CRAM_ARCHIVE CRAM_TO_BAM; do + body=$(module_script "$process") + rc=0 + in_genome "$SAMTOOLS_IMAGE" bash -euo pipefail -c "$body" || rc=$? + check_eq "${process}: faithful module script passes" "$rc" 0 + rc=0 + in_genome "$SAMTOOLS_IMAGE" bash -euo pipefail -c "${EMPTY_CHECKSUM} +${body}" || rc=$? + check "${process}: empty checksum reports fail" test "$rc" -ne 0 +done finish From d70b5252c181a3313ed5e17917509c1bdd85a78c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:26:52 +0100 Subject: [PATCH 115/255] fix(setup): install clinical reference data by default Bead: D11 --- CHANGELOG.md | 2 + docs/00-reference-setup.md | 51 ++++++----- docs/13-vep-annotation.md | 4 +- docs/17-cpsr.md | 25 +++--- docs/getting-started.md | 12 ++- docs/hardware-requirements.md | 6 +- docs/troubleshooting.md | 17 ++++ scripts/ci/fake-docker/lib.sh | 43 ++++----- scripts/lib/common.sh | 64 +++++++++++++- scripts/setup.sh | 96 ++++++++++++--------- scripts/validate-setup.sh | 13 ++- tests/fake-docker/base-setup.sh | 1 + tests/fake-docker/setup-clinical-default.sh | 11 +++ versions.env | 1 + 14 files changed, 229 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..f78b639b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin and warns without stopping the remaining installs. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/docs/00-reference-setup.md b/docs/00-reference-setup.md index dfa88ad7d..b8f108743 100644 --- a/docs/00-reference-setup.md +++ b/docs/00-reference-setup.md @@ -4,6 +4,20 @@ One-time downloads required before running the pipeline. Each heading below give > **Estimated time:** 1-3 hours depending on internet speed. The two VEP caches (26 GB for step 13, 24 GB for step 17) are the largest downloads of a default run. +A plain `./scripts/setup.sh "$GENOME_DIR"` installs both VEP caches and the PCGR/CPSR bundle after AnnotSV. +These clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. +Setup checks each install separately: VEP 116 needs 56 GB peak, the bundle 14 GB, and VEP 115 needs 52 GB. +Each check adds a 5 GB margin. An item that does not fit is skipped with a warning. Setup continues. + +```bash +./scripts/setup.sh --no-clinical-data "$GENOME_DIR" # skip all three clinical downloads +./scripts/setup.sh --clinical-data "$GENOME_DIR" # install or retry only these three +``` + +An interrupted transfer resumes. Existing caches and bundles are kept. +Without these data, `run-all.sh` skips the affected steps and records the reason. +A bare `nextflow run` still needs its tools and data parameters. Setup does not change that selection. + ## GRCh38 Reference Genome The foundation for everything. All tools need this. The pipeline uses NCBI's **GRCh38 no-ALT analysis set**: chr1-22, X, Y and M, the unplaced and unlocalized scaffolds and the EBV genome, 195 sequences, with UCSC names (`chr1`, `chrM`). `setup.sh` downloads it, checks it and stores it as `reference/GRCh38_no_alt_analysis_set.fasta`: @@ -143,7 +157,7 @@ Required for step 5. `setup.sh` downloads them (about 5 GB, about 20 GB unpacked ## VEP Cache (~26 GB) -Ensembl Variant Effect Predictor annotation database. Required for step 13 and the pipeline's VEP. Without it, a run skips VEP, vcfanno, the clinical filter and slivar. Install it before the first run: +Ensembl Variant Effect Predictor annotation database. Required for step 13 and the pipeline's VEP. Setup installs it by default. Without it, a run skips VEP, vcfanno, the clinical filter and slivar. To install only this cache: ```bash ./scripts/setup.sh --vep-cache ${GENOME_DIR} @@ -178,38 +192,21 @@ Required for step 17 (CPSR cancer predisposition screening). Includes ClinVar, g > **Important:** PCGR 2.3.2 bundles VEP 115, which requires a **release-115** cache — different from the release-116 cache used by step 13 above. See the next section for the VEP 115 download. The bundle date (`PCGR_DATA_BUNDLE`) and the cache release (`PCGR_VEP_CACHE_RELEASE`) are in `versions.env` and move with `PCGR_IMAGE`. -```bash -mkdir -p ${GENOME_DIR}/pcgr_data -cd ${GENOME_DIR}/pcgr_data - -# Download (~7 GB) -wget -c https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.20260620.grch38.tgz +Setup installs it by default. To add the clinical data to an existing setup: -# Extract and organize into version-stamped directory -tar xzf pcgr_ref_data.20260620.grch38.tgz -mkdir -p 20260620 && mv data/ 20260620/ -# Creates: ${GENOME_DIR}/pcgr_data/20260620/data/ - -# Optional: delete the tarball to save 7 GB -# rm pcgr_ref_data.20260620.grch38.tgz +```bash +./scripts/setup.sh --clinical-data "$GENOME_DIR" ``` +The installer checks the archive against `PCGR_DATA_SHA256` in `versions.env`, then extracts it into a temporary folder. +Only a complete tree is moved to `pcgr_data/20260620/data/`. The archive is deleted after installation. +The [PCGR resource list](https://sigven.github.io/pcgr/articles/annotation_resources.html) names the bundle's third-party databases. Their own terms apply. Setup prints the resource link before downloading. + ## VEP 115 Cache for CPSR (~24 GB) PCGR 2.3.2 (step 17) bundles VEP 115 internally, which needs the **release-115** cache. This is separate from the release-116 cache used by step 13. Both coexist in the same `vep_cache/` directory under different subdirectories (`116_GRCh38/` and `115_GRCh38/`). A `113_GRCh38/` cache left from PCGR 2.2.5 is no longer read and can be deleted. -```bash -mkdir -p ${GENOME_DIR}/vep_cache/tmp -cd ${GENOME_DIR}/vep_cache/tmp - -# Download VEP 115 cache (~24 GB) -wget -c https://ftp.ensembl.org/pub/release-115/variation/indexed_vep_cache/homo_sapiens_vep_115_GRCh38.tar.gz - -# Extract alongside the existing release-116 cache -cd ${GENOME_DIR}/vep_cache -tar xzf tmp/homo_sapiens_vep_115_GRCh38.tar.gz -# Creates: ${GENOME_DIR}/vep_cache/homo_sapiens/115_GRCh38/ -``` +Setup installs this cache by default, after the PCGR bundle. `--clinical-data` retries it using Ensembl's `CHECKSUMS` file and a temporary extraction folder. > If you only run step 13 (VEP annotation) and skip step 17 (CPSR), you only need the release-116 cache. If you only run step 17, you only need release-115. @@ -510,7 +507,7 @@ wget -c https://giab.s3.amazonaws.com/release/AshkenazimTrio/HG002_NA24385_son/N [Hardware and storage requirements](hardware-requirements.md#shared-reference-data-one-time) has the one table of every download above, with the totals. -> **Tip:** If disk space is tight, you can skip the VEP cache (step 13) and PCGR bundle (step 17) initially. The core pipeline (steps 2-3-6-7) only needs the reference FASTA and ClinVar (~3.5 GB unpacked). +> **Tip:** If disk space is tight, use `setup.sh --no-clinical-data` to skip both VEP caches and the PCGR bundle initially. The core pipeline (steps 2-3-6-7) only needs the reference FASTA and ClinVar (~3.5 GB unpacked). ## Verifying Your Setup diff --git a/docs/13-vep-annotation.md b/docs/13-vep-annotation.md index c1e859f6e..b3c0ca43d 100644 --- a/docs/13-vep-annotation.md +++ b/docs/13-vep-annotation.md @@ -16,7 +16,7 @@ Pinned in `versions.env`; [Image versions](versions.md) lists the current tag. ## Prerequisites - The VCF from step 3 **and its `.tbi`**: the step refuses a VCF without its index, which may be half written -- Offline VEP cache: step 13 downloads, checks and unpacks it the first time it runs (see step 00-reference-setup) +- Offline VEP cache: setup installs it by default. To retry, run `./scripts/setup.sh --clinical-data "$GENOME_DIR"` ([reference setup](00-reference-setup.md#vep-cache-26-gb)). The pipeline skips VEP when the cache is missing. - Cache size: see [Hardware and storage requirements](hardware-requirements.md#shared-reference-data-one-time) (about 26 GB to download, 30 GB unpacked) ## Command @@ -112,4 +112,4 @@ After annotation, use step 23 (clinical filter) which automatically detects avai - `--dir_cache /opt/vep/.vep`: the cache is mounted there, not in the home directory VEP looks in by default - `--fasta`: running `--offline` without a FASTA file disables HGVS notation (`INFO: Disabling --hgvs`), so the script always passes the reference ([`REF_FASTA`](00-reference-setup.md#the-reference-path-on-every-page) is the reference path) - VEP does NOT assess variant pathogenicity in ClinVar context — combine with step 6 (ClinVar screen) for full picture -- **Upgrading from an older release:** VEP reads the cache directory named after the release passed in `--cache_version` (`homo_sapiens/116_GRCh38/` for the pinned release 116). When that directory is missing, VEP stops with an error instead of annotating from an older cache such as `112_GRCh38/`; step 13 downloads the matching cache on its first run. +- **Upgrading from an older release:** VEP reads the cache directory named after the release passed in `--cache_version` (`homo_sapiens/116_GRCh38/` for the pinned release 116). When that directory is missing, VEP stops with an error instead of annotating from an older cache such as `112_GRCh38/`. Setup installs the matching cache by default. The standalone step script can download a missing cache, but `run-all.sh` skips VEP until the cache is installed. diff --git a/docs/17-cpsr.md b/docs/17-cpsr.md index e542e1feb..e0284dda8 100644 --- a/docs/17-cpsr.md +++ b/docs/17-cpsr.md @@ -22,22 +22,21 @@ The image is PCGR 2.3.2 from `ghcr.io/sigven/pcgr` (Docker Hub's `sigven/pcgr` s ### 1. VEP Cache PCGR 2.3.2 bundles VEP 115, which requires the **release-115** cache (~24 GB). This is different from the release-116 cache used by step 13. Both coexist in the same `vep_cache/` directory under different subdirectories (`116_GRCh38/` and `115_GRCh38/`). -```bash -mkdir -p ${GENOME_DIR}/vep_cache -wget -c -P ${GENOME_DIR}/vep_cache https://ftp.ensembl.org/pub/release-115/variation/indexed_vep_cache/homo_sapiens_vep_115_GRCh38.tar.gz -tar xzf ${GENOME_DIR}/vep_cache/homo_sapiens_vep_115_GRCh38.tar.gz -C ${GENOME_DIR}/vep_cache -``` -### 2. PCGR Ref Data Bundle -PCGR 2.x uses a ref data bundle (~7 GB for `20260620`) separate from VEP: +### 2. PCGR ref data bundle +PCGR 2.x uses a separate ref data bundle (`20260620`, about 7 GB). +A plain setup installs the VEP 115 cache and this bundle by default, alongside VEP 116 for step 13. +For an existing setup, install or retry all three with: + ```bash -mkdir -p ${GENOME_DIR}/pcgr_data -cd ${GENOME_DIR}/pcgr_data -wget -c https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.20260620.grch38.tgz -tar xzf pcgr_ref_data.20260620.grch38.tgz -mkdir -p 20260620 && mv data/ 20260620/ +./scripts/setup.sh --clinical-data "$GENOME_DIR" ``` -This creates a `20260620/data/` directory with ClinVar, CancerMine, UniProt, and other databases. VEP cache is now mounted separately. + +The bundle goes to `pcgr_data/20260620/data/`; the caches coexist under `vep_cache/homo_sapiens/`. +Setup checks Ensembl's cache checksums and the bundle's pinned SHA-256 before extraction, then removes each archive. +It checks free space for each install and continues with a warning if one fails or does not fit. +`--no-clinical-data` skips all three. Without either CPSR item, `run-all.sh` skips step 17. +See [reference setup](00-reference-setup.md) for disk peaks and the [PCGR resource list](https://sigven.github.io/pcgr/articles/annotation_resources.html) for the bundled databases. Review each source's terms before use. ### Upgrading from PCGR 2.2.5 PCGR 2.2.5 read bundle `20250314` and the release-113 cache. 2.3.2 needs both new ones: about 31 GB to download (the 7 GB bundle and the 24 GB cache). Once step 17 runs on 2.3.2, `pcgr_data/20250314/` and `vep_cache/homo_sapiens/113_GRCh38/` are no longer read and can be deleted. diff --git a/docs/getting-started.md b/docs/getting-started.md index 6067bc415..e72239787 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -18,7 +18,7 @@ | **CPU** | 4 cores | 8-16 cores | `run-all.sh` runs DeepVariant with at most 8 shards, fewer when `--max_cpus` is lower; more cores run other steps beside it ([CPU](hardware-requirements.md#cpu-requirements)) | | **RAM** | 16 GB | 32 GB | Every container has a hard memory limit; the 8-CPU steps ask for 32 GB, capped at the machine's RAM | | **Disk** | 500 GB free from a BAM, about 700 GB from FASTQ | 1 TB+ | See [detailed breakdown](hardware-requirements.md#disk-space-breakdown) | -| **Internet** | Broadband | 100+ Mbps | ~70-75 GB core downloads + ~175 GB optional annotation databases | +| **Internet** | Broadband | 100+ Mbps | ~73-78 GB core downloads + ~175 GB optional annotation databases | | **OS** | Linux (amd64) | Ubuntu 22.04+ | macOS/ARM works but slower (see below) | > **Disk space is the #1 surprise.** A single 30X WGS sample produces 60-90 GB of FASTQ, 80-120 GB of BAM, plus reference genomes and databases. See [docs/hardware-requirements.md](hardware-requirements.md) for the full breakdown. @@ -46,11 +46,17 @@ Every analysis tool runs inside Docker: no conda environments, no Python version | PCGR/CPSR data bundle + VEP 115 cache | ~31 GB | Step 17 (cancer predisposition) | | Docker images (all steps) | ~10-15 GB | All steps | | Annotation databases (CADD, SpliceAI, REVEL, AlphaMissense) | ~175 GB | Steps 30-31 (optional) | -| **Total one-time setup (core)** | **~70-75 GB** | | +| **Total one-time setup (core)** | **~73-78 GB** | | | **Total with annotation enrichment** | **~250 GB** | | See [docs/00-reference-setup.md](00-reference-setup.md) for download instructions. +Plain setup includes VEP 116, the PCGR/CPSR bundle and VEP 115: about 57 GB of clinical downloads and 65 GB unpacked. +Allow 1 to 3 hours. Setup checks each install's disk peak plus a 5 GB margin and warns if it skips one. +Use `./scripts/setup.sh --no-clinical-data "$GENOME_DIR"` to opt out. +Use `./scripts/setup.sh --clinical-data "$GENOME_DIR"` to add or retry only these data later. +`run-all.sh` skips steps whose data are still missing; inspect its plan and setup warnings. + ## Platform Notes ### Linux (Recommended) @@ -120,7 +126,7 @@ Then, once per machine and once per sample: ```bash export GENOME_DIR=/path/to/your/data -./scripts/setup.sh $GENOME_DIR # reference, ClinVar, small data files, images and og (once) +./scripts/setup.sh $GENOME_DIR # reference, ClinVar, VEP caches, CPSR bundle, images and og (once) ./scripts/validate-setup.sh your_name # what is installed and what is missing ./scripts/run-all.sh your_name male # or female: sets the chrX/chrY ploidy and the sex check ``` diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index ba4ef1c61..0077edce2 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -43,7 +43,7 @@ A run from FASTQ needs more space than a run from a BAM. Until you delete ` "${CASE_WORK}/cache/homo_sapiens/${rel}_GRCh38/info.txt" - tar -czf "${FAKE_DOWNLOAD_DIR}/homo_sapiens_vep_${rel}_GRCh38.tar.gz" -C "${CASE_WORK}/cache" "homo_sapiens/${rel}_GRCh38" -done -for rel in 116 115; do - name="homo_sapiens_vep_${rel}_GRCh38.tar.gz" - printf '%s %s\n' "$(sum -r "${FAKE_DOWNLOAD_DIR}/${name}" | awk '{print $1, $2}')" "$name" -done > "${FAKE_DOWNLOAD_DIR}/CHECKSUMS" -mkdir -p "${CASE_WORK}/bundle/data/grch38" -printf 'fake PCGR data\n' > "${CASE_WORK}/bundle/data/grch38/README" -tar -czf "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" -C "${CASE_WORK}/bundle" data -if command -v sha256sum >/dev/null 2>&1; then - PCGR_DATA_SHA256=$(sha256sum "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') -else - PCGR_DATA_SHA256=$(shasum -a 256 "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') -fi -export PCGR_DATA_SHA256 +seed_clinical_downloads() { + local rel name + export FAKE_DOWNLOAD_DIR="${CASE_WORK}/downloads" + mkdir -p "$FAKE_DOWNLOAD_DIR" + for rel in 116 115; do + mkdir -p "${CASE_WORK}/cache/homo_sapiens/${rel}_GRCh38" + printf 'fake VEP cache\n' > "${CASE_WORK}/cache/homo_sapiens/${rel}_GRCh38/info.txt" + tar -czf "${FAKE_DOWNLOAD_DIR}/homo_sapiens_vep_${rel}_GRCh38.tar.gz" -C "${CASE_WORK}/cache" "homo_sapiens/${rel}_GRCh38" + done + for rel in 116 115; do + name="homo_sapiens_vep_${rel}_GRCh38.tar.gz" + printf '%s %s\n' "$(sum -r "${FAKE_DOWNLOAD_DIR}/${name}" | awk '{print $1, $2}')" "$name" + done > "${FAKE_DOWNLOAD_DIR}/CHECKSUMS" + mkdir -p "${CASE_WORK}/bundle/data/grch38" + printf 'fake PCGR data\n' > "${CASE_WORK}/bundle/data/grch38/README" + tar -czf "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" -C "${CASE_WORK}/bundle" data + if command -v sha256sum >/dev/null 2>&1; then + PCGR_DATA_SHA256=$(sha256sum "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') + else + PCGR_DATA_SHA256=$(shasum -a 256 "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" | awk '{print $1}') + fi + export PCGR_DATA_SHA256 +} fail() { echo "ASSERT FAIL: $*" >&2 diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 74f3cfa2e..2088b40a6 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -632,6 +632,10 @@ install_vep_cache() { url=$(vep_cache_url "$rel") tarball="${dir}/$(basename "$url")" final="${dir}/homo_sapiens/${rel}_GRCh38" + if [ -f "${final}/info.txt" ]; then + echo "[OK] VEP ${rel} cache already present." + return 0 + fi if [ -e "$final" ]; then echo "ERROR: ${final} exists but has no info.txt (an interrupted extraction?)." >&2 echo " Remove it and run again." >&2 @@ -646,10 +650,68 @@ install_vep_cache() { echo "ERROR: could not unpack ${tarball} into a ${rel}_GRCh38 cache." >&2 return 1 fi - mv "${tmp}/homo_sapiens/${rel}_GRCh38" "$final" + if ! mv "${tmp}/homo_sapiens/${rel}_GRCh38" "$final"; then + rm -rf "$tmp" + return 1 + fi rm -rf "$tmp" "$tarball" } +# clinical_disk_ok DIR PEAK_GB LABEL: GB as du counts them (2^30 bytes), +# plus a 5 GB margin. +# Use the destination's filesystem, creating the directory when it is new. +# An unreadable df is a failed preflight, never permission to fill the disk. +clinical_disk_ok() { + local dir=$1 peak=$2 label=$3 free + mkdir -p "$dir" || return 1 + free=$(df -Pk "$dir" 2>/dev/null | awk 'NR == 2 {print $4}') || free="" + case "$free" in + ''|*[!0-9]*) echo "[WARN] Cannot read free space for ${label}; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\""; return 1 ;; + esac + if [ "$free" -lt "$(( (peak + 5) * 1048576 ))" ]; then + echo "[WARN] Skipping ${label}: needs ${peak} GB peak plus a 5 GB margin; $(awk -v k="$free" 'BEGIN {printf "%.1f", k / 1048576}') GB free." + echo " Install later: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" + return 1 + fi +} + +# install_pcgr_bundle DIR: check the pinned archive before extracting into a +# staging tree. Upstream ships data/ at the root; CPSR reads BUNDLE/data/. +# The shared fetch helper resumes interrupted transfers and deletes bad ones. +# shellcheck disable=SC2154 # PCGR_DATA_BUNDLE and PCGR_DATA_SHA256 are pins in versions.env +install_pcgr_bundle() { + local dir=$1 url tarball tmp + if [ -d "${dir}/${PCGR_DATA_BUNDLE}/data" ]; then + echo "[OK] PCGR/CPSR ${PCGR_DATA_BUNDLE} bundle already present." + return 0 + fi + echo "PCGR/CPSR reference data includes third-party databases with their own terms:" + echo " https://sigven.github.io/pcgr/articles/annotation_resources.html" + echo " Review those terms before use; --no-clinical-data skips this download." + url="https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" + tarball="${dir}/${url##*/}" + mkdir -p "$dir" || return 1 + fetch "$url" "$tarball" sha256 "$PCGR_DATA_SHA256" || return 1 + tmp=$(mktemp -d "${dir}/.extract.XXXXXX") || return 1 + if ! tar xzf "$tarball" -C "$tmp" || [ ! -d "${tmp}/data/grch38" ]; then + rm -rf "$tmp" + echo "ERROR: could not unpack ${tarball} into PCGR data/grch38." >&2 + return 1 + fi + # An incomplete destination is never mistaken for a completed bundle. + if [ -e "${dir}/${PCGR_DATA_BUNDLE}" ]; then + rm -rf "$tmp" + echo "ERROR: ${dir}/${PCGR_DATA_BUNDLE} exists without data; remove it and retry." >&2 + return 1 + fi + if ! mv "$tmp" "${dir}/${PCGR_DATA_BUNDLE}"; then + rm -rf "$tmp" + return 1 + fi + rm -f "$tarball" + echo "[OK] PCGR/CPSR ${PCGR_DATA_BUNDLE} bundle installed." +} + # lock_acquire DIR / lock_release DIR: a lock shared by concurrent runs. The # lock is a directory holding the owner's PID. It is stale when that PID is # gone (a killed run), or when it has no PID a minute after it was made (a run diff --git a/scripts/setup.sh b/scripts/setup.sh index 326dfeddd..272808b4f 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # setup.sh — One-stop setup: download references, pull Docker images, validate # Usage: ./scripts/setup.sh +# ./scripts/setup.sh --no-clinical-data +# ./scripts/setup.sh --clinical-data # ./scripts/setup.sh --pull-only pull the Docker images and exit # ./scripts/setup.sh --og put OwnGenome's og binaries in bin/.og # (Linux, and macOS for a live-site host): @@ -82,8 +84,8 @@ # checkout of pgsc_calc and the Nextflow plugin it needs, so step 25 and # the PRS process run without the network # -# The VEP cache (~26 GB, --vep-cache) and PCGR ref data (~7 GB) are -# downloaded separately because they take a long time. +# 7. VEP 116, PCGR/CPSR data and VEP 115 (~57 GB download, ~65 GB unpacked). +# --no-clinical-data skips them; --clinical-data installs only these three. # # Every download goes through fetch (scripts/lib/common.sh): it is written to # .part, checked, and only then renamed, so a file that exists is whole. @@ -99,6 +101,9 @@ # existing samples. set -euo pipefail +# Keep an explicit checksum override for fixture tests and verified mirrors. +PCGR_SHA_OVERRIDE=${PCGR_DATA_SHA256:-} + # The usage text below names data releases, so read them before it prints. # shellcheck source=../versions.env . "$(dirname "$0")/../versions.env" @@ -107,12 +112,19 @@ PULL_ONLY=false REFRESH="" SAMPLE_QC_ONLY=false OPT_IN="" +CLINICAL_DATA=true case "${1:-}" in --og) exec "$(dirname "$0")/../rust/og.sh" install ;; --cyrius|--parascopy-data|--kir-data|--ancestry-panel|--ancestry-fine-panel|--yleaf-data|--consumer-sites|--ancient-panel|--viral-data|--masked-reference|--exomiser-data|--chip-data|--strchive-data|--megane-kmers|--mocha-data|--locityper-data|--vep-cache|--annotsv-data|--archaic) OPT_IN=${1#--} shift ;; + --clinical-data) + OPT_IN=clinical-data + shift ;; + --no-clinical-data) + CLINICAL_DATA=false + shift ;; --pull-only) PULL_ONLY=true shift ;; @@ -132,6 +144,8 @@ esac GENOME_DIR=${1:-${GENOME_DIR:-""}} if [ -z "$GENOME_DIR" ] && ! $PULL_ONLY; then echo "Usage: $0 " + echo " $0 --no-clinical-data " + echo " $0 --clinical-data " echo " $0 --pull-only" echo " $0 --og" echo " $0 --refresh clinvar " @@ -140,6 +154,8 @@ if [ -z "$GENOME_DIR" ] && ! $PULL_ONLY; then echo "" echo " Where to store reference data and sample outputs." echo " Needs at least 500 GB free space per sample." + echo " --no-clinical-data Skip both VEP caches and the PCGR/CPSR bundle." + echo " --clinical-data Install only those three downloads for an existing setup." echo " --pull-only Pull the Docker images listed in versions.env and exit." echo " --og Put OwnGenome's og binaries in bin/.og, for Linux and for a macOS live-site host" echo " (built from rust/, or a release's) and exit." @@ -210,6 +226,7 @@ export GENOME_DIR # Image tags, data versions and the helpers (run_in, fetch, pipeline_images) # shellcheck source=lib/common.sh . "$(dirname "$0")/lib/common.sh" +if [ -n "$PCGR_SHA_OVERRIDE" ]; then PCGR_DATA_SHA256=$PCGR_SHA_OVERRIDE; fi # NCBI's GRCh38 analysis set without ALT contigs: chr1-22, X, Y, M, the # unplaced and unlocalized scaffolds and chrEBV, 195 sequences. NCBI lists the @@ -2065,6 +2082,7 @@ install_vep_cache_opt_in() { echo "[OK] VEP ${VEP_CACHE_RELEASE} cache already present: ${dir}/homo_sapiens/${VEP_CACHE_RELEASE}_GRCh38" return 0 fi + clinical_disk_ok "$dir" 56 "VEP ${VEP_CACHE_RELEASE} cache" || return 1 echo "Installing the VEP ${VEP_CACHE_RELEASE} cache: ~26 GB download, ~30 GB unpacked, both on disk until it ends." if ! install_vep_cache "$dir" "$VEP_CACHE_RELEASE"; then echo "[WARN] Could not install the VEP ${VEP_CACHE_RELEASE} cache. Run this command again: the download resumes." @@ -2073,6 +2091,33 @@ install_vep_cache_opt_in() { echo "[OK] VEP ${VEP_CACHE_RELEASE} cache: ${dir}/homo_sapiens/${VEP_CACHE_RELEASE}_GRCh38" } +# Each missing item gets its own disk check. A skipped or failed item must +# not prevent the smaller bundle or the other cache from installing. +install_clinical_data() { + local dir="${GENOME_DIR}/vep_cache" rel peak + echo "Clinical data: about 57 GB to download, 65 GB unpacked; allow 1 to 3 hours." + for rel in "$VEP_CACHE_RELEASE" bundle "$PCGR_VEP_CACHE_RELEASE"; do + if [ "$rel" = bundle ]; then + [ ! -d "${GENOME_DIR}/pcgr_data/${PCGR_DATA_BUNDLE}/data" ] || { echo "[OK] PCGR/CPSR bundle already present."; continue; } + if clinical_disk_ok "${GENOME_DIR}/pcgr_data" 14 'PCGR/CPSR bundle'; then + install_pcgr_bundle "${GENOME_DIR}/pcgr_data" || echo "[WARN] Could not install PCGR/CPSR. Step 17 needs it; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" + fi + else + [ ! -f "${dir}/homo_sapiens/${rel}_GRCh38/info.txt" ] || { echo "[OK] VEP ${rel} cache already present."; continue; } + peak=56 + [ "$rel" != "$PCGR_VEP_CACHE_RELEASE" ] || peak=52 + if clinical_disk_ok "$dir" "$peak" "VEP ${rel} cache"; then + if ! install_vep_cache "$dir" "$rel"; then + echo "[WARN] Could not install VEP ${rel}. Run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\" (interrupted downloads resume)." + if [ "$rel" = "$VEP_CACHE_RELEASE" ]; then + echo " To retry only this cache: ./scripts/setup.sh --vep-cache \"${GENOME_DIR}\"" + fi + fi + fi + fi + done +} + case "$OPT_IN" in ancestry-panel) install_ancestry_panel; exit $? ;; ancestry-fine-panel) install_ancestry_fine_panel; exit $? ;; @@ -2090,6 +2135,7 @@ case "$OPT_IN" in megane-kmers) ensure_image MEGANE_IMAGE 50 && install_megane_kmers; exit $? ;; mocha-data) install_mocha_data; exit $? ;; locityper-data) install_locityper_data; exit $? ;; + clinical-data) install_clinical_data; exit $? ;; vep-cache) install_vep_cache_opt_in; exit $? ;; annotsv-data) install_annotsv_data; exit $? ;; archaic) install_archaic; exit $? ;; @@ -2459,47 +2505,15 @@ echo "=== Phase 5: AnnotSV annotation data (~5.3 GB + ~1.7 GB download, ~20 GB u install_annotsv_data || true ############################################################################### -# Phase 6: Optional Downloads (instructions only) +# Phase 6: Clinical reference data (default) ############################################################################### echo "" -echo "=== Phase 6: Optional Downloads (manual) ===" -echo "" -echo "The following are only needed for specific steps and are large downloads:" -echo "" - -VEPDIR="${GENOME_DIR}/vep_cache" -if [ -f "${VEPDIR}/homo_sapiens/${VEP_CACHE_RELEASE}_GRCh38/info.txt" ]; then - echo "[OK] VEP ${VEP_CACHE_RELEASE} cache already present." -else - echo "[SKIP] VEP ${VEP_CACHE_RELEASE} cache (~26 GB) — without it a run skips VEP, vcfanno, the clinical filter and slivar" - echo " Install it before the first run (downloads, checks and unpacks it; ~30 GB unpacked):" - echo " ./scripts/setup.sh --vep-cache ${GENOME_DIR}" -fi -echo "" - -PCGRDIR="${GENOME_DIR}/pcgr_data" -if [ -d "${PCGRDIR}/${PCGR_DATA_BUNDLE}/data" ]; then - echo "[OK] PCGR/CPSR ref data bundle already present." -else - echo "[SKIP] PCGR/CPSR ref data (~7 GB download) — needed for step 17 (cancer predisposition)" - echo " Download:" - echo " mkdir -p ${PCGRDIR} && cd ${PCGRDIR}" - echo " curl -fL -C - -O https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" - echo " tar xzf pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" - echo " mkdir -p ${PCGR_DATA_BUNDLE} && mv data/ ${PCGR_DATA_BUNDLE}/" -fi -echo "" - -# CPSR runs the VEP release inside the PCGR image, not the one step 13 uses. -if [ -f "${VEPDIR}/homo_sapiens/${PCGR_VEP_CACHE_RELEASE}_GRCh38/info.txt" ]; then - echo "[OK] VEP ${PCGR_VEP_CACHE_RELEASE} cache (for CPSR) already present." +echo "=== Phase 6: Clinical reference data ===" +if $CLINICAL_DATA; then + install_clinical_data else - echo "[SKIP] VEP ${PCGR_VEP_CACHE_RELEASE} cache (~24 GB) — needed for step 17 (CPSR runs VEP ${PCGR_VEP_CACHE_RELEASE} inside the PCGR image)" - echo " This is separate from the release-${VEP_CACHE_RELEASE} cache used by step 13. Both coexist in vep_cache/." - echo " Download:" - echo " mkdir -p ${VEPDIR}" - echo " curl -fL -C - -o ${VEPDIR}/$(basename "$(vep_cache_url "$PCGR_VEP_CACHE_RELEASE")") $(vep_cache_url "$PCGR_VEP_CACHE_RELEASE")" - echo " tar xzf ${VEPDIR}/$(basename "$(vep_cache_url "$PCGR_VEP_CACHE_RELEASE")") -C ${VEPDIR}" + echo "[SKIP] Clinical data (--no-clinical-data): steps 13, 30, 23, 31 and 17 need the caches or bundle." + echo " Install later: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" fi ############################################################################### @@ -2520,4 +2534,4 @@ echo " 2. Run: export GENOME_DIR=${GENOME_DIR}" echo " 3. Run: ./scripts/validate-setup.sh " echo " 4. Run: ./scripts/run-all.sh " echo "" -echo "For optional VEP and CPSR setup, see the instructions above." +echo "Review any warnings above; use --clinical-data to retry missing clinical downloads." diff --git a/scripts/validate-setup.sh b/scripts/validate-setup.sh index e8c19b6e4..60beff37c 100755 --- a/scripts/validate-setup.sh +++ b/scripts/validate-setup.sh @@ -588,13 +588,14 @@ else echo " Install it: ./scripts/setup.sh --sample-qc-data ${GENOME_DIR}" fi - # --- VEP cache of the VEP image's release, for step 13 (optional) --- + # --- Clinical data installed by default by setup.sh --- VEP_DIR="${GENOME_DIR}/vep_cache/homo_sapiens" if [ -f "${VEP_DIR}/${VEP_CACHE_RELEASE}_GRCh38/info.txt" ]; then pass "VEP ${VEP_CACHE_RELEASE} cache (step 13): present" else warn "VEP ${VEP_CACHE_RELEASE} cache not found at: ${VEP_DIR}/${VEP_CACHE_RELEASE}_GRCh38" - echo " Required for step 13 (VEP annotation), which downloads it (~26 GB) the first time it runs." + echo " Missing cache skips steps 13 (VEP), 30 (vcfanno), 23 (clinical filter) and 31 (slivar)." + echo " Install it: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" echo " See docs/00-reference-setup.md for full instructions." fi @@ -604,7 +605,7 @@ else else warn "VEP ${PCGR_VEP_CACHE_RELEASE} cache not found at: ${VEP_DIR}/${PCGR_VEP_CACHE_RELEASE}_GRCh38" echo " Required for step 17 (CPSR). ${PCGR_IMAGE} needs VEP ${PCGR_VEP_CACHE_RELEASE}, separate from step 13's VEP ${VEP_CACHE_RELEASE}." - echo " curl -fL -C - -O $(vep_cache_url "$PCGR_VEP_CACHE_RELEASE")" + echo " Missing cache skips step 17. Install it: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" echo " See docs/17-cpsr.md for full instructions." fi @@ -614,10 +615,8 @@ else pass "PCGR/CPSR ref data bundle: present" else warn "PCGR ref data bundle not found at: ${PCGR_DIR}" - echo " Required for step 17 (CPSR cancer predisposition). Download ~7 GB:" - echo " cd ${GENOME_DIR}/pcgr_data" - echo " curl -fL -C - -O https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" - echo " tar xzf pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz && mkdir -p ${PCGR_DATA_BUNDLE} && mv data/ ${PCGR_DATA_BUNDLE}/" + echo " Missing bundle skips step 17 (CPSR cancer predisposition)." + echo " Install it: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" echo " See docs/17-cpsr.md for full instructions." fi diff --git a/tests/fake-docker/base-setup.sh b/tests/fake-docker/base-setup.sh index 6e5412647..82c7ee9dd 100755 --- a/tests/fake-docker/base-setup.sh +++ b/tests/fake-docker/base-setup.sh @@ -4,6 +4,7 @@ # finish with exit 0. # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" +seed_clinical_downloads G="${CASE_WORK}/genome" mkdir -p "$G" diff --git a/tests/fake-docker/setup-clinical-default.sh b/tests/fake-docker/setup-clinical-default.sh index 4c289cdde..83a674b03 100644 --- a/tests/fake-docker/setup-clinical-default.sh +++ b/tests/fake-docker/setup-clinical-default.sh @@ -2,6 +2,7 @@ # Default clinical downloads, explicit opt-out, disk gates and corrupt input. # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" +seed_clinical_downloads export FETCH_TRIES=1 FETCH_WAIT=0 G="${CASE_WORK}/without" run_expect 0 without "${SCRIPTS}/setup.sh" --no-clinical-data "$G" @@ -17,6 +18,16 @@ for rel in 116 115; do done [ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'PCGR bundle missing' output_lacks clinical 'Phase 1:|Docker Images' +# Installed clinical data turns the three default launch-plan rows on. +seed_reference "$G" +seed_clinvar "$G" +seed_sample "$G" sample1 +use_output_hook +GENOME_DIR="$G" SKIP_VALIDATION=true run_expect 0 plan "${SCRIPTS}/run-all.sh" sample1 male +output_has plan '^ 13 VEP +runs$' +output_has plan '^ 23 Clinical filter +runs$' +output_has plan '^ 17 CPSR +runs$' + # A second install does no network work, even with too little space. cp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" run_expect 0 again "${SCRIPTS}/setup.sh" --clinical-data "$G" diff --git a/versions.env b/versions.env index 018f5c605..5c9595f9d 100644 --- a/versions.env +++ b/versions.env @@ -192,6 +192,7 @@ PCGR_VEP_CACHE_RELEASE="115" PCGR_DATA_BUNDLE="20260620" # Exact archive size from the server; SHA-256 is measured from the full download. PCGR_DATA_BYTES="7664082467" +PCGR_DATA_SHA256="f9b8149e37c7e63bd7d877713cdb11a1a609c492b3f1a055fb7e90cd80273e54" # pypgx-bundle git tag (step 32): must equal the version of PYPGX_IMAGE. PYPGX_BUNDLE_VERSION="0.26.0" # Cyrius (step 21, opt-in; legacy: no automated bumps) is installed by From 7be974dbfeeee0ba1639bf2e09c38eeb82a3c122 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:27:04 +0100 Subject: [PATCH 116/255] test(ancestry): use a verb no plan adds for the unknown-verb case og ancestry-fine paint is now a planned verb, so the test of the unknown-verb refusal names one that never will be. Bead: D14 --- rust/crates/og-ancestry/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/crates/og-ancestry/src/lib.rs b/rust/crates/og-ancestry/src/lib.rs index 5ae862961..743ea611a 100644 --- a/rust/crates/og-ancestry/src/lib.rs +++ b/rust/crates/og-ancestry/src/lib.rs @@ -730,9 +730,9 @@ mod tests { #[test] fn unknown_verbs_and_missing_options_are_usage_errors() { - let (rc, _, err) = testutil::run(&["paint"]); + let (rc, _, err) = testutil::run(&["no-such-verb"]); assert_eq!(rc, 2); - assert!(err.contains("invalid choice: 'paint'"), "{err}"); + assert!(err.contains("invalid choice: 'no-such-verb'"), "{err}"); let (rc, _, err) = testutil::run(&["order", "--pop", "x"]); assert_eq!(rc, 2); assert!( From 4fb1beaaac40ddaef02e35a4268f438ba791bac1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:27:29 +0100 Subject: [PATCH 117/255] fix(bloodtype): read every record that covers a marker, in any order The first record at a position won, so an all-sites reference record or a split SNV record before the ABO c.261 insertion made the site reference and ABO read O. Each record now gives a reading with a weight (a call of the marker ALT, a no call of a record naming it, a reference call, any other no call) and the strongest counts, so order does not matter. A reference-only record shorter than a deletion marker says nothing, so the reads decide. A record that spans the marker from upstream with an allele that changes its base makes the site a no call. The index path now keeps the records that span a position too. Bead: trait-40 Bead: trait-41 Bead: trait-29 --- rust/crates/og-bloodtype/src/genotypes.rs | 396 ++++++++++++++++++---- rust/crates/og-bloodtype/tests/report.rs | 18 +- 2 files changed, 344 insertions(+), 70 deletions(-) diff --git a/rust/crates/og-bloodtype/src/genotypes.rs b/rust/crates/og-bloodtype/src/genotypes.rs index a04dfbee2..6eac1aec9 100644 --- a/rust/crates/og-bloodtype/src/genotypes.rs +++ b/rust/crates/og-bloodtype/src/genotypes.rs @@ -6,14 +6,30 @@ //! index: a few hundred BGZF blocks instead of the whole file. Without a //! usable index (a plain or gzip file, no index, an index older than the //! VCF, or one that cannot be read) the file is read once, line by line, -//! and a line is split only when its CHROM and POS are a marker's. Both +//! and a line is split only when its REF covers a marker's position. Both //! ways apply the same records in the same order, so they give the same //! sites. +//! +//! Several records can cover one site: an all-sites VCF writes a reference +//! record and then the indel at one position, a split multi-allelic record +//! is two lines, and a deletion that starts upstream spans the site. Every +//! one of them is read, and the answer does not depend on their order: +//! +//! - a record with a non-reference allele that changes the site's base +//! from upstream makes the site a no call, whatever the site's own +//! records say; +//! - else, of the records at the site's position, a call of the marker's +//! ALT wins, then a no call of a record that names the marker's ALT, +//! then a reference call, then any other no call; the first of equals +//! counts, and another allele a losing call names stays in the note; +//! - a reference-only record whose REF is shorter than the marker's (an +//! all-sites record or a gVCF block starting at a deletion marker) says +//! nothing past its own bases, so the site reads as if it had no record. use crate::Error; use crate::markers::{Marker, Sites}; use indexmap::IndexMap; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::path::Path; /// Reads (MAPQ >= 20) before a site missing from the VCF counts as reference. @@ -97,11 +113,11 @@ pub fn norm_chrom(c: &str) -> String { } /// The sites at each (chrom, pos), in table order. -struct ByPos(HashMap>>); +struct ByPos(HashMap>>); impl ByPos { fn new(sites: &Genotyped) -> ByPos { - let mut m: HashMap>> = HashMap::new(); + let mut m: HashMap>> = HashMap::new(); for (i, s) in sites.values().enumerate() { m.entry(s.row.chrom.clone()) .or_default() @@ -136,10 +152,108 @@ fn check_header(line: &str, path: &str) -> Result<(), Error> { Ok(()) } -/// Applies one record line (no line ending) to the sites at its position. -fn apply_record(line: &str, by_pos: &ByPos, sites: &mut Genotyped) { - let mut c = line.splitn(3, '\t'); - let (Some(chrom), Some(pos), Some(_)) = (c.next(), c.next(), c.next()) else { +/// How much a record at a site's own position says about the marker's +/// ALT; the strongest one read counts. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Weight { + /// A no call of a record that does not name the marker's ALT. + Other, + /// A call with no copy of the marker's ALT. + Reference, + /// A no call of a record that names the marker's ALT (`./.`, a + /// failed FILTER): the ALT may be there. + Doubt, + /// A call with the marker's ALT. + Alt, +} + +/// What one record at a site's position says: status, alt copies, note. +#[derive(Debug, Clone, PartialEq)] +struct Reading { + weight: Weight, + status: Status, + n: Option, + note: String, +} + +impl Reading { + fn no_call(weight: Weight, note: String) -> Reading { + Reading { + weight, + status: Status::NoCall, + n: None, + note, + } + } +} + +/// Everything the records said about one site so far. +#[derive(Debug, Default)] +struct Evidence { + /// The strongest reading at the site's position. + best: Option, + /// The notes of losing calls (another allele at the position). + others: Vec, + /// The first upstream record that changes the site's base. + span: Option, +} + +impl Evidence { + fn add(&mut self, r: Reading) { + let loser = match &mut self.best { + Some(b) if r.weight > b.weight => Some(std::mem::replace(b, r)), + Some(_) => Some(r), + None => { + self.best = Some(r); + None + } + }; + if let Some(l) = loser + && l.status == Status::Called + && !l.note.is_empty() + && !self.others.contains(&l.note) + { + self.others.push(l.note); + } + } + + /// The site as the records leave it; `Absent` when none said anything. + fn settle(self, site: &mut Site) { + if let Some(note) = self.span { + site.status = Status::NoCall; + site.n = None; + site.note = note; + } else if let Some(b) = self.best { + site.status = b.status; + site.n = b.n; + site.note = if b.status == Status::Called { + let others = self.others.into_iter().filter(|o| *o != b.note); + std::iter::once(b.note.clone()) + .chain(others) + .filter(|p| !p.is_empty()) + .collect::>() + .join("; ") + } else { + b.note + }; + } + } +} + +/// Applies one record line (no line ending) to the sites it covers: those +/// at its position and those its REF spans. `only` keeps the sites at one +/// position (the index path reads each position's records on their own). +fn apply_record( + line: &str, + by_pos: &ByPos, + sites: &Genotyped, + ev: &mut [Evidence], + only: Option, +) { + let mut c = line.splitn(5, '\t'); + let (Some(chrom), Some(pos), Some(_), Some(rf), Some(_)) = + (c.next(), c.next(), c.next(), c.next(), c.next()) + else { return; }; let chrom = if chrom.starts_with("chr") { @@ -150,14 +264,22 @@ fn apply_record(line: &str, by_pos: &ByPos, sites: &mut Genotyped) { let Some(here) = by_pos.0.get(chrom.as_ref()) else { return; }; - let Some(names) = py_pos(pos).and_then(|p| here.get(&p)) else { + let Some(p) = py_pos(pos) else { return; }; + let span = i64::try_from(rf.len().max(1)).unwrap_or(1); + let mut hits = here + .range(p..p.saturating_add(span)) + .filter(|(at, _)| only.is_none_or(|o| o == **at)) + .peekable(); + if hits.peek().is_none() { + return; + } let f: Vec<&str> = line.split('\t').collect(); if f.len() < 10 { return; } - let (rf, alts, filt) = (f[3], f[4].split(',').collect::>(), f[6]); + let (alts, filt) = (f[4].split(',').collect::>(), f[6]); let fmt: Vec<&str> = f[8].split(':').collect(); let smp: Vec<&str> = f[9].split(':').collect(); let gt = fmt @@ -167,18 +289,57 @@ fn apply_record(line: &str, by_pos: &ByPos, sites: &mut Genotyped) { .copied() .unwrap_or("."); let alleles: Vec<&str> = gt.split(['/', '|']).collect(); - for &i in names { - let site = &mut sites[i]; - if site.status == Status::Called { - continue; // a second record at the position: the first one counts + for (&at, names) in hits { + if at == p { + for &i in names { + if let Some(r) = read_record(&sites[i].row, rf, &alts, filt, &alleles) { + ev[i].add(r); + } + } + continue; + } + let offset = usize::try_from(at - p).unwrap_or(0); + if changes_base(rf, &alts, &alleles, offset) { + for &i in names { + ev[i].span.get_or_insert_with(|| { + format!( + "a record at {}:{} ({rf}>{}, genotype {}) spans this site", + f[0], + f[1], + f[4], + alleles.join("/") + ) + }); + } } - genotype_site(site, rf, &alts, filt, &alleles); } } -/// The site's status, alt copies and note from one record. -pub fn genotype_site(site: &mut Site, rf: &str, alts: &[&str], filt: &str, alleles: &[&str]) { - let r = &site.row; +/// Whether a record that starts `offset` bases before a site and spans it +/// calls an allele that changes the site's base: one of another length +/// (an indel, a symbolic or `*` allele) or with another base there. A +/// reference or missing genotype changes nothing. +fn changes_base(rf: &str, alts: &[&str], alleles: &[&str], offset: usize) -> bool { + alleles.iter().any(|a| { + let Some(k) = a.parse::().ok().filter(|&k| k > 0) else { + return false; + }; + alts.get(k - 1).is_none_or(|alt| { + alt.len() != rf.len() || alt.as_bytes().get(offset) != rf.as_bytes().get(offset) + }) + }) +} + +/// What one record at the site's position says about it; `None` when it +/// says nothing: a reference-only record whose REF is shorter than the +/// marker's, which reads no base past its own. +fn read_record( + r: &Marker, + rf: &str, + alts: &[&str], + filt: &str, + alleles: &[&str], +) -> Option { // A null row may list several ALTs (null alleles at one position): any of them counts. let wants: Vec<&str> = if r.ref_ == rf { r.alt.split(',').collect() @@ -191,42 +352,58 @@ pub fn genotype_site(site: &mut Site, rf: &str, alts: &[&str], filt: &str, allel .filter(|(_, a)| wants.contains(a)) .map(|(i, _)| (i + 1).to_string()) .collect(); + let doubt = if idx.is_empty() { + Weight::Other + } else { + Weight::Doubt + }; let joined = alleles.join("/"); if alleles.contains(&".") || alleles.len() != 2 { - site.status = Status::NoCall; - site.note = format!("genotype {joined}"); - return; + return Some(Reading::no_call(doubt, format!("genotype {joined}"))); } - // A record whose REF is not the marker's REF says nothing about the - // marker's ALT, unless it is a reference call over a span that starts - // with the marker's REF. - if rf != r.ref_ && !(rf.starts_with(r.ref_.as_str()) && alleles.iter().all(|a| *a == "0")) { - site.note = format!("REF {rf} at this position, the marker's REF is {}", r.ref_); - site.status = Status::NoCall; - return; + let all_ref = alleles.iter().all(|a| *a == "0"); + if rf != r.ref_ { + if all_ref && rf.len() < r.ref_.len() && r.ref_.starts_with(rf) { + return None; + } + // A record whose REF is not the marker's REF says nothing about + // the marker's ALT, unless it is a reference call over a span that + // starts with the marker's REF. + if !(all_ref && rf.starts_with(r.ref_.as_str())) { + return Some(Reading::no_call( + Weight::Other, + format!("REF {rf} at this position, the marker's REF is {}", r.ref_), + )); + } } - if filt != "PASS" && filt != "." && alleles.iter().any(|a| *a != "0") { - site.status = Status::NoCall; - site.note = format!("FILTER {filt}"); - return; + if filt != "PASS" && filt != "." && !all_ref { + return Some(Reading::no_call(doubt, format!("FILTER {filt}"))); } - site.status = Status::Called; - site.n = Some( - alleles - .iter() - .filter(|a| idx.iter().any(|i| i == *a)) - .count() as i64, - ); - site.note.clear(); - if alleles + let n = alleles + .iter() + .filter(|a| idx.iter().any(|i| i == *a)) + .count() as i64; + let note = if alleles .iter() .any(|a| *a != "0" && !idx.iter().any(|i| i == a)) { - site.note = format!( + format!( "another allele at this position ({rf}>{}, genotype {joined})", alts.join(",") - ); - } + ) + } else { + String::new() + }; + Some(Reading { + weight: if n > 0 { + Weight::Alt + } else { + Weight::Reference + }, + status: Status::Called, + n: Some(n), + note, + }) } /// Genotypes the sites from the sample's VCF (its one sample column). @@ -236,10 +413,12 @@ pub fn read_vcf(path: &str, markers: Sites) -> Result<(Genotyped, Access), Error .map(|(k, m)| (k, Site::new(m))) .collect(); let by_pos = ByPos::new(&sites); + let mut ev: Vec = sites.values().map(|_| Evidence::default()).collect(); if let Some(records) = indexed_records(path, &sites)? { - for line in &records { - apply_record(line, &by_pos, &mut sites); + for (pos, line) in &records { + apply_record(line, &by_pos, &sites, &mut ev, Some(*pos)); } + settle(&mut sites, ev); return Ok((sites, Access::Index)); } let mut lines = og_io::Lines::new( @@ -261,8 +440,9 @@ pub fn read_vcf(path: &str, markers: Sites) -> Result<(Genotyped, Access), Error header = true; continue; } - apply_record(l, &by_pos, &mut sites); + apply_record(l, &by_pos, &sites, &mut ev, None); } + settle(&mut sites, ev); if !header { return Err(Error(format!( "{path} has no #CHROM header line: not a VCF" @@ -271,10 +451,18 @@ pub fn read_vcf(path: &str, markers: Sites) -> Result<(Genotyped, Access), Error Ok((sites, Access::Stream)) } -/// The record lines at the marker positions, read through the VCF's -/// index, those of each position in file order; `None` when the file is to +/// Each site takes what its records said. +fn settle(sites: &mut Genotyped, ev: Vec) { + for (s, e) in sites.values_mut().zip(ev) { + e.settle(s); + } +} + +/// The record lines that cover each marker position (start there or span +/// it), read through the VCF's index, with the position they were read +/// for, those of each position in file order; `None` when the file is to /// be read whole. The `#CHROM` line is checked as a full pass checks it. -fn indexed_records(path: &str, sites: &Genotyped) -> Result>, Error> { +fn indexed_records(path: &str, sites: &Genotyped) -> Result>, Error> { let Some(mut reader) = usable_index(path) else { return Ok(None); }; @@ -319,15 +507,14 @@ fn indexed_records(path: &str, sites: &Genotyped) -> Result>, let Ok(raw) = got else { return Ok(None); }; - let l = String::from_utf8_lossy(&raw).into_owned(); - if l.split('\t').nth(1).and_then(py_pos) == Some(*pos) { - out.push(l); - } + // The index also returns a record whose INFO/END reaches + // the position; apply_record keeps those whose REF does. + out.push((*pos, String::from_utf8_lossy(&raw).into_owned())); } } } - // A site reads only the records at its own position, and those come in - // file order, so the order between positions does not matter. + // A site reads only the records that cover its own position, and those + // come in file order, so the order between positions does not matter. Ok(Some(out)) } @@ -421,6 +608,23 @@ mod tests { }) } + /// The site as these records, in this order, leave it. + fn genotype(s: &mut Site, rf: &str, alts: &[&str], filt: &str, alleles: &[&str]) { + records(s, &[(rf, alts, filt, alleles)]); + } + + type Rec<'a> = (&'a str, &'a [&'a str], &'a str, &'a [&'a str]); + + fn records(s: &mut Site, recs: &[Rec]) { + let mut e = Evidence::default(); + for (rf, alts, filt, alleles) in recs { + if let Some(r) = read_record(&s.row, rf, alts, filt, alleles) { + e.add(r); + } + } + e.settle(s); + } + #[test] fn chroms_normalise() { assert_eq!(norm_chrom("chr7"), "chr7"); @@ -432,7 +636,7 @@ mod tests { #[test] fn a_het_call_counts_one_alt_copy() { let mut s = site("G", "A"); - genotype_site(&mut s, "G", &["A"], "PASS", &["0", "1"]); + genotype(&mut s, "G", &["A"], "PASS", &["0", "1"]); assert_eq!( (s.status, s.n, s.note.as_str()), (Status::Called, Some(1), "") @@ -442,7 +646,7 @@ mod tests { #[test] fn the_second_alt_of_a_multiallelic_record() { let mut s = site("G", "A"); - genotype_site(&mut s, "G", &["C", "A"], "PASS", &["1", "2"]); + genotype(&mut s, "G", &["C", "A"], "PASS", &["1", "2"]); assert_eq!((s.status, s.n), (Status::Called, Some(1))); assert_eq!( s.note, @@ -453,24 +657,24 @@ mod tests { #[test] fn a_filtered_alt_call_is_no_call_but_a_refcall_is_reference() { let mut s = site("G", "A"); - genotype_site(&mut s, "G", &["A"], "LowQual", &["1", "1"]); + genotype(&mut s, "G", &["A"], "LowQual", &["1", "1"]); assert_eq!( (s.status, s.note.as_str()), (Status::NoCall, "FILTER LowQual") ); let mut s = site("G", "A"); - genotype_site(&mut s, "G", &["A"], "RefCall", &["0", "0"]); + genotype(&mut s, "G", &["A"], "RefCall", &["0", "0"]); assert_eq!((s.status, s.n), (Status::Called, Some(0))); } #[test] fn another_ref_is_no_call_unless_a_reference_span() { let mut s = site("G", "A"); - genotype_site(&mut s, "C", &["T"], "PASS", &["1", "1"]); + genotype(&mut s, "C", &["T"], "PASS", &["1", "1"]); assert_eq!(s.status, Status::NoCall); assert_eq!(s.note, "REF C at this position, the marker's REF is G"); let mut s = site("G", "A"); - genotype_site(&mut s, "GT", &["G"], "PASS", &["0", "0"]); + genotype(&mut s, "GT", &["G"], "PASS", &["0", "0"]); assert_eq!((s.status, s.n), (Status::Called, Some(0))); } @@ -478,11 +682,77 @@ mod tests { fn a_haploid_or_missing_genotype_is_no_call() { for gt in [&["1"][..], &[".", "."], &["0", "1", "1"]] { let mut s = site("G", "A"); - genotype_site(&mut s, "G", &["A"], "PASS", gt); + genotype(&mut s, "G", &["A"], "PASS", gt); assert_eq!(s.status, Status::NoCall, "{gt:?}"); } } + #[test] + fn the_strongest_record_at_a_position_counts_in_any_order() { + let r: Rec = ("T", &["."], ".", &["0", "0"]); + let ins: Rec = ("T", &["TC"], "PASS", &["0", "1"]); + let snv: Rec = ("T", &["G"], "PASS", &["1", "0"]); + let ins_nocall: Rec = ("T", &["TC"], "PASS", &[".", "."]); + let other_ref: Rec = ("TA", &["T"], "PASS", &["0", "1"]); + for (recs, want) in [ + (vec![r, ins], (Status::Called, Some(1), "")), + (vec![ins, r], (Status::Called, Some(1), "")), + ( + vec![snv, ins], + ( + Status::Called, + Some(1), + "another allele at this position (T>G, genotype 1/0)", + ), + ), + (vec![r, ins_nocall], (Status::NoCall, None, "genotype ./.")), + (vec![ins_nocall, r], (Status::NoCall, None, "genotype ./.")), + (vec![other_ref, r], (Status::Called, Some(0), "")), + ( + vec![other_ref], + ( + Status::NoCall, + None, + "REF TA at this position, the marker's REF is T", + ), + ), + (vec![ins, ins], (Status::Called, Some(1), "")), + ] { + let mut s = site("T", "TC"); + records(&mut s, &recs); + assert_eq!((s.status, s.n, s.note.as_str()), want, "{recs:?}"); + } + } + + #[test] + fn a_shorter_reference_record_says_nothing() { + let mut s = site("CG", "C"); + genotype(&mut s, "C", &["."], ".", &["0", "0"]); + assert_eq!((s.status, s.n), (Status::Absent, None)); + let mut s = site("CG", "C"); + genotype(&mut s, "C", &["<*>"], ".", &["0", "0"]); + assert_eq!(s.status, Status::Absent); + let mut s = site("CG", "C"); + genotype(&mut s, "C", &["T"], "PASS", &["0", "1"]); + assert_eq!(s.status, Status::NoCall); + } + + #[test] + fn a_spanning_allele_changes_the_base_when_its_length_or_base_differs() { + // CG at the record, the site at offset 1 (the G). + assert!(changes_base("CG", &["CA"], &["1", "1"], 1)); + assert!(changes_base("CG", &["C"], &["0", "1"], 1)); + assert!(changes_base("CG", &["C", "CA"], &["1", "2"], 1)); + assert!(changes_base("CG", &["*"], &["0", "1"], 1)); + assert!( + changes_base("CG", &["A"], &["0", "2"], 1), + "an ALT index the record lacks" + ); + assert!(!changes_base("CG", &["TG"], &["1", "1"], 1)); + assert!(!changes_base("CG", &["C"], &["0", "0"], 1)); + assert!(!changes_base("CG", &["C"], &[".", "."], 1)); + } + #[test] fn depth_decides_a_site_with_no_record() { let mut g: Genotyped = IndexMap::new(); diff --git a/rust/crates/og-bloodtype/tests/report.rs b/rust/crates/og-bloodtype/tests/report.rs index 893169aa8..6cb81053f 100644 --- a/rust/crates/og-bloodtype/tests/report.rs +++ b/rust/crates/og-bloodtype/tests/report.rs @@ -249,10 +249,7 @@ fn both_ways(d: &Path, records: &[String]) -> (PathBuf, PathBuf) { } at = end; } - let names: Vec = seqs - .iter() - .flat_map(|s| s.0.bytes().chain([0])) - .collect(); + let names: Vec = seqs.iter().flat_map(|s| s.0.bytes().chain([0])).collect(); let mut tbi: Vec = b"TBI\x01".to_vec(); let i32s = |v: &mut Vec, xs: &[i32]| xs.iter().for_each(|x| v.extend(x.to_le_bytes())); // n_ref, VCF preset, columns 1 2 0, meta '#', skip 0, names. @@ -328,7 +325,10 @@ fn a_later_record_of_the_marker_allele_that_is_no_call_wins_over_a_reference_rec let d = dir("abo261-nocall"); let r = line("chr9", 133257521, "T", ".", ".", "0/0"); for (later, note) in [ - (line("chr9", 133257521, "T", "TC", "PASS", "./."), "genotype ./."), + ( + line("chr9", 133257521, "T", "TC", "PASS", "./."), + "genotype ./.", + ), ( line("chr9", 133257521, "T", "TC", "LowQual", "0/1"), "FILTER LowQual", @@ -388,7 +388,11 @@ fn a_record_that_spans_a_marker_with_another_base_there_is_no_call() { // KEL c.578C>T (K) is chr7:142957921 G>A. A padded record one base // upstream carries the same change (CG>CA), or a deletion of that // base with it (DeepVariant's CG>C,CA 1/2). - for (r, alt, gt) in [("CG", "CA", "1/1"), ("CG", "C,CA", "1/2"), ("CG", "C", "0/1")] { + for (r, alt, gt) in [ + ("CG", "CA", "1/1"), + ("CG", "C,CA", "1/2"), + ("CG", "C", "0/1"), + ] { let recs = [line("chr7", 142957920, r, alt, "PASS", gt)]; let (status, n, note) = site_both_ways(&d, &recs, "KEL_578"); assert_eq!( @@ -401,7 +405,7 @@ fn a_record_that_spans_a_marker_with_another_base_there_is_no_call() { ) ); let t = run(&d, &recs, reads()); - assert_eq!(t["Kell"][0], "K/k not determined", "{recs:?}"); + assert!(t["Kell"][0].starts_with("K/k not determined"), "{recs:?}"); assert_ne!(t["Kell"][2], "high", "{recs:?}"); } // Split (bcftools norm -m-any): the deletion before the SNV record From f446ec41265d70af97bc19a9e59b9937cad264a9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:27:30 +0100 Subject: [PATCH 118/255] docs(bloodtype): say how overlapping records at a marker are read Bead: trait-40 Bead: trait-41 Bead: trait-29 --- CHANGELOG.md | 7 +++++++ docs/41-bloodtype.md | 13 ++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..ab677d5bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 41 (blood groups) reads every VCF record that covers a marker site, and their order no longer changes the answer. Before, the first record at a position won. An all-sites VCF writes a reference record and then the indel at one position (`T . 0/0`, then `T TC 0/1` at ABO c.261), so a person with an A or B allele read O. A split multi-allelic record with the SNV first did the same. No column, file or option changes; only the values of the affected sites do. What changes: + - A call of the marker's ALT now wins over the other records at the position. Another allele a second record calls there is named in the note, as when one record holds both. + - A no call of a record that names the marker's ALT (`./.`, or a failed FILTER) now wins over a reference record at the same position, so the site is a no call instead of a confident reference. + - A 0/0 record over a REF shorter than the marker's (an all-sites `C . 0/0`, or a gVCF block that starts at the position) now counts as no record, so the reads decide. Before it was a no call: ABO read "A, A1 or A2 not determined", and Vel, HPA-14 and every system with a deletion marker lost confidence. + - A record that starts upstream and spans a marker with an allele that changes the marker's base (a deletion over it, or a padded `CG CA`, as DeepVariant writes `CG C,CA` when a deletion and an SNV overlap) now makes the site a no call, with a note that names the record. Before, the site had no record at its own position and 8 reads made it a confident reference: a K/K person read k/k at high confidence. + - A VCF with one record per position and no overlapping records, as DeepVariant writes, gives the same output as before: the 34 parity cases still agree with the Python oracle byte for byte. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/docs/41-bloodtype.md b/docs/41-bloodtype.md index 20f2cb000..2456d51b5 100644 --- a/docs/41-bloodtype.md +++ b/docs/41-bloodtype.md @@ -33,7 +33,7 @@ Pinned in `versions.env`; [Image versions](versions.md) lists the current tags. ## Input -- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample, left-aligned records (DeepVariant writes them so; split multi-allelic records are read as they are) +- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample, left-aligned records (DeepVariant writes them so). Split multi-allelic records, all-sites VCFs (a reference record and then the variant at one position) and gVCF reference blocks are read too: every record that covers a site counts, whatever their order (see below) - `${SAMPLE}/aligned/${SAMPLE}_sorted.bam` with its `.bai` (`ALIGN_DIR` as for the other BAM steps). Optional: without it RhD is not determined, and a site the VCF has no record for is read as the reference allele at confidence medium. ## Command @@ -62,7 +62,14 @@ With `run-all.sh`: `TOOLS=...,bloodtype`. It is opt-in and needs no data, so a r | HPA-7 to -14, -16 to -35 | one variant each in ITGB3, ITGA2B, ITGA2, GP1BB or GP9 | one allele per copy; the a antigen is on the reference allele, the low-prevalence b antigen on the other | | HNA-3, -4, -5 | SLC44A2 c.461G>A, ITGAM c.230G>A, ITGAL c.2372G>C | one allele per copy; GRCh38 carries HNA-3b | -A site the VCF has no record for is the reference allele when the BAM has at least 8 reads there at MAPQ 20, and not determined below that. Without the BAM it is read as reference, and the confidence says so. The step leaves out of `samtools bedcov` the rows on a contig the BAM has no `@SQ` line for (a BAM cut to a few chromosomes), because `bedcov` would stop on them. Those sites then have no depth row: the ones with no VCF record are not determined, and RhD is not determined without chr1. A record at a site whose REF is not the table's REF is a no call, because it says nothing about the table's ALT; the one exception is a 0/0 call over a longer REF that starts with the table's REF, which is the reference allele. +A site the VCF has no record for is the reference allele when the BAM has at least 8 reads there at MAPQ 20, and not determined below that. Without the BAM it is read as reference, and the confidence says so. The step leaves out of `samtools bedcov` the rows on a contig the BAM has no `@SQ` line for (a BAM cut to a few chromosomes), because `bedcov` would stop on them. Those sites then have no depth row: the ones with no VCF record are not determined, and RhD is not determined without chr1. A record at a site whose REF is not the table's REF is a no call, because it says nothing about the table's ALT. There are two exceptions. A 0/0 call over a longer REF that starts with the table's REF is the reference allele. A 0/0 record over a shorter REF (an all-sites VCF's `C . 0/0`, or a gVCF block that starts at a deletion marker such as ABO c.1061delC) says nothing past its own base, so the site reads as one with no record: the reads decide. + +When several records cover one site, each one is read and their order does not matter: + +- A call of the table's ALT wins over the other records at the site's position. An all-sites VCF writes `T . 0/0` and then `T TC 0/1` at ABO c.261, so the insertion counts. Another allele that a second record calls there (a split `T G 1/0`) is named in the note. +- A no call of a record that names the table's ALT (`./.`, or a failed FILTER) comes next: the ALT may be there, so a reference record beside it does not make the site reference. +- Then a reference call, then any other no call. +- A record that starts upstream and spans the site with an allele that changes the site's base (a deletion over it, or a padded record such as `CG CA`) makes the site a no call, whatever the site's own records say. A reference call or a no call upstream changes nothing, and neither does an allele that keeps the site's base. ## Output @@ -94,7 +101,7 @@ Lacking an antigen is common (most people lack HPA-1b, for example). It means a ## Runtime -Not measured on a whole genome. The step reads the VCF once, line by line, and samtools reads the BAM at 284 small regions. +Not measured on a whole genome. The step reads the marker positions through the VCF's index (the VCF once, line by line, without one), and samtools reads the BAM at 284 small regions. ## Notes From 875196f849f2b6e40bea7637af672e2e45b016e5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:37:19 +0100 Subject: [PATCH 119/255] test(live): inspect alignment rendering inside the viewer shadow root Bead: pgp-iqon.11 --- tests/test_live_viewer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index 803d7e33f..df50d0d61 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -272,9 +272,13 @@ def loaded(): check("map: moving updates the same iframe to the map's locus", b.wait("savedFrame===document.querySelector('.gm-reads iframe') && savedFrame.contentWindow.location.hash==='#chr1:1-49001'")) check("viewer: alignments remain visible at 49 kb", - b.wait("(function(){var v=savedFrame.contentDocument.querySelector('.igv-viewport[data-track-type=alignment]');" + b.wait("(function(){var h=savedFrame.contentDocument.getElementById('igv');" + "var v=h && h.shadowRoot && h.shadowRoot.querySelector('.igv-viewport[data-track-type=alignment]');" "var n=v && v.querySelector('.igv-zoom-in-notice-container');" - "return !!(v && v.querySelector('canvas') && n && n.style.display!=='flex');})()")) + "return !!(v && v.querySelector('canvas') && n && n.style.display!=='flex');})()"), + b.eval("JSON.stringify(Array.from(savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelectorAll('.igv-viewport')).map(function(v){" + "var n=v.querySelector('.igv-zoom-in-notice-container');return {type:v.dataset.trackType," + "notice:n && n.style.display,canvas:v.querySelectorAll('canvas').length,text:v.textContent.slice(0,180)};}))")) if os.environ.get("MAP_SCREENS"): os.makedirs(os.environ["MAP_SCREENS"], exist_ok=True) for width, height in ((1440, 900), (390, 844)): From 67856fddf4a13fd2df1808a6a7f76aa309279701 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:38:44 +0100 Subject: [PATCH 120/255] fix(build): mount and fingerprint the embedded imprinting table The supported builder stages rust at /src, so the table needs a read-only mount at /assets. Include its blob in the source stamp so an asset edit makes old binaries stale. Keep the stamp format and release-install checks. The new build checks failed five assertions against the previous builder and all pass with this change. The original missing-mount failure is upstream run 43658012836. Bead: lic-41 Bead: lin-17 --- CHANGELOG.md | 2 +- docs/rust.md | 2 +- rust/README.md | 2 +- rust/og.sh | 35 ++++++++++++++++++++++++++--------- tests/test_og_check.sh | 24 +++++++++++++++++++++--- 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78de0ffed..ab95d832e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased - ROH summaries now divide F_ROH by the GRCh38 length outside the bands of the assessed autosomes. Per-site rows identify those chromosomes, including chromosomes with no ROH. A new `roh.data.scope` records the chromosomes, denominator and scope source. Relationship labels and UPD flags require sites on all 22 autosomes and the full unmasked autosome length. Segment-only files use their observed chromosomes as an unverified lower bound and get neither label. Empty input has null F_ROH. Segment counts and the step 11 summary table keep their format. -- A one-chromosome ROH flag now names regions and citations without syndrome names. Its note includes deletion, acquired loss of heterozygosity and shared ancestry as alternatives. The maternal and paternal JSON keys remain as `"-"` for reader compatibility. The attributed `assets/imprinted_regions.tsv` is embedded at build time, replacing the duplicated Rust table. Every listed chromosome uses the same lookup, including chr11 and chr15. A region hint does not establish overlap with its imprinted locus or parental origin. Broad cytoband names remain reference labels, not diagnostic intervals. +- A one-chromosome ROH flag now names regions and citations without syndrome names. Its note includes deletion, acquired loss of heterozygosity and shared ancestry as alternatives. The maternal and paternal JSON keys remain as `"-"` for reader compatibility. The attributed `assets/imprinted_regions.tsv` is embedded at build time, replacing the duplicated Rust table. The container builder mounts it, and the source stamp includes its blob hash, so a table edit makes an old binary stale. The stamp format and `tree-hash` command name stay the same. Every listed chromosome uses the same lookup, including chr11 and chr15. A region hint does not establish overlap with its imprinted locus or parental origin. Broad cytoband names remain reference labels, not diagnostic intervals. - The ROH collector now streams per-site rows instead of keeping the whole file in memory. On 1.1 million synthetic site rows, peak RSS fell from 65.7-67.0 MB to 3.0-3.5 MB. CPU time stayed near 0.2 seconds. Under shared host load, wall time ranged from 1.40-21.46 seconds before and 7.89-38.16 seconds after; these timings do not establish a speedup. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: diff --git a/docs/rust.md b/docs/rust.md index 0e0de2374..a9da46f20 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -70,7 +70,7 @@ The binary is static, so it runs in any image that has `/bin/sh`: the Python ima `bin/.og` is not in git, so after a pull or an edit it can hold a binary built from other code. Results would then come from code the checkout does not show. To stop that: -- The build stamps the binary with the git tree hash of `rust/`: `git rev-parse HEAD:rust` on a clean checkout, or for an edited `rust/` the tree those edits would commit. Any edit, committed or not, changes the hash. +- The build stamps the binary with a source hash of two git object IDs: the tree of `rust/` as it is on disk and the blob of `assets/imprinted_regions.tsv`. The table is embedded at compile time. Any edit to either input, committed or not, changes the hash. The `tree-hash` verb and stamp format retain their names. - `rust/og.sh check` reads the stamp from each binary with `grep`, without running it, and compares it with the checkout's. The macOS pair is optional, because no pipeline step runs it, but a stale one is refused like a Linux one. - `run-all.sh` (even with `SKIP_VALIDATION=true`), `main.nf` and `validate-setup.sh` run that check. A binary built from another `rust/` stops them with `run scripts/setup.sh --og`. - A missing binary is still a note, which names steps 04, 05, 07, 09b, 21, 25, 32 and 33. Those steps, step 06 with `CLINVAR_SCREEN` and `CLINVAR_CDNA_CHECK`, step 17, and steps 23 and 31 with their modules, stop without it: `bin/og` says `ERROR: og is not built for ; run scripts/setup.sh --og` (exit 127). Step 32 does not read that as an unmeasured share: it stops. `REQUIRED` in `rust/og.sh` stays `false` until the fake-docker and Nextflow stub tests, which start runs without `og`, build it first; then a missing binary stops a run before any step. diff --git a/rust/README.md b/rust/README.md index ce5fae19c..a4fa66be8 100644 --- a/rust/README.md +++ b/rust/README.md @@ -33,7 +33,7 @@ cargo clippy --all-targets -- -D warnings cargo build --release && ../tests/rust/pycompat-diff.sh target/release/og # pycompat against CPython ``` -A binary from a plain `cargo build` is stamped `unknown`, and `rust/og.sh check` refuses it: only `scripts/setup.sh --og` stamps the tree hash the pipeline checks. +A binary from a plain `cargo build` is stamped `unknown`, and `rust/og.sh check` refuses it: only `scripts/setup.sh --og` stamps the source hash of `rust/` and its embedded imprinting table that the pipeline checks. CI readers also run through `og dev pharmcat diplotypes`, `og dev nf-trace requests|attempts` and `og dev smoke`. See [the Rust command notes](../docs/rust.md#ci-readers). `og traits table`, `og traits phase-sites` and `og traits check` read step 42's diff --git a/rust/og.sh b/rust/og.sh index 70f5af058..0df99a92f 100755 --- a/rust/og.sh +++ b/rust/og.sh @@ -19,15 +19,15 @@ # --required, or once REQUIRED below is true) no # Linux pair. The macOS pair is optional, since no # pipeline step runs it, but a stale one is refused -# rust/og.sh tree-hash the git tree hash of rust/ as it is on disk +# rust/og.sh tree-hash the source hash of rust/ and its embedded table # rust/og.sh built the tree hash the binaries were built from, for # run_manifest.tsv ("not built" when there are none) # -# The tree hash is git's: `git rev-parse HEAD:rust` on a clean checkout, and -# for an edited rust/ the tree those edits would commit (a throwaway index), -# so any edit, committed or not, makes the built binary stale. The build -# stamps it into the binary as og-tree-hash:; so the check -# reads it with grep and never runs a binary the host may not be able to. +# The source stamp hashes two git object IDs: rust/'s tree as it is on +# disk and the blob of assets/imprinted_regions.tsv. The latter is embedded +# at compile time, so an edit to it must also make a binary stale. The +# legacy tree-hash verb and og-tree-hash:; stamp keep their format. +# Checks read the stamp without executing a binary for another platform. set -euo pipefail # Steps 04, 05, 07, 09b, 21, 25, 32 and 33 (and the modules of 09b, 21 and 32: @@ -57,7 +57,7 @@ names() { } # tree_hash: exit 3 when this is not a git checkout, 1 when git fails. -tree_hash() { +rust_tree_hash() { local idx rc=0 git -C "$ROOT" rev-parse --verify -q HEAD:rust > /dev/null 2>&1 || return 3 if [ -z "$(git -C "$ROOT" status --porcelain --untracked-files=all -- rust 2> /dev/null)" ]; then @@ -72,6 +72,22 @@ tree_hash() { [ "$rc" -eq 0 ] || return 1 } +# Hash the two source object IDs without writing to the checkout's object +# store. Both committed and uncommitted table edits change the stamp. +source_hash() { printf '%s\n%s\n' "$1" "$2" | git -C "$ROOT" hash-object --stdin; } +tree_hash() { + local rust asset + rust=$(rust_tree_hash) || return $? + asset=$(git -C "$ROOT" hash-object -- assets/imprinted_regions.tsv) || return 1 + source_hash "$rust" "$asset" +} +committed_hash() { + local rust asset + rust=$(git -C "$ROOT" rev-parse HEAD:rust) || return 1 + asset=$(git -C "$ROOT" rev-parse HEAD:assets/imprinted_regions.tsv) || return 1 + source_hash "$rust" "$asset" +} + # embedded FILE: the hash stamped into a binary, or "unknown". embedded() { local h @@ -176,7 +192,8 @@ build() { zigbuild() { "$ENGINE" run --rm --user "$(id -u):$(id -g)" \ -e HOME=/cache/home -e CARGO_HOME=/cache/cargo -e CARGO_TARGET_DIR=/cache/target -e OG_TREE_HASH="$hash" \ - -v "${ROOT}/rust:/src:ro" -v "${cache}:/cache" -w /src "$OG_BUILDER" \ + -v "${ROOT}/rust:/src:ro" \ + -v "${ROOT}/assets/imprinted_regions.tsv:/assets/imprinted_regions.tsv:ro" -v "${cache}:/cache" -w /src "$OG_BUILDER" \ cargo zigbuild --release --locked -p og "$@" } zigbuild "${targets[@]}" || return @@ -250,7 +267,7 @@ install() { local tag rc=0 hash hash=$(tree_hash) || { echo "ERROR: ${ROOT} is not a git checkout; clone the repository to install og" >&2; return 1; } tag=$(git -C "$ROOT" describe --exact-match --tags --match 'v[0-9]*.[0-9]*.[0-9]*' HEAD 2> /dev/null || true) - if [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [ "$hash" = "$(git -C "$ROOT" rev-parse HEAD:rust)" ]; then + if [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] && [ "$hash" = "$(committed_hash)" ]; then echo "Release ${tag}: downloading its og binaries" fetch "$tag" || rc=$? [ "$rc" -ne 0 ] || return 0 diff --git a/tests/test_og_check.sh b/tests/test_og_check.sh index e4f2e6b43..0657ab3f3 100755 --- a/tests/test_og_check.sh +++ b/tests/test_og_check.sh @@ -27,7 +27,7 @@ expect() { # expect NAME RC WANT_RC OUTPUT PATTERN C="${T}/repo" mkdir -p "$C" -cp -R "${REPO}/scripts" "${REPO}/bin" "${REPO}/rust" "${REPO}/versions.env" "${REPO}/nextflow.config" "${REPO}/.gitignore" "$C/" +cp -R "${REPO}/scripts" "${REPO}/bin" "${REPO}/rust" "${REPO}/assets" "${REPO}/versions.env" "${REPO}/nextflow.config" "${REPO}/.gitignore" "$C/" rm -rf "${C}/bin/.og" "${C}/rust/target" g() { git -C "$C" -c user.name=t -c user.email=t@example.invalid "$@"; } g init -q && g add -A && g commit -qm base @@ -50,8 +50,11 @@ plant_mac() { # plant_mac HASH: the macOS pair, stamped with HASH done } +committed_hash() { + printf '%s\n%s\n' "$(g rev-parse HEAD:rust)" "$(g rev-parse HEAD:assets/imprinted_regions.tsv)" | g hash-object --stdin +} h0=$(bash "$OG" tree-hash) -[ "$h0" = "$(g rev-parse HEAD:rust)" ] && ok "tree-hash of a clean rust/ is HEAD:rust" || bad "tree-hash ${h0} != HEAD:rust" +[ "$h0" = "$(committed_hash)" ] && ok "source hash covers rust/ and the embedded table" || bad "source hash omits an input" out=$(bash "$OG" check); rc=$? expect "no binary: a note, exit 0" "$rc" 0 "$out" "NOTE: og is not built for x86_64 aarch64*setup.sh --og*" @@ -60,6 +63,14 @@ expect "no binary with --required: refused" "$rc" 1 "$out" "ERROR: og is not bui out=$(bash "$OG" built); [ "$out" = "not built" ] && ok "built without binaries: not built" || bad "built printed '${out}'" plant "$h0" +cp "${C}/assets/imprinted_regions.tsv" "${T}/imprinted.saved" +printf '\n# edited source\n' >> "${C}/assets/imprinted_regions.tsv" +ha=$(bash "$OG" tree-hash) +[ "$ha" != "$h0" ] && ok "an embedded table edit changes the source hash" || bad "embedded table edit did not change the hash" +out=$(bash "$OG" check); rc=$? +expect "an embedded table edit makes the binary stale" "$rc" 1 "$out" "ERROR: og is stale:*" +cp "${T}/imprinted.saved" "${C}/assets/imprinted_regions.tsv" +[ "$(bash "$OG" tree-hash)" = "$h0" ] && ok "restoring the table restores the hash" || bad "restored table has a different hash" out=$(bash "$OG" check); rc=$? expect "binaries of this rust/: OK" "$rc" 0 "$out" "OK: og is built for x86_64, aarch64*${h0}*" out=$(bash "$OG" built); [ "$out" = "$h0" ] && ok "built prints the one hash" || bad "built printed '${out}'" @@ -91,7 +102,7 @@ h1=$(bash "$OG" tree-hash) out=$(bash "$OG" check); rc=$? expect "uncommitted edit to rust/: stale, refused" "$rc" 1 "$out" "ERROR: og is stale:*${h0}*${h1}*run scripts/setup.sh --og" g add rust/og/src/main.rs && g commit -qm edit -[ "$(bash "$OG" tree-hash)" = "$h1" ] && [ "$h1" = "$(g rev-parse HEAD:rust)" ] \ +[ "$(bash "$OG" tree-hash)" = "$h1" ] && [ "$h1" = "$(committed_hash)" ] \ && ok "committing the edit keeps the hash the build would stamp" || bad "hash after commit differs" out=$(bash "$OG" check); rc=$? expect "committed edit to rust/: stale, refused" "$rc" 1 "$out" "ERROR: og is stale:*" @@ -213,6 +224,13 @@ else echo "SKIP release download cases: no python3 or curl" fi +# The builder must mount the compile-time table beside /src. The fake +# engine refuses its absence and stops before any cargo invocation. +printf '#!/usr/bin/env bash\nfor arg in "$@"; do\n case $arg in *"/assets/imprinted_regions.tsv:/assets/imprinted_regions.tsv:ro") echo EMBEDDED_TABLE_MOUNT_OK; exit 99 ;; esac\ndone\necho EMBEDDED_TABLE_MOUNT_MISSING\nexit 98\n' > "${T}/fake/embed-engine" +chmod 755 "${T}/fake/embed-engine" +out=$(PATH="${T}/fake:${PATH}" CONTAINER_ENGINE=embed-engine bash "$OG" build 2>&1); rc=$? +expect "the builder mounts its embedded table" "$rc" 99 "$out" "*EMBEDDED_TABLE_MOUNT_OK*" + # Outside git the hash cannot be checked: a note, never a silent OK. N="${T}/nogit" plant "$h1" From a5e5dbfaaac7d859d9ffce7476aa7bd97df00fd9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:39:18 +0100 Subject: [PATCH 121/255] fix(archaic): name the missing gVCF when no site was read Bead: lin-01 --- rust/crates/og-ancestry/src/archaic.rs | 17 ++++- .../tests/i411_archaic_callable.rs | 76 ++++++++++++++----- 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index f31e29c0b..89fa026d1 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -1172,9 +1172,13 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { let names: Vec<&str> = tables.iter().map(|(c, _)| c.as_str()).collect(); let mut sorted = names.clone(); sorted.sort_unstable(); + let why = if callable.is_some() { + "no call there, and no reference block of the gVCF over a table site" + } else { + "no call at a table site (without --gvcf a site with no call is unknown)" + }; return die(format!( - "{vcf_path} has no site read on a chromosome of the archaic table ({}): \ - no call there, and no reference block of the gVCF over a table site", + "{vcf_path} has no site read on a chromosome of the archaic table ({}): {why}", sorted.join(", ") )); } @@ -1821,7 +1825,14 @@ const STRICT: Spec = Spec { const GENOTYPES: Spec = Spec { verb: "genotypes", usage: "usage: og archaic genotypes --table-dir TABLE_DIR --vcf VCF [--gvcf GVCF] [--pop POP] [--pseudo PSEUDO] --out-dir OUT_DIR\n", - options: &["--table-dir", "--vcf", "--gvcf", "--pop", "--pseudo", "--out-dir"], + options: &[ + "--table-dir", + "--vcf", + "--gvcf", + "--pop", + "--pseudo", + "--out-dir", + ], required: &["--table-dir", "--vcf", "--out-dir"], }; const REPORT: Spec = Spec { diff --git a/rust/crates/og-ancestry/tests/i411_archaic_callable.rs b/rust/crates/og-ancestry/tests/i411_archaic_callable.rs index 4f319b623..78721e7fe 100644 --- a/rust/crates/og-ancestry/tests/i411_archaic_callable.rs +++ b/rust/crates/og-ancestry/tests/i411_archaic_callable.rs @@ -11,8 +11,7 @@ use std::collections::BTreeMap; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; -const TABLE_HEAD: &str = - "chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tAF_ALL\tAF_AFR\tAF_AMR\tAF_EAS\tAF_EUR\tAF_SAS\n"; +const TABLE_HEAD: &str = "chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tAF_ALL\tAF_AFR\tAF_AMR\tAF_EAS\tAF_EUR\tAF_SAS\n"; const VCF_HEAD: &str = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n"; @@ -63,7 +62,10 @@ fn run(d: &Path, vcf: &str, gvcf: Option<&str>) -> (u8, String) { .to_vec(); if let Some(g) = gvcf { std::fs::write(d.join("s.g.vcf"), format!("{VCF_HEAD}{g}")).unwrap(); - args.extend(["--gvcf".to_owned(), d.join("s.g.vcf").to_str().unwrap().to_owned()]); + args.extend([ + "--gvcf".to_owned(), + d.join("s.g.vcf").to_str().unwrap().to_owned(), + ]); } let mut err = Vec::new(); let code = cli(&args, &mut Vec::new(), &mut err); @@ -102,7 +104,13 @@ fn spans(d: &Path) -> Vec> { #[test] fn a_gap_in_coverage_is_unknown_not_the_reference() { let d = dir("gap"); - table(&d, "chr22", (100_000..=300_000).step_by(1000).chain((301_000..=351_000).step_by(1000))); + table( + &d, + "chr22", + (100_000..=300_000) + .step_by(1000) + .chain((301_000..=351_000).step_by(1000)), + ); let vcf = "chr22\t50000\t.\tC\tT\t50\tPASS\t.\tGT\t0/1\n"; let gvcf = block("chr22", 1, 99_999, ".", "GT:GQ:MIN_DP", "0/0:50:30") + &block("chr22", 300_500, 360_000, ".", "GT:GQ:MIN_DP", "0/0:50:30"); @@ -111,8 +119,14 @@ fn a_gap_in_coverage_is_unknown_not_the_reference() { let c = codes(&d, "chr22"); let gap: Vec<&String> = c.range(100_000..=300_000).map(|(_, v)| v).collect(); assert_eq!(gap.len(), 201); - assert!(gap.iter().all(|v| *v == "9"), "the gap must be unknown: {gap:?}"); - assert!(c.range(301_000..).all(|(_, v)| v == "0"), "a covered site is the reference"); + assert!( + gap.iter().all(|v| *v == "9"), + "the gap must be unknown: {gap:?}" + ); + assert!( + c.range(301_000..).all(|(_, v)| v == "0"), + "a covered site is the reference" + ); assert_eq!(spans(&d), [["chr22", "301000", "351000", "51", "0", "0"]]); assert!(err.contains("51 sites read, 201 not read"), "{err}"); let _ = std::fs::remove_dir_all(&d); @@ -147,14 +161,14 @@ fn only_a_confident_reference_block_reads_as_the_reference() { table(&d, "chr22", (1..=12).map(|i| i * 1000)); let gvcf = [ block("chr22", 1000, 1000, ".", "GT:GQ:MIN_DP", "0/0:20:10"), // reference - block("chr22", 2000, 2000, "PASS", "GT", "0|0"), // reference, phased, no GQ - block("chr22", 3000, 3000, ".", "GT:GQ:MIN_DP", "0/0:0:10"), // GQ 0 - block("chr22", 4000, 4000, ".", "GT:GQ:MIN_DP", "0/0:30:0"), // no reads - block("chr22", 5000, 5000, ".", "GT:GQ:DP", "0/0:30:0"), // no reads (DP) - block("chr22", 6000, 6000, "RefCall", "GT:GQ", "0/0:30"), // filtered - block("chr22", 7000, 7000, ".", "GT:GQ", "./.:0"), // no call + block("chr22", 2000, 2000, "PASS", "GT", "0|0"), // reference, phased, no GQ + block("chr22", 3000, 3000, ".", "GT:GQ:MIN_DP", "0/0:0:10"), // GQ 0 + block("chr22", 4000, 4000, ".", "GT:GQ:MIN_DP", "0/0:30:0"), // no reads + block("chr22", 5000, 5000, ".", "GT:GQ:DP", "0/0:30:0"), // no reads (DP) + block("chr22", 6000, 6000, "RefCall", "GT:GQ", "0/0:30"), // filtered + block("chr22", 7000, 7000, ".", "GT:GQ", "./.:0"), // no call "chr22\t8000\t.\tA\tG,<*>\t30\tPASS\t.\tGT:GQ\t0/1:30\n".to_owned(), // a variant - block("chr22", 9000, 9000, ".", "GT:GQ:MIN_DP", "0:20:10"), // haploid reference + block("chr22", 9000, 9000, ".", "GT:GQ:MIN_DP", "0:20:10"), // haploid reference // 10000 is in no record; 11000 is a VCF call outside every block. block("chr22", 12000, 12000, ".", "GT:GQ:MIN_DP", "0/0:.:."), // missing values ] @@ -192,7 +206,11 @@ fn a_site_is_covered_up_to_the_furthest_end() { table(&d, "chr22", [1000, 1500, 2500, 3500]); let gvcf = block("chr22", 900, 3000, ".", "GT:GQ", "0/0:40") + &block("chr22", 1200, 1300, ".", "GT:GQ", "./.:0"); - let (code, err) = run(&d, "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n", Some(&gvcf)); + let (code, err) = run( + &d, + "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n", + Some(&gvcf), + ); assert_eq!(code, 0, "{err}"); let c = codes(&d, "chr22"); assert_eq!(c[&1500], "0"); @@ -209,11 +227,19 @@ fn the_gvcf_is_read_in_the_vcf_order() { let d = dir("order"); table(&d, "chr21", [1000, 2000]); table(&d, "chr22", [1000, 2000]); - let vcf = "chr21\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\nchr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n"; + let vcf = + "chr21\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\nchr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n"; let chr21 = block("chr21", 1, 5000, ".", "GT:GQ", "0/0:40"); let chr22 = block("chr22", 1, 5000, ".", "GT:GQ", "0/0:40"); let other = block("chrM", 1, 100, ".", "GT:GQ", "0/0:40"); - let (code, err) = run(&d, vcf, Some(&format!("{other}{chr21}{other_x}{chr22}", other_x = block("chrX", 1, 9, ".", "GT", "0/0")))); + let (code, err) = run( + &d, + vcf, + Some(&format!( + "{other}{chr21}{other_x}{chr22}", + other_x = block("chrX", 1, 9, ".", "GT", "0/0") + )), + ); assert_eq!(code, 0, "{err}"); assert_eq!(codes(&d, "chr21")[&2000], "0"); assert_eq!(codes(&d, "chr22")[&2000], "0"); @@ -222,12 +248,18 @@ fn the_gvcf_is_read_in_the_vcf_order() { assert_eq!(code, 2, "{err}"); assert!(err.contains("one chromosome order"), "{err}"); - let back = format!("{chr21}{chr22}{}", block("chr21", 9000, 9100, ".", "GT", "0/0")); + let back = format!( + "{chr21}{chr22}{}", + block("chr21", 9000, 9100, ".", "GT", "0/0") + ); let (code, err) = run(&d, vcf, Some(&back)); assert_eq!(code, 2, "{err}"); assert!(err.contains("comes back"), "{err}"); - let unsorted = format!("{}{chr21}{chr22}", block("chr21", 3000, 3100, ".", "GT", "0/0")); + let unsorted = format!( + "{}{chr21}{chr22}", + block("chr21", 3000, 3100, ".", "GT", "0/0") + ); let (code, err) = run(&d, vcf, Some(&unsorted)); assert_eq!(code, 2, "{err}"); assert!(err.contains("is not sorted"), "{err}"); @@ -242,7 +274,13 @@ fn a_gvcf_of_one_sample_only() { let vcf = "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n"; std::fs::write(d.join("s.vcf"), format!("{VCF_HEAD}{vcf}")).unwrap(); let two = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n"; - for (gvcf, want) in [(two, "holds 2 samples"), ("chr22\t1\t.\tA\t<*>\t0\t.\tEND=9\tGT\t0/0\n", "no #CHROM header")] { + for (gvcf, want) in [ + (two, "holds 2 samples"), + ( + "chr22\t1\t.\tA\t<*>\t0\t.\tEND=9\tGT\t0/0\n", + "no #CHROM header", + ), + ] { std::fs::write(d.join("x.g.vcf"), gvcf).unwrap(); let args: Vec = [ "genotypes", From fb997035926419410a06cb6440b33d3a9036d1c2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:39:18 +0100 Subject: [PATCH 122/255] fix(archaic): give step 43 the gVCF, and skip a sample without one Both callers hand og archaic genotypes the gVCF of step 3. The Nextflow pipeline skips a samplesheet row without a gVCF with a warning, as the traits and ancient steps do; the bash step stops with the reason. The unit test runs real IBDmix on a stretch with no call: it gave a 149 kb segment read as the reference, and gives none now. Parity goldens record the deliberate change: a site with no call is unknown without a gVCF. Bead: lin-01 --- main.nf | 19 ++++- modules/local/archaic/main.nf | 8 +- nextflow_schema.json | 2 +- scripts/43-archaic.sh | 19 +++-- tests/parity/cases.tsv | 2 + .../archaic-genotypes/afr-one/gt/chr21.gt.gz | Bin 0 -> 332 bytes .../archaic-genotypes/afr-one/gt/chr22.gt.gz | Bin 0 -> 453 bytes .../archaic-genotypes/afr-one/gt/spans.tsv | 3 + .../golden/archaic-genotypes/afr-one/stderr | 1 + .../archaic-genotypes/default/gt/chr21.gt.gz | Bin 0 -> 694 bytes .../archaic-genotypes/default/gt/chr22.gt.gz | Bin 0 -> 961 bytes .../archaic-genotypes/default/gt/spans.tsv | 3 + .../golden/archaic-genotypes/default/stderr | 1 + .../archaic-genotypes/gvcf-order/stderr | 1 + .../parity/golden/archaic-genotypes/gvcf/exit | 1 + .../gvcf/gt/chr21.Altai.mask.bed | 4 + .../gvcf/gt/chr21.Denisova.mask.bed | 9 +++ .../gvcf/gt/chr21.Vindija.mask.bed | 5 ++ .../archaic-genotypes/gvcf/gt/chr21.gt.gz | Bin 0 -> 365 bytes .../gvcf/gt/chr22.Altai.mask.bed | 7 ++ .../gvcf/gt/chr22.Denisova.mask.bed | 15 ++++ .../gvcf/gt/chr22.Vindija.mask.bed | 8 ++ .../archaic-genotypes/gvcf/gt/chr22.gt.gz | Bin 0 -> 512 bytes .../archaic-genotypes/gvcf/gt/samples.txt | 4 + .../archaic-genotypes/gvcf/gt/spans.tsv | 3 + .../golden/archaic-genotypes/gvcf/stderr | 1 + .../archaic-genotypes/no-table-chrom/stderr | 1 + .../archaic-genotypes/plain/gt/chr21.gt.gz | Bin 0 -> 367 bytes .../archaic-genotypes/plain/gt/chr22.gt.gz | Bin 0 -> 514 bytes .../archaic-genotypes/plain/gt/spans.tsv | 3 + .../golden/archaic-genotypes/plain/stderr | 1 + .../inputs/archaic-genotypes/order.g.vcf | 10 +++ tests/parity/inputs/archaic-genotypes/s.g.vcf | 11 +++ tests/test_archaic_report.py | 72 ++++++++++++++++-- 34 files changed, 193 insertions(+), 21 deletions(-) create mode 100644 tests/parity/golden/archaic-genotypes/afr-one/gt/chr21.gt.gz create mode 100644 tests/parity/golden/archaic-genotypes/afr-one/gt/chr22.gt.gz create mode 100644 tests/parity/golden/archaic-genotypes/afr-one/gt/spans.tsv create mode 100644 tests/parity/golden/archaic-genotypes/afr-one/stderr create mode 100644 tests/parity/golden/archaic-genotypes/default/gt/chr21.gt.gz create mode 100644 tests/parity/golden/archaic-genotypes/default/gt/chr22.gt.gz create mode 100644 tests/parity/golden/archaic-genotypes/default/gt/spans.tsv create mode 100644 tests/parity/golden/archaic-genotypes/default/stderr create mode 100644 tests/parity/golden/archaic-genotypes/gvcf-order/stderr create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/exit create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Altai.mask.bed create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Denisova.mask.bed create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Vindija.mask.bed create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.gt.gz create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Altai.mask.bed create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Denisova.mask.bed create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Vindija.mask.bed create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.gt.gz create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/gt/samples.txt create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/gt/spans.tsv create mode 100644 tests/parity/golden/archaic-genotypes/gvcf/stderr create mode 100644 tests/parity/golden/archaic-genotypes/no-table-chrom/stderr create mode 100644 tests/parity/golden/archaic-genotypes/plain/gt/chr21.gt.gz create mode 100644 tests/parity/golden/archaic-genotypes/plain/gt/chr22.gt.gz create mode 100644 tests/parity/golden/archaic-genotypes/plain/gt/spans.tsv create mode 100644 tests/parity/golden/archaic-genotypes/plain/stderr create mode 100644 tests/parity/inputs/archaic-genotypes/order.g.vcf create mode 100644 tests/parity/inputs/archaic-genotypes/s.g.vcf diff --git a/main.nf b/main.nf index 78939b7c1..62de6de75 100644 --- a/main.nf +++ b/main.nf @@ -953,8 +953,10 @@ workflow { // ─── Archaic ancestry (opt-in: archaic, step 43) ──────────────────── // IBDmix against the Altai and Vindija Neanderthals and the Altai - // Denisovan, from the VCF and the table setup.sh --archaic builds; the - // genes of the segments come from --hla_genes (GENCODE's gene lines). + // Denisovan, from the VCF, the gVCF and the table setup.sh --archaic + // builds; the genes of the segments come from --hla_genes (GENCODE's gene + // lines). The gVCF's reference blocks tell a reference match from a site + // without reads, so a sample without a gVCF is skipped with a warning. // Equivalent to scripts/43-archaic.sh. ch_archaic = Channel.empty() ch_archaic_versions = Channel.empty() @@ -965,8 +967,19 @@ workflow { if (!(params.archaic_pop in ['ALL', 'AFR', 'AMR', 'EAS', 'EUR', 'SAS'])) { error "--archaic_pop must be ALL, AFR, AMR, EAS, EUR or SAS, got '${params.archaic_pop}'." } + ch_vcf + .map { meta, _vcf, _idx -> [meta.id, meta] } + .join(ch_gvcf.map { meta, _gvcf, _gidx -> [meta.id, true] }, remainder: true) + .filter { _id, meta, has_gvcf -> meta != null && !has_gvcf } + .subscribe { id, _meta, _has_gvcf -> + log.warn "archaic skipped for ${id}: it reads the gVCF's reference blocks, and the samplesheet row has none " + + "(add the gvcf and gvcf_index columns, or start from the BAM or FASTQ)." + } ARCHAIC_GENOTYPES( - ch_vcf.map { meta, vcf, idx -> [meta, vcf] }, + ch_vcf + .map { meta, vcf, _idx -> [meta.id, meta, vcf] } + .join(ch_gvcf.map { meta, gvcf, _gidx -> [meta.id, gvcf] }) + .map { _id, meta, vcf, gvcf -> [meta, vcf, gvcf] }, Channel.value(file(params.archaic_data, checkIfExists: true)) ) ARCHAIC_IBDMIX(ARCHAIC_GENOTYPES.out.genotypes) diff --git a/modules/local/archaic/main.nf b/modules/local/archaic/main.nf index bf68a4653..d9cf600cf 100644 --- a/modules/local/archaic/main.nf +++ b/modules/local/archaic/main.nf @@ -7,7 +7,9 @@ ARCHAIC_GENOTYPES og archaic genotypes: IBDmix's genotype file of each chromosome (the archaic genomes, the sample's VCF and the 1000 Genomes stand-ins of --archaic_pop), and each - archaic genome's mask + archaic genome's mask; a site with no call is the + reference only inside a reference block of the gVCF, + and unknown to IBDmix otherwise ARCHAIC_IBDMIX IBDmix against each archaic genome, LOD 4, the options of Chen et al. 2020, with that genome's mask ARCHAIC_REPORT og archaic report: the segments of 50 kb or @@ -22,7 +24,7 @@ process ARCHAIC_GENOTYPES { label 'process_single' input: - tuple val(meta), path(vcf) + tuple val(meta), path(vcf), path(gvcf) path(archaic_data) // the folder setup.sh --archaic writes output: @@ -34,7 +36,7 @@ process ARCHAIC_GENOTYPES { script: """ - og archaic genotypes --table-dir ${archaic_data} --vcf ${vcf} --pop ${params.archaic_pop} --out-dir gt + og archaic genotypes --table-dir ${archaic_data} --vcf ${vcf} --gvcf ${gvcf} --pop ${params.archaic_pop} --out-dir gt cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/nextflow_schema.json b/nextflow_schema.json index d162509ac..2e27db16a 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -104,7 +104,7 @@ "properties": { "tools": { "type": "string", - "description": "Comma-separated list of tools to run; an unknown name stops the run. DB-heavy tools (vep, slivar, clinical_filter, cpsr, clinvar, expansion_hunter, sample_qc) and extras (stranger, ancestry (needs --ancestry_ref and prs), ancestry_fine (needs --ancestry_fine_panel), ancient (needs --ancient_panel; samples with a gVCF), pypgx, SV callers, cram_archive, cyrius, parascopy, paraphase (PacBio HiFi BAMs only), y_haplogroup (male samples), y_str (Y-STR repeat counts, male samples), consumer_export (samples with a gVCF), bloodtype (blood group antigens; RhD needs the BAM), lpa (the LPA KIV-2 repeat count; needs the BAM), vntyper (MUC1 VNTR frameshifts behind ADTKD-MUC1; needs the BAM), viral (viral DNA in the unmapped reads; needs --viral_ref and a BAM), exomiser (runs when --hpo is set), chip (clonal haematopoiesis from the BAM; needs --chip_gff), archaic (needs --archaic_data), mei (Alu, L1 and SVA insertions with MEGAnE; needs --megane_kmers and a BAM), mocha (mosaic chromosomal alterations; needs --mocha_data), locityper (a second opinion at HLA class II, KIR, CYP2D6 and FCGR; samples with a BAM; needs --locityper_data)) are opt-in. prs and vcfanno are skipped with a warning until --pgs_scoring or a score file is set. With pharmcat, PharmCAT also gets the HLA types of hla_typing and a CYP2D6 call pypgx and cyrius agree on (PGX_CONSENSUS).", + "description": "Comma-separated list of tools to run; an unknown name stops the run. DB-heavy tools (vep, slivar, clinical_filter, cpsr, clinvar, expansion_hunter, sample_qc) and extras (stranger, ancestry (needs --ancestry_ref and prs), ancestry_fine (needs --ancestry_fine_panel), ancient (needs --ancient_panel; samples with a gVCF), pypgx, SV callers, cram_archive, cyrius, parascopy, paraphase (PacBio HiFi BAMs only), y_haplogroup (male samples), y_str (Y-STR repeat counts, male samples), consumer_export (samples with a gVCF), bloodtype (blood group antigens; RhD needs the BAM), lpa (the LPA KIV-2 repeat count; needs the BAM), vntyper (MUC1 VNTR frameshifts behind ADTKD-MUC1; needs the BAM), viral (viral DNA in the unmapped reads; needs --viral_ref and a BAM), exomiser (runs when --hpo is set), chip (clonal haematopoiesis from the BAM; needs --chip_gff), archaic (needs --archaic_data; samples with a gVCF), mei (Alu, L1 and SVA insertions with MEGAnE; needs --megane_kmers and a BAM), mocha (mosaic chromosomal alterations; needs --mocha_data), locityper (a second opinion at HLA class II, KIR, CYP2D6 and FCGR; samples with a BAM; needs --locityper_data)) are opt-in. prs and vcfanno are skipped with a warning until --pgs_scoring or a score file is set. With pharmcat, PharmCAT also gets the HLA types of hla_typing and a CYP2D6 call pypgx and cyrius agree on (PGX_CONSENSUS).", "default": "pharmcat,cpic,vcfanno,roh,prs,mito_haplogroup,telomere_hunter,mosdepth,mito_variants,traits,html_report,multiqc", "fa_icon": "fas fa-toolbox" } diff --git a/scripts/43-archaic.sh b/scripts/43-archaic.sh index 9a98b9392..af68c891a 100755 --- a/scripts/43-archaic.sh +++ b/scripts/43-archaic.sh @@ -8,10 +8,14 @@ # It needs the archaic data setup.sh --archaic builds (reference/archaic/): # 1. og archaic genotypes writes IBDmix's genotype file of each # chromosome: the archaic genotypes, yours from the VCF, and 100 -# stand-ins carrying the 1000 Genomes allele count of ARCHAIC_POP at each -# site (IBDmix takes the modern frequency from the modern columns, and -# one genome alone gives it none; docs/43-archaic.md says why this is -# the same score); +# stand-ins carrying round(AF x 200) ALT alleles of ARCHAIC_POP at each +# site (IBDmix takes the modern frequency from the modern columns, yours +# included, and one genome alone gives it none; docs/43-archaic.md says +# what that does to the score). A site the VCF has no record for is the +# reference only inside a reference block of the gVCF of step 3 +# (vcf/.g.vcf.gz), and unknown to IBDmix otherwise: a +# variant-only VCF cannot tell a reference match from a site without +# reads; # 2. IBDmix scores your column against each archaic genome (LOD 4); # 3. og archaic report keeps the segments of 50 kb or more, # merges the two Neanderthals, and writes the share, the segments and @@ -35,6 +39,7 @@ DATA=${ARCHAIC_DIR:-${GENOME_DIR}/reference/archaic/${ARCHAIC_DATA_VERSION}} POP=${ARCHAIC_POP:-ALL} S="${GENOME_DIR}/${SAMPLE}" VCF="${S}/vcf/${SAMPLE}.vcf.gz" +GVCF="${S}/vcf/${SAMPLE}.g.vcf.gz" OUT="${S}/archaic" WORK="${OUT}/work" @@ -44,6 +49,7 @@ if [ "${ARCHAIC_DIR:-}" = none ] || [ ! -s "${DATA}/DONE" ]; then fi case "$POP" in ALL|AFR|AMR|EAS|EUR|SAS) ;; *) echo "ERROR: ARCHAIC_POP must be ALL, AFR, AMR, EAS, EUR or SAS, got '${POP}'" >&2; exit 2 ;; esac [ -f "$VCF" ] || { echo "ERROR: File not found: ${VCF}" >&2; exit 1; } +[ -f "$GVCF" ] || { echo "ERROR: File not found: ${GVCF}. Step 43 reads the gVCF of step 3 beside the VCF (a variant-only VCF cannot tell a reference match from a site without reads)." >&2; exit 1; } D=$(cpath "$DATA") echo "============================================" @@ -51,6 +57,7 @@ echo " Step 43: Archaic ancestry (IBDmix)" echo " Archaic genomes: Altai and Vindija 33.19 Neanderthals, Altai Denisovan" echo " Modern frequencies: 1000 Genomes ${POP}" echo " Sample: ${SAMPLE}" +echo " Input: ${VCF} and ${GVCF}" echo " Output: ${OUT}" echo "============================================" echo "" @@ -59,10 +66,10 @@ rm -rf "$WORK" "${OUT}/${SAMPLE}_archaic".* "${OUT}/${SAMPLE}_archaic_segments.t mkdir -p "${WORK}/ibdmix" W=$(cpath "$WORK") -echo "[1/3] Writing IBDmix's genotype files (your VCF, the archaic genomes, the stand-ins)..." +echo "[1/3] Writing IBDmix's genotype files (your VCF and gVCF, the archaic genomes, the stand-ins)..." run_in --cpus 1 --memory 4g -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$PYTHON_IMAGE" \ /pgp-bin/og archaic genotypes --table-dir "$D" --vcf "/genome/${SAMPLE}/vcf/${SAMPLE}.vcf.gz" \ - --pop "$POP" --out-dir "${W}/gt" + --gvcf "/genome/${SAMPLE}/vcf/${SAMPLE}.g.vcf.gz" --pop "$POP" --out-dir "${W}/gt" # One IBDmix run per chromosome and archaic genome, THREADS at a time, with the # options of Chen et al. 2020 (their j.cell.2020.01.012-workflow/config.yaml), diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 7fbfa6087..8303b81bc 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -1014,6 +1014,8 @@ archaic-genotypes bad-code gen:archaic-inputs.sh --table-dir badcode --vcf s.vcf archaic-genotypes bad-pop gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pop XYZ --out-dir gt - bytes PYTHON_IMAGE archaic-genotypes bad-pseudo gen:archaic-inputs.sh --table-dir . --vcf s.vcf --pseudo 0 --out-dir gt - bytes PYTHON_IMAGE archaic-genotypes no-table gen:archaic-inputs.sh --table-dir emptydir --vcf s.vcf --out-dir gt - bytes PYTHON_IMAGE +archaic-genotypes gvcf gen:archaic-inputs.sh,archaic-genotypes/s.g.vcf --table-dir . --vcf s.vcf --gvcf s.g.vcf --pop EUR --pseudo 3 --out-dir gt gt/samples.txt,gt/spans.tsv,gt/chr21.gt.gz,gt/chr22.gt.gz *.gz=gz-content bytes PYTHON_IMAGE +archaic-genotypes gvcf-order gen:archaic-inputs.sh,archaic-genotypes/order.g.vcf --table-dir . --vcf s.vcf --gvcf order.g.vcf --out-dir gt - *.gz=gz-content bytes PYTHON_IMAGE archaic-report genes gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans spans.tsv --genes genes.gtf --pop EUR --out-dir rep rep/S_archaic.tsv,rep/S_archaic.txt,rep/S_archaic_segments.tsv bytes PYTHON_IMAGE archaic-report no-genes gen:archaic-inputs.sh --sample S --ibdmix-dir . --spans spans.tsv --out-dir rep rep/S_archaic.tsv,rep/S_archaic.txt,rep/S_archaic_segments.tsv bytes PYTHON_IMAGE archaic-report missing-output gen:archaic-inputs.sh --sample S --ibdmix-dir rep --spans spans.tsv --out-dir rep - bytes PYTHON_IMAGE diff --git a/tests/parity/golden/archaic-genotypes/afr-one/gt/chr21.gt.gz b/tests/parity/golden/archaic-genotypes/afr-one/gt/chr21.gt.gz new file mode 100644 index 0000000000000000000000000000000000000000..a815987d49f01ff27fd7aaa44b8346bee09b7378 GIT binary patch literal 332 zcmV-S0ki%eiwFP!00004|4ow1PQx$|Mc2dE%(%R=v<&J7q)JrmBPb%Mw5YB4{vP`# zX;rzBC*w2sIQjTK?0@ZVf3(Bq%lhrf^7hmZ_CD-Bho9b_HoM`t|L^TBzph@MZRH*} zDJ(qPiY+b7S;@v~a%yX1Zyi5Z@4{_unZ4>`_138s&O7~`jn%t}8$9VOiEux0Q|le8 z58^uHw$%G5Zap9MiK?BrO2B`shKtJ_yYvQbHLsH(&4j=$;~aF>X)+Gn3eGiGU9t$= zmN?hGwlod5$_=wRemBw|ba}_sn<2o5`;wqWmX2LDQRWj(! eIN?^{*7E^bc^9}VgFhHiBPu?>)$`3J zc-TOCduyt)d3gESz1!#R+rFNjZS(eH_3dY~wfpVI?4$|~g1c%Wt zP0vKg&B!~NE+WVwhFQ4G{yPK}Y34f1UfqppPd_2H0p4|H`ZM$E~)EQY!3nz z$z{t9W^p24nPDO>ZOP?3yU;bILWE?jzfPf|V#XE@%V;Z9G|c3{ekT+v(v0tLm*HEe zNUj|C)Giejt{iwmQz}~LI;X`-ph8?G`{F`{xakH*gz<)Cwx80L+o=RINCI2k zSO2Jqii(+YQrW-biHe3icX_KTqc-lI*&IL`7~1zX4=G++n7?3gy;^-Fq^5Un8{&3rJ`XbhZt{urXtN8 vLKY7$72-ypk0}+kbJKx4y<(=K!S!^-Srx}rB$q8_Zsq6?b6wz$DFgrj5`^H0 literal 0 HcmV?d00001 diff --git a/tests/parity/golden/archaic-genotypes/afr-one/gt/spans.tsv b/tests/parity/golden/archaic-genotypes/afr-one/gt/spans.tsv new file mode 100644 index 000000000..dcf5acd63 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/afr-one/gt/spans.tsv @@ -0,0 +1,3 @@ +chrom first last sites sites_with_your_alt sites_other_allele +chr21 1010 1290 6 6 0 +chr22 2010 2500 26 21 4 diff --git a/tests/parity/golden/archaic-genotypes/afr-one/stderr b/tests/parity/golden/archaic-genotypes/afr-one/stderr new file mode 100644 index 000000000..c5b08c8d6 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/afr-one/stderr @@ -0,0 +1 @@ +2 chromosomes, 32 sites read, 44 not read (unknown to IBDmix) diff --git a/tests/parity/golden/archaic-genotypes/default/gt/chr21.gt.gz b/tests/parity/golden/archaic-genotypes/default/gt/chr21.gt.gz new file mode 100644 index 0000000000000000000000000000000000000000..0d9988f8eb4bce97b1ae380e3e626b0571de4aec GIT binary patch literal 694 zcmb2|=3oE;mj8EWoXrz46lgjBEc4B`KR+#`0`wGIlaAK1I_hvmPI0lZ|GKmMsNn?B z%~IN%o=@_BEEQf_{^{rK#~l;wGPgt zCM1d}P=U*(QA2=rk%LGFQ=rQdiTf|if_w~h9;z)7*s3#QUg7W4TTd37C#w5uiu_&h zu717fzPZ`KR%MCnRt0Q7RDHdmH9NR3eW#GGTC0t8#5!Nm^p%y$g{i&yyN+4M%+6Z; zymyM|)4u1|uCMf4)irs0#NQiTW}mBr_s^VaE^{+RH!wYBiaEE&;cniLNe|}v7<%3G z>-)2quXTOA#e-M(msc5O>V4efdUfKqS1FfbjIQrdy_%MF^6LJb=L$bc{<&u|U8m^j zYug|1pBz=MkKCJmImW5p`g?=uz2Bcs>iy@vzHDd9o`)VuTXp%RZT>88n|x2sywj7H?Mzu$>1_C2Y-!== z)%Rv+>P1d5f6w@0cS+&@@c(A(e2Y0NAG@sm$^f+T-Z|R?Q3X$4z6WxQwDVPZt9M_S zSbuo^gNW}-o=sZ%O!u1H22e<~0p0hg(z_}>+18=B^Et1%=`!EqlWX==@+*{P-1+rs r=4qgRPD%Iqc-^@cI`>@s=V_+XCz!AP&z*IA@_xpY9~}yBq!}0htTs%L literal 0 HcmV?d00001 diff --git a/tests/parity/golden/archaic-genotypes/default/gt/chr22.gt.gz b/tests/parity/golden/archaic-genotypes/default/gt/chr22.gt.gz new file mode 100644 index 0000000000000000000000000000000000000000..f95e23eac050e5bfac65d000562295f4e60d529a GIT binary patch literal 961 zcmb2|=3oE;mj8EWoX!(56mY$M?)r{9{`%3pr`VQwxV&)}II>{E0+mO*Wr$Km2=k_x6WsZpmIgqQPgc7;N>C4L(1`G%}^hfTJ0XJOntE99244CMpQIGyyq_ znpP-ib#Nh3SshY~+EyqcQH>c2QXOoI5L8r$*rL`9MI@^Eg@RQF-y#H6)gia2{e>ce zQgTk|;F+jsxzOY)2R zS-+jzdQx_K;)Hb~F?X4?v-#8Gj&419ym{B8t&>DP?QDK{tK?D7(R>TNv-=mHUi08q z$)$^vH(t@vf3R)m-I#grZAsz2spgUgq~E?>2XyU^v*$g36@9gxzvxn&dFpqC{SQsJ%^dwV**|xm z^XTn)dypfKKiaMlTQ={;x&OP&Qn&xkHQ%+O0ch;*`!5#HeR}()IqP|bnD%39`S(v( zx$FHwS~j!j&b83F_x68DS^8#~`bvqP(_UPA%YS{@8@VZrfmJ`=eqHk0Udx01!rOEQZ#!zVC?v0L2gqd)zAiF0U*CNT#HssSd`a}2t^S5;U^v>I$XDCCZTq~Cvi?ue zlh&=-cwk z>b>bvYoH^(ui!b5sdj#;=Da=sjDd#F=2;MR`Oeh;ch0Uc{alcHD)0O3%X#TmIuKalPIKy)~K)3;^vz B@NobD literal 0 HcmV?d00001 diff --git a/tests/parity/golden/archaic-genotypes/default/gt/spans.tsv b/tests/parity/golden/archaic-genotypes/default/gt/spans.tsv new file mode 100644 index 000000000..dcf5acd63 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/default/gt/spans.tsv @@ -0,0 +1,3 @@ +chrom first last sites sites_with_your_alt sites_other_allele +chr21 1010 1290 6 6 0 +chr22 2010 2500 26 21 4 diff --git a/tests/parity/golden/archaic-genotypes/default/stderr b/tests/parity/golden/archaic-genotypes/default/stderr new file mode 100644 index 000000000..c5b08c8d6 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/default/stderr @@ -0,0 +1 @@ +2 chromosomes, 32 sites read, 44 not read (unknown to IBDmix) diff --git a/tests/parity/golden/archaic-genotypes/gvcf-order/stderr b/tests/parity/golden/archaic-genotypes/gvcf-order/stderr new file mode 100644 index 000000000..259f51299 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf-order/stderr @@ -0,0 +1 @@ +ERROR: order.g.vcf lists chr22 before a chromosome the VCF lists first; give the gVCF and the VCF in one chromosome order diff --git a/tests/parity/golden/archaic-genotypes/gvcf/exit b/tests/parity/golden/archaic-genotypes/gvcf/exit new file mode 100644 index 000000000..573541ac9 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf/exit @@ -0,0 +1 @@ +0 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Altai.mask.bed b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Altai.mask.bed new file mode 100644 index 000000000..b595cf405 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Altai.mask.bed @@ -0,0 +1,4 @@ +21 1039 1060 +21 1109 1130 +21 1179 1200 +21 1249 1270 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Denisova.mask.bed b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Denisova.mask.bed new file mode 100644 index 000000000..92b0d8f12 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Denisova.mask.bed @@ -0,0 +1,9 @@ +21 1009 1010 +21 1039 1050 +21 1079 1080 +21 1109 1120 +21 1149 1150 +21 1179 1190 +21 1219 1220 +21 1249 1260 +21 1289 1290 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Vindija.mask.bed b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Vindija.mask.bed new file mode 100644 index 000000000..94fa2937a --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.Vindija.mask.bed @@ -0,0 +1,5 @@ +21 1009 1030 +21 1079 1100 +21 1149 1170 +21 1219 1240 +21 1289 1300 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.gt.gz b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr21.gt.gz new file mode 100644 index 0000000000000000000000000000000000000000..043acafc07ab0abe08d964301b1eb4f09a5a56cf GIT binary patch literal 365 zcmV-z0h0b7iwFP!00004|5cLBPQyS9gwN(_Y?r?rR}10-QY9+x5fl+rTGUoNKcCm# zsyXZ`^2GB^MtT3*?|$TWcaZ)1Q^w6v%H}vu@-}ThrteW6*4yc@`y1u8yeyv|WvOa$ zd0dMtH!(GpDox$Cnvn(Fz=~{^b?uSCwJ)JiGF(Grlvv%!7+RyU2{x@4ho!1ec6uDL4}j4!;7ul zp@ZZoUC(ytAUOu_R5+R1+3OsqgW!bMJhjt7aGZ9AS#M4U1x^y; zhSyR%9n{9rmYTWbU*&X=9D_bfO$W(|w`U3+6gZhWIOo)KP~n7^K~g;POBZwyoO$Yh LRsbxbO9KD^t(~uR literal 0 HcmV?d00001 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Altai.mask.bed b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Altai.mask.bed new file mode 100644 index 000000000..e56e6b80f --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Altai.mask.bed @@ -0,0 +1,7 @@ +22 2039 2050 +22 2109 2130 +22 2189 2200 +22 2249 2270 +22 2319 2340 +22 2389 2410 +22 2459 2480 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Denisova.mask.bed b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Denisova.mask.bed new file mode 100644 index 000000000..e366fc41b --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Denisova.mask.bed @@ -0,0 +1,15 @@ +22 2009 2010 +22 2039 2050 +22 2079 2080 +22 2109 2120 +22 2149 2150 +22 2189 2190 +22 2219 2220 +22 2249 2260 +22 2289 2290 +22 2319 2330 +22 2359 2360 +22 2389 2400 +22 2429 2430 +22 2459 2470 +22 2499 2500 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Vindija.mask.bed b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Vindija.mask.bed new file mode 100644 index 000000000..7e55d6902 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.Vindija.mask.bed @@ -0,0 +1,8 @@ +22 2009 2030 +22 2079 2100 +22 2149 2170 +22 2219 2240 +22 2289 2310 +22 2359 2380 +22 2429 2450 +22 2499 2500 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.gt.gz b/tests/parity/golden/archaic-genotypes/gvcf/gt/chr22.gt.gz new file mode 100644 index 0000000000000000000000000000000000000000..6113c674a4753a344b11318a4effa40980fdd93b GIT binary patch literal 512 zcmV+b0{{IViwFP!00004|5cOCZq!f=h1c%W*f&!k%P633K&nK=egs7fDjn2Teg1yu zo@8{paH9Iy=f~;%{PT4F7yr-C@pS(?jt?)fJiHuF@#FOP>-6^+pYI<}&*#@;d|$rZ ze*F@+nWab5W4dZBT2{syH&Kz4o*8MRCH=c-^k-0pqZyep?qZ2~!v^G#^jD@6Oj{SW z>s_Zn8+AcGnl0g@HzJ?Rrj6QBJDAPZLj6LEnT}N2wY}xQRwR{c_2gJ@CoPgLT%lmg zUb)?%NV;+b%}cAltwYiquCTiv5Ryu#(3U&6cD4`BBc}OXp5g}5v1@U$WgU{5O?-4= zwd+RG1=pHxu!(g;NV?+M-nPBY)<`PX>Zv-P5t2$rEOId*Bwf%^x42H%^A1VZu9N)> z3G40LZC~52rG^LZa0BUxt58bubz~x`*fcFv?tO!aqzkS!tLx~! zAtYUKU1Z5#xqY9Bq;f51O&TiekW@PMX57DpiKGiU#caXWG4GIcMW@l7DO7eHl1gWH zJs>2tPLfkNSZ6zIU%NWfjM@d^2G;4?BSKQK6+IOjn}Y*M7hK!hJ2sN8xR!QxEw=1{ zq;k#QcuNXBBP6v>p+r}h5t1%lM_r+88zJe6jxRbQ*CdsWda4FzUcCV!oo1Zt1ONc< CuKue4 literal 0 HcmV?d00001 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/gt/samples.txt b/tests/parity/golden/archaic-genotypes/gvcf/gt/samples.txt new file mode 100644 index 000000000..5d70469ba --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf/gt/samples.txt @@ -0,0 +1,4 @@ +SAMPLE +P001 +P002 +P003 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/gt/spans.tsv b/tests/parity/golden/archaic-genotypes/gvcf/gt/spans.tsv new file mode 100644 index 000000000..eef401ec3 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf/gt/spans.tsv @@ -0,0 +1,3 @@ +chrom first last sites sites_with_your_alt sites_other_allele +chr21 1010 1290 13 6 0 +chr22 2010 2500 34 21 4 diff --git a/tests/parity/golden/archaic-genotypes/gvcf/stderr b/tests/parity/golden/archaic-genotypes/gvcf/stderr new file mode 100644 index 000000000..2cb4e17fe --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/gvcf/stderr @@ -0,0 +1 @@ +2 chromosomes, 47 sites read, 29 not read (unknown to IBDmix) diff --git a/tests/parity/golden/archaic-genotypes/no-table-chrom/stderr b/tests/parity/golden/archaic-genotypes/no-table-chrom/stderr new file mode 100644 index 000000000..ff2124a91 --- /dev/null +++ b/tests/parity/golden/archaic-genotypes/no-table-chrom/stderr @@ -0,0 +1 @@ +ERROR: none.vcf has no site read on a chromosome of the archaic table (chr21, chr22): no call at a table site (without --gvcf a site with no call is unknown) diff --git a/tests/parity/golden/archaic-genotypes/plain/gt/chr21.gt.gz b/tests/parity/golden/archaic-genotypes/plain/gt/chr21.gt.gz new file mode 100644 index 0000000000000000000000000000000000000000..b1d14e4f356cd416f7558ff50a98f1baf98f37ff GIT binary patch literal 367 zcmV-#0g(P5iwFP!00004|6P*JPQyS92G8bcY?r?rR{?PWsS*|U2#N?QEov*CpYb{= zT@Kku_O9pKNnStpyD$0P9b~_Lmtk|1vN;Z;yo}qo@pF*7^>#e$eg=6iPs_)9S*ltr z9?RmwO-xOtmj^LX9%oh2?6^{2iBs>SW+zXJ#(1C8IN9aZjz%^^I=|gZj%$UA_4W8(M@Z$^^9q9;wI7iJ7YMA`$VNp zZ8#|SF{ZT%3q)uoY>uTa8Tf6>fn$eIXLLy#4LlP%*>lEa1fj} N^)H1NYT-)*005_dv0DHD literal 0 HcmV?d00001 diff --git a/tests/parity/golden/archaic-genotypes/plain/gt/chr22.gt.gz b/tests/parity/golden/archaic-genotypes/plain/gt/chr22.gt.gz new file mode 100644 index 0000000000000000000000000000000000000000..a08708fe7f73b2a659751d6eef23092e0e659225 GIT binary patch literal 514 zcmV+d0{#6TiwFP!00004|6P;8Zqz^!M9-bC=^k$YA4dk|0#YO*?kfmlQOKes%J=70 z_t^60;I+~1VoX&6QkpZrv&MWz<*>C!Ims>N*nx7YwY<1hIgpl{ z{Pr7dkak+@bi+-q#nT{NXkFcOZ*^;ssjeC{D4wNIWAj##K$^@z0bXq)WUptr}?X=ce zZAb5+K)TR6VDWr>bekYmYc+d!2U6u!x6VS==i6LbpPQmbx|F65}wj6%wq1F3SV zb(#ml-rK6_XVSMdKr +##FORMAT= +##FORMAT= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT S +chr22 2000 . N <*> 0 . END=2600 GT:GQ:MIN_DP 0/0:40:25 +chr21 1000 . N <*> 0 . END=1150 GT:GQ:MIN_DP 0/0:30:20 +chr21 1151 . N <*> 0 . END=1200 GT:GQ:MIN_DP 0/0:0:5 +chr21 1201 . N <*> 0 . END=1250 GT:GQ:MIN_DP ./.:0:0 diff --git a/tests/parity/inputs/archaic-genotypes/s.g.vcf b/tests/parity/inputs/archaic-genotypes/s.g.vcf new file mode 100644 index 000000000..f0bfb94f9 --- /dev/null +++ b/tests/parity/inputs/archaic-genotypes/s.g.vcf @@ -0,0 +1,11 @@ +##fileformat=VCFv4.2 +##INFO= +##FORMAT= +##FORMAT= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT S +chr1 1 . N <*> 0 . END=100 GT:GQ:MIN_DP 0/0:50:30 +chr21 1000 . N <*> 0 . END=1150 GT:GQ:MIN_DP 0/0:30:20 +chr21 1151 . N <*> 0 . END=1200 GT:GQ:MIN_DP 0/0:0:5 +chr21 1201 . N <*> 0 . END=1250 GT:GQ:MIN_DP ./.:0:0 +chr22 2000 . N <*> 0 . END=2600 GT:GQ:MIN_DP 0/0:40:25 diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py index cffe87e77..31a89956b 100644 --- a/tests/test_archaic_report.py +++ b/tests/test_archaic_report.py @@ -14,7 +14,9 @@ 2. genotypes: a masked archaic call (10 to 12) goes to IBDmix as its ALT count and its position into that genome's mask file; the stand-in columns carry the population's ALT count; your column is your ALT - count, 0 where your VCF has no record, 9 for a non-PASS record; a site where your VCF has another ALT is left out; a + count, 9 for a non-PASS record, and where your VCF has no record 0 + inside a reference block of the gVCF and 9 (unknown) outside every + one; the span runs over the sites read; a site where your VCF has another ALT is left out; a chromosome your VCF has no record on is not analysed; in a split VCF an SNV keeps its call beside an insertion, a multi-allelic record or the 0/0 half of a split site, in either order, a usable record of the same @@ -33,7 +35,11 @@ where the person contradicts both Neanderthals inside the planted stretch and their calls there are masked, the segment stops at the masked mismatches, while the same calls written as unknown (9, the - table before the fix) let it run through. + table before the fix) let it run through; a stretch where the person + has no call matches archaic genomes that carry the common reference + allele, and gives a segment only when a gVCF reference block says the + stretch was read (the step before that fix read it as the reference + either way). Set ARCHAIC_TEST_NO_DOCKER=1 to skip part 4 on a machine without docker; CI never sets it. @@ -224,7 +230,17 @@ def table_info(out, extra=()): "chr22\t3000\t.\tG\tA\t50\tRefCall\t.\tGT\t1/1\n" "chr22\t5000\t.\tA\tT\t50\tPASS\t.\tGT\t1/1\n") write(f"{TMP}/s.vcf", vcf) -r = run("genotypes", "--table-dir", tdir, "--vcf", f"{TMP}/s.vcf", "--pop", "EUR", "--pseudo", "10", "--out-dir", f"{TMP}/gt") +GVCF_HEAD = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + + +def ref_block(chrom, start, end): + return f"{chrom}\t{start}\t.\tN\t<*>\t0\t.\tEND={end}\tGT:GQ:MIN_DP\t0/0:50:30\n" + + +# A reference block over 1-2500: 2000 was read as the reference; 4000 was not read. +write(f"{TMP}/s.g.vcf", GVCF_HEAD + ref_block("chr22", 1, 2500)) +r = run("genotypes", "--table-dir", tdir, "--vcf", f"{TMP}/s.vcf", "--gvcf", f"{TMP}/s.g.vcf", + "--pop", "EUR", "--pseudo", "10", "--out-dir", f"{TMP}/gt") check("genotypes exits 0", r.returncode == 0, r.stderr) lines = gzip.open(f"{TMP}/gt/chr22.gt.gz", "rt").read().splitlines() g = {int(l.split("\t")[1]): l.split("\t") for l in lines[1:]} @@ -233,7 +249,8 @@ def table_info(out, extra=()): and len(lines[0].split("\t")) == 18, lines[0]) check("the chromosome is an integer for IBDmix", g[1000][0] == "22", g[1000][0]) check("your heterozygous call is 1", g[1000][7] == "1", g[1000]) -check("no record is 0 (reference)", g[2000][7] == "0", g[2000]) +check("no record inside a gVCF reference block is 0 (reference)", g[2000][7] == "0", g[2000]) +check("no record outside every reference block is 9 (unknown)", g[4000][7] == "9", g[4000]) check("a non-PASS record is 9", g[3000][7] == "9", g[3000]) check("a site where you carry another ALT is left out", 5000 not in g, sorted(g)) check("stand-ins carry EUR's ALT count (0.5 of 20 alleles = 10)", sum(int(x) for x in g[1000][8:]) == 10, g[1000][8:]) @@ -241,7 +258,7 @@ def table_info(out, extra=()): check("each stand-in column is one character", all(len(x) == 1 for x in g[1000][4:]), g[1000]) check("a chromosome the VCF has no record on is not analysed", not os.path.exists(f"{TMP}/gt/chr21.gt.gz")) spans = open(f"{TMP}/gt/spans.tsv").read().splitlines() -check("spans: chr22 from 1000 to 4000, four sites", spans[1].split("\t")[:4] == ["chr22", "1000", "4000", "4"], spans) +check("spans: the sites read, chr22 from 1000 to 2000, two sites", spans[1].split("\t")[:4] == ["chr22", "1000", "2000", "2"], spans) samples = open(f"{TMP}/gt/samples.txt").read().split() check("samples.txt: SAMPLE and the stand-ins", samples[0] == "SAMPLE" and len(samples) == 11, samples) write(f"{TMP}/two.vcf", "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n") @@ -296,7 +313,9 @@ def table_info(out, extra=()): f.write(hdr + "".join(f"chr22\t{p}\t{ra}\t{c}\t0.2\t0\t0\t0\t0\t0\n" for p, (ra, c) in mrows.items())) write(f"{TMP}/m.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" "chr22\t500\t.\tA\tT\t50\tPASS\t.\tGT\t0/1\n") # another ALT: row 500 is left out -r = run("genotypes", "--table-dir", mdir, "--vcf", f"{TMP}/m.vcf", "--pseudo", "10", "--out-dir", f"{TMP}/gtm") +write(f"{TMP}/m.g.vcf", GVCF_HEAD + ref_block("chr22", 1, 1000)) +r = run("genotypes", "--table-dir", mdir, "--vcf", f"{TMP}/m.vcf", "--gvcf", f"{TMP}/m.g.vcf", "--pseudo", "10", + "--out-dir", f"{TMP}/gtm") check("genotypes on a table with masked calls exits 0", r.returncode == 0, r.stderr) gm = {int(l.split("\t")[1]): l.split("\t")[4:7] for l in gzip.open(f"{TMP}/gtm/chr22.gt.gz", "rt").read().splitlines()[1:]} check("IBDmix gets each masked call as its ALT count (12, 10, 11 are 2, 0, 1)", @@ -461,8 +480,12 @@ def synthetic(d, plant, mid=None): write(f"{d}/s.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + "\n".join(calls) + "\n") -def ibdmix_run(d): - r = run("genotypes", "--table-dir", f"{d}/table", "--vcf", f"{d}/s.vcf", "--out-dir", f"{d}/gt") +def ibdmix_run(d, gvcf=None): + if gvcf is None: + gvcf = GVCF_HEAD + ref_block("chr22", 1, 6000000) # every site read + write(f"{d}/s.g.vcf", gvcf) + r = run("genotypes", "--table-dir", f"{d}/table", "--vcf", f"{d}/s.vcf", "--gvcf", f"{d}/s.g.vcf", + "--out-dir", f"{d}/gt") if r.returncode: return r os.makedirs(f"{d}/ibd") @@ -529,6 +552,39 @@ def ibdmix_run(d): check("calls written as masked: IBDmix's mask keeps the mismatches, and no segment covers the middle", "masked" in cover_mid and cover_mid["masked"] <= 0.1 * span, cover_mid) + # A stretch with no call (lin-01 of the 1.0.0 review). 150 sites where + # all three archaic genomes carry the reference allele and modern people + # carry the ALT at 0.9, and a person with one call far away: read as the + # reference, the stretch matches the archaic genomes. Control: with a + # reference block over it (the person was read and matches the + # reference) IBDmix calls a segment there. With no block over it, + # nothing was read, and no segment may come out. + gap_lo, gap_hi = 200000, 349000 + gap_segment = {} + for covered in (True, False): + d = f"{TMP}/real_gap_{covered}" + os.makedirs(f"{d}/table") + os.chmod(d, 0o777) + with gzip.open(f"{d}/table/chr22.tsv.gz", "wt") as f: + f.write("\t".join(TABLE_HEADER) + "\n" + "".join( + f"chr22\t{p}\tC\tT\t0\t0\t0\t" + "\t".join(["0.9"] * 6) + "\n" + for p in range(gap_lo, gap_hi + 1, 1000))) + write(f"{d}/s.vcf", GVCF_HEAD + "chr22\t10000000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n") + # Without the block the person has one read site left, at the start. + block = ref_block("chr22", 1, 400000) if covered else ref_block("chr22", 1, gap_lo) + r = ibdmix_run(d, GVCF_HEAD + block) + check(f"IBDmix chain exits 0 (stretch {'read' if covered else 'with no call'})", r.returncode == 0, + r.stderr + r.stdout) + if r.returncode: + continue + body = [l.split("\t") for l in open(f"{d}/out/S_archaic_segments.tsv").read().splitlines()[1:]] + print(f" segments (stretch {'read' if covered else 'with no call'}): {[(b[1], b[2], b[4]) for b in body]}") + gap_segment[covered] = sum(int(b[3]) for b in body if b[4] == "Neanderthal") + check("control: a stretch read as the reference gives a Neanderthal segment of 100 kb or more", + gap_segment.get(True, 0) >= 100000, gap_segment) + check("a stretch with no call and no reference block gives no segment (it scored 149 kb when read as the reference)", + gap_segment.get(False, 1) == 0, gap_segment) + shutil.rmtree(TMP, ignore_errors=True) print(f"{FAILS} check(s) failed" if FAILS else "all checks passed") sys.exit(1 if FAILS else 0) From 8ecac42a0994c00b227da0eba4b363bdffd3e53c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:41:18 +0100 Subject: [PATCH 123/255] test(archaic): give the e2e samples a gVCF of reference blocks HG002 and the plant get GIAB's confident regions on chr22, COMMON one block over the chromosome, and the Nextflow samplesheet names the gVCF. The case checks that sites outside the confident regions are not read. Bead: lin-01 --- tests/e2e/archaic-ancestry-1.sh | 49 +++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/tests/e2e/archaic-ancestry-1.sh b/tests/e2e/archaic-ancestry-1.sh index 4baa9348e..1f9b813b4 100755 --- a/tests/e2e/archaic-ancestry-1.sh +++ b/tests/e2e/archaic-ancestry-1.sh @@ -8,8 +8,14 @@ # 1000 Genomes GRCh38 strict mask, in a segmental duplication or within # 5 bp of a 1000 Genomes indel is marked masked (code 10 to 12), which # step 43 hands IBDmix as its mask; +# - every sample gets a gVCF of reference blocks beside its VCF, since +# step 43 reads a site with no call as the reference only inside one: +# HG002 and PLANT get GIAB's v4.2.1 confident regions on chr22 (outside +# them GIAB makes no claim, so a site there is unknown, not a match), +# COMMON one block over the chromosome; # - HG002 (GIAB's v4.2.1 genotypes on chr22): the step runs, the shares are -# numbers, the analysed span is most of the chromosome, and the +# numbers, the table sites outside the confident regions are not read, +# the analysed span is most of the chromosome, and the # Neanderthal total lies between the lowest value (51 kb) of the range # Chen et al.'s own released calls give Europeans on chr22 and a 1,200 kb # regression guard (the guard that catches the unmasked table calling @@ -30,6 +36,7 @@ G="$GENOME_DIR" T="$CASE_TMP" DATA="${G}/reference/archaic/${ARCHAIC_DATA_VERSION}" TRUTH="${FIXTURE_DIR}/HG002_GRCh38_1_22_v4.2.1_benchmark.vcf.gz" +CONFIDENT="${FIXTURE_DIR}/HG002_GRCh38_1_22_v4.2.1_benchmark_noinconsistent.bed" W_FROM=28000000 W_TO=29000000 dk() { local img=$1; shift; docker run --rm -u "$(id -u):$(id -g)" -e HOME=/tmp -v "${E2E_WORK}:${E2E_WORK}" -w "$T" "$img" "$@"; } # kv SAMPLE KEY: one value of the sample's summary @@ -40,7 +47,25 @@ covered() { s = ($2 > f ? $2 : f); e = ($3 < t ? $3 : t); if (e > s) n += e - s } END {print n + 0}' \ "${G}/$1/archaic/$1_archaic_segments.tsv" 2>/dev/null } -sample() { rm -rf "${G:?}/$1"; mkdir -p "${G}/$1/vcf"; cp "$2" "${G}/$1/vcf/$1.vcf.gz"; dk "$BCFTOOLS_IMAGE" bcftools index -t -f "${G}/$1/vcf/$1.vcf.gz"; } +# gvcf BED OUT: a one-sample gVCF with a confident reference block over each +# chr22 interval of the BED (0-based, half-open). +gvcf() { + { printf '##fileformat=VCFv4.2\n##contig=\n##ALT=\n' + printf '##INFO=\n' + printf '##FORMAT=\n##FORMAT=\n' + printf '##FORMAT=\n' + printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n' + awk -F'\t' 'BEGIN {OFS = "\t"} $1 == "chr22" {print "chr22", $2 + 1, ".", "N", "<*>", 0, ".", "END=" $3, "GT:GQ:MIN_DP", "0/0:50:30"}' "$1" | sort -k2,2n + } > "${2%.gz}" + dk "$BCFTOOLS_IMAGE" bcftools view -Oz -o "$2" "${2%.gz}" +} +# sample NAME VCF GVCF: the sample's VCF and gVCF, indexed, where step 43 reads them. +sample() { + rm -rf "${G:?}/$1"; mkdir -p "${G}/$1/vcf" + cp "$2" "${G}/$1/vcf/$1.vcf.gz"; cp "$3" "${G}/$1/vcf/$1.g.vcf.gz" + dk "$BCFTOOLS_IMAGE" bcftools index -t -f "${G}/$1/vcf/$1.vcf.gz" + dk "$BCFTOOLS_IMAGE" bcftools index -t -f "${G}/$1/vcf/$1.g.vcf.gz" +} step() { run_step 43-archaic.sh "$1"; check_step_exit "43-archaic.sh ($1)"; } GENES=$(bash -c '. "$1/scripts/lib/common.sh" && install_data_file gencode_genes >/dev/null && data_file gencode_genes' _ "$REPO") @@ -112,9 +137,17 @@ printf '#### Step 43 archaic table (GRCh37 chr22)\n\n%s s to install, %s on disk # --- HG002 ------------------------------------------------------------------------------ dk "$BCFTOOLS_IMAGE" bcftools view -r chr22 -Oz -o "${T}/hg002.vcf.gz" "$TRUTH" +gvcf "$CONFIDENT" "${T}/hg002.g.vcf.gz" H="${SAMPLE}ar" -sample "$H" "${T}/hg002.vcf.gz" +sample "$H" "${T}/hg002.vcf.gz" "${T}/hg002.g.vcf.gz" step "$H" +# og archaic genotypes counts the table sites it read and those it did not. +# GIAB's confident regions leave part of chr22 out, so some are not read; a +# run that read every site with no call as the reference has none. +read -r READ_SITES UNREAD_SITES < <(sed -n 's/^.* chromosomes\{0,1\}, \([0-9]*\) sites read, \([0-9]*\) not read.*$/\1 \2/p' "$STEP_LOG" | tail -1) +check_ge "HG002: table sites read on chr22" "${READ_SITES:-0}" 100000 +check_ge "HG002: table sites outside the confident regions are not read" "${UNREAD_SITES:-0}" 1000 +echo " chr22 table sites read: ${READ_SITES}, not read: ${UNREAD_SITES}" cat "${G}/${H}/archaic/${H}_archaic.txt" 2>/dev/null check "Neanderthal share is a number" has '^[0-9]+\.[0-9]+$' "$(kv "$H" neanderthal_percent)" check "Denisovan share is a number" has '^[0-9]+\.[0-9]+$' "$(kv "$H" denisovan_percent)" @@ -146,7 +179,7 @@ check "the segments file has its header" has '^chrom start end length_bp ancestr } > "${T}/plant.vcf" dk "$BCFTOOLS_IMAGE" bcftools sort -Oz -o "${T}/plant.vcf.gz" "${T}/plant.vcf" P="${SAMPLE}arp" -sample "$P" "${T}/plant.vcf.gz" +sample "$P" "${T}/plant.vcf.gz" "${T}/hg002.g.vcf.gz" step "$P" cat "${G}/${P}/archaic/${P}_archaic_segments.tsv" 2>/dev/null | head -20 check_ge "a Neanderthal segment covers 70% of the planted window (bp)" "$(covered "$P" Neanderthal "$W_FROM" "$W_TO")" 700000 @@ -171,8 +204,11 @@ check "with the Neanderthal line of the summary" has "Neanderthal: $(kv "$P" nea gzip -dc "$TABLE" | awk -F'\t' 'BEGIN {OFS = "\t"} NR > 1 && $8 >= 0.5 {print "chr22", $2, ".", $3, $4, 50, "PASS", ".", "GT", "1/1"}' } > "${T}/common.vcf" dk "$BCFTOOLS_IMAGE" bcftools sort -Oz -o "${T}/common.vcf.gz" "${T}/common.vcf" +# COMMON is the reference wherever it carries no ALT: one block over chr22. +printf 'chr22\t0\t50818468\n' > "${T}/chr22.bed" +gvcf "${T}/chr22.bed" "${T}/common.g.vcf.gz" C="${SAMPLE}arc" -sample "$C" "${T}/common.vcf.gz" +sample "$C" "${T}/common.vcf.gz" "${T}/common.g.vcf.gz" step "$C" # COMMON carries no allele rarer than half the population, so almost nothing # should look archaic in it. Not nothing at all: the masks leave fewer sites @@ -187,7 +223,8 @@ check "and no more Denisovan bp than HG002" test "$(kv "$C" denisovan_bp)" -le " # --- Nextflow against the bash step ---------------------------------------------------------- command -v nextflow >/dev/null || { fail "nextflow is not on PATH"; finish; } SHEET="${T}/samplesheet.csv" NF_OUT="${G}/nf-archaic" -printf 'sample,vcf,vcf_index,sex\n%s,%s,%s,male\n' "$P" "${G}/${P}/vcf/${P}.vcf.gz" "${G}/${P}/vcf/${P}.vcf.gz.tbi" > "$SHEET" +printf 'sample,vcf,vcf_index,gvcf,gvcf_index,sex\n%s,%s,%s,%s,%s,male\n' "$P" "${G}/${P}/vcf/${P}.vcf.gz" "${G}/${P}/vcf/${P}.vcf.gz.tbi" \ + "${G}/${P}/vcf/${P}.g.vcf.gz" "${G}/${P}/vcf/${P}.g.vcf.gz.tbi" > "$SHEET" echo "+ nextflow run main.nf -profile docker --tools archaic" ( cd "$T" && nextflow run "${REPO}/main.nf" -profile docker -ansi-log false -work-dir "${E2E_WORK}/nf-work-archaic" \ --input "$SHEET" --reference "${G}/reference/GRCh38_no_alt_analysis_set.fasta" --tools archaic \ From 38283e7485796a462e550968f821b03370ff9f04 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:43:01 +0100 Subject: [PATCH 124/255] docs(archaic): state the gVCF rule, the stand-in rounding, the test scope and the data terms Bead: lin-01 Bead: lin-02 Bead: lin-03 Bead: lin-06 Bead: lin-07 --- CHANGELOG.md | 2 ++ docs/43-archaic.md | 36 +++++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f880347a2..35a8087ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 43 (archaic ancestry) no longer counts a site without reads as a match with the reference genome. It wrote every table site the VCF had no record for as 0/0, so a stretch with no reads, where the archaic genomes carry the common reference allele, could score as a long Neanderthal segment, and one call anywhere on a chromosome was enough to analyse all of it. On a synthetic chromosome with one call far away and no reads over 150 table sites, the pinned IBDmix called a 149 kb segment and a 100% Neanderthal share; it now calls none. `og archaic genotypes` takes the sample's gVCF (`--gvcf`) and reads such a site as the reference only inside a confident reference block: FILTER PASS or `.`, every allele of GT 0, and GQ and depth (MIN_DP, else DP) of 1 or more where given. Any other site with no record goes to IBDmix as unknown (9). The analysed span and `analysed_sites` now run over the sites read, and the command prints how many sites it read and how many it did not. Step 43 now needs the gVCF of step 3, as steps 42 and 44 do: the bash step stops without it and says why, and the Nextflow pipeline skips a samplesheet row without one, with a warning. Without `--gvcf`, a site with no record is unknown; a VCF with a record at every site still needs no gVCF. No option, output file or column is renamed or removed; the values of `_archaic.tsv` and the segments move for a sample with gaps in its coverage. The parity goldens of `archaic-genotypes` record the change. `docs/43-archaic.md` now also says how the 100 stand-ins carry round(AF × 200) ALT alleles with your own genotype counted in the frequency, as Chen et al. counted each person in their group's, what the tests check and what they do not, and the terms of the MPI-EVA archaic genomes, which the pipeline does not redistribute. + - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. - The genome map on each person's page now zooms past the chromosome, to a region and down to a gene's exons. From the region level a genes lane draws every gene that fits, with the genes a finding names in bold; at the gene level each gene is a model with its strand and exon blocks, and a finding is a dashed tick with its change under it. A search box (`/`) finds a gene by name, a band, a finding on the map or a typed place; a click on a finding opens its card beside the map; a Nearest button flies to the closest finding when none is in view; Esc steps out one level at a time. Search, the card and Nearest offer only what the map draws, so a closed result is never suggested. The map's address keeps 3 decimals of Mb under 2 Mb and plain positions under 60 kb. A click on a finding on the map now opens its card instead of going straight to the finding's page; the card links there. `docs/genome-map.md` describes the map. diff --git a/docs/43-archaic.md b/docs/43-archaic.md index d9689a32c..2d25574e5 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -19,14 +19,22 @@ IBDmix needs no unadmixed reference population. It scores each site from the mod ### One genome and the stand-ins -IBDmix takes the modern allele frequency of each site from the modern genomes in its input. With one genome the frequency is that person's own genotype, and IBDmix drops every site below its minimum allele count, so nothing scores. The step therefore adds 100 stand-in people whose genotypes carry the 1000 Genomes phase 3 allele count of the chosen population at each site (`ARCHAIC_POP`, default `ALL`; `AFR`, `AMR`, `EAS`, `EUR` or `SAS`). IBDmix scores each modern column from the site's frequency and that column's genotype only, so your column gets the score it would get inside a population sample with that frequency. The stand-ins' own rows are dropped. +IBDmix takes the modern allele frequency of each site from every modern column in its input, and your column is one of them. With one genome the frequency is that person's own genotype, and IBDmix drops nearly every site at its minimum allele count, so almost nothing scores. The step therefore adds 100 stand-in people. Together they carry round(AF × 200) ALT alleles at each site, where AF is the 1000 Genomes phase 3 frequency of the chosen population (`ARCHAIC_POP`, default `ALL`; `AFR`, `AMR`, `EAS`, `EUR` or `SAS`). The stand-ins' own segments are dropped. + +So the frequency IBDmix uses at a site is (the stand-ins' ALT count + yours) / 202, or / 200 where your genotype is unknown. That is you inside a sample of about 100 people, about the size of one 1000 Genomes group (they hold 61 to 113 people). Chen et al. ran IBDmix the same way: once per 1000 Genomes group, with the scored person counted in the group's frequency. What follows from it: + +- With `-m 1`, IBDmix keeps a site only when the modern columns hold at least 2 ALT and at least 2 REF alleles. Your own alleles count, so your genotype can add a site or remove one. +- An AF at or under 0.0025 gives the stand-ins 0 ALT alleles (the rounding is half to even, so 0.0025 gives 0). A heterozygous call scores nothing there, and a homozygous ALT call is kept only by your own two copies. +- An AF from just over 0.0025 to just under 0.0075 gives the stand-ins 1 ALT allele, so the site is kept whenever you carry the ALT. Each step of one allele covers an AF band about 0.005 wide: 0.0026 and 0.0074 score the same. +- An AF at or over 0.9975 gives the stand-ins 200 ALT alleles. A homozygous ALT call then removes the site, and a 0/0 call keeps it. +- Chen et al.'s group count is a random sample, and the stand-ins carry the rounded expected count. Near an AF of 0.002 a real group would keep the site some of the time, and step 43 never does. This is a small difference in one direction, and the stand-ins carry a frequency of the whole population or continent, not of one group (see the same-person comparison under Runtime). ### The calls - IBDmix with the options of Chen et al.: LOD 4 (`-d 4`), minor allele count 1 (`-m 1`), archaic error 0.01 (`-a`), modern error 0.002 (`-e`), `-c 2` - Segments with LOD 4 or more and 50 kb or longer, the cutoffs of the paper - **Neanderthal**: the union of the segments against Altai and against Vindija. **Denisovan**: the segments against the Denisovan that overlap no Neanderthal segment, because the Neanderthal and Denisovan genomes share much of their sequence. -- The share is the segment length over the analysed span: from the first to the last site of the table on each autosome the VCF has calls on. +- The share is the segment length over the analysed span: from the first to the last site read on each autosome the VCF has calls on. A site is read when the VCF has a usable call there, or when a confident reference block of the gVCF covers it (see Input). ## Docker Image @@ -48,6 +56,8 @@ Pinned in `versions.env`; [Image versions](versions.md) lists the current tags. | 1000 Genomes | the GRCh38 strict mask, `20160622.allChr.mask.bed` (128 MB) | sha256 | | UCSC | `hg19ToHg38.over.chain.gz` | sha256 | +> **Terms:** MPI-EVA makes the snpAD genotypes and the FilterBed masks freely available under the terms in https://cdna.eva.mpg.de/neandertal/Vindija/README: users are asked to observe the Fort Lauderdale principles, and the data can be used freely for studies of individual genes or other features of the genome. All three genomes' files come from that folder. Cite the genome papers in References. MPI-EVA gives no standard licence such as an SPDX one. The pipeline does not redistribute these files: `setup.sh --archaic` downloads them from MPI-EVA onto your disk, and deletes each chromosome's files once its table rows are built. + MPI-EVA publishes md5 lists, not sha256 sums, for 66 genotype files of about 2.5 GB each. Pinning the sha256 of the three lists pins every file's md5 without downloading 150 GB to hash it. The FilterBed files have no checksum file beside them and are about 1.5 MB each, so each one's sha256 is pinned directly. ### The masks @@ -69,14 +79,16 @@ The table is built on GRCh37 and lifted once to GRCh38 with Picard `LiftoverVcf` ## Input -- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample. A site with no record is read as the reference allele, a record whose FILTER is not PASS as unknown, and a site where you carry another ALT than the table's is left out. +- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample. A record whose FILTER is not PASS is read as unknown, and a site where you carry another ALT than the table's is left out. +- `${SAMPLE}/vcf/${SAMPLE}.g.vcf.gz`: the gVCF of step 3, one sample. A variant-only VCF cannot tell a site that matches the reference from a site without reads, so a table site with no record in the VCF is read as the reference only where a confident reference block of the gVCF covers it. A block is confident when its FILTER is PASS or `.`, every allele of its GT is 0, and its GQ and its depth (MIN_DP, else DP) are 1 or more where it gives them. Any other site with no record goes to IBDmix as unknown (9), which IBDmix skips. Without the gVCF the bash step stops and says why, and the Nextflow pipeline skips the sample with a warning (add the `gvcf` and `gvcf_index` columns to its samplesheet row, or start from the BAM or the reads, as for steps 42 and 44). The gVCF must list the chromosomes in the VCF's order. - The GENCODE gene lines `setup.sh` installs, for the genes of each segment (without them the segments are listed without genes) -`og archaic genotypes` reads the VCF and table together, keeping one -duplicate-position window. Positions within each chromosome must be in -ascending order in both inputs. A position that goes backwards stops the -command with exit 2. Mask runs are written as they close. Duplicate calls, -clashes, masked sites and the output columns keep their previous meaning. +`og archaic genotypes` reads the VCF, the gVCF (`--gvcf`) and the table +together, keeping one duplicate-position window of the VCF and one record of +the gVCF. Positions within each chromosome must be in ascending order in all +three inputs. A position that goes backwards, or a gVCF in another +chromosome order, stops the command with exit 2. Mask runs are written as +they close. It prints how many table sites it read and how many it did not. ## Command @@ -105,6 +117,8 @@ Measured on public HG002 by the `archaic-hg002` jobs of `public-evidence.yml`, o - Install: the archaic files were 150.5 GB in all, from 1.8 GB (chromosome 22) to 13.8 GB (chromosome 2). Downloading them took 2 h 32 min summed over the 22 chromosomes, from 3 to 32 minutes each, at the speed MPI-EVA served that day. Building the table took 3 h 15 min summed, from 2 to 20 minutes per chromosome. The finished table is 0.44 GB with 51,812,908 sites. - Step 43, one population: 19 to 88 seconds per chromosome, 18 minutes summed over the 22. IBDmix runs once per chromosome and archaic genome, `THREADS` chromosomes at a time. +Every figure in this section was measured before step 43 read the gVCF, when each site GIAB's variant-only VCF had no call for was read as the reference, inside GIAB's confident regions or not. The e2e case now gives GIAB's confident regions as the reference blocks and prints its chr22 figure on every run. The whole-genome figure needs a new release run of `public-evidence.yml` with a gVCF. + That run was made before the masks, from a table that held every lifted site. It gave HG002 115.2 Mb of Neanderthal sequence over the 22 autosomes, about twice what Chen et al. report for Europeans (a median of 51.6 Mb per person, and 42.0 to 62.0 Mb across the released per-person calls of their 503 Europeans; see the same-person comparison below for what their method is). The cause was the missing masks: the table carried the positions each archaic genome's own filters exclude, and IBDmix scored them. On chr22, 43% of that run's Neanderthal sequence lay outside the 1000 Genomes GRCh38 strict mask alone. With the masks dropping sites, a run on chr22 (run 42089578470, before the masked codes) gave HG002 1.3 Mb of Neanderthal sequence with the `ALL` frequencies and 1.7 Mb with `EUR`, still at the top of the published spread below. Keeping the masked sites as IBDmix's mask, as the published workflow does, lets the mismatches there end a segment; the chr22 figure on this table is printed by the e2e case on every run. The e2e case compares it with the spread of Chen et al.'s own released calls for the 497 Europeans of CEU, TSI, GBR, FIN and IBS (51 kb at the lowest, 541 kb at the median, 1,132 kb at the 95th percentile, 1,716 kb at the highest). The pre-mask run gave 1,748 kb on chr22, more than all 497 of them. With the masked codes but the strict-mask rows still dropped, the fork runs gave 1,177 to 1,190 kb. Keeping those rows marked, the last difference from the published workflow in how masked sites are handled, gave 1,138 kb on chr22 (run 42335592918). One chromosome of one person varies widely, so the case gates on a regression guard of 1,200 kb (and on the lowest published value, 51 kb), and prints the published figures beside the measured one. It does not claim the figure is inside the published range. The whole-genome total is measured by the release run of `public-evidence.yml`, which installs the 150 GB of archaic data once per release, not by the e2e case, and printed beside the published 42.0 to 62.0 Mb; it is not gated on it, for the reasons in the same-person comparison below. The case prints where HG002 falls inside the spread; it does not gate on the median or the 95th percentile, because one person's chromosome is one draw from that distribution. The same-person comparison below shows the input's coverage is not why the step finds more than the published calls do. @@ -132,10 +146,10 @@ What this means for the figure step 43 reports. It is the union of the Altai and - Part of what IBDmix finds in anyone is old ancestry shared with the archaic genomes, not gene flow from them. Chen et al. find Neanderthal sequence in every population they studied, Africans included. - A segment counts when either of your two copies carries it. The share is of the analysed span, not of the whole genome. - The table has autosomes only: chrX and chrY are not assessed. -- Sites the VCF has no record for are read as the reference allele. A VCF that holds only variants (no reference calls) cannot tell a reference call from a gap in coverage. -- The share is of the analysed span, and the span still runs from the first to the last table site of each chromosome whatever the masks remove. A segment runs from its first site to the site after its last, so it can cross masked sequence and count those bases (a homozygous mismatch at a masked site still ends it, as in the published workflow); a wrong genotype at a site that was kept can raise the share as well. The number is an estimate in both directions, not a floor. Compare it with the published per-person figures, which come from masked data too. +- A site the VCF has no record for is read as the reference allele only inside a confident reference block of the gVCF. Elsewhere it is unknown, so a gap in coverage cannot make a segment, and a missed homozygous ALT call cannot end one as a false mismatch. Before 1.0.0 every such site was read as the reference: one call anywhere on a chromosome was enough to analyse all of it, and a stretch with no reads where the archaic genomes carry the common reference allele could score as a long segment. +- The share is of the analysed span, and the span runs from the first to the last site read on each chromosome whatever the masks remove. A segment runs from its first site to the site after its last, so it can cross masked sequence and count those bases (a homozygous mismatch at a masked site still ends it, as in the published workflow); a wrong genotype at a site that was kept can raise the share as well. The number is an estimate in both directions, not a floor. Compare it with the published per-person figures, which come from masked data too. - The CpG sites Chen et al. exclude are kept here. A C to T change at a CpG site is the commonest ancient-DNA error, so a few of the archaic genotypes at those sites are wrong in both directions, and the sites are over-represented among the matches a method like this finds. Their own filter needs data this pipeline does not install, so the share carries that error; it is one of the reasons a number from this step is an estimate. -- On the e2e fixture the case builds the table for GRCh37 chr22 and checks HG002's GIAB v4.2.1 genotypes against the published chr22 spread, a copy of HG002 with one chromosome over chr22:28-29 Mb taken from Vindija (a Neanderthal segment must cover it), a person built from the table who carries only the majority allele (a quarter of HG002's Neanderthal sequence at most), that every table row outside the strict mask carries only masked calls, and that the FilterBeds and the exclusions marked calls masked. +- What the tests check. The unit tests check the table build, the masks, the stand-in counts, the gVCF rule and the report's merging on synthetic data. With the pinned IBDmix image they also check a planted Neanderthal and Denisovan stretch, a masked mismatch that must end a segment, and a stretch with no call that must give no segment. They do not check that the share is calibrated against published per-person totals (by design it is not; see Runtime), and the stand-in rounding is pinned, not compared with a sampled population. On the e2e fixture the case builds the table for GRCh37 chr22 and checks HG002's GIAB v4.2.1 genotypes, with GIAB's confident regions as the gVCF's reference blocks, against the published chr22 spread, a copy of HG002 with one chromosome over chr22:28-29 Mb taken from Vindija (a Neanderthal segment must cover it), a person built from the table who carries only the majority allele (a quarter of HG002's Neanderthal sequence at most), that every table row outside the strict mask carries only masked calls, and that the FilterBeds and the exclusions marked calls masked. ## References From ec8e1e4f4b2770b6ec03dd5077c4dc5bf352ad92 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:51:58 +0100 Subject: [PATCH 125/255] test(archaic): planted control, read every site with no call as the reference (reverted next) --- rust/crates/og-ancestry/src/archaic.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 89fa026d1..de6136da0 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -1327,13 +1327,12 @@ fn write_chrom( // there, which IBDmix must see as unknown, not as a match. None => match callable.as_deref_mut() { Some(c) => { - if c.covers(pos)? { - 0 - } else { - 9 - } + // PLANTED CONTROL: the rule before the fix, a site with + // no record read as the reference whatever the gVCF says. + c.covers(pos)?; + 0 } - None => 9, + None => 0, }, Some(record) => { let call = &record.call; From 54d7e98093124ba68a9fb13e0325cb2c7c9a4e59 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:56:42 +0100 Subject: [PATCH 126/255] Revert "test(archaic): planted control, read every site with no call as the reference (reverted next)" This reverts commit ec8e1e4f4b2770b6ec03dd5077c4dc5bf352ad92. --- rust/crates/og-ancestry/src/archaic.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index de6136da0..89fa026d1 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -1327,12 +1327,13 @@ fn write_chrom( // there, which IBDmix must see as unknown, not as a match. None => match callable.as_deref_mut() { Some(c) => { - // PLANTED CONTROL: the rule before the fix, a site with - // no record read as the reference whatever the gVCF says. - c.covers(pos)?; - 0 + if c.covers(pos)? { + 0 + } else { + 9 + } } - None => 0, + None => 9, }, Some(record) => { let call = &record.call; From e6fe655b8ababf325feea5e37c08b135c458ad38 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:23:24 +0100 Subject: [PATCH 127/255] fix(docs): link the imprinting table through the public repository The built site cannot link outside its own root. The strict local build passes, and site-check reports 97 pages with zero errors. Bead: lic-41 --- docs/11-roh-analysis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/11-roh-analysis.md b/docs/11-roh-analysis.md index 07fc1e338..c9fe48202 100644 --- a/docs/11-roh-analysis.md +++ b/docs/11-roh-analysis.md @@ -85,7 +85,7 @@ In uniparental disomy (UPD) both copies of a chromosome come from one parent. Wh The reports flag a chromosome when its segments of 5 Mb or more (band parts left out) add up to at least 20 Mb **and** to more than half of the length of all such autosomal segments. Both numbers are our thresholds, not a published cut-off: 20 Mb on one chromosome is the long-standing sign of isodisomy (see the uniparental disomy note under [Important Notes](#important-notes)), and the share rule keeps out a child of related parents, whose runs spread over many chromosomes. The flag says "may": a run that long can also be chance in a child of related parents. -A flagged chromosome is looked up in [assets/imprinted_regions.tsv](../assets/imprinted_regions.tsv). That file is the only imprinting table and is embedded into `og` at build time. Its header credits [Eggermann et al. 2023](https://doi.org/10.1038/s41572-023-00443-4), with per-row PMIDs for the reference facts. +A flagged chromosome is looked up in [assets/imprinted_regions.tsv](https://github.com/GeiserX/OwnGenome/blob/main/assets/imprinted_regions.tsv). That file is the only imprinting table and is embedded into `og` at build time. Its header credits [Eggermann et al. 2023](https://doi.org/10.1038/s41572-023-00443-4), with per-row PMIDs for the reference facts. The flag lists the chromosome, run length, share, region names and citations. It names no syndrome. The existing `maternal_upd` and `paternal_upd` JSON keys stay as `"-"` for reader compatibility. The asset retains their reference facts. Those facts cannot be inferred from a region match in one genome. From 2bb5acc7d7bc1da22318852c398e1c210af963fd Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:34:21 +0100 Subject: [PATCH 128/255] fix(archive): regenerate MD/NM when restoring a CRAM without stored tags Restore and CRAM_TO_BAM turned MD/NM generation off for every CRAM, so a CRAM written without store_md/store_nm (an older step 34, or sarek) lost the tags samtools drops where they match the reference. Turn generation off only when the CRAM's last writer stored both tags, and decode the CRAM the same way on both sides of the content check. The e2e case now restores a plain CRAM in bash and Nextflow and checks NM, and proves the module content checks fail on a changed CRAM-side checksum. --- CHANGELOG.md | 2 +- docs/34-cram-archive.md | 2 +- modules/local/cram_archive/main.nf | 14 +++++++++++--- scripts/34-cram-archive.sh | 27 ++++++++++++++++++++++----- tests/e2e/qc-3-cram-checksum.sh | 29 +++++++++++++++++++++++++++++ 5 files changed, 64 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e3a75bfd..0ab34acd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- CRAM archiving now compares full-content samtools checksums before `--delete-bam` can remove the BAM. Matching read counts alone could accept changed or unreadable read data. Archive and restore in bash and Nextflow now check all records, alignment fields and tags, independent of order. Explicit MD/NM tags are preserved and missing tags stay missing. A mismatch or checksum failure keeps the source and removes the partial output. The step adds `.cram.checksum` and `.bam.checksum` reports beside the alignments; Nextflow publishes the CRAM report. Existing output names and formats stay the same. +- CRAM archiving now compares full-content samtools checksums before `--delete-bam` can remove the BAM. Matching read counts alone could accept changed or unreadable read data. Archive and restore in bash and Nextflow now check all records, alignment fields and tags, independent of order. Explicit MD/NM tags are preserved and missing tags stay missing. An older or external CRAM written without stored MD/NM gets them regenerated on restore, as before. A mismatch or checksum failure keeps the source and removes the partial output. The step adds `.cram.checksum` and `.bam.checksum` reports beside the alignments; Nextflow publishes the CRAM report. Existing output names and formats stay the same. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/34-cram-archive.md b/docs/34-cram-archive.md index d4d74b062..c9fc9b1bb 100644 --- a/docs/34-cram-archive.md +++ b/docs/34-cram-archive.md @@ -36,7 +36,7 @@ samtools checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' --input-fmt-option The CRAM must pass `samtools quickcheck`, the two `flagstat` reports must match, and the content checksums must match. The checksum reads every record, including secondary and supplementary alignments. It covers read names, sequences, qualities, flags, positions, CIGARs, mate fields and all auxiliary tags. A changed base quality or tag can leave `flagstat` counts unchanged and still fail this check. A decoding error also fails the step. -The checksums use samtools' order-independent aggregation. The comparison ignores the filename header and sorts the report rows, so read and tag order do not affect it. samtools sanitizes format-level differences, such as an unmapped read's MAPQ and equivalent CIGARs. Explicit `MD` and `NM` tags are stored in the CRAM. They are not regenerated during the check or restore, so missing tags stay missing. These CRC-based content checks detect conversion faults. They are not a cryptographic identity proof. See the [samtools 1.24 checksum manual](https://www.htslib.org/doc/1.24/samtools-checksum.html). +The checksums use samtools' order-independent aggregation. The comparison ignores the filename header and sorts the report rows, so read and tag order do not affect it. samtools sanitizes format-level differences, such as an unmapped read's MAPQ and equivalent CIGARs. Explicit `MD` and `NM` tags are stored in the CRAM. For a CRAM this step wrote, they are not regenerated during the check or restore, so missing tags stay missing. An older or external CRAM, written without stored `MD`/`NM`, keeps only the tags that differ from the reference. Restoring it regenerates `MD` and `NM` from the reference, and the check decodes the CRAM the same way. These CRC-based content checks detect conversion faults. They are not a cryptographic identity proof. See the [samtools 1.24 checksum manual](https://www.htslib.org/doc/1.24/samtools-checksum.html). Only after all checks pass does the CRAM lose its `.part` name. A mismatch or a failed or empty checksum command removes the partial CRAM, keeps the BAM and its index, and exits non-zero. This also applies with `--delete-bam`. An older published CRAM stays in place if a new attempt fails. Every archive attempt writes a fresh CRAM from the current BAM before it can delete anything. The checksum adds a full read of each file to the archive and restore checks. diff --git a/modules/local/cram_archive/main.nf b/modules/local/cram_archive/main.nf index 069e98eaa..74912d13c 100644 --- a/modules/local/cram_archive/main.nf +++ b/modules/local/cram_archive/main.nf @@ -117,7 +117,15 @@ process CRAM_TO_BAM { script: """ - samtools view -@ ${task.cpus} -b --reference ${reference} --input-fmt-option decode_md=0 -o ${meta.id}_from_cram.bam ${cram} + # A CRAM whose last writer stored MD and NM (CRAM_ARCHIVE, step 34) decodes + # with generation off, so a tag the BAM lacked stays missing. Any other + # CRAM (an older step 34, sarek) holds only the MD/NM that differ from the + # reference and decodes with the samtools default, which regenerates them. + decode="" + if samtools view -H --no-PG ${cram} | awk '/^@PG/ { last = \$0 } END { exit !(last ~ /store_md=1/ && last ~ /store_nm=1/) }'; then + decode="--input-fmt-option decode_md=0" + fi + samtools view -@ ${task.cpus} -b --reference ${reference} \$decode -o ${meta.id}_from_cram.bam ${cram} samtools index -@ ${task.cpus} ${meta.id}_from_cram.bam samtools quickcheck -v ${meta.id}_from_cram.bam samtools flagstat -@ ${task.cpus} --input-fmt-option reference=${reference} ${cram} > cram.flagstat @@ -128,13 +136,13 @@ process CRAM_TO_BAM { fi # Match every record field and tag, independent of read/tag/group order. - # MD/NM are stored explicitly and never regenerated while checking. + # The CRAM decodes the way the BAM was written from it. # pipefail must propagate decoder failures through tail and sort. set -o pipefail checksum_rows() { samtools checksum -@ ${task.cpus} -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "\$@" | tail -n +2 | LC_ALL=C sort } - checksum_rows --input-fmt-option reference=${reference} --input-fmt-option decode_md=0 ${cram} > cram.checksum + checksum_rows --input-fmt-option reference=${reference} \$decode ${cram} > cram.checksum checksum_rows ${meta.id}_from_cram.bam > bam.checksum if [ ! -s cram.checksum ] || [ ! -s bam.checksum ] || ! diff cram.checksum bam.checksum >&2; then echo "ERROR: the restored BAM content differs from ${cram}" >&2 diff --git a/scripts/34-cram-archive.sh b/scripts/34-cram-archive.sh index 0b50a0e3b..b0da7efa0 100755 --- a/scripts/34-cram-archive.sh +++ b/scripts/34-cram-archive.sh @@ -89,15 +89,28 @@ same_reads() { fi } +# How a CRAM is decoded. This step's encoder stores MD and NM explicitly, so +# its CRAMs decode with generation off and a tag the BAM lacked stays missing. +# A CRAM written without them (an older step 34, or one made elsewhere) holds +# only the MD/NM that differ from the reference: it decodes with samtools' +# default, which regenerates them. Both sides of a check decode the same way. +CRAM_DECODE=(--input-fmt-option decode_md=0) + +# md_stored CRAM_IN_CONTAINER: the last program that wrote the CRAM stored MD +# and NM (its @PG command line has store_md=1 and store_nm=1). +md_stored() { + run_in "$SAMTOOLS_IMAGE" samtools view -H --no-PG "$1" \ + | awk '/^@PG/ { last = $0 } END { exit !(last ~ /store_md=1/ && last ~ /store_nm=1/) }' +} + # The checksum reads every block, unlike flagstat. Use all record fields and # tags, including secondary/supplementary records, with order-independent # aggregation. Normalize the format-level repairs CRAM makes to unmapped -# records and CIGARs. Keep explicit MD/NM in the encoder and disable their -# generation on decode, so missing tags stay missing. Only the filename -# header is removed; sorting makes read-group row order immaterial too. +# records and CIGARs. Only the filename header is removed; sorting makes +# read-group row order immaterial too. checksum_rows() { local -a ref=() - case "$1" in *.cram) ref=(--input-fmt-option "reference=${REF_FASTA_C}" --input-fmt-option decode_md=0) ;; esac + case "$1" in *.cram) ref=(--input-fmt-option "reference=${REF_FASTA_C}" ${CRAM_DECODE[@]+"${CRAM_DECODE[@]}"}) ;; esac run_in --cpus "$THREADS" --memory 2g "$SAMTOOLS_IMAGE" \ samtools checksum -@ "$THREADS" -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' \ ${ref[@]+"${ref[@]}"} "$1" | tail -n +2 | LC_ALL=C sort @@ -126,8 +139,12 @@ if [ "$MODE" = --restore ]; then exit 1 } rm -f "${ALN}/${SAMPLE}_sorted.part.bam" "${ALN}/${SAMPLE}_sorted.part.bam.bai" + if ! md_stored "${C}.cram"; then + CRAM_DECODE=() + echo " The CRAM has no stored MD/NM tags: the BAM gets them regenerated from the reference." + fi run_in --cpus "$THREADS" --memory 4g "$SAMTOOLS_IMAGE" \ - samtools view -@ "$THREADS" -b --reference "$REF_FASTA_C" --input-fmt-option decode_md=0 \ + samtools view -@ "$THREADS" -b --reference "$REF_FASTA_C" ${CRAM_DECODE[@]+"${CRAM_DECODE[@]}"} \ -o "${C}.part.bam" "${C}.cram" || fail_restore "samtools could not write the BAM from the CRAM." run_in --cpus "$THREADS" --memory 2g "$SAMTOOLS_IMAGE" \ samtools index -@ "$THREADS" "${C}.part.bam" || fail_restore "samtools could not index the BAM." diff --git a/tests/e2e/qc-3-cram-checksum.sh b/tests/e2e/qc-3-cram-checksum.sh index acb4ce323..be74b9957 100755 --- a/tests/e2e/qc-3-cram-checksum.sh +++ b/tests/e2e/qc-3-cram-checksum.sh @@ -112,6 +112,20 @@ check "faithful restore writes BAM and index" test -s "${A}/${C}_sorted.bam" -a sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/original.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/original.checksum" sam checksum -PCMc -b 0xfff -f0 -F0 -z all,cigarx -t '*' "${C}/aligned/${C}_sorted.bam" | tail -n +2 | LC_ALL=C sort > "${CASE_TMP}/restored.checksum" check "round trip preserves read content and explicit tags" cmp -s "${CASE_TMP}/original.checksum" "${CASE_TMP}/restored.checksum" +# A CRAM written without store_md/store_nm (an older step 34, or one made +# elsewhere) drops MD and NM wherever they match the reference. Restoring it +# regenerates them, so read0's NM:i:0 is back in the BAM. +plain_cram() { + sam view -C -T reference/GRCh38_no_alt_analysis_set.fasta -o "$1" "${C}/aligned/original.bam" + sam index "$1" +} +nm_of_read0() { sam view "$1" | awk -F '\t' '$1 == "read0" { for (i = 12; i <= NF; i++) if ($i ~ /^NM:i:/) print $i }'; } +plain_cram "${C}/aligned/${C}_sorted.cram" +check "the plain CRAM stores no NM for read0" test -z "$(sam view -T reference/GRCh38_no_alt_analysis_set.fasta --input-fmt-option decode_md=0 "${C}/aligned/${C}_sorted.cram" | awk -F '\t' '$1 == "read0" && /NM:i:/')" +rm -f "${A}/${C}_sorted.bam" "${A}/${C}_sorted.bam.bai" +run_step 34-cram-archive.sh "$C" --restore +check_step_exit 34-cram-archive.sh +check_eq "restore of a plain CRAM regenerates NM" "$(nm_of_read0 "${C}/aligned/${C}_sorted.bam")" "NM:i:0" # Run the module's actual script blocks on the same synthetic records. Only # Nextflow interpolation and version metadata are omitted by this harness. # Literal Nextflow placeholders are substituted by sed, not by this shell. @@ -135,6 +149,10 @@ module_script() { # The function expands its arguments in the container shell. # shellcheck disable=SC2016 EMPTY_CHECKSUM='samtools() { if [ "$1" = checksum ]; then return 0; else command samtools "$@"; fi; }' +# A checksum that differs on the CRAM side only: the last report row's +# combined value is replaced. The module must fail on the diff alone. +# shellcheck disable=SC2016 +CHANGED_CHECKSUM='samtools() { if [ "$1" = checksum ]; then case "${@: -1}" in *.cram) command samtools "$@" | sed "\$s/[0-9a-f]\{8\}\$/00000000/" ;; *) command samtools "$@" ;; esac; else command samtools "$@"; fi; }' for process in CRAM_ARCHIVE CRAM_TO_BAM; do body=$(module_script "$process") rc=0 @@ -144,5 +162,16 @@ for process in CRAM_ARCHIVE CRAM_TO_BAM; do in_genome "$SAMTOOLS_IMAGE" bash -euo pipefail -c "${EMPTY_CHECKSUM} ${body}" || rc=$? check "${process}: empty checksum reports fail" test "$rc" -ne 0 + rc=0 + in_genome "$SAMTOOLS_IMAGE" bash -euo pipefail -c "${CHANGED_CHECKSUM} +${body}" > "${CASE_TMP}/changed-${process}.log" 2>&1 || rc=$? + check "${process}: a changed CRAM checksum fails" test "$rc" -ne 0 + check "${process}: the failure is the content check" has "content .*differs" "$(cat "${CASE_TMP}/changed-${process}.log")" done +# CRAM_TO_BAM on a CRAM without stored MD/NM regenerates them, like --restore. +plain_cram module_sorted.cram +rc=0 +in_genome "$SAMTOOLS_IMAGE" bash -euo pipefail -c "$(module_script CRAM_TO_BAM)" || rc=$? +check_eq "CRAM_TO_BAM: plain CRAM passes" "$rc" 0 +check_eq "CRAM_TO_BAM: plain CRAM regenerates NM" "$(nm_of_read0 module_from_cram.bam)" "NM:i:0" finish From beb57c97425e9bbcfd689e46fdd8135cff2ac3c4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:34:24 +0100 Subject: [PATCH 129/255] fix(recheck): count failed items, not failed lookups, in the last line An item also fails when the run's manifest lacks the value to compare, which no later lookup fixes, so the closing line counts looked-up items that failed and points at each item's own line for why. Bead: plat-26 --- CHANGELOG.md | 2 +- docs/recheck.md | 2 +- rust/crates/og-recheck/src/lib.rs | 10 ++++++---- rust/crates/og-recheck/src/tests.rs | 11 ++++++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 574c7d7a1..0fef8601b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- `og recheck` no longer says "Nothing here is newer than what the run used." when a lookup failed. With no network it said that line, and a cron log then read as up to date. Now the last line says how many lookups failed and that the check is incomplete, or "Could not check: all N lookups failed" when none answered. The exit code was already 1 and stays 1. A failed lookup also no longer hides a newer file or pin that is already installed here: a newer ClinVar file on disk stays `rerun-ready`, with its steps, in `state.json`, the reports and the live page, and the item keeps its `error`. `og recheck --rerun` no longer replays the stored command through `bash -c`. It reads `meta command` of `logs/run_status.tsv` back word by word as the call `run-all.sh` records (`cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS`) and starts that `run-all.sh` with its arguments one by one. A row of any other shape, such as one edited to add `; rm ...`, or a `cd` folder that is gone, is refused with exit 2 and nothing runs. A row that `run-all.sh` wrote replays as before. No option, output file or format changes. [docs/recheck.md](docs/recheck.md) +- `og recheck` no longer says "Nothing here is newer than what the run used." when a lookup failed. With no network it said that line, and a cron log then read as up to date. Now the last line says how many of the looked-up items failed and that the check is incomplete, or "Could not check: all N looked-up items failed" when none could be checked. The exit code was already 1 and stays 1. A failed lookup also no longer hides a newer file or pin that is already installed here: a newer ClinVar file on disk stays `rerun-ready`, with its steps, in `state.json`, the reports and the live page, and the item keeps its `error`. `og recheck --rerun` no longer replays the stored command through `bash -c`. It reads `meta command` of `logs/run_status.tsv` back word by word as the call `run-all.sh` records (`cd DIR && [SWITCH=VALUE ...] .../scripts/run-all.sh ARGS`) and starts that `run-all.sh` with its arguments one by one. A row of any other shape, such as one edited to add `; rm ...`, or a `cd` folder that is gone, is refused with exit 2 and nothing runs. A row that `run-all.sh` wrote replays as before. No option, output file or format changes. [docs/recheck.md](docs/recheck.md) - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/recheck.md b/docs/recheck.md index 0d149ec73..bfc471eb2 100644 --- a/docs/recheck.md +++ b/docs/recheck.md @@ -51,7 +51,7 @@ Each item gets one class: A newer file or pin installed here stays `rerun-ready` when its lookup fails, because that comparison needs no network. The item then keeps its `error` too, and the command still exits 1. -The last line of the output sums up the check. When something is `rerun-ready` or `refresh-available`, it names the steps a rerun would redo. When a lookup failed, it says how many of how many failed and that the check is incomplete, or "Could not check: all N lookups failed" when none answered. "Nothing here is newer than what the run used." comes only when every lookup answered and nothing is newer. +The last line of the output sums up the check. When something is `rerun-ready` or `refresh-available`, it names the steps a rerun would redo. When an item failed, because its lookup failed or the run's manifest lacks the value to compare, it says how many of the looked-up items failed and that the check is incomplete, or "Could not check: all N looked-up items failed" when none could be checked. Each item's own line says why. "Nothing here is newer than what the run used." comes only when every looked-up item was checked and nothing is newer. The 2-day margin for ClinVar is there because NCBI dates the file a day or two after the release in its `##fileDate`. A newer PGS file date means the catalog wrote the harmonised file again; it does not prove the weights changed. diff --git a/rust/crates/og-recheck/src/lib.rs b/rust/crates/og-recheck/src/lib.rs index b966f3159..aca5f5bd4 100644 --- a/rust/crates/og-recheck/src/lib.rs +++ b/rust/crates/og-recheck/src/lib.rs @@ -792,7 +792,9 @@ pub fn print_check(c: &Check, out: &mut dyn std::io::Write) -> std::io::Result<( )?; } // A failed lookup is never read as "nothing newer". An item that was - // looked up has an answer or an error; the others were never asked. + // looked up has an answer or an error; the others were never asked. An + // error is a failed lookup or a run value missing from the manifest, so + // the count is of items, and each item's line says which. let asked = c .items .iter() @@ -806,13 +808,13 @@ pub fn print_check(c: &Check, out: &mut dyn std::io::Write) -> std::io::Result<( } else if failed >= asked { writeln!( out, - "Could not check: all {asked} lookups failed (each line above says why). Check again later." + "Could not check: all {asked} looked-up items failed (each line above says why)." )?; } else { writeln!( out, - "{failed} of {asked} lookups failed: this check is incomplete, so it cannot say \ - nothing else is newer. Check again later." + "{failed} of {asked} looked-up items failed (each line above says why): this check is \ + incomplete, so it cannot say nothing else is newer." )?; } Ok(()) diff --git a/rust/crates/og-recheck/src/tests.rs b/rust/crates/og-recheck/src/tests.rs index fb1fd6bea..a19d8dd4f 100644 --- a/rust/crates/og-recheck/src/tests.rs +++ b/rust/crates/og-recheck/src/tests.rs @@ -320,6 +320,11 @@ fn what_cannot_be_compared_is_an_error() { let cv = by_item(&c)["clinvar_file_date"]; assert_eq!(cv.class, Class::Error); assert!(cv.error.as_deref().unwrap().contains("which ClinVar file")); + // A run value missing from the manifest is no "nothing newer" either, + // and is not called a failed lookup. + let out = printed(&c); + assert!(!out.contains("Nothing here is newer"), "{out}"); + assert!(out.contains("1 of 7 looked-up items failed"), "{out}"); let g = genome( "nodate", Opt { @@ -1133,7 +1138,7 @@ fn a_failed_lookup_never_prints_nothing_newer_and_keeps_a_newer_file_here_rerun_ assert!(!out.contains("Nothing here is newer"), "{out}"); assert!(out.contains("the lookup failed: no answer"), "{out}"); assert!(out.contains("A rerun would redo steps 06,"), "{out}"); - assert!(out.contains("1 of 7 lookups failed"), "{out}"); + assert!(out.contains("1 of 7 looked-up items failed"), "{out}"); } #[test] @@ -1158,7 +1163,7 @@ fn when_every_lookup_is_refused_the_check_says_it_could_not_check() { let out = printed(&c); assert!(!out.contains("Nothing here is newer"), "{out}"); assert!( - out.contains("Could not check: all 7 lookups failed"), + out.contains("Could not check: all 7 looked-up items failed"), "{out}" ); // The command: every answer an error, exit 1, the same closing line. @@ -1182,7 +1187,7 @@ fn when_every_lookup_is_refused_the_check_says_it_could_not_check() { assert_eq!(rc, 1, "{out}"); assert!(!out.contains("Nothing here is newer"), "{out}"); assert!( - out.contains("Could not check: all 7 lookups failed"), + out.contains("Could not check: all 7 looked-up items failed"), "{out}" ); // Control: with every answer given, nothing newer is said. From def428aee61fb8900e710e4260790531d00147bc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:40:32 +0100 Subject: [PATCH 130/255] fix(live): clear reads links and labels when the panel closes Bead: pgp-iqon.11 --- CHANGELOG.md | 2 +- assets/live/map.js | 4 ++++ docs/viewer.md | 2 +- tests/test_live_viewer.py | 4 +++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9382a20d1..fdf953ca5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer; Esc closes it and returns focus to the map. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. +- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/assets/live/map.js b/assets/live/map.js index 0af553b75..43105cd81 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -1251,6 +1251,10 @@ if (reads.hidden) { return; } reads.hidden = true; readsBody.textContent = ''; // stop reads requests, and discard a result that can close again + readsPlace.textContent = ''; + readsWho.textContent = ''; + readsFull.removeAttribute('href'); + readsFull.hidden = true; readsFrame = null; readsId = null; readsLocus = ''; readsView = ''; readsPill.setAttribute('aria-expanded', 'false'); inview.hidden = !!failure; diff --git a/docs/viewer.md b/docs/viewer.md index 8b30532c8..b6e6d1f49 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -84,7 +84,7 @@ Then open `http://127.0.0.1:8765/` on your computer. The report's links work the On the genome map, zoom below 50 kb and press **R** or select **See the reads**. The **Reads** gauge opens a 2 kb view. A finding's map card also offers **See the reads**, centred on its position with 50 bases on each side. The panel otherwise starts at the nearest open finding in view, or at the view itself. Moving the map updates the viewer's locus. Alignments are visible up to 50 kb. -The panel takes the place of the In view table. Press **Esc**, including from inside the viewer, or select **Back to the map** to close it. Focus returns to the map at the same view. **Open full viewer** opens that locus in a new tab. A closed result has no reads button, and closing an opened result removes its reads panel. When the viewer is off, the panel shows the server's reason. The static demo explains that it has no viewer. +The panel takes the place of the In view table. Press **Esc**, including from inside the viewer, or select **Back to the map** to close it. Focus returns to the map at the same view. Closing the panel clears its previous locus, title, and link to the full viewer. **Open full viewer** opens that locus in a new tab. A closed result has no reads button, and closing an opened result removes its reads panel. When the viewer is off, the panel shows the server's reason. The static demo explains that it has no viewer. The live viewer allows framing only by the same origin: it sends `frame-ancestors 'self'` and `X-Frame-Options: SAMEORIGIN`. Live pages allow `frame-src 'self'`. Other routes keep `frame-ancestors 'none'` and `X-Frame-Options: DENY`. The standalone local viewer keeps its original policy, which blocks all framing. diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index df50d0d61..288bd61b4 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -308,7 +308,9 @@ def loaded(): check("map: an opened finding offers reads", b.wait("document.querySelector('.gm-reads iframe').contentWindow.location.hash==='#chr1:501-601'")) b.eval("window.closedResult=true;testMap.refresh('one')") check("map: closing a result removes its open reads panel and link", - b.eval("document.querySelector('.gm-reads').hidden && !document.querySelector('.gm-card-reads')")) + b.eval("document.querySelector('.gm-reads').hidden && !document.querySelector('.gm-card-reads') && " + "!document.querySelector('.gm-reads a[href]') && " + "!document.querySelector('.gm-reads').textContent.includes('Closed finding')")) # No finding in view: use the view itself, and no viewer flags: show the server's reason. b.eval("location.hash='chr1:10000-59000'") From 604fd42ec5c55be579c7a09ee8a94f207797b044 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:54:07 +0100 Subject: [PATCH 131/255] test(pgx): a split 1/2 site in a recessive gene must warn, not read as a carrier --- rust/crates/og-pgx/src/sections.rs | 4 ++-- .../og-pgx/tests/i411_contraindications.rs | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/rust/crates/og-pgx/src/sections.rs b/rust/crates/og-pgx/src/sections.rs index f2e59a628..7ce151301 100644 --- a/rust/crates/og-pgx/src/sections.rs +++ b/rust/crates/og-pgx/src/sections.rs @@ -785,10 +785,10 @@ mod tests { #[test] fn genotypes_with_both_copies_changed() { - for gt in ["1/1", "1|1", "1/2", "2|1", "1"] { + for gt in ["1/1", "1|1", "1/2", "2|1", "1", "./1", "1/.", ".|1"] { assert!(both_copies(gt), "{gt}"); } - for gt in ["0/1", "1|0", "./1", "1/.", "0", "", "./."] { + for gt in ["0/1", "1|0", "0/.", "./0", "0", "", ".", "./."] { assert!(!both_copies(gt), "{gt}"); } } diff --git a/rust/crates/og-pgx/tests/i411_contraindications.rs b/rust/crates/og-pgx/tests/i411_contraindications.rs index 3d31046d3..208a15b8a 100644 --- a/rust/crates/og-pgx/tests/i411_contraindications.rs +++ b/rust/crates/og-pgx/tests/i411_contraindications.rs @@ -59,8 +59,8 @@ fn one_heterozygous_urea_cycle_hit_is_a_note() { "{gene}: {out:#?}" ); } - // A phased or partly missing genotype is still one changed copy. - for gt in ["0|1", "1|0", "./1"] { + // A phased genotype is still one changed copy. + for gt in ["0|1", "1|0"] { let out = lines(&[hit("ASS1", "40000000", "A", gt)]); assert!(warnings(&out).is_empty(), "{gt}: {out:#?}"); } @@ -76,6 +76,19 @@ fn both_copies_or_two_variants_in_a_urea_cycle_gene_warn() { "{gt}: {out:#?}" ); } + // Step 06 splits a 1/2 record into 1/. and ./1: the missing allele is + // the other ALT, so both copies are changed. + for gt in ["1/.", "./1"] { + let out = lines(&[hit("ASS1", "40000000", "A", gt)]); + assert_eq!( + warnings(&out), + [format!( + "WARNING: valproic acid -- ASS1 chr13:40000000 G>A, genotype {gt}, ClinVar Pathogenic, both copies changed in this recessive gene (the other copy is not the reference); ON THE MEDICATION LIST." + )], + "{gt}: {out:#?}" + ); + assert!(!out.iter().any(|l| l.contains("NOTE:")), "{gt}: {out:#?}"); + } // Two variants in trans, at two positions or as the two alleles of one // split multi-allelic site: one WARNING for each. for (pos2, alt2) in [("40000500", "A"), ("40000000", "C")] { From 83d39d68c91b88799f1c258bd02ce50b483d6a2e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:54:07 +0100 Subject: [PATCH 132/255] fix(pgx): count 1/. and ./1 as both copies changed in a recessive gene --- CHANGELOG.md | 2 +- docs/27-cpic-lookup.md | 2 +- rust/crates/og-pgx/src/sections.rs | 26 +++++++++++++++++--------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05209c768..25fe8aa67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- A single pathogenic variant in a recessive urea cycle gene (ASL, ASS1, CPS1 or NAGS) no longer gives a valproate `WARNING:` in step 27's drug contraindications. One copy makes a carrier, so the line is now a `NOTE:` that says so, and it stays out of the summary. A genotype with both copies changed, or two variants in the gene, still warns, and the line says which; the hits are not phased, so two variants on one copy still warn. POLG, OTC and HPRT1 warn on one variant, as before. The table `assets/pgx_contraindications.tsv` gets a `Report_if` column (`any` or `biallelic`) that holds this rule. The SCN1A rows for carbamazepine and phenytoin are gone: no drug label or guideline names SCN1A, and its gain-of-function variants make a gene-wide warning wrong. Each drug line now prints its CPIC level as provisional ("provisional CPIC level B: no CPIC guideline covers this pair"), which is what CPIC says of every pair in the table. Part of #411. +- A single pathogenic variant in a recessive urea cycle gene (ASL, ASS1, CPS1 or NAGS) no longer gives a valproate `WARNING:` in step 27's drug contraindications. One copy makes a carrier, so the line is now a `NOTE:` that says so, and it stays out of the summary. A genotype with both copies changed (`1/.` and `./1`, the two alleles of a split `1/2` site, count as both copies), or two variants in the gene, still warns, and the line says which; the hits are not phased, so two variants on one copy still warn. POLG, OTC and HPRT1 warn on one variant, as before. The table `assets/pgx_contraindications.tsv` gets a `Report_if` column (`any` or `biallelic`) that holds this rule. The SCN1A rows for carbamazepine and phenytoin are gone: no drug label or guideline names SCN1A, and its gain-of-function variants make a gene-wide warning wrong. Each drug line now prints its CPIC level as provisional ("provisional CPIC level B: no CPIC guideline covers this pair"), which is what CPIC says of every pair in the table. Part of #411. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/27-cpic-lookup.md b/docs/27-cpic-lookup.md index 180009f50..c27c4dfb7 100644 --- a/docs/27-cpic-lookup.md +++ b/docs/27-cpic-lookup.md @@ -44,7 +44,7 @@ MEDICATIONS=/path/to/medications.txt ./scripts/27-cpic-lookup.sh your_name 5. With the pypgx summary it writes the comparison table and, for a gene PharmCAT reports as not called or ambiguous but pypgx called (CYP2D6 is the usual one), a warning in the recommendations naming every drug PharmCAT links to that gene. When pypgx's call is normal the line is a note instead. 6. With step 36's consensus table it writes a section "Calls From Other Tools": each gene passed to PharmCAT as an outside call, from which tool, and whether PharmCAT's report shows it as one (`callSource` `OUTSIDE`); and each gene held back, with the reason and what each caller said. For a held-back CYP2D6 it names the drugs CYP2D6 affects and says no guidance is given for them, and the pypgx-only warning of point 5 is not printed for it: one caller's call is what the consensus refused. Genes PharmCAT reports as outside calls carry `[outside call]` in the gene results. An outside call of CYP2D6, HLA-A or HLA-B in PharmCAT's report that the table does not pass on, with the same diplotype, comes from an earlier step 36 run that step 07 has not replaced: it gets no drug guidance, is listed as not called, and a warning says to rerun step 07. When the table is missing (a caller step ran again since) but PharmCAT's report has outside calls of those genes, the step passes the missing table, so none of them is confirmed. MT-RNR1's outside call comes from step 20's chrM calls through step 07, never from step 36: the table never lists it, so it is not checked against it and keeps its drug guidance. 7. With a medication list, each line is matched by its first word (a space, comma, semicolon, slash or plus ends a word, so `lopinavir/ritonavir` is matched as lopinavir; leading commas, semicolons, slashes and pluses are ignored, and a line of only those is skipped) against `assets/pgx_inhibitors.tsv`: the FDA's strong and moderate inhibitors of CYP2D6 and CYP2C19, its moderate CYP2C9 inhibitors (it lists no strong one) and its strong CYP3A inhibitors, one row each for CYP3A4 and CYP3A5, as the FDA's table of clinical CYP inhibitors classes them, each row with its source. A ritonavir-boosted combination the FDA classes as one (`lopinavir and ritonavir`) has a row under ritonavir and one under its partner. For CYP2D6 the step applies CPIC's rule from its opioid guideline: "for patients on strong CYP2D6 inhibitors, the CYP2D6 activity score is adjusted to 0 and the predicted phenotype is a poor metabolizer", and for a moderate inhibitor "the activity score is multiplied by 0.5 and then converted to the predicted phenotype" (CPIC's bands: 0 poor, under 1.25 intermediate, 1.25 to 2.25 normal, above 2.25 ultrarapid). The activity score is the one PharmCAT gives the called diplotype. With several inhibitors of one gene the strongest counts. The result is written as `Poor Metabolizer (phenoconverted by paroxetine)` beside the genotype phenotype, never in its place, with a `WARNING:` line in the section "Medications and Phenoconversion", so the summary carries it. Nothing is converted for a gene that is not called, ambiguous or unclassified, or for a moderate inhibitor when the report gives no activity score: the line says why. CPIC has no activity score for CYP2C19, and gives no rule that converts the phenotype of CYP2C9, CYP3A4 or CYP3A5, so an inhibitor of one of these genes gets a note and a `not computed: ...` value, with no converted phenotype. For a poor metabolizer of these genes (the most common CYP3A5 result) the note does not say the inhibitor can lower the activity further. A line that names no drug of the table is listed as "Not in the inhibitor table", never dropped. A list that names no drug converts nothing and leaves the phenotype table as it is without a list. An unreadable list or table is reported in that section; the rest of the report is written as usual. The FDA's table lists no strong or moderate CYP2B6 inhibitor, so CYP2B6 has no rows. -8. With step 06's ClinVar hits, each hit classed `Pathogenic` or `Likely_pathogenic` (one part of `CLNSIG`; a conflicting classification or `risk_factor` alone does not count) in a gene of `assets/pgx_contraindications.tsv` adds a line in the section "Drug Contraindications From Disease Variants" for each drug the table names for the gene, with the label's note, the provisional CPIC level and the label; a drug whose name is on your medication list is marked `ON THE MEDICATION LIST`. The table's `Report_if` column says when the line is a `WARNING:`. For POLG, OTC and HPRT1 (`any`) one variant gives it. For the recessive urea cycle genes ASL, ASS1, CPS1 and NAGS (`biallelic`) it needs a genotype with both copies changed (`1/1`, `1/2`, a hemizygous `1`) or two distinct variants in the gene, and the line says which. The hits are not phased, so two variants on the same copy still give the warning. One heterozygous variant in such a gene makes a carrier: it gives a `NOTE:` line instead, which says so, and only `WARNING:` lines reach the summary. The table holds the CPIC gene-drug pairs at level A, B or A/B (CPIC API, queried 2026-11-03) whose gene is a disease gene PharmCAT does not call, whose drug the variant makes risky and whose risk a drug label names: POLG with valproic acid (the valproate label contraindicates it in POLG mitochondrial disorders), ASL, ASS1, CPS1, NAGS and OTC with valproic acid (the label contraindicates it in urea cycle disorders), and HPRT1 with mycophenolic acid (the label says to avoid it in hereditary HGPRT deficiency). Every one of these pairs is provisional in CPIC: no CPIC guideline covers it, and the line says so. SCN1A with carbamazepine and phenytoin is left out: no label or guideline names the gene, and SCN1A also has gain-of-function disease variants, for which a warning against these drugs does not follow. Without the hits table the section lists the table's genes as not checked. +8. With step 06's ClinVar hits, each hit classed `Pathogenic` or `Likely_pathogenic` (one part of `CLNSIG`; a conflicting classification or `risk_factor` alone does not count) in a gene of `assets/pgx_contraindications.tsv` adds a line in the section "Drug Contraindications From Disease Variants" for each drug the table names for the gene, with the label's note, the provisional CPIC level and the label; a drug whose name is on your medication list is marked `ON THE MEDICATION LIST`. The table's `Report_if` column says when the line is a `WARNING:`. For POLG, OTC and HPRT1 (`any`) one variant gives it. For the recessive urea cycle genes ASL, ASS1, CPS1 and NAGS (`biallelic`) it needs a genotype with both copies changed (`1/1`, a hemizygous `1`, or `1/.` and `./1`, which step 06 writes for the two alleles of a `1/2` site: the missing copy holds the other allele, not the reference) or two distinct variants in the gene, and the line says which. The hits are not phased, so two variants on the same copy still give the warning. One heterozygous variant in such a gene makes a carrier: it gives a `NOTE:` line instead, which says so, and only `WARNING:` lines reach the summary. The table holds the CPIC gene-drug pairs at level A, B or A/B (CPIC API, queried 2026-11-03) whose gene is a disease gene PharmCAT does not call, whose drug the variant makes risky and whose risk a drug label names: POLG with valproic acid (the valproate label contraindicates it in POLG mitochondrial disorders), ASL, ASS1, CPS1, NAGS and OTC with valproic acid (the label contraindicates it in urea cycle disorders), and HPRT1 with mycophenolic acid (the label says to avoid it in hereditary HGPRT deficiency). Every one of these pairs is provisional in CPIC: no CPIC guideline covers it, and the line says so. SCN1A with carbamazepine and phenytoin is left out: no label or guideline names the gene, and SCN1A also has gain-of-function disease variants, for which a warning against these drugs does not follow. Without the hits table the section lists the table's genes as not checked. 9. A report that cannot be read, or that yields no gene, writes a "PARSING FAILED" report and the step exits 1. It never writes an all-clear report from a format it could not read. A field of the report with an unexpected type (a number where a name belongs, say) reads as missing. 10. The `Date:` line of the recommendations is today's date in UTC. The output files are written under a temporary name and renamed into place, so a step that stops half way leaves no half-written table. diff --git a/rust/crates/og-pgx/src/sections.rs b/rust/crates/og-pgx/src/sections.rs index 7ce151301..8bd5cd0b3 100644 --- a/rust/crates/og-pgx/src/sections.rs +++ b/rust/crates/og-pgx/src/sections.rs @@ -643,13 +643,14 @@ pub fn hit_genes(geneinfo: Option<&str>) -> BTreeSet { .collect() } -/// Every allele of the genotype is called and none is the reference: both -/// copies changed (`1/1`, `1|2`), or the one copy of a hemizygous call (`1`). -/// `0/1`, `1|0` and `./1` are one changed copy. +/// No allele of the genotype is the reference and at least one is a called +/// ALT: both copies changed (`1/1`, `1|2`), the one copy of a hemizygous +/// call (`1`), or one allele of a split multi-allelic site (`1/.`, `./1`). +/// Step 06 splits a `1/2` record into `1/.` and `./1`: the missing allele +/// is the other ALT, not the reference. `0/1`, `1|0` and `./.` are not. fn both_copies(genotype: &str) -> bool { - genotype - .split(['/', '|']) - .all(|a| !matches!(a.trim(), "" | "." | "0")) + let alleles = || genotype.split(['/', '|']).map(str::trim); + !alleles().any(|a| matches!(a, "" | "0")) && alleles().any(|a| a != ".") } /// The lines for each pathogenic or likely pathogenic hit in a gene of the @@ -707,9 +708,16 @@ pub fn contraindication_lines( ); match r.report_if { ReportIf::Any => lines.push(format!(" WARNING: {head}{on}.")), - ReportIf::Biallelic if both_copies(&h.genotype) => lines.push(format!( - " WARNING: {head}, both copies changed in this recessive gene{on}." - )), + ReportIf::Biallelic if both_copies(&h.genotype) => { + let why = if h.genotype.split(['/', '|']).any(|a| a.trim() == ".") { + " (the other copy is not the reference)" + } else { + "" + }; + lines.push(format!( + " WARNING: {head}, both copies changed in this recessive gene{why}{on}." + )); + } ReportIf::Biallelic if n >= 2 => lines.push(format!( " WARNING: {head}, {n} variants in {gene}, a recessive gene (phase not known){on}." )), From bdea2e905712d8636de5b02c5228ad9da32d8c41 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:10:09 +0100 Subject: [PATCH 133/255] test(setup): require clinical installs to resume in low space and report a skip --- tests/fake-docker/setup-clinical-default.sh | 40 +++++++++++++++++---- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/tests/fake-docker/setup-clinical-default.sh b/tests/fake-docker/setup-clinical-default.sh index 83a674b03..e83b145d9 100644 --- a/tests/fake-docker/setup-clinical-default.sh +++ b/tests/fake-docker/setup-clinical-default.sh @@ -28,11 +28,7 @@ output_has plan '^ 13 VEP +runs$' output_has plan '^ 23 Clinical filter +runs$' output_has plan '^ 17 CPSR +runs$' -# A second install does no network work, even with too little space. -cp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" -run_expect 0 again "${SCRIPTS}/setup.sh" --clinical-data "$G" -cmp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" || fail 'installed clinical data downloaded again' - +# A fake df that reports 40 GiB free. mkdir -p "${CASE_WORK}/low-bin" cat > "${CASE_WORK}/low-bin/df" <<'DF' #!/usr/bin/env bash @@ -40,19 +36,49 @@ echo 'Filesystem 1024-blocks Used Available Capacity Mounted on' echo 'fakefs 41943040 0 41943040 0% /' DF chmod +x "${CASE_WORK}/low-bin/df" + +# A second install does no network work, even with too little space. +cp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" +PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 0 again "${SCRIPTS}/setup.sh" --clinical-data "$G" +cmp "$FAKE_DOCKER_LOG" "${CASE_WORK}/installed.log" || fail 'installed clinical data downloaded again' + +# A skipped item makes --clinical-data exit 1; the other items still install. G="${CASE_WORK}/low" -PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 0 low "${SCRIPTS}/setup.sh" --clinical-data "$G" +PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 1 low "${SCRIPTS}/setup.sh" --clinical-data "$G" output_has low '56 GB.*5 GB margin' output_has low '52 GB.*5 GB margin' [ ! -e "${G}/vep_cache/homo_sapiens/116_GRCh38" ] || fail 'low-space VEP installed' [ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'small bundle skipped with 40 GB free' +# A partial download counts as space already taken: 40 GiB free plus a +# 22 GiB .part covers VEP 116's 56 GB peak and margin, so it resumes. +G="${CASE_WORK}/resume" +mkdir -p "${G}/vep_cache" +dd if=/dev/null of="${G}/vep_cache/homo_sapiens_vep_116_GRCh38.tar.gz.part" bs=1048576 seek=22528 2>/dev/null +PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 1 resume "${SCRIPTS}/setup.sh" --clinical-data "$G" +output_lacks resume 'Skipping VEP 116' +[ -s "${G}/vep_cache/homo_sapiens/116_GRCh38/info.txt" ] || fail 'partial VEP 116 download not resumed with low space' +[ ! -e "${G}/vep_cache/homo_sapiens_vep_116_GRCh38.tar.gz.part" ] || fail 'resumed VEP 116 left its .part' + +# An install stopped during extraction keeps its checked archive: the next +# run unpacks it without downloading again and removes the old staging tree. +G="${CASE_WORK}/extract" +mkdir -p "${G}/vep_cache/.extract.old" "${G}/pcgr_data/.extract.old" +cp "${FAKE_DOWNLOAD_DIR}/homo_sapiens_vep_116_GRCh38.tar.gz" "${G}/vep_cache/" +cp "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" "${G}/pcgr_data/" +FAKE_DOWNLOAD_FAIL='vep_116|pcgr_ref_data' run_expect 0 extract "${SCRIPTS}/setup.sh" --clinical-data "$G" +[ -s "${G}/vep_cache/homo_sapiens/116_GRCh38/info.txt" ] || fail 'checked VEP 116 archive downloaded again' +[ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'checked PCGR archive downloaded again' +[ ! -e "${G}/vep_cache/.extract.old" ] || fail 'stale VEP staging tree kept' +[ ! -e "${G}/pcgr_data/.extract.old" ] || fail 'stale PCGR staging tree kept' + # The digest belongs to the original fixture. Serve a different valid tarball. printf 'corrupt\n' >> "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" G="${CASE_WORK}/corrupt" -run_expect 0 corrupt "${SCRIPTS}/setup.sh" --clinical-data "$G" +run_expect 1 corrupt "${SCRIPTS}/setup.sh" --clinical-data "$G" output_has corrupt 'sha256 checksum.*expected' output_has corrupt '\[WARN\].*PCGR' [ ! -e "${G}/pcgr_data/20260620" ] || fail 'corrupt bundle published' [ ! -e "${G}/pcgr_data/pcgr_ref_data.20260620.grch38.tgz.part" ] || fail 'bad bundle kept' [ -s "${G}/vep_cache/homo_sapiens/115_GRCh38/info.txt" ] || fail 'bundle failure stopped remaining install' + From d80c29ba06dbc117667fb0f286bb9b0257598b20 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:10:27 +0100 Subject: [PATCH 134/255] fix(live): give adult pages main's answer order, close a child on every family page and never show a closed count as nothing flagged --- assets/live/core.js | 99 ++++++++++++++++++++++++++--------- assets/live/family-page.js | 6 ++- assets/live/map.js | 4 +- bin/owngenome_live.py | 10 ++-- bin/owngenome_live_family.py | 33 ++++++++++-- tests/demo/build_live_demo.py | 3 +- 6 files changed, 118 insertions(+), 37 deletions(-) diff --git a/assets/live/core.js b/assets/live/core.js index 5c5e0fd7f..0728fb4a8 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -186,17 +186,45 @@ } // A file the gate reads before the views, with the options the view that reads it uses: its answer is kept once, // whatever its status (a 404 or a failed request too), so the view's own first read of it costs no request. The - // view gets the response itself; the gate reads a copy of an answer that is ok and leaves any other unread, as - // the views do. - function sharedJSON(url, options) { + // view gets the response itself, and the kept answers reach the views in the order they arrived (inTurn), as the + // network would have handed them without the gate. + var arrivals = 0, claimed = []; + function keep(url, options) { url = absolute(url); - return originalFetch(url, options).then(function (r) { - shared[url] = { response: r.ok ? r.clone() : r }; - return r.ok ? read(r) : read({ status: r.status, ok: false }); - }, function () { shared[url] = { failed: true }; return null; }).catch(function () { return null; }); - } - function keptResponse(kept) { - return kept.failed ? Promise.reject(new TypeError('Failed to fetch')) : Promise.resolve(kept.response); + var entry = shared[url] = { at: null }; + entry.kept = originalFetch(url, options).then(function (r) { + entry.response = r.ok ? r.clone() : r; + return r.ok ? r.text().then(function (t) { entry.text = t; }) : null; + }).catch(function () { entry.failed = true; }).then(function () { entry.at = ++arrivals; return entry; }); + return entry; + } + // What the gate itself reads of a kept answer: its JSON when it is ok, else null (a 401 goes to the login). + function parsedOf(entry) { + return entry.kept.then(function (e) { + if (e.failed) { return null; } + if (!e.response.ok) { return read({ status: e.response.status, ok: false }); } + return JSON.parse(e.text); + }).catch(function () { return null; }); + } + function sharedJSON(url, options) { return parsedOf(keep(url, options)); } + // A kept answer reaches its view in a task of its own, after every kept answer the views asked for that arrived + // earlier. Its json() gives the body the gate read, at once, so no answer overtakes another while it is parsed. + function inTurn(entry) { + claimed.push(entry); + return entry.kept.then(function () { + return new Promise(function (go, fail) { + (function wait() { + setTimeout(function () { + if (claimed.some(function (c) { return !c.handed && c.at !== null && c.at < entry.at; })) { wait(); return; } + entry.handed = true; + if (entry.failed) { fail(new TypeError('Failed to fetch')); return; } + var r = entry.response, text = entry.text; + if (r.ok) { r.json = function () { return Promise.resolve().then(function () { return JSON.parse(text); }); }; } + go(r); + }, 0); + }()); + }); + }); } function random() { var b = new Uint32Array(1); crypto.getRandomValues(b); return b[0] / 4294967296; } // The results to act on, one record each: the findings of level "act" (bin/live_results.py). null when unreadable. @@ -220,7 +248,8 @@ } function makeRecord(url, child, r, old) { var acts = actsOf(r), rec = { url: url, child: child, acts: acts, opened: {}, order: [], perm: [], - ids: r && Array.isArray(r.findings) ? r.findings.map(function (f) { return f && f.id; }) : [] }; + ids: r && Array.isArray(r.findings) ? r.findings.map(function (f) { return f && f.id; }) : [], + attention: r && typeof r.attention === 'number' ? r.attention : null }; if (!acts) { return rec; } // An act whose record changed is closed again; an unchanged one keeps its opening for this page. rec.order = old ? old.order.filter(function (id) { return same(byId(old, id), byId(rec, id)); }) : []; @@ -229,9 +258,10 @@ rec.perm = old && same(ids, (old.acts || []).map(function (f) { return f.id; })) ? old.perm : shuffled(ids); return rec; } - function record(url, child, cached) { + // PRE: the kept answer of results.json the gate asked for already (preparePerson), else it is read here. + function record(url, child, cached, pre) { url = absolute(url); - return (cached ? sharedJSON(url + 'results.json', NO_STORE) : jsonAt(url + 'results.json')).then(function (r) { + return (pre ? parsedOf(pre) : cached ? sharedJSON(url + 'results.json', NO_STORE) : jsonAt(url + 'results.json')).then(function (r) { var old = records[url]; var rec = makeRecord(url, child !== undefined ? child : old ? old.child : true, r, old); records[url] = rec; @@ -351,8 +381,10 @@ // The person and finding pages: the run's state, its family's roles and, for a child, its results, read in the // order the views read them (the finding page asks for its results at once, the person page once the state has // answered) and with the views' options. The family is read once the page's scripts have run, so live.js has - // opened its event stream first, as it does without the gate. An adult's results are not read here: the views read - // them as they would without the gate. + // opened its event stream first, as it does without the gate. In live mode the person page asks for the family and + // then its results on its first state, both at once: the gate asks for both in that order before it knows whether + // the person is a child, so the two answers race as they do without the gate and reach the views in the order they + // arrived. Only a child's results are read by the gate; an adult's answer goes to the views unread. var parsed = new Promise(function (done) { if (document.readyState === 'complete') { done(); } else { document.addEventListener('DOMContentLoaded', done); window.addEventListener('load', done); } }); @@ -361,21 +393,24 @@ var replaying = root.getAttribute('data-mode') === 'replay'; // live.js reads a replay's frames.json with no options; finding.js reads it, and state.json, as JSON (NO_STORE). var stateRead = sharedJSON(url + (replaying ? 'frames.json' : 'state.json'), replaying && !findingPath ? undefined : NO_STORE); - var results = findingPath ? record(url, undefined, true) : null; + var results = findingPath ? record(url, undefined, true) : null, pre = null; return Promise.all([stateRead, parsed]).then(function (x) { var raw = x[0]; var state = replaying ? raw && raw.frames && raw.frames.slice(-1)[0] : raw; state = replaying ? state && state.state : state; var fam = state && state.plugins && state.plugins.family; var entry = fam && fam.kind === 'member' && (fam.families || [])[0]; - return entry ? sharedJSON(absolute(entry.url, url) + 'family.json', NO_STORE).then(function (f) { + if (!entry) { return !state || !state.plugins || !!(fam && fam.unknown); } + var famRead = sharedJSON(absolute(entry.url, url) + 'family.json', NO_STORE); + if (!replaying && !findingPath) { pre = keep(url + 'results.json', NO_STORE); } + return famRead.then(function (f) { var member = f && f.state && (f.state.members || []).filter(function (m) { return m.sample === root.getAttribute('data-run'); })[0]; // A family that cannot be read, or does not list this person, cannot say they are not a child. return !member || !!(member.father || member.mother); - }) : !state || !state.plugins || !!(fam && fam.unknown); + }); }).then(function (child) { if (!child && !results) { return { url: url, child: false, acts: [], opened: {}, order: [], perm: [] }; } - return (results || record(url, child, true)).then(function (rec) { rec.child = child; return rec; }); + return (results || record(url, child, true, pre)).then(function (rec) { rec.child = child; return rec; }); }).then(function (rec) { current = records[url] = rec; return consume(current); @@ -386,20 +421,31 @@ current = { child: true, acts: null, opened: {}, order: [], perm: [] }; ownsPage = true; renderGate(); }); } - // A family's children with a run here; a member who was never sequenced, or has no run, has no report to close. + // A family's children with a run here, by the one rule (childIn, bin/owngenome_live.py child_of): family.json + // marks each member run `child` as the server decides it over every family, so a person who is a child in another + // family is one here too. A member who was never sequenced, or has no run, has no report to close. + function childRun(f, s) { var r = f.runs && f.runs[s]; return !!(r && r.url && r.child !== false); } function familyRecords(f, base) { - return Promise.all(((f.state && f.state.members) || []).filter(function (m) { - return (m.father || m.mother) && f.runs && f.runs[m.sample] && f.runs[m.sample].url; - }).map(function (m) { + return Promise.all(((f.state && f.state.members) || []).filter(function (m) { return childRun(f, m.sample); }).map(function (m) { return record(absolute(f.runs[m.sample].url, base), true).then(function (r) { return [m.sample, r]; }); })).then(function (entries) { var out = {}; entries.forEach(function (e) { out[e[0]] = e[1]; }); return out; }); } // A run's progress on the home and family pages: how far the run is, never what it found. var PROGRESS = ['name', 'kind', 'url', 'state', 'started_utc', 'last_heard_utc', 'finished_utc', 'planned', 'done', 'running', 'failed']; function progressOf(r) { var out = {}; PROGRESS.forEach(function (k) { if (r[k] !== undefined) { out[k] = r[k]; } }); return out; } + // A released child's flagged-card count: family.json leaves it out (null) while the server keeps the child's report + // closed, so the page takes it from the results it read, once every result to act on is opened. + function released(raw, recs) { + Object.keys(recs).forEach(function (k) { + var r = raw.runs && raw.runs[k]; + if (r && r.attention === null && !shut(recs[k]) && recs[k].attention !== null) { r.attention = recs[k].attention; } + }); + return raw; + } function projectFamily(raw, recs) { var closed = Object.keys(recs).filter(function (k) { return shut(recs[k]); }); - if (!closed.length) { return raw; } + if (!closed.length) { return released(raw, recs); } + raw = released(raw, recs); function gated(s) { return closed.indexOf(s) >= 0; } // An ownership allowlist, not a classifier of genes, prose or individual report fields: a field a producer adds // tomorrow stays out until it is listed here. @@ -409,7 +455,8 @@ layout: raw.layout ? { width: raw.layout.width, height: raw.layout.height, size: raw.layout.size, lines: copy(raw.layout.lines), people: (raw.layout.people || []).map(function (m) { return { sample: m.sample, sex: m.sex, sequenced: m.sequenced, x: m.x, y: m.y }; }) } : null, couple_note: raw.couple_note, chrom_lengths: copy(raw.chrom_lengths), gated: closed, runs: {} }; - f.state.children = (raw.state.members || []).filter(function (m) { return m.father || m.mother; }).map(function (m) { + // A gated person this family lists as a parent or partner (a child in another family) gets their buttons here too. + f.state.children = (raw.state.members || []).filter(function (m) { return m.father || m.mother || gated(m.sample); }).map(function (m) { return gated(m.sample) ? { child: m.sample, gated: true } : copy((raw.state.children || []).filter(function (c) { return c.child === m.sample; })[0]); }).filter(Boolean); // A row belongs to a gated child when it names that child as one of its members, by exact name. @@ -502,7 +549,7 @@ if (ownsPage || closedRun(requestURL)) { return new Promise(function () {}); } var kept = shared[requestURL]; delete shared[requestURL]; - var got = kept ? keptResponse(kept) : originalFetch(url, options); + var got = kept ? inTurn(kept) : originalFetch(url, options); return got.then(function (response) { if (ownsPage) { return new Promise(function () {}); } var path = new URL(requestURL).pathname; diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 8c93c9557..26a10a130 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -138,7 +138,11 @@ var state = !m.sequenced ? 'Not sequenced' : !r ? 'No run on this server' : r.state === 'finished' ? 'Finished' : (r.state === 'stopped' ? 'Stopped, ' : '') + r.done + ' of ' + r.planned + ' steps'; add(txt, el('span', { 'class': 'person-state' }, state)); - if (r && r.results) { + // A child whose report is closed gets the fixed closed line. A count left out (null: the server keeps the + // child's report closed and the page has not opened it) is never read as nothing flagged. + if (r && r.gated) { + add(txt, el('span', { 'class': 'person-look' }, 'Report closed')); + } else if (r && r.results && r.attention !== null && r.attention !== undefined) { add(txt, el('span', { 'class': 'person-look' + (r.attention ? ' attn' : '') }, r.attention ? r.attention + ' worth a look' : 'Nothing flagged; see the caveats')); } diff --git a/assets/live/map.js b/assets/live/map.js index f12499d5b..d66dd078c 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -165,7 +165,9 @@ add(head, kicker, add(el('div', { 'class': 'gm-titlerow' }), title, gauge, zoom), crumb, finder, chips); var stage = el('div', { 'class': 'gm-stage' }); var canvas = el('canvas', { 'class': 'gm-canvas', tabindex: '0', role: 'img', 'aria-describedby': 'gm-narration' }); - var tip = el('div', { 'class': 'gm-tip', role: 'tooltip', hidden: 'hidden' }); + // hidden as hideTip() writes it (tip.hidden = true gives hidden=""), so the drawing is the same whether or not a + // refresh hid the tooltip before the document arrived. + var tip = el('div', { 'class': 'gm-tip', role: 'tooltip', hidden: '' }); var toast = el('div', { 'class': 'gm-toast', role: 'status', hidden: 'hidden' }); add(stage, canvas, tip, toast); // The marks drawn, as links: the keyboard and screen-reader path to each finding (Tab steps through them). diff --git a/bin/owngenome_live.py b/bin/owngenome_live.py index 9af6f86ad..05125276b 100755 --- a/bin/owngenome_live.py +++ b/bin/owngenome_live.py @@ -909,10 +909,14 @@ def runs_index(self): lo = (hit[0], None) self.live_overview[name] = lo o = lo[1] or o - # A child's results to act on decide whether its own page is closed: count them from the summary - # its results.json reads, so the home page closes the child whenever the child's page does. + # A child's results to act on decide whether its own page is closed: the summary its results.json + # reads counts them. When that summary or the overview above has one, that summary's overview + # replaces it, so the home page closes the child exactly when the child's page does, with its count. + # A child with none in either keeps the numbers it had without the gate. if self.child(name, runs): - o = self.summary_overview(name, r["dir"] or os.path.join(self.root, name), st) or o + so = self.summary_overview(name, r["dir"] or os.path.join(self.root, name), st) + if so and any(((x or {}).get("levels") or {}).get("act") for x in (so, o)): + o = so item["attention"] = o["attention"] if o else None item["results"] = mtime is not None if o: # the Journal home: counts by level, the first findings, the lead diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py index 277a64f35..a55c0f616 100644 --- a/bin/owngenome_live_family.py +++ b/bin/owngenome_live_family.py @@ -21,11 +21,14 @@ compound_het.tsv) with whether each child was assessed and why not (trio_status; a file linking out of the folder is "rejected" in trio_file_state), - each member's run progress and attention count, + each member's run progress, attention count and + whether their pages treat them as a child (child), and the members side by side: CYP2D6, the CPIC genes, HLA, the haplogroups and every blood group system, each value with its stale flag (from each - summary.json only) + summary.json only); 500 with a fixed line when a + file of the family cannot be read, and the family + then lists no members until it can state.json gets plugins.family: {"kind": "family", ...} on a family's page, {"kind": "member", "families": [...]} on the page of a sample in a family, @@ -514,15 +517,34 @@ def runs(app_): "sequenced": members[s]["sequenced"]} for s in order] except (OSError, ValueError): people = [] + # A family whose family.json could not be built has no readable members either: the server then decides + # who is a child as the page does when family.json fails (bin/owngenome_live.py child_of). + if n in unreadable: + people = [] out.append({"name": n, "dir": d, "kind": "family", "members": people}) return out + # Families whose last family.json failed: listed with no members until one is built again. + unreadable = set() + def page_data(req, m): name = m.group(1) d = req.run_dir(name) if not d or not is_family(d) or not inside(d, root()): app.log("family", name, "family.json", 404) return req.send_body(404, "text/plain; charset=utf-8", "not found\n") + try: + out = page_json(req, name, d) + except Exception as e: # an unreadable file is a fixed error, never a dropped connection + unreadable.add(name) + print(f"owngenome_live: family.json of {name} not built: {type(e).__name__}", file=sys.stderr) + app.log("family", name, "family.json", 500) + return req.send_body(500, "text/plain; charset=utf-8", "family not readable\n") + unreadable.discard(name) + app.log("family", name, "family.json", 200) + return req.send_json(out) + + def page_json(req, name, d): out = family_page(name, d) live = getattr(req, "live", None) runs = {} @@ -539,9 +561,11 @@ def page_data(req, m): # with which cards the unopened results flag, and a parent's page would show it. if live.gated(p["sample"], rd, st, listed): n = None + # child: whether the person's pages treat them as a child, by the one rule over every family (child_of), + # so the family page closes the same people as their own pages do. runs[p["sample"]] = {"url": "/runs/%s/" % p["sample"], "state": st["run"]["state"], "planned": len(planned), "done": sum(1 for s in planned if s["state"] == "done"), - "attention": n, "results": mtime is not None} + "attention": n, "results": mtime is not None, "child": live.child(p["sample"], listed)} out["runs"] = runs given = {} comp_p = os.path.join(d, "compound_het.tsv") @@ -559,8 +583,7 @@ def page_data(req, m): r = live.run(s_) if live else None sums[s_] = member_summary(live, r["dir"] or os.path.join(live.root, r["name"])) if r else None out["side_by_side"] = side_by_side(seq, sums) if seq else None - app.log("family", name, "family.json", 200) - return req.send_json(out) + return out def hook(run_dir, state, events): if not run_dir: diff --git a/tests/demo/build_live_demo.py b/tests/demo/build_live_demo.py index 5eac6c200..8327144f7 100644 --- a/tests/demo/build_live_demo.py +++ b/tests/demo/build_live_demo.py @@ -350,7 +350,8 @@ def family_json(root, index): for p in out["state"].get("members") or [] if p["sample"] in by} # As the family plugin serves it: no count of flagged cards for a child whose own page keeps the report closed. for name, r in out["runs"].items(): - if owngenome_live.child_of(index.get("families"), name) and (by[name].get("levels") or {}).get("act"): + r["child"] = owngenome_live.child_of(index.get("families"), name) + if r["child"] and (by[name].get("levels") or {}).get("act"): r["attention"] = None seq = [p["sample"] for p in out["state"].get("members") or [] if p.get("sequenced")] sums = {} From c8f0c64d44c75b0b29b414b2a0c48a06af9b4271 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:10:27 +0100 Subject: [PATCH 135/255] test(live): check section addresses in live mode, the family page's count and roles, a family that cannot be read, and the release and cleanup guards --- tests/test_live_demo.py | 190 +++++++++++++++++++--------- tests/test_owngenome_live_family.py | 179 +++++++++++++++++++++++++- 2 files changed, 303 insertions(+), 66 deletions(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index cfed65d67..485079f6b 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -17,6 +17,7 @@ import gzip import json import os +import re import shutil import subprocess import sys @@ -174,9 +175,13 @@ def log_message(self, *a): pass def end_headers(self): - self.send_header("Cache-Control", "no-store") + if self.path.split("?")[0] not in Static.BFCACHE: + self.send_header("Cache-Control", "no-store") super().end_headers() + # Paths answered without no-store, so Chrome may keep their page in the back-forward cache (gate_controls). + BFCACHE = set() + # A path answered after a wait, once (the gate's slow-reload check): {path: seconds}. DELAY = {} @@ -1117,8 +1122,8 @@ def map_find(b, q): def find_problems(b, base, out): """The search box: / puts the cursor in it; a gene, a band and a finding are found and Enter flies there (a - finding with its card); on the son's closed page his closed result is never a line, by its gene, its id or - its title, while his open finding still is.""" + finding with its card); APOE, closed on every page until opened, is never a line, by its gene, its id or its + title, while an open finding still is. (The son's page is closed whole by the report gate: it has no map.)""" bad = [] map_page(b, base, S) map_key(b, "/", target="body") @@ -1138,27 +1143,27 @@ def find_problems(b, base, out): bad.append(f"{q}: Enter flies to {c['chrom']} at {c['level']}, want {chrom} at {lv}") if kind == "finding" and "DEMOC" not in n["card"]: bad.append(f"{q}: Enter opens no card for it: {n['card']!r}") - son = OTHERS[1] - doc = read_map(os.path.join(out, "runs", son, "map.json")) - f = doc["per"][son]["findings"] - i = [j for j, lv in enumerate(f["level"]) if lv == "act" and not f["gate"][j]][0] + doc = read_map(os.path.join(out, "runs", S, "map.json")) + f = doc["per"][S]["findings"] + i = [j for j in range(len(f["id"])) if f["gate"][j]][0] fid, gene, title = f["id"][i], f["gene"][i], f["title"][i] - map_page(b, base, son) + map_page(b, base, S) for q in (gene, fid, title): found = map_find(b, q) - named = [x for x in found if (x["kind"] == "finding" and gene in x["text"]) or fid in x["text"] or title in x["text"]] + # The gene itself stays findable (a line of kind "gene", from the gene names); its finding never is. + named = [x for x in found if x["kind"] == "finding" and (gene in x["text"] or fid in x["text"] or title in x["text"])] if named: - bad.append(f"the son's closed result is a line for {q!r}: {named}") + bad.append(f"{gene}, closed, is a line for {q!r}: {named}") found = map_find(b, "DEMOC") if not any(x["kind"] == "finding" for x in found): - bad.append(f"the son's open finding DEMOC is not found: {found}") + bad.append(f"the open finding DEMOC is not found: {found}") return bad def nearest_problems(b, base, out): """Nearest: with no finding of the open chromosome in view it names the closest one (else the first on the - next chromosome that has one) and flies there with its card; Esc closes the card; on the son's closed page - it never leads to his closed result.""" + next chromosome that has one) and flies there with its card; Esc closes the card; it never leads to APOE, + closed on every page until opened.""" bad = [] for link, words, chrom in (("#chr11:100.0-101.0M", "Nearest: DEMOB at 108.3 Mb ›", "chr11"), ("#chr3", "Nearest: DEMOB on chr11 ›", "chr11")): @@ -1183,19 +1188,18 @@ def nearest_problems(b, base, out): time.sleep(0.2) if json.loads(b.eval(NEAR_JS))["card"] or map_now(b)["level"] == "chromosome": bad.append(f"{link}: Esc leaves the card open or steps out with a card open") - son = OTHERS[1] - doc = read_map(os.path.join(out, "runs", son, "map.json")) - f = doc["per"][son]["findings"] - i = [j for j, lv in enumerate(f["level"]) if lv == "act" and not f["gate"][j]][0] + doc = read_map(os.path.join(out, "runs", S, "map.json")) + f = doc["per"][S]["findings"] + i = [j for j in range(len(f["id"])) if f["gate"][j]][0] others = [f["s"][j] for j in range(len(f["id"])) if f["chrom"][j] == f["chrom"][i] and j != i] - # A view next to his closed result, with every other finding of that chromosome farther away. + # A view next to the closed result, with every other finding of that chromosome farther away. a = max(0, f["s"][i] - 3_000_000) if not any(abs(x - f["s"][i]) < 6_000_000 for x in others): - n = map_page(b, base, son, f"#{f['chrom'][i]}:{a + 1}-{a + 2_000_000}") and json.loads(b.eval(NEAR_JS)) + n = map_page(b, base, S, f"#{f['chrom'][i]}:{a + 1}-{a + 2_000_000}") and json.loads(b.eval(NEAR_JS)) if f["gene"][i] in n["near"] or f["gene"][i] in n["card"]: - bad.append(f"the son's Nearest leads to his closed result: {n['near']!r}") + bad.append(f"Nearest leads to {f['gene'][i]}, closed: {n['near']!r}") else: - bad.append("the son's closed result has another finding within 6 Mb: no view to check Nearest on") + bad.append(f"{f['gene'][i]}, closed, has another finding within 6 Mb: no view to check Nearest on") return bad @@ -1479,6 +1483,12 @@ def plant_long_run(out, og): return keep +def unplant(keep): + for p, data in keep.items(): + with open(p, "wb") as fh: + fh.write(data) + + def predicate_problems(b, base, out, og, runs=None): """drawsAt() of map.js against og map build --predicate-table, row by row, on the three demo runs (DEMO-001 with a planted long run; RUNS only when given): the same rows and the same answer at each level; and the @@ -1573,7 +1583,7 @@ def count_words(n): (S, "#chr22:40.5-43.5M", "region"), (S, "#chr11:4.0-7.0M", "region"), (S, "#chr19:43.5-46.5M", "region"), (S, "#chr11:5.150-5.350M", "gene"), (S, "#chr22:42.000-42.200M", "gene"), (S, "#chr11:5225001-5250000", "reads"), (S, "#chr22:42110001-42140000", "reads"), - (OTHERS[1], "", "genome"), (OTHERS[1], "#chr16", "chromosome")) + (OTHERS[0], "", "genome"), (OTHERS[0], "#chr16", "chromosome")) def inview_problems(b, base, views=INVIEW_VIEWS): @@ -1619,29 +1629,18 @@ def inview_problems(b, base, views=INVIEW_VIEWS): return bad -def inview_veil_problems(b, base, adult=True): - """A closed result is never a row of the table or a tick of the strip: APOE on DEMO-001's page (unless not - ADULT), the son's closed result to act on on his; once the son's is opened from the chip it is both, so the - check sees it.""" - bad = [] +def inview_veil_problems(b, base): + """A closed result is never a row of the table or a tick of the strip: APOE on DEMO-001's page. The son's page, + closed whole by the report gate while a result to act on is unopened, has no map, table or strip at all.""" + bad = inview_apoe_problems(b, base) son = OTHERS[1] - if adult: - bad += inview_apoe_problems(b, base) - c, t = inview_page(b, base, son, "#chr7") - closed = [k for k in t["rows"] + (t["ticks"] or []) if k.startswith("findings\tchr7\t")] - if closed or "KCNH2" in t["text"]: - bad.append(f"{son}: his closed result shows at chr7: {closed}") - # The chip opens his closed results one per click, in the page's order: open them until the one at chr7 is. - for _ in range(4): - _, cbad = chip_open(b) - bad += cbad - time.sleep(0.5) - t = inview_now(b) - if any(k.startswith("findings\tchr7\t") for k in t["rows"]) or cbad: - break - if not any(k.startswith("findings\tchr7\t") for k in t["rows"]) or \ - not any(k.startswith("findings\tchr7\t") for k in (t["ticks"] or [])): - bad.append(f"{son}: opened, his result is not a row and a tick at chr7: {t['rows']}, {t['ticks']}") + b.go("about:blank") + b.wait("location.href === 'about:blank'") + b.go(f"{base}/runs/{son}/#chr7") + if not b.wait("document.documentElement.dataset.pgpGated === 'true'"): + bad.append(f"{son}: his page is not closed by the report gate") + elif b.eval("!!document.querySelector('#genome-map, .gm-inview, .gm-strip, canvas')"): + bad.append(f"{son}: his closed page has a map, a table or a strip") return bad @@ -2027,15 +2026,15 @@ def planted(desc, a, z, fn, more=()): " if (!drawsAt('findings', r, 'genome') || r.chrom !== chrom().n) { return; }", lambda: inview_problems(b, base, INVIEW_VIEWS[:2])) bad = inview_veil_problems(b, base) - check("map: a closed result is no row of the In view table and no tick of the strip (APOE on an adult's page, " - "the son's result to act on), until opened", not bad, bad) + check("map: a closed result is no row of the In view table and no tick of the strip (APOE on an adult's page); " + "the son's page, closed by the report gate, has no map", not bad, bad) planted("map.js whose table lists every finding, closed or not", "drawn().forEach(function (r) { if (meets(", "M.findings.forEach(function (r) { if (meets(", - lambda: inview_veil_problems(b, base, adult=False)) + lambda: inview_apoe_problems(b, base)) planted("map.js whose strip ticks every finding, closed or not", " drawn().forEach(function (r) {\n if (!drawsAt('findings', r, 'genome'))", " M.findings.forEach(function (r) {\n if (!drawsAt('findings', r, 'genome'))", - lambda: inview_veil_problems(b, base, adult=False)) + lambda: inview_apoe_problems(b, base)) bad = sort_problems(b, base) check("map: the In view table sorts by place with the findings first (aria-sort ascending on Where), Where again " "reverses it, and Size sorts by size", not bad, bad) @@ -2243,7 +2242,7 @@ def map_checks(b, base, out): fh.write(keep_mjs) bad = find_problems(b, base, out) check("map: / opens the search; a gene, a band and a finding are found and Enter flies there (a finding with " - "its card); the son's closed result is never a line, by its gene, id or title", not bad, bad) + "its card); APOE, closed, is never a line, by its gene, id or title", not bad, bad) drawn_ = "var fs = drawn();\n fs.filter(" try: with open(mjs, "w") as fh: @@ -2255,7 +2254,7 @@ def map_checks(b, base, out): fh.write(keep_mjs) bad = nearest_problems(b, base, out) check("map: Nearest names the closest finding (else the first on the next chromosome) and flies there with its " - "card; Esc closes the card first; it never leads to the son's closed result", not bad, bad) + "card; Esc closes the card first; it never leads to APOE, closed", not bad, bad) near_ = "var ch = chrom(), v = view(), fs = drawn()," try: with open(mjs, "w") as fh: @@ -2275,6 +2274,7 @@ def map_checks(b, base, out): finally: with open(mjs, "w") as fh: fh.write(keep_mjs) + og = og_binary() or build_live_demo.OG inview_checks(b, base, out, og) perf_checks(b, base, out) b.call("Emulation.setDeviceMetricsOverride", {"width": 390, "height": 844, "deviceScaleFactor": 2, "mobile": False}, b.session) @@ -2434,7 +2434,9 @@ def payloads(count, changed=False, keep_first=False): sb['rows'].append(copy.deepcopy(sb['rows'][0])) sb['rows'].reverse() if son in d.get('runs', {}): - d['runs'][son] = dict(changed_family['runs'][son], **dict(progress(d['runs'][son]), url=d['runs'][son]['url'])) + # Whether the son is a child (child, the server's one rule) is a routing control, as his url is. + d['runs'][son] = dict(changed_family['runs'][son], **dict(progress(d['runs'][son]), url=d['runs'][son]['url'], + child=d['runs'][son].get('child'))) d['future_child_payload'] = changed_family['future_payload_field'] d = {k: d[k] for k in reversed(d)} write(p, d) @@ -2559,7 +2561,7 @@ def snap(): b.eval("document.querySelector('[data-opening=closed-1]').click(); true") check('gate handles follow their act after another act opens', b.eval("location.hash") == '#f/closed-2') gate_addresses(b, base, out, acts, visit, snap) - gate_slow_release(b, base, acts, visit, payloads) + gate_slow_release(b, base, out, acts, visit, payloads) finally: b.call('Page.removeScriptToEvaluateOnNewDocument', {'identifier': seed}, b.session) for p, raw in dict(saved, **html_saved).items(): @@ -2619,19 +2621,40 @@ def gate_addresses(b, base, out, acts, visit, snap): check('gate addresses: the opened report links and keeps #f/, as the full report does', got == [True, False] and kept, (got, kept)) -def gate_slow_release(b, base, acts, visit, payloads): - """Opening the last act releases the report however slowly the reloaded page's results arrive.""" +def gate_slow_release(b, base, out, acts, visit, payloads): + """Opening the last act releases the report however slowly the reloaded page and its results arrive: the opening + never depends on the time. A gate that accepts the opening only within 1.5 s must fail here.""" son = OTHERS[1] payloads(2) - visit('/runs/'+son+'/') - open_act(b, acts[0]['title']) - Static.DELAY['/runs/'+son+'/results.json'] = 6 + + def release(): + visit('/runs/'+son+'/') + open_act(b, acts[0]['title']) + Static.DELAY['/runs/'+son+'/'] = 4 + Static.DELAY['/runs/'+son+'/results.json'] = 6 + try: + b.eval("document.querySelector('[data-opening^=closed]').click(); true") + return b.wait("!document.documentElement.dataset.pgpGated && !!document.querySelector('#results .finding')", timeout=25) + finally: + Static.DELAY.clear() + released = release() + check('gate release: a reload whose page takes 4 s and whose results.json takes 6 s still releases the report', + released, released) + js = os.path.join(out, 'assets', 'core.js') + source = open(js).read() + pairs = (("sessionStorage.setItem(TRANSFER, JSON.stringify({ path: target,", "sessionStorage.setItem(TRANSFER, JSON.stringify({ at: Date.now(), path: target,"), + ("t.path === location.pathname) { ticket = t; }", "t.path === location.pathname && Date.now() - t.at < 1500) { ticket = t; }")) try: - b.eval("document.querySelector('[data-opening^=closed]').click(); true") - released = b.wait("!document.documentElement.dataset.pgpGated && !!document.querySelector('#results .finding')", timeout=25) + planted = source + for x, y in pairs: + assert source.count(x) == 1, 'mutation anchor moved: ' + x + planted = planted.replace(x, y) + with open(js, 'w') as f: + f.write(planted) + control('gate release: an opening accepted only within 1.5 s', bool(release())) finally: - Static.DELAY.clear() - check('gate release: a reload whose results.json takes 6 s still releases the report', released, released) + with open(js, 'w') as f: + f.write(source) def gate_order(b, base, out): @@ -2825,6 +2848,42 @@ def ticket(): seen['ticket'] = left[0] return left[0] is None and not left[1].startswith('og-report-opening:') + # What may keep once a child's open report is cleared: the gate's own two attributes. + LEFT = ("Array.prototype.map.call(document.documentElement.attributes, function (a) { return a.name; }).filter(function (n) {" + " return (n.indexOf('data-pgp-') === 0 && n !== 'data-pgp-status' && n !== 'data-pgp-gated') || n === 'data-finding'; })") + + def opened(): + visit(); open_gate(b) + assert b.wait("!!document.querySelector('#results .finding') && !!document.documentElement.dataset.pgpResults"), 'full report never loaded' + + def close_attrs(): + # Read in the same turn as the close, before the document is replaced. + opened() + fid = next(f['id'] for f in data['findings'] if f['level'] == 'act') + seen['attrs'] = b.eval('PGPUI.closeAct(' + json.dumps(fid) + '); ' + LEFT) + return not seen['attrs'] + + def pagehide_attrs(): + opened() + seen['attrs'] = b.eval("window.dispatchEvent(new PageTransitionEvent('pagehide', { persisted: false })); " + LEFT) + return not seen['attrs'] and b.eval("!document.querySelector('#results')") + + def back(): + # Leave the open report for another page, then Back: from the back-forward cache the page loads afresh, closed. + # The page is served without no-store here, so Chrome keeps it in that cache (a no-store page never enters it). + Static.BFCACHE.add('/runs/%s/' % son) + try: + opened() + b.go(base + '/') + assert b.wait("document.documentElement.dataset.pgpStatus === 'ready' && !!document.documentElement.dataset.pgpPeople"), 'home never drawn' + b.eval('history.back(); true') + b.wait("location.pathname === '/runs/%s/' && document.readyState === 'complete'" % son) + got = b.wait("document.documentElement.dataset.pgpGated === 'true' && performance.getEntriesByType('navigation')[0].type", timeout=10) + seen['back'] = got or b.eval("[performance.getEntriesByType('navigation')[0].type, document.body.textContent.length]") + finally: + Static.BFCACHE.clear() + return got == 'reload' + gate_open = "if (ownsPage || closedRun(requestURL)) { return new Promise(function () {}); }" gate_late = "return got.then(function (response) {\n if (ownsPage) { return new Promise(function () {}); }" mutations = [ @@ -2845,7 +2904,14 @@ def ticket(): ('skip cleanup after closing', ['ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return true;'], ['return true;'], cleanup, lambda: seen.get('query') == 'KCNH2'), ('keep the opening ticket after reading it', [' sessionStorage.removeItem(TRANSFER);\n'], [''], ticket, - lambda: seen.get('ticket') is not None)] + lambda: seen.get('ticket') is not None), + ('keep the page attributes after closing', ["if (a.name.indexOf('data-pgp-') === 0 || a.name === 'data-finding') { root.removeAttribute(a.name); }"], + [''], close_attrs, lambda: bool(seen.get('attrs'))), + ('keep the open report on pagehide', ['ownsPage = true; cleaned = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); }'], + ['ownsPage = true; cleaned = true; eventSources.forEach(function (es) { es.close(); }); }'], pagehide_attrs, + lambda: bool(seen.get('attrs'))), + ('show the cleared report again from the back-forward cache', ["if (e.persisted && cleaned) { location.reload(); }"], [''], + back, lambda: seen.get('back') != 'reload')] try: for desc, befores, afters, probe, broke in mutations: for before in befores: diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index dd63bb54a..277003634 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -35,7 +35,13 @@ child's results to act on from the summary its results.json reads (a stale summary.json too); family.json has no flagged-card count for a child whose report is closed, so a parent's page is the same whichever card holds the child's results; and an adult's and a no-act child's pages make the requests of the page without - the gate, in its order, the event stream included. + the gate, in its order, the event stream included, and are the same page (the genome map drawn with OG); an + adult's page at a section address is the page without the gate whichever of the family and the results + answers first; each member run of family.json says whether it is a child by the one rule; a no-act child + with a stale summary.json keeps its numbers on the home page; a family file that cannot be read gives a + fixed 500 and the server then reads that family as unreadable; on the family page a gated child reads + "Report closed" and, once opened there, the real count; a child of one family is closed on another family's + page where it is a partner. Checks 2 and 4 are run again on a copy of the fixture with two parents swapped and must fail there: a check that cannot fail is no check. @@ -50,6 +56,7 @@ import sys import tempfile import threading +import time import types import xml.etree.ElementTree as ET @@ -523,6 +530,21 @@ def index(): run("father_a", NO_ACT) n = family()["runs"]["father_a"]["attention"] check("family.json: a child with no result to act on keeps its count", n == live_results.overview(NO_ACT)["attention"], n) + flags = {k: v.get("child") for k, v in family()["runs"].items()} + check("family.json: each member run says whether its pages treat it as a child, by the one rule", + flags.get("father_a") is True and flags.get("gf_a") is False, flags) + # A no-act child while summary.json is older than a finished step: the home page keeps the numbers it had + # without the gate (summary.json's), as long as neither summary has a result to act on. + stale, now = live_results.overview(STALE_SUMMARY), live_results.overview(NO_ACT) + assert (stale["levels"], stale["top"], stale["lead"]) != (now["levels"], now["top"], now["lead"]) + run("father_a", STALE_SUMMARY) + live.summary = lambda name, d, st: NO_ACT if name == "father_a" else real(name, d, st) + got = {k: (index().get("father_a") or {}).get(k) for k in ("attention", "levels", "top", "lead")} + live.summary = real + check("home: a no-act child with a stale summary.json keeps the numbers it has without the gate", + got == {k: stale[k] for k in got}, got) + run("father_a", NO_ACT) + family_failure_checks(live, get, root) live_browser_checks(live, port, work, run, real) finally: live.summary = real @@ -531,6 +553,66 @@ def index(): run("child_a") +def family_failure_checks(live, get, root): + """A family file that cannot be read: family.json answers a fixed error, never a dropped connection, and the server + then reads that family as unreadable, as the page does when family.json fails (one rule of who is a child).""" + rel = os.path.join(root, "family_a", "relationships.tsv") + mode = os.stat(rel).st_mode + os.chmod(rel, 0) + try: + try: + code, _, body = get("/runs/family_a/family.json") + except (OSError, http.client.HTTPException) as e: + code, body = type(e).__name__, b"" + check("family.json with an unreadable file: a fixed error", code == 500 and body == b"family not readable\n", + (code, body[:80])) + check("...and the server then reads that family as unreadable: an adult it lists is a child until it reads again", + live.child("gf_a"), live.child("gf_a")) + finally: + os.chmod(rel, mode) + code = get("/runs/family_a/family.json")[0] + check("...and once the file reads again, family.json answers and the adult is an adult", code == 200 and not live.child("gf_a"), + (code, live.child("gf_a"))) + + +# The map's per-load timings, the live free-disk figure and the run's elapsed time change between two loads of the +# same code: blanked before two pages are compared. +PAGE_JS = """(function () { + var clone = document.documentElement.cloneNode(true); + clone.querySelectorAll('[data-pgp-map-paint-ms],[data-pgp-map-parse-ms]').forEach(function (n) { + n.removeAttribute('data-pgp-map-paint-ms'); n.removeAttribute('data-pgp-map-parse-ms'); }); + clone.querySelectorAll('.ops-disk, #timing .v, #timing .r').forEach(function (n) { n.textContent = ''; }); + return clone.outerHTML; +}())""" + + +def differences(x, y, n=4): + """Up to N places where two pages differ, a little of each side.""" + import difflib + a, b = x.split("><"), y.split("><") + out = [] + for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, a, b, autojunk=False).get_opcodes(): + if tag != "equal" and len(out) < n: + out.append(("><".join(a[i1:i2])[:160], "><".join(b[j1:j2])[:160])) + return out + + +def slow_routes(live, delays): + """The server's routes, each answer whose path ends with a key of DELAYS held that many seconds: fixes which of two + answers arrives first. Returns the routes to put back.""" + import urllib.parse + keep = list(live.app.routes) + + def wrap(fn): + def run_(req, m): + path = urllib.parse.urlsplit(req.path).path + time.sleep(sum(v for k, v in delays.items() if path.endswith(k))) + return fn(req, m) + return run_ + live.app.routes[:] = [(a, rx, wrap(fn), auth) for a, rx, fn, auth in keep] + return keep + + def live_browser_checks(live, port, work, run, real): """In live mode (state.json, then the event stream): an adult's and a no-act child's page make the requests of the page without the gate, in its order; a parent's page is the same whichever card holds a child's results.""" @@ -548,6 +630,14 @@ def live_browser_checks(live, port, work, run, real): source = open(js).read() keep = live.cfg.assets live.cfg.assets = assets + # The map plugin runs og from bin/; CI's og is OG (scripts/setup.sh --og puts one in bin/.og), so the pages + # compared below have a map. + # The server loads its plugins itself: the map plugin is the module behind its routes. + maps = [fn.__globals__ for _, _, fn, _ in live.app.routes if "NO_OG" in getattr(fn, "__globals__", {})][:1] + og_keep = [m["OG"] for m in maps] + if os.environ.get("OG") and os.access(os.environ["OG"], os.X_OK): + for m in maps: + m["OG"] = os.environ["OG"] # gf_a's own report, so its page draws the contents and the family beside them. adult = live.summary = lambda name, d, st: NO_ACT if name == "gf_a" else real(name, d, st) b = demo.Chrome(chrome, os.path.join(work, "gate-chrome")) @@ -565,9 +655,10 @@ def capture(route, text): b.go(base + route) assert b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " "!!document.querySelector('#rail a[href*=family_a]')"), route + " never drawn" + b.wait("/^(drawn|failed|nodecompress|unavailable)$/.test((document.getElementById('genome-map') || {dataset: {}}).dataset.pgpMap || '')", 15) demo.time.sleep(1.0) - return b.eval("[document.documentElement.outerHTML, location.href, (window.ogRequests || []).map(function (r) { " - "return r.join(' '); })]") + return b.eval("[%s, location.href, (window.ogRequests || []).map(function (r) { " + "return r.join(' '); }), (document.getElementById('genome-map') || {dataset: {}}).dataset.pgpMap || '']" % PAGE_JS) pages = {} for label, route, summary in (("an adult", "/runs/gf_a/", None), ("a no-act child", "/runs/father_a/", NO_ACT)): # The child's results come from its summary as results.json reads it (a step file read now). @@ -578,11 +669,38 @@ def capture(route, text): live.summary = adult pages[label] = pair check("live mode: the page of %s makes the requests of the page without the gate, in its order, and is the " - "same page" % label, pair[0] == pair[1], [p[2] for p in pair] if pair[0][2] != pair[1][2] else "the pages differ") + "same page" % label, pair[0][:3] == pair[1][:3], + [p[2] for p in pair] if pair[0][2] != pair[1][2] else differences(pair[0][0], pair[1][0])) + drawn = pages["an adult"][1][3] + if os.environ.get("CI") == "true" or os.environ.get("OG"): + check("live mode: the adult's page compared has its genome map drawn", drawn == "drawn", drawn) + else: + print("[SKIP] no og (OG) here: the live pages are compared without a map (map %r)" % drawn) first = [r.split(" ")[1].rsplit("/", 1)[-1] for r in pages["an adult"][0][2]][:3] print(" live mode, an adult's reads without the gate: " + ", ".join(pages["an adult"][0][2])) check("live mode: without the gate the page asks for state.json, then the event stream, then family.json", first == ["state.json", "events", "family.json"], first) + # A section address on an adult's page: the marks it sets (the contents' aria-current, the card's .target) + # depend on whether the family or the results answer first, without the gate as with it. Each order is forced + # by holding the other answer back, twice each; the map's document, asked for once the results are in, comes + # last, as it does when nothing is held. + marks = [] + for first_, held in (("results.json", "/family.json"), ("family.json", "/results.json")): + keep_routes = slow_routes(live, {held: 0.6, "/map.json": 1.5}) + try: + for route in ("/runs/gf_a/#s3", "/runs/gf_a/#s/clinvar"): + for _ in range(2): + pair = [capture(route, demo.bare_core(source)), capture(route, source)] + marks.append((first_, route, pair[0][0].count('aria-current="true"'), pair[0][:3] == pair[1][:3])) + if pair[0][:3] != pair[1][:3]: + print(" %s first, %s: %r" % (first_, route, differences(pair[0][0], pair[1][0]) or + [p[1:3] for p in pair])) + finally: + live.app.routes[:] = keep_routes + check("live mode: an adult's page at a section address is the page without the gate, whichever of the family " + "and the results answers first", all(m[3] for m in marks), marks) + check("...and the two orders give the page two outcomes, so the check sees the order", + len({(m[0], m[2] > 0) for m in marks}) == 2 and len({m[2] > 0 for m in marks}) == 2, marks) # A parent's page, whichever card holds the child's two results to act on. seen = [] for summary in (TWO_IN_ONE_CARD, TWO_CARDS): @@ -591,14 +709,67 @@ def capture(route, text): check("live mode: a parent's page is the same whichever card holds a child's results to act on", seen[0] == seen[1], "the pages differ") b.call("Page.removeScriptToEvaluateOnNewDocument", {"identifier": log}, b.session) + family_gate_checks(b, base, live, run) finally: live.summary = real live.cfg.assets = keep + for m, o in zip(maps, og_keep): + m["OG"] = o b.close() shutil.rmtree(assets, ignore_errors=True) print(" live gate browser checks: %.2f s" % (demo.time.time() - t0)) +def family_gate_checks(b, base, live, run): + """The family page: a gated child's member line is the fixed closed line, never "Nothing flagged"; once the child's + results are opened there, the line and the pedigree's mark give the real count, as without the gate. A person who + is a child in one family and a partner in another is closed on both family pages (one rule of who is a child).""" + import live_results + run("father_a", TWO_CARDS) + prev = live.summary + live.summary = lambda name, d, st: TWO_CARDS if name == "father_a" else prev(name, d, st) + n = live_results.overview(TWO_CARDS)["attention"] + assert n, "the released child needs a count to show" + z = os.path.join(live.root, "family_z") + os.makedirs(z) + with open(os.path.join(z, "members.tsv"), "w") as f: + f.write("sample\tsex\tfather\tmother\tpartner\tin_pedigree\tsequenced\n" + "father_a\tmale\t\t\tzara\tyes\tyes\nzara\tfemale\t\t\tfather_a\tyes\tno\n") + line = ("Array.prototype.filter.call(document.querySelectorAll('.person-card'), function (c) { " + "return c.querySelector('.person-name').textContent === 'father_a'; }).map(function (c) { " + "var l = c.querySelector('.person-look'); return l ? l.textContent : ''; })[0]") + flag = ("Array.prototype.filter.call(document.querySelectorAll('.ped-person'), function (g) { " + "return g.querySelector('.ped-name').textContent === 'father_a'; }).map(function (g) { " + "var f = g.querySelector('.ped-flag'); return f ? f.textContent : ''; })[0]") + try: + b.go("about:blank") + assert b.wait("location.href === 'about:blank'") + b.go(base + "/runs/family_a/") + assert b.wait("!!document.querySelector('[data-gate-child=father_a] [data-opening^=closed]')"), "father_a never gated" + closed = [b.eval(line), b.eval(flag)] + for _ in range(4): + if not b.eval("document.querySelectorAll('[data-gate-child=father_a] [data-opening^=closed]').length"): + break + b.eval("document.querySelector('[data-gate-child=father_a] [data-opening^=closed]').click(); true") + b.wait("!document.querySelector('[data-gate-child=father_a]') || " + "!!document.querySelector('[data-gate-child=father_a] [data-opening^=opened]')", 10) + b.wait("!document.querySelector('[data-gate-child=father_a]') && /worth a look|Nothing flagged/.test(%s || '')" % line, 15) + opened = [b.eval(line), b.eval(flag)] + check("family page: a gated child's member line is the fixed closed line; once opened, the real count shows on " + "the line and the pedigree", closed == ["Report closed", ""] and opened == ["%d worth a look" % n, "▲ %d" % n], + (closed, opened)) + b.go("about:blank") + assert b.wait("location.href === 'about:blank'") + b.go(base + "/runs/family_z/") + got = b.wait("document.documentElement.dataset.pgpStatus === 'ready' && !!document.querySelector('.person-card') && " + "(document.querySelector('[data-gate-child=father_a]') ? 'closed' : 'open')", 15) + check("family page: a person who is a child in another family is closed on this family's page too", got == "closed", got) + finally: + live.summary = prev + shutil.rmtree(z, ignore_errors=True) + run("father_a", NO_ACT) + + # A stale blood-group section with two systems past ABO and RhD, for child_a's summary.json. STALE_SUMMARY = {"sections": {"bloodtype": {"state": "stale", "note": "older than its input", "data": { "abo": "A1", "rhd": "D+", "systems": [{"system": "ABO", "phenotype": "A1"}, {"system": "RhD", "phenotype": "D+"}, From 34c4b64e0998aa7a072cecb98a457247bfeba7f7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:10:27 +0100 Subject: [PATCH 136/255] docs(live): describe the answer order, the family page's closed line and child flag, and the family error --- CHANGELOG.md | 2 +- docs/live-site.md | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a838bf3e..5b8f13116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -186,7 +186,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Each output is written whole or not at all: a run that stops leaves no part-written table behind. A read-only `family_plan.tsv` is replaced; a folder that cannot be written still stops the plan. - Step 38, step 39 with a families file, `run-all.sh` with `FAMILIES` and the family workflow now need `og` (`scripts/setup.sh --og`). -- A child's live report now has one gate where the page reads its data, instead of a veil on each place that could name a result. While any of a child's results to act on is unopened, the page shows a fixed explanation, the number of such results, one "Show the result" button each, and the own record of each result opened on purpose; nothing else of the child's report reaches the page, and the page is the same whichever result is closed. Which result a button opens is drawn at random on each load, and the address names buttons (`#f/closed-1`), never a result. The server answers every finding address of such a child with the same page. Opening the last result shows the full report with its own addresses. Closing one again closes the report and that result, and the others stay opened; a reload or Back closes them all. The cost: such a child sees none of the rest of their report until every result to act on is opened. The home page shows a gated child's tile with fixed text, the count and the run's progress; the family page shows the child's buttons and leaves out the child's rows, stretches, couple screen and cells (the side by side reads "closed"), while the parents' values stay. Adults and children with no result to act on see the same pages as before, with the same requests in the same order. One exception: another member's page, a parent's for example, no longer shows the flagged-card count of a child whose report is closed, because that count changes with which cards the unopened results flag. APOE keeps its own warning. Two outputs change, and no field is renamed or removed: `family.json` gives `attention: null` for such a child, and `/runs.json` counts a child's results from the summary its `results.json` reads, also while `summary.json` is older than a finished step. `results.json` keeps its `veil` field, which the gate does not read. The gate guards the page in the browser: the static files still reach a browser that asks for them. +- A child's live report now has one gate where the page reads its data, instead of a veil on each place that could name a result. While any of a child's results to act on is unopened, the page shows a fixed explanation, the number of such results, one "Show the result" button each, and the own record of each result opened on purpose; nothing else of the child's report reaches the page, and the page is the same whichever result is closed. Which result a button opens is drawn at random on each load, and the address names buttons (`#f/closed-1`), never a result. The server answers every finding address of such a child with the same page. Opening the last result shows the full report with its own addresses. Closing one again closes the report and that result, and the others stay opened; a reload or Back closes them all. The cost: such a child sees none of the rest of their report until every result to act on is opened. The home page shows a gated child's tile with fixed text, the count and the run's progress; the family page shows the child's buttons and leaves out the child's rows, stretches, couple screen and cells (the side by side reads "closed"), while the parents' values stay. Adults and children with no result to act on see the same pages as before, with the same requests in the same order; their pages draw once the family has answered. One exception: another member's page, a parent's for example, no longer shows the flagged-card count of a child whose report is closed, because that count changes with which cards the unopened results flag. The family page reads "Report closed" for that child instead, and shows the count again once it opens the child's results. A person who is a child in one family is closed on every family page that lists them. APOE keeps its own warning. Three outputs change, and no field is renamed or removed: `family.json` gives `attention: null` for such a child and a new `child` field for each member run; `family.json` answers 500 with a fixed line when a family file cannot be read, where it dropped the connection; and `/runs.json` counts a child's results from the summary its `results.json` reads when either summary has a result to act on, also while `summary.json` is older than a finished step. `results.json` keeps its `veil` field, which the gate does not read. The gate guards the page in the browser: the static files still reach a browser that asks for them. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. diff --git a/docs/live-site.md b/docs/live-site.md index 33d0e7822..325c49121 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -62,7 +62,7 @@ A stopped run's page says so under the name and in the summary line, with the da While the run goes, the page still opens on the report being written. A short band under the name says how far the run is ("23 of 27 are done, about 8–13 hours remain"), with the steps done as a number and a bar, the time elapsed, the time to go from the usual step times and the retries, and links to the step timeline. Each section and each line of the contents says whether it is written, being written or waiting. The run itself, below, closes the page after section 10, folded until opened (or reached through `#progress`, the band or the contents); a page with no results yet keeps it open. -**A child's report gate.** A child's report stays closed while any of the child's results to act on in adult life (the findings of level `act`) is unopened. A child is a person their family lists with a father or a mother. When several families list the person, the first by name decides. A person whose family roles cannot be read is treated as a child, and so is a person no readable family lists while some family cannot be read. A child whose results cannot be read stays closed too. The person page, the home page and the server decide this by the same rule. While the gate is shut, the page shows only: +**A child's report gate.** A child's report stays closed while any of the child's results to act on in adult life (the findings of level `act`) is unopened. A child is a person their family lists with a father or a mother. When several families list the person, the first by name decides. A person whose family roles cannot be read is treated as a child, and so is a person no readable family lists while some family cannot be read. When a file of a family folder cannot be read, `family.json` answers 500 with a fixed line, and the server lists that family with no members until it builds again, so the server decides as the page does when `family.json` fails. A child whose results cannot be read stays closed too. The person page, the home page and the server decide this by the same rule. While the gate is shut, the page shows only: - a fixed explanation and the number of results to act on ("1 result to act on", "2 results to act on"); - one "Show the result" button per closed result, all in one block; @@ -74,13 +74,13 @@ Which result a button opens is drawn at random on each page load, so neither a b Opening the last result loads the page again, and the full report shows, as an adult's does, with its own addresses (`#f/`). A map or section address returns to the default view. On a finding address, the finding's page opens when the results have that finding; any other id opens the run's page, as the server answers 404 for it once the report is open. The open report keeps results closed until opened, as before the gate, so its search says "Results this page keeps closed are left out of the search until they are opened." under the hits. Only which results were opened crosses that load: a one-use ticket in session storage with a digest of the results' records, bound to that one load by a random value in the window's name. The page reads and removes it as it starts, before any network read, so a slow server does not lose the opening. Closing a result again (the genome map's "Close it again") clears the page at once and loads it again closed, without its search, caches or views; that result is closed, and the others stay opened. A result to act on that appears or changes while the report is open closes it the same way. A reload, or a page shown again with Back, closes every result again. -The cost: a child with an unopened result to act on sees none of the rest of their report until every one is opened. Adults and children with no result to act on never meet the gate. The gate reads the files their page reads anyway, with the same options, and hands each view its first answer for that file, whatever its status. So each file is fetched as often, in the same order and with the same options as without the gate. The event stream opens when `live.js` asks for it, right after `state.json`; its messages wait until the gate knows whether the page is a child's, then reach the page in order. The views get their answers once the gate has decided, after the family has answered. Without the gate, an answer can reach the page earlier or later, and a later redraw can then replace a mark the address set (the contents' `aria-current`). The requests and the page's data are the same; only the moment a view gets an answer differs. +The cost: a child with an unopened result to act on sees none of the rest of their report until every one is opened. Adults and children with no result to act on never meet the gate. The gate reads the files their page reads anyway, with the same options, and hands each view its first answer for that file, whatever its status. So each file is fetched as often, in the same order and with the same options as without the gate. In live mode the person page asks for the family and for its results at once, on its first state. The gate asks for both, in that order, as soon as the state has answered, so the two answers race as they do without the gate, and they reach the views in the order they arrived. The marks a section address sets (the contents' `aria-current`, the card's highlight) therefore come out as without the gate: they stay when the family answers first, and the redraw that the family's answer starts clears them when the results answer first. The event stream opens when `live.js` asks for it, right after `state.json`; its messages wait until the gate knows whether the page is a child's, then reach the page in order. One thing differs: the views get their first answer once the family has answered, so the page draws later by the time the family takes. -One count is left out for a child whose report is closed. Other members' pages, a parent's for example, list the family's members with how many cards each report flags. That count changes with which cards the child's unopened results flag, so `family.json` gives `attention: null` for that child, and the page shows the name without a count. For adults and children with no result to act on, `family.json` is as before. +One count is left out for a child whose report is closed. Other members' pages, a parent's for example, list the family's members with how many cards each report flags. That count changes with which cards the child's unopened results flag, so `family.json` gives `attention: null` for that child. While the child is gated on the family page, the child's member line reads "Report closed" and the pedigree has no count, never "Nothing flagged". Once every result to act on of the child is opened there, the page takes the count from the child's `results.json` and shows it as before. For adults and children with no result to act on, the count in `family.json` is as before. Each member run in `family.json` also has `child`: whether that person's pages treat them as a child, by the rule above over every family. So a person who is a child in one family is closed on every family page that lists them, a partner's family too. -The home page decides from `/runs.json` alone. A gated child is a child (the rule above, read from the families of `/runs.json`) whose entry counts or names a result to act on. The server counts a child's results from the summary its `results.json` reads. While `summary.json` is older than a finished step, that is the step files read again, so the home page closes a child whenever the child's page does. Its tile and attention rows show the fixed text and the count of results to act on, and link to the run's page. The run's progress (its state, the steps done, planned, running and failed, and its times) stays, because it says how far the run is, never what it found. +The home page decides from `/runs.json` alone. A gated child is a child (the rule above, read from the families of `/runs.json`) whose entry counts or names a result to act on. The server counts a child's results from the summary its `results.json` reads. While `summary.json` is older than a finished step, that is the step files read again, so the home page closes a child whenever the child's page does. A child with no result to act on in either summary keeps the numbers of `summary.json`, as before. Its tile and attention rows show the fixed text and the count of results to act on, and link to the run's page. The run's progress (its state, the steps done, planned, running and failed, and its times) stays, because it says how far the run is, never what it found. -On the family page, chapter IV shows a gated child's buttons only, and each child opens on its own. The family view gets the family without that child's report data: no de novo or compound-het rows, stretches, trio status or trio files of the child, no relationship or shared-segment row with the child, and no couple screen of a couple the child is part of. The side by side keeps a row only when an open member has a value in it, in a fixed order, and the child's cell reads "closed". Plan rows that name the child are left out. The parents' other plan rows and the couple reports of couples with no gated partner stay. Other reports of the family folder are left out while a child is gated, because they can hold the child's data. A member with no run here has no report, so the gate does not close anything of theirs. The pedigree and the members' progress stay. +On the family page, chapter IV shows a gated child's buttons only, and each child opens on its own. A gated person whom this family lists as a parent or partner, because another family lists them as a child, gets their buttons in chapter IV too. The family view gets the family without that child's report data: no de novo or compound-het rows, stretches, trio status or trio files of the child, no relationship or shared-segment row with the child, and no couple screen of a couple the child is part of. The side by side keeps a row only when an open member has a value in it, in a fixed order, and the child's cell reads "closed". Plan rows that name the child are left out. The parents' other plan rows and the couple reports of couples with no gated partner stay. Other reports of the family folder are left out while a child is gated, because they can hold the child's data. A member with no run here has no report, so the gate does not close anything of theirs. The pedigree and the members' progress stay. The gate holds the page's `fetch` and `EventSource`. A view or plugin reads a run's files through them. `XMLHttpRequest`, an `` and `import()` are not held, so the views use them only on what the gate has released. The gate guards the page in the browser: its document, attributes, accessibility text, title, address, canvas and drawings, storage and the requests its views make. The static files (`results.json`, `map.json`, the charts, the search index and `family.json`) still reach a browser that asks for them whole: the gate is not an access control for them. @@ -96,7 +96,7 @@ The header search finds a gene or rsID across the results and opens the card tha **A finding** (`/runs//finding//`). One finding on a page of its own: its level and how many others share it, the title, the gene and variant, a plain explanation, and **What to do** as numbered steps. Then the evidence (its numbers, how common the change is on a log scale from one in a million to everyone, the caveat in the margin), the reads with a link to the genome viewer, the rest of the family, the related findings of the same gene or step, and the section of the report it comes from. The contents on the left lead back to the report and on to the next finding. The page is 404 unless the id is a finding of that run's `results.json` now. For a child with a result to act on, the server answers every well-formed id with the page, which stays closed while the report gate is shut. -**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags (never for a child whose report is closed), beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. While a child's report gate is shut, chapter IV shows that child's buttons only (the report gate above). V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. While a child's report gate is shut, the child's cell reads "closed" in every row, and a row stays only when an open member has a value in it. On a phone the grid becomes a list of pairs. +**A family** (`/runs//`). A report in five chapters, opened by a lead: how many genomes are read together, whether the couple shares a carrier gene, whether the measured relationships match the tree, and how many children. I, the pedigree, drawn from `members.tsv` or `pedigree.tsv` in the members' colours, where each sequenced person with a run here opens their page and shows how many cards their report flags (a child whose report is closed reads "Report closed" instead), beside the members and the analyses step 38 planned. II, who is related how: step 38's relationship check beside the shared segments, as a grid of every pair, with ⚑ where the measured relationship differs from the declared one. III, each couple's carrier screen in full (genes where both carry a finding, drawn as four children with the chance for each, X-linked genes, genes where one partner does), with the screen's caveat and a link to the couple report. IV, each child: the stretches inherited from each parent painted per chromosome (coloured by grandparent where step 38 could tell), the de novo candidates and the compound-het pairs, in trans first, or why the child was not assessed. While a child's report gate is shut, chapter IV shows that child's buttons only (the report gate above). V, the members side by side: their CYP2D6 calls, CPIC phenotypes, HLA types, haplogroups and every blood group system, one column each, with a value from an older result file marked in its cell. While a child's report gate is shut, the child's cell reads "closed" in every row, and a row stays only when an open member has a value in it. On a phone the grid becomes a list of pairs. On a phone each page is one column and each table row becomes a short list of label and value. On a person's report the contents give way to a tab bar at the bottom (To act, Medicines, Family, Trust, More); under 700 px the report is short: the findings of sections 1 to 4 (to act on, medicines, family, worth a look) stay open, while the cards behind them, every information card and the bodies of sections 5 to 10 start closed, each behind one line that says what it holds (section 5 stays open when it holds a finding about data quality, or a card the report flags with no finding of its own). The report gate is the same at every width. A fold you open stays open when the page is drawn again, the phone's folds too when you turn the phone to a wide screen and back. The counts read one line per level, and the "How to read this" note and the links between findings are left out. A link to a card, a section or a finding, a search hit and a tab open the folds around it (a Ctrl, Cmd or Shift click leaves this page as it is); a finding page drops its contents (the header links back to the report), and a family page keeps its chapter list above the chapters. The pages are dark. `tests/demo/render_pictures.py --out DIR --only live-phone` draws the picture below from the demo. @@ -110,7 +110,7 @@ Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with it The reads cross the network to the browser that logged in: only the byte ranges igv.js asks for, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS); without it the password and the reads cross the network in clear. Opening the viewer writes one access-log row; the byte ranges igv.js reads write none. -`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. For a child these fields come from the summary the child's `results.json` reads, even when `summary.json` is older. For a gated child the page replaces `levels`, `top` and `lead` with fixed text before the home page draws them (the report gate above). +`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. For a child with a result to act on in either summary, these fields come from the summary the child's `results.json` reads, even when `summary.json` is older. For a gated child the page replaces `levels`, `top` and `lead` with fixed text before the home page draws them (the report gate above). The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (the rows outside a result's own card that name a result to act on, and `spans`, the results to act on with their gene and position; the report gate reads the findings of level `act` instead; on a page that keeps results closed until opened, a child's or one whose family roles are not known, the charts and the search box still leave out a mark or hit whose tag names no finding of these results). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. From 91eb7722fa8a919bc3eefbbcb7bbef01e1d35634 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:15:25 +0100 Subject: [PATCH 137/255] fix(setup): let an interrupted clinical download resume in tight space and report a skipped item --- CHANGELOG.md | 2 +- docs/00-reference-setup.md | 5 +++-- docs/hardware-requirements.md | 2 +- docs/troubleshooting.md | 3 ++- scripts/lib/common.sh | 27 +++++++++++++++++++++------ scripts/setup.sh | 21 ++++++++++++++------- 6 files changed, 42 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f78b639b2..da7194bc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin and warns without stopping the remaining installs. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. +- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin, counts a partial download already on disk, and warns without stopping the remaining installs; `--clinical-data` exits 1 when one is still missing. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/00-reference-setup.md b/docs/00-reference-setup.md index b8f108743..b00f2c3b9 100644 --- a/docs/00-reference-setup.md +++ b/docs/00-reference-setup.md @@ -6,8 +6,8 @@ One-time downloads required before running the pipeline. Each heading below give A plain `./scripts/setup.sh "$GENOME_DIR"` installs both VEP caches and the PCGR/CPSR bundle after AnnotSV. These clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. -Setup checks each install separately: VEP 116 needs 56 GB peak, the bundle 14 GB, and VEP 115 needs 52 GB. -Each check adds a 5 GB margin. An item that does not fit is skipped with a warning. Setup continues. +Setup checks each install separately: VEP 116 needs 56 GB peak, the bundle 15 GB, and VEP 115 needs 52 GB. +Each check adds a 5 GB margin and counts a partial download already on disk. An item that does not fit is skipped with a warning. Setup continues. ```bash ./scripts/setup.sh --no-clinical-data "$GENOME_DIR" # skip all three clinical downloads @@ -15,6 +15,7 @@ Each check adds a 5 GB margin. An item that does not fit is skipped with a warni ``` An interrupted transfer resumes. Existing caches and bundles are kept. +`--clinical-data` exits 1 when any of the three is still missing after it runs. Without these data, `run-all.sh` skips the affected steps and records the reason. A bare `nextflow run` still needs its tools and data parameters. Setup does not change that selection. diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index 0077edce2..a895b392a 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -57,7 +57,7 @@ One row per download, in GB as `wget` and `du -h` count them (1 GB = 2^30 bytes) | **Total, default run** | **~73-78 GB** | **~98-103 GB** | | | With `--no-clinical-data` (first three rows + Docker images) | ~16-21 GB | ~33-38 GB | | -While a VEP cache installs, the tarball and the unpacked tree are on disk together until the tarball is deleted (`install_vep_cache` in `scripts/lib/common.sh`): 56 GB peak for release 116 and 52 GB for release 115. The bundle needs 14 GB peak. Setup adds a 5 GB margin to each check and warns when it skips an item. The three clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. To retry only these downloads, run `./scripts/setup.sh --clinical-data "$GENOME_DIR"`. +While a VEP cache installs, the tarball and the unpacked tree are on disk together until the tarball is deleted (`install_vep_cache` in `scripts/lib/common.sh`): 56 GB peak for release 116 and 52 GB for release 115. The bundle needs 15 GB peak. Setup adds a 5 GB margin to each check and warns when it skips an item. The three clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. To retry only these downloads, run `./scripts/setup.sh --clinical-data "$GENOME_DIR"`. **Optional** (only for the step named): diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 120af6ba0..e052fb407 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -739,7 +739,8 @@ ExpansionHunter \ Setup installs VEP 116, the PCGR/CPSR bundle and VEP 115 by default. If one is missing, read the warning above `Setup complete!`. -The disk checks require 56 GB, 14 GB and 52 GB peak respectively, each plus a 5 GB margin. +The disk checks require 56 GB, 15 GB and 52 GB peak respectively, each plus a 5 GB margin. +A partial download on disk counts toward that peak. A failed download also leaves a warning. Other installs continue. Free space, then retry only the clinical data: diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 2088b40a6..923f082dc 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -642,8 +642,11 @@ install_vep_cache() { return 1 fi mkdir -p "${dir}/homo_sapiens" - fetch "$url" "$tarball" sum "$(dirname "$url")/CHECKSUMS" || return 1 + # fetch publishes the tarball only after its check, so one already here is + # whole: an install stopped during extraction unpacks it again. + [ -f "$tarball" ] || fetch "$url" "$tarball" sum "$(dirname "$url")/CHECKSUMS" || return 1 echo " Extracting $(basename "$tarball")..." + rm -rf "${dir}"/.extract.* tmp=$(mktemp -d "${dir}/.extract.XXXXXX") if ! tar xzf "$tarball" -C "$tmp" || [ ! -f "${tmp}/homo_sapiens/${rel}_GRCh38/info.txt" ]; then rm -rf "$tmp" @@ -657,18 +660,28 @@ install_vep_cache() { rm -rf "$tmp" "$tarball" } -# clinical_disk_ok DIR PEAK_GB LABEL: GB as du counts them (2^30 bytes), -# plus a 5 GB margin. +# clinical_disk_ok DIR PEAK_GB LABEL [ARCHIVE]: GB as du counts them +# (2^30 bytes), plus a 5 GB margin. # Use the destination's filesystem, creating the directory when it is new. +# The bytes of ARCHIVE, or of its interrupted ARCHIVE.part, are already part +# of the peak, so a resumed download needs only the rest. # An unreadable df is a failed preflight, never permission to fill the disk. clinical_disk_ok() { - local dir=$1 peak=$2 label=$3 free + local dir=$1 peak=$2 label=$3 archive=${4:-} free have=0 f mkdir -p "$dir" || return 1 free=$(df -Pk "$dir" 2>/dev/null | awk 'NR == 2 {print $4}') || free="" case "$free" in ''|*[!0-9]*) echo "[WARN] Cannot read free space for ${label}; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\""; return 1 ;; esac - if [ "$free" -lt "$(( (peak + 5) * 1048576 ))" ]; then + if [ -n "$archive" ]; then + for f in "$archive" "${archive}.part"; do + if [ -f "$f" ]; then + have=$(( $(wc -c < "$f") / 1024 )) + break + fi + done + fi + if [ "$(( free + have ))" -lt "$(( (peak + 5) * 1048576 ))" ]; then echo "[WARN] Skipping ${label}: needs ${peak} GB peak plus a 5 GB margin; $(awk -v k="$free" 'BEGIN {printf "%.1f", k / 1048576}') GB free." echo " Install later: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" return 1 @@ -691,7 +704,9 @@ install_pcgr_bundle() { url="https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" tarball="${dir}/${url##*/}" mkdir -p "$dir" || return 1 - fetch "$url" "$tarball" sha256 "$PCGR_DATA_SHA256" || return 1 + # fetch publishes the archive only after its check, so one already here is whole. + [ -f "$tarball" ] || fetch "$url" "$tarball" sha256 "$PCGR_DATA_SHA256" || return 1 + rm -rf "${dir}"/.extract.* tmp=$(mktemp -d "${dir}/.extract.XXXXXX") || return 1 if ! tar xzf "$tarball" -C "$tmp" || [ ! -d "${tmp}/data/grch38" ]; then rm -rf "$tmp" diff --git a/scripts/setup.sh b/scripts/setup.sh index 272808b4f..20d0fb79e 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -2082,7 +2082,7 @@ install_vep_cache_opt_in() { echo "[OK] VEP ${VEP_CACHE_RELEASE} cache already present: ${dir}/homo_sapiens/${VEP_CACHE_RELEASE}_GRCh38" return 0 fi - clinical_disk_ok "$dir" 56 "VEP ${VEP_CACHE_RELEASE} cache" || return 1 + clinical_disk_ok "$dir" 56 "VEP ${VEP_CACHE_RELEASE} cache" "${dir}/$(basename "$(vep_cache_url "$VEP_CACHE_RELEASE")")" || return 1 echo "Installing the VEP ${VEP_CACHE_RELEASE} cache: ~26 GB download, ~30 GB unpacked, both on disk until it ends." if ! install_vep_cache "$dir" "$VEP_CACHE_RELEASE"; then echo "[WARN] Could not install the VEP ${VEP_CACHE_RELEASE} cache. Run this command again: the download resumes." @@ -2092,30 +2092,37 @@ install_vep_cache_opt_in() { } # Each missing item gets its own disk check. A skipped or failed item must -# not prevent the smaller bundle or the other cache from installing. +# not prevent the smaller bundle or the other cache from installing; it makes +# the return status 1. install_clinical_data() { - local dir="${GENOME_DIR}/vep_cache" rel peak + local dir="${GENOME_DIR}/vep_cache" rel peak missing=0 echo "Clinical data: about 57 GB to download, 65 GB unpacked; allow 1 to 3 hours." for rel in "$VEP_CACHE_RELEASE" bundle "$PCGR_VEP_CACHE_RELEASE"; do if [ "$rel" = bundle ]; then [ ! -d "${GENOME_DIR}/pcgr_data/${PCGR_DATA_BUNDLE}/data" ] || { echo "[OK] PCGR/CPSR bundle already present."; continue; } - if clinical_disk_ok "${GENOME_DIR}/pcgr_data" 14 'PCGR/CPSR bundle'; then - install_pcgr_bundle "${GENOME_DIR}/pcgr_data" || echo "[WARN] Could not install PCGR/CPSR. Step 17 needs it; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" + if clinical_disk_ok "${GENOME_DIR}/pcgr_data" 15 'PCGR/CPSR bundle' "${GENOME_DIR}/pcgr_data/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz"; then + install_pcgr_bundle "${GENOME_DIR}/pcgr_data" || { missing=1; echo "[WARN] Could not install PCGR/CPSR. Step 17 needs it; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\""; } + else + missing=1 fi else [ ! -f "${dir}/homo_sapiens/${rel}_GRCh38/info.txt" ] || { echo "[OK] VEP ${rel} cache already present."; continue; } peak=56 [ "$rel" != "$PCGR_VEP_CACHE_RELEASE" ] || peak=52 - if clinical_disk_ok "$dir" "$peak" "VEP ${rel} cache"; then + if clinical_disk_ok "$dir" "$peak" "VEP ${rel} cache" "${dir}/$(basename "$(vep_cache_url "$rel")")"; then if ! install_vep_cache "$dir" "$rel"; then + missing=1 echo "[WARN] Could not install VEP ${rel}. Run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\" (interrupted downloads resume)." if [ "$rel" = "$VEP_CACHE_RELEASE" ]; then echo " To retry only this cache: ./scripts/setup.sh --vep-cache \"${GENOME_DIR}\"" fi fi + else + missing=1 fi fi done + return "$missing" } case "$OPT_IN" in @@ -2510,7 +2517,7 @@ install_annotsv_data || true echo "" echo "=== Phase 6: Clinical reference data ===" if $CLINICAL_DATA; then - install_clinical_data + install_clinical_data || true else echo "[SKIP] Clinical data (--no-clinical-data): steps 13, 30, 23, 31 and 17 need the caches or bundle." echo " Install later: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\"" From b4be807a36c3c0d0201275b6fa574ee0e7664d3d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:15:50 +0100 Subject: [PATCH 138/255] fix(live): preserve reads history and locus across map navigation Replace iframe fragment navigation, keep the requested locus on resize, and add only one camera history entry from the Reads gauge. Bead: pgp-iqon.11 --- CHANGELOG.md | 2 +- assets/live/map.js | 10 ++++++---- assets/viewer/viewer.js | 9 ++++++++- docs/viewer.md | 2 ++ tests/test_live_viewer.py | 30 ++++++++++++++++++++++++++++-- 5 files changed, 45 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdf953ca5..5de2d8186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. +- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Back follows the map's history, and resizing the viewer keeps its requested locus. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/assets/live/map.js b/assets/live/map.js index 43105cd81..86f0c83df 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -1244,7 +1244,9 @@ readsFull.href = readsBase() + '#' + locus; if (readsFrame) { // Changing only the fragment lets viewer.js search without reloading igv.js. - readsFrame.src = readsFull.href; + if (readsFrame.parentNode && readsFrame.contentWindow) { + readsFrame.contentWindow.location.replace(readsFull.href); + } else { readsFrame.src = readsFull.href; } } } function closeReads() { @@ -1693,14 +1695,14 @@ c = f ? (f.s + f.e) / 2 : M.chroms[ci].len / 2; } if (lv === 'chromosome') { openChrom(ci, undefined, undefined, true); return; } - openChrom(ci, c, (lv === 'region' ? REGION_STOP : GENE_STOP) * openW(), true); + openChrom(ci, c, lv === 'reads' ? 2000 : (lv === 'region' ? REGION_STOP : GENE_STOP) * openW(), true); } stopC.addEventListener('click', function () { stopTo('chromosome'); }); stopR.addEventListener('click', function () { stopTo('region'); }); stopN.addEventListener('click', function () { stopTo('gene'); }); stopReads.addEventListener('click', function () { - stopTo('gene'); - openChrom(tgt.ci, tgt.c, 2000, true); + if (!M || failure) { return; } + stopTo('reads'); // The target is already at reads level; finish the camera motion before selecting its nearest row. cam.ci = tgt.ci; cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; openReads(); diff --git a/assets/viewer/viewer.js b/assets/viewer/viewer.js index aadfcf506..558ea23cd 100644 --- a/assets/viewer/viewer.js +++ b/assets/viewer/viewer.js @@ -58,11 +58,18 @@ // igv.js otherwise fetches its genome list from igv.org at start. loadDefaultGenomes: false, }); - window.addEventListener("hashchange", function () { + function followLocus() { const locus = locusFromHash(); if (locus) { browser.search(locus); } + } + window.addEventListener("hashchange", followLocus); + // igv.js keeps bases per pixel on resize. Keep the requested absolute locus instead. + let resizeTimer; + window.addEventListener("resize", function () { + window.clearTimeout(resizeTimer); + resizeTimer = window.setTimeout(followLocus, 100); }); root.dataset.pgpTracks = String(cfg.tracks.length); root.dataset.pgpStatus = "ready"; diff --git a/docs/viewer.md b/docs/viewer.md index b6e6d1f49..09b8e3680 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -84,6 +84,8 @@ Then open `http://127.0.0.1:8765/` on your computer. The report's links work the On the genome map, zoom below 50 kb and press **R** or select **See the reads**. The **Reads** gauge opens a 2 kb view. A finding's map card also offers **See the reads**, centred on its position with 50 bases on each side. The panel otherwise starts at the nearest open finding in view, or at the view itself. Moving the map updates the viewer's locus. Alignments are visible up to 50 kb. +Back follows the map's history. Moving the map creates no extra history entry for the iframe, and the Reads gauge adds one camera view. Resizing the viewer keeps its requested locus. + The panel takes the place of the In view table. Press **Esc**, including from inside the viewer, or select **Back to the map** to close it. Focus returns to the map at the same view. Closing the panel clears its previous locus, title, and link to the full viewer. **Open full viewer** opens that locus in a new tab. A closed result has no reads button, and closing an opened result removes its reads panel. When the viewer is off, the panel shows the server's reason. The static demo explains that it has no viewer. The live viewer allows framing only by the same origin: it sends `frame-ancestors 'self'` and `X-Frame-Options: SAMEORIGIN`. Live pages allow `frame-src 'self'`. Other routes keep `frame-ancestors 'none'` and `X-Frame-Options: DENY`. The standalone local viewer keeps its original policy, which blocks all framing. diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index 288bd61b4..666a8b9f0 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -268,9 +268,12 @@ def loaded(): check("map: a closed result has no card or reads link", b.eval("!document.querySelector('.gm-marks [data-id=closed]') && !document.querySelector('.gm-card-reads[data-id=closed]')")) # A moved view must update the existing frame, with no page reload. - b.eval("window.savedFrame=document.querySelector('.gm-reads iframe');location.hash='chr1:1-49001'") + b.eval("window.savedFrame=document.querySelector('.gm-reads iframe');window.savedViewerDocument=savedFrame.contentDocument;" + "window.savedHistoryLength=history.length;location.hash='chr1:1-49001'") check("map: moving updates the same iframe to the map's locus", b.wait("savedFrame===document.querySelector('.gm-reads iframe') && savedFrame.contentWindow.location.hash==='#chr1:1-49001'")) + check("map: a camera move keeps the viewer document and adds no iframe history entry", + b.eval("savedViewerDocument===savedFrame.contentDocument && history.length===savedHistoryLength+1")) check("viewer: alignments remain visible at 49 kb", b.wait("(function(){var h=savedFrame.contentDocument.getElementById('igv');" "var v=h && h.shadowRoot && h.shadowRoot.querySelector('.igv-viewport[data-track-type=alignment]');" @@ -279,11 +282,31 @@ def loaded(): b.eval("JSON.stringify(Array.from(savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelectorAll('.igv-viewport')).map(function(v){" "var n=v.querySelector('.igv-zoom-in-notice-container');return {type:v.dataset.trackType," "notice:n && n.style.display,canvas:v.querySelectorAll('canvas').length,text:v.textContent.slice(0,180)};}))")) + def viewer_locus(): + return b.eval("savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelector('input.igv-search-input').value") + + def resized(): + b.call("Runtime.evaluate", {"expression": "new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))", + "awaitPromise": True}, b.session) + + expected_locus = viewer_locus() + js("savedFrame.style.width='50%'") + resized() + check("viewer: resizing keeps the full requested locus", + b.wait("savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelector('input.igv-search-input').value===" + + json.dumps(expected_locus)), viewer_locus()) + js("savedFrame.style.width=''") + resized() + b.wait("savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelector('input.igv-search-input').value===" + + json.dumps(expected_locus)) if os.environ.get("MAP_SCREENS"): os.makedirs(os.environ["MAP_SCREENS"], exist_ok=True) for width, height in ((1440, 900), (390, 844)): b.call("Emulation.setDeviceMetricsOverride", {"width": width, "height": height, "deviceScaleFactor": 1, "mobile": False}, b.session) + resized() + b.wait("savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelector('input.igv-search-input').value===" + + json.dumps(expected_locus)) b.eval("document.querySelector('.gm-reads').scrollIntoView({block:'center'})") png = b.call("Page.captureScreenshot", {"format": "png"}, b.session)["data"] with open(os.path.join(os.environ["MAP_SCREENS"], f"map-reads-{width}.png"), "wb") as out: @@ -296,9 +319,12 @@ def loaded(): "document.querySelector('.gmap').dataset.view==='chr1:1-49001'")) b.eval("document.querySelector('.gm-canvas').dispatchEvent(new KeyboardEvent('keydown',{key:'r',bubbles:true}))") check("map: R reopens the panel", loaded()) - b.eval("document.querySelector('.gm-reads-close').click();document.querySelector('[data-stop=reads]').click()") + b.eval("document.querySelector('.gm-reads-close').click();window.gaugeHistoryLength=history.length;" + "document.querySelector('[data-stop=reads]').click()") check("map: Reads gauge opens a 2 kb view and the panel", b.wait("!document.querySelector('.gm-reads').hidden && " "document.querySelector('.gmap').dataset.level==='reads'")) + check("map: the Reads gauge adds one camera history entry", + b.eval("history.length===gaugeHistoryLength+1")) js("document.querySelector('.gm-reads-close').click();location.hash='chr1:1-2000'") b.wait("document.querySelector('.gmap').dataset.view==='chr1:1-2000'") js("document.querySelector('.gm-marks [data-id=open]').focus();document.querySelector('.gm-card-reads').click()") From 1c83b9953982a93f6dac3f4fb164b10b313ece26 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 04:17:51 +0100 Subject: [PATCH 139/255] test(live): preserve the targeted reload control guards Keep the Reads navigation guard distinct from the two drawing guards the existing failed-reload control replaces. Bead: pgp-iqon.11 --- assets/live/map.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assets/live/map.js b/assets/live/map.js index 86f0c83df..990d02b28 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -1701,7 +1701,7 @@ stopR.addEventListener('click', function () { stopTo('region'); }); stopN.addEventListener('click', function () { stopTo('gene'); }); stopReads.addEventListener('click', function () { - if (!M || failure) { return; } + if (failure || !M) { return; } stopTo('reads'); // The target is already at reads level; finish the camera motion before selecting its nearest row. cam.ci = tgt.ci; cam.c = tgt.c; cam.ls = tgt.ls; cam.k = tgt.k; From 4a52aabac50fb6e3c0325ab73e3b5ffbf0c1f8bf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:15:35 +0100 Subject: [PATCH 140/255] fix(ancestry): list the four planned verbs in og ancestry-fine --help The dispatcher already routes freqs, project, paint and simulate, but the usage line named only the six ported verbs. A test now checks that the help names every verb. --- rust/crates/og-ancestry/src/lib.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/rust/crates/og-ancestry/src/lib.rs b/rust/crates/og-ancestry/src/lib.rs index 743ea611a..633066b3d 100644 --- a/rust/crates/og-ancestry/src/lib.rs +++ b/rust/crates/og-ancestry/src/lib.rs @@ -551,8 +551,7 @@ fn is_negative_number(s: &str) -> bool { /// A verb's body: the parsed options and stdout. type Run = fn(&Args, &mut dyn io::Write) -> Result<()>; -const USAGE: &str = - "usage: og ancestry-fine {panel-map,check-map,order,sample-bed,chroms,report} ...\n"; +const USAGE: &str = "usage: og ancestry-fine {panel-map,check-map,order,sample-bed,chroms,report,freqs,project,paint,simulate} ...\n"; /// `og ancestry-fine [args]`: the exit code. Output goes to `out`, /// messages to `err`. @@ -750,4 +749,24 @@ mod tests { assert_eq!(rc, 0); assert!(out.starts_with("usage: og ancestry-fine report")); } + + #[test] + fn help_names_every_verb() { + let (rc, out, _) = testutil::run(&["--help"]); + assert_eq!(rc, 0); + for verb in [ + "panel-map", + "check-map", + "order", + "sample-bed", + "chroms", + "report", + "freqs", + "project", + "paint", + "simulate", + ] { + assert!(out.contains(verb), "{verb} missing from {out}"); + } + } } From ba3105dc277d49825bc3c173f8d6600f3c0fe233 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:15:35 +0100 Subject: [PATCH 141/255] build(live): keep og-live's results module visible to git The root .gitignore drops every results/ folder (Nextflow run output), so a new file under rust/crates/og-live/src/results/ was silently ignored. A crate-level .gitignore re-includes that one source folder. --- rust/crates/og-live/.gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 rust/crates/og-live/.gitignore diff --git a/rust/crates/og-live/.gitignore b/rust/crates/og-live/.gitignore new file mode 100644 index 000000000..73b89169b --- /dev/null +++ b/rust/crates/og-live/.gitignore @@ -0,0 +1,3 @@ +# The repo root ignores results/ (Nextflow run output); this crate keeps a +# source module of that name. +!/src/results/ From eade22a68121042f80862696fb5fa7aeb59a89d6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:16:36 +0100 Subject: [PATCH 142/255] test(setup): catch stale extraction space and unverified retained bundles --- tests/fake-docker/setup-clinical-default.sh | 41 ++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/fake-docker/setup-clinical-default.sh b/tests/fake-docker/setup-clinical-default.sh index e83b145d9..4c6b49e46 100644 --- a/tests/fake-docker/setup-clinical-default.sh +++ b/tests/fake-docker/setup-clinical-default.sh @@ -50,6 +50,16 @@ output_has low '52 GB.*5 GB margin' [ ! -e "${G}/vep_cache/homo_sapiens/116_GRCh38" ] || fail 'low-space VEP installed' [ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'small bundle skipped with 40 GB free' +# Model an allocated 22 GiB partial download without writing 22 GiB in CI. +cat > "${CASE_WORK}/low-bin/du" <<'DU' +#!/usr/bin/env bash +case "$2" in + */resume/vep_cache/*.part) printf '23068672\t%s\n' "$2" ;; + *) /usr/bin/du "$@" ;; +esac +DU +chmod +x "${CASE_WORK}/low-bin/du" + # A partial download counts as space already taken: 40 GiB free plus a # 22 GiB .part covers VEP 116's 56 GB peak and margin, so it resumes. G="${CASE_WORK}/resume" @@ -66,12 +76,41 @@ G="${CASE_WORK}/extract" mkdir -p "${G}/vep_cache/.extract.old" "${G}/pcgr_data/.extract.old" cp "${FAKE_DOWNLOAD_DIR}/homo_sapiens_vep_116_GRCh38.tar.gz" "${G}/vep_cache/" cp "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" "${G}/pcgr_data/" -FAKE_DOWNLOAD_FAIL='vep_116|pcgr_ref_data' run_expect 0 extract "${SCRIPTS}/setup.sh" --clinical-data "$G" +# Model space held by each stale tree. Only cleanup can make its preflight pass. +mkdir -p "${CASE_WORK}/staging-bin" +cat > "${CASE_WORK}/staging-bin/df" <<'DF' +#!/usr/bin/env bash +echo 'Filesystem 1024-blocks Used Available Capacity Mounted on' +if [ -d "${@: -1}/.extract.old" ]; then + echo 'fakefs 2147483648 2141192192 6291456 99% /' +else + echo 'fakefs 2147483648 0 2147483648 0% /' +fi +DF +chmod +x "${CASE_WORK}/staging-bin/df" +PATH="${CASE_WORK}/staging-bin:${PATH}" FAKE_DOWNLOAD_FAIL='vep_116|pcgr_ref_data' run_expect 0 extract "${SCRIPTS}/setup.sh" --clinical-data "$G" [ -s "${G}/vep_cache/homo_sapiens/116_GRCh38/info.txt" ] || fail 'checked VEP 116 archive downloaded again' [ -s "${G}/pcgr_data/20260620/data/grch38/README" ] || fail 'checked PCGR archive downloaded again' [ ! -e "${G}/vep_cache/.extract.old" ] || fail 'stale VEP staging tree kept' [ ! -e "${G}/pcgr_data/.extract.old" ] || fail 'stale PCGR staging tree kept' +# A retained archive with a valid layout but wrong data must be replaced. +G="${CASE_WORK}/retained-bad" +mkdir -p "${G}/pcgr_data" "${CASE_WORK}/altered/data/grch38" +printf 'modified reference\n' > "${CASE_WORK}/altered/data/grch38/README" +tar -czf "${G}/pcgr_data/pcgr_ref_data.20260620.grch38.tgz" -C "${CASE_WORK}/altered" data +run_expect 0 retained_bad "${SCRIPTS}/setup.sh" --clinical-data "$G" +cmp "${G}/pcgr_data/20260620/data/grch38/README" "${CASE_WORK}/bundle/data/grch38/README" || fail 'unverified retained PCGR archive published' +[ ! -e "${G}/pcgr_data/pcgr_ref_data.20260620.grch38.tgz" ] || fail 'replaced archive kept' + +# Sparse holes occupy no disk: they must not grant space for an installation. +G="${CASE_WORK}/sparse" +mkdir -p "${G}/vep_cache" +dd if=/dev/null of="${G}/vep_cache/homo_sapiens_vep_116_GRCh38.tar.gz.part" bs=1048576 seek=22528 2>/dev/null +PATH="${CASE_WORK}/low-bin:${PATH}" run_expect 1 sparse "${SCRIPTS}/setup.sh" --clinical-data "$G" +output_has sparse 'Skipping VEP 116' +[ ! -e "${G}/vep_cache/homo_sapiens/116_GRCh38" ] || fail 'sparse holes counted as allocated space' + # The digest belongs to the original fixture. Serve a different valid tarball. printf 'corrupt\n' >> "${FAKE_DOWNLOAD_DIR}/pcgr_ref_data.20260620.grch38.tgz" G="${CASE_WORK}/corrupt" From 690e8ae13c117645a1c4579a6eb179c6cac6df5d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:20:16 +0100 Subject: [PATCH 143/255] fix(setup): reclaim staging space and verify retained clinical archives --- CHANGELOG.md | 2 +- docs/00-reference-setup.md | 3 ++- docs/troubleshooting.md | 3 ++- scripts/lib/common.sh | 19 +++++++++++++++---- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da7194bc5..38f38ef96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin, counts a partial download already on disk, and warns without stopping the remaining installs; `--clinical-data` exits 1 when one is still missing. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. +- Setup now installs VEP 116, the PCGR/CPSR reference bundle and VEP 115 by default so `run-all.sh` can plan the clinical steps. `--no-clinical-data` skips them; `--clinical-data` installs only these data. Each install checks its disk peak plus a 5 GB margin, credits the allocated space of a partial download, and warns without stopping the remaining installs. Stale extraction files are removed before the disk check. Retained PCGR archives are checked against the pinned digest before reuse. `--clinical-data` exits 1 when one item is still missing. The bundle is checked against a pinned SHA-256, resumes interrupted transfers, and is published only after extraction. Validation and setup docs now name the affected steps and the repair command. Freshness checks the three archive URLs with HEAD, both Ensembl checksum entries and the bundle's byte count. No analysis output format changes. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/00-reference-setup.md b/docs/00-reference-setup.md index b00f2c3b9..fa2236f7d 100644 --- a/docs/00-reference-setup.md +++ b/docs/00-reference-setup.md @@ -7,7 +7,8 @@ One-time downloads required before running the pipeline. Each heading below give A plain `./scripts/setup.sh "$GENOME_DIR"` installs both VEP caches and the PCGR/CPSR bundle after AnnotSV. These clinical downloads add about 57 GB to transfer and 65 GB on disk. Allow 1 to 3 hours. Setup checks each install separately: VEP 116 needs 56 GB peak, the bundle 15 GB, and VEP 115 needs 52 GB. -Each check adds a 5 GB margin and counts a partial download already on disk. An item that does not fit is skipped with a warning. Setup continues. +Each check adds a 5 GB margin and credits the disk space occupied by a partial download. +Stale extraction files are removed before the check. Retained PCGR archives are checked against the pinned SHA-256 before reuse. An item that does not fit is skipped with a warning. Setup continues. ```bash ./scripts/setup.sh --no-clinical-data "$GENOME_DIR" # skip all three clinical downloads diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e052fb407..b75d734d2 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -740,7 +740,8 @@ ExpansionHunter \ Setup installs VEP 116, the PCGR/CPSR bundle and VEP 115 by default. If one is missing, read the warning above `Setup complete!`. The disk checks require 56 GB, 15 GB and 52 GB peak respectively, each plus a 5 GB margin. -A partial download on disk counts toward that peak. +The disk space occupied by a partial download counts toward that peak. Sparse file holes do not count. +Setup removes stale extraction files before checking space and checks a retained PCGR archive against its pinned SHA-256 before reuse. A failed download also leaves a warning. Other installs continue. Free space, then retry only the clinical data: diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh index 923f082dc..17374e740 100644 --- a/scripts/lib/common.sh +++ b/scripts/lib/common.sh @@ -663,12 +663,15 @@ install_vep_cache() { # clinical_disk_ok DIR PEAK_GB LABEL [ARCHIVE]: GB as du counts them # (2^30 bytes), plus a 5 GB margin. # Use the destination's filesystem, creating the directory when it is new. -# The bytes of ARCHIVE, or of its interrupted ARCHIVE.part, are already part -# of the peak, so a resumed download needs only the rest. +# Allocated blocks of ARCHIVE, or of its interrupted ARCHIVE.part, are already +# part of the peak, so a resumed download needs only the rest. # An unreadable df is a failed preflight, never permission to fill the disk. clinical_disk_ok() { local dir=$1 peak=$2 label=$3 archive=${4:-} free have=0 f mkdir -p "$dir" || return 1 + # A stopped extraction is disposable. Reclaim it before measuring space, + # otherwise its staging tree can prevent every retry from reaching cleanup. + rm -rf "${dir}"/.extract.* || return 1 free=$(df -Pk "$dir" 2>/dev/null | awk 'NR == 2 {print $4}') || free="" case "$free" in ''|*[!0-9]*) echo "[WARN] Cannot read free space for ${label}; run: ./scripts/setup.sh --clinical-data \"${GENOME_DIR}\""; return 1 ;; @@ -676,7 +679,10 @@ clinical_disk_ok() { if [ -n "$archive" ]; then for f in "$archive" "${archive}.part"; do if [ -f "$f" ]; then - have=$(( $(wc -c < "$f") / 1024 )) + have=$(du -k "$f" 2>/dev/null | awk 'NR == 1 {print $1}') || have="" + case "$have" in + ''|*[!0-9]*) echo "[WARN] Cannot read allocated space for ${label}; retry the clinical install."; return 1 ;; + esac break fi done @@ -704,7 +710,12 @@ install_pcgr_bundle() { url="https://insilico.hpc.uio.no/pcgr/pcgr_ref_data.${PCGR_DATA_BUNDLE}.grch38.tgz" tarball="${dir}/${url##*/}" mkdir -p "$dir" || return 1 - # fetch publishes the archive only after its check, so one already here is whole. + # A retained archive may have changed since fetch checked it. Verify it + # against the pin again before reuse, replacing a mismatch through fetch. + if [ -f "$tarball" ] && ! _fetch_check sha256 "$PCGR_DATA_SHA256" "$tarball" "$tarball" "${url##*/}"; then + echo "[WARN] Removing PCGR/CPSR archive that failed its checksum; downloading it again." + rm -f "$tarball" || return 1 + fi [ -f "$tarball" ] || fetch "$url" "$tarball" sha256 "$PCGR_DATA_SHA256" || return 1 rm -rf "${dir}"/.extract.* tmp=$(mktemp -d "${dir}/.extract.XXXXXX") || return 1 From b7e3f47c4f3155cc04e7c51b384f7c8f9ed4d551 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:24:04 +0100 Subject: [PATCH 144/255] fix(live): keep the reader's igv place through a resize, and document the reads level A resize applied the URL's locus again even after the reader moved inside igv, so any resize threw the reader back. The viewer now follows the URL only until the reader pans, zooms or types a locus, and again after the next locus from the map. docs/genome-map.md now names the Reads stop, the R key, the Esc order and the card's See the reads button. --- CHANGELOG.md | 2 +- assets/viewer/viewer.js | 22 +++++++++++++++++++--- docs/genome-map.md | 15 ++++++++++----- docs/viewer.md | 2 +- tests/test_live_viewer.py | 18 ++++++++++++++++++ 5 files changed, 49 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5de2d8186..1ea353a9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Back follows the map's history, and resizing the viewer keeps its requested locus. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. +- Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Back follows the map's history, and resizing the viewer keeps its requested locus until you move inside the viewer. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/assets/viewer/viewer.js b/assets/viewer/viewer.js index 558ea23cd..7edf7248a 100644 --- a/assets/viewer/viewer.js +++ b/assets/viewer/viewer.js @@ -64,12 +64,28 @@ browser.search(locus); } } - window.addEventListener("hashchange", followLocus); - // igv.js keeps bases per pixel on resize. Keep the requested absolute locus instead. + // igv.js keeps bases per pixel on resize. While igv still shows the locus the URL asks + // for, a resize applies that locus again. Once the reader pans, zooms or types a locus in + // igv, igv keeps the reader's place, until the URL asks for a new locus. The events + // come out of igv's shadow root, because pointer, wheel and key events are composed. + let follow = true; + ["pointerdown", "wheel", "keydown"].forEach(function (type) { + document.getElementById("igv").addEventListener(type, function () { + follow = false; + }, { capture: true, passive: true }); + }); + window.addEventListener("hashchange", function () { + follow = true; + followLocus(); + }); let resizeTimer; window.addEventListener("resize", function () { window.clearTimeout(resizeTimer); - resizeTimer = window.setTimeout(followLocus, 100); + resizeTimer = window.setTimeout(function () { + if (follow) { + followLocus(); + } + }, 100); }); root.dataset.pgpTracks = String(cfg.tracks.length); root.dataset.pgpStatus = "ready"; diff --git a/docs/genome-map.md b/docs/genome-map.md index f7e3296c1..413c93064 100644 --- a/docs/genome-map.md +++ b/docs/genome-map.md @@ -12,7 +12,7 @@ The map has five levels. The level follows how many bases one screen pixel cover | Chromosome | more than 16,667 bases per pixel | one chromosome open in place: an overview with the part in view, a ruler, the bands with their names, a findings lane with each gene and its level's words, and the runs of homozygosity with their lengths | | Region | 833 to 16,667 bases per pixel | the same lanes, with each band named with its chromosome (`16p13.3`), and the genes lane: every gene that fits in three rows, its name beside it; the genes a finding names come first, in bold, then the protein-coding genes, then the rest | | Gene | 833 bases per pixel or fewer, a view of 50 kb or more | the genes as models: the span as a thin line with its strand's chevrons and the exon blocks inside it; each finding is a dashed tick down through every lane, with its change (HGVS) under its gene | -| Reads | a view under 50 kb | drawn as the gene level | +| Reads | a view under 50 kb | drawn as the gene level, with a brass band under the ruler, a line that says the reads are one key away, and a **See the reads** button on the map | A dashed line marks the centre of an open chromosome. The margin beside the map says what is under it: the chromosome, the position and the band, the findings in view and the runs of homozygosity. @@ -21,9 +21,9 @@ The exon blocks are the union of the exons of every basic GENCODE transcript of ## Moving around - Click a chromosome to open it, and drag to move along it. Scroll, the + and − buttons or the + and − keys zoom. -- The zoom stops above the map (Genome, Chromosome, Region, Gene) jump to a level and keep the place in view. From the whole genome they go to the first finding by level. +- The zoom stops above the map (Genome, Chromosome, Region, Gene, Reads) jump to a level and keep the place in view. From the whole genome they go to the first finding by level. Reads opens a 2 kb view and the reads panel. - The path under the title (`Genome › chr16 › 16p13.3 › 2.050-2.300M`) shows where the map is; each step up is a button. -- Click a gene to zoom to it. Click a finding to open its card in the margin: its level, title, gene, position, change and the map's sentence about it, with a link to the finding's own page. +- Click a gene to zoom to it. Click a finding to open its card in the margin: its level, title, gene, position, change and the map's sentence about it, with a link to the finding's own page and a **See the reads** button. - When an open chromosome has no finding in view, a "Nearest" button names the closest finding on that chromosome, or the first one on the next chromosome that has one, and flies there. - The address follows the map: `#genome`, `#chr7`, `#chr7:117.4-137.4M`, `#chr7:117.480-117.669M` or `#chr7:117559001-117599000` (1-based, both ends included). A link to one of them opens that view, and Back returns to the view before. @@ -36,12 +36,17 @@ The exon blocks are the union of the exons of every basic GENCODE transcript of | + − | zoom in or out 2×, at the centre | | `[` `]` | the chromosome before or after | | `0` | the whole genome | -| Esc | closes the card, then steps out one level: reads to gene, gene to region, region to the whole chromosome, chromosome to the genome | +| Esc | closes the reads panel, then the card, then steps out one level: reads to gene, gene to region, region to the whole chromosome, chromosome to the genome | | Tab | steps through the findings in view, each with a ring on the map and its card, then leaves the map | | `L` | opens the layers and moves to them | | `T` | folds or opens the In view table | +| `R` | opens the reads panel, in a view under 50 kb | -The keys work with the map focused (`/`, `L` and `T` anywhere on the page) and never inside a text box or with Cmd, Ctrl or Alt. +The keys work with the map focused (`/`, `L`, `T` and `R` anywhere on the page) and never inside a text box or with Cmd, Ctrl or Alt. + +### The reads + +Under 50 kb, the run's reads open in a panel under the map, in the genome viewer. `docs/viewer.md` describes the panel. ### The whole-genome strip diff --git a/docs/viewer.md b/docs/viewer.md index 09b8e3680..add80039b 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -84,7 +84,7 @@ Then open `http://127.0.0.1:8765/` on your computer. The report's links work the On the genome map, zoom below 50 kb and press **R** or select **See the reads**. The **Reads** gauge opens a 2 kb view. A finding's map card also offers **See the reads**, centred on its position with 50 bases on each side. The panel otherwise starts at the nearest open finding in view, or at the view itself. Moving the map updates the viewer's locus. Alignments are visible up to 50 kb. -Back follows the map's history. Moving the map creates no extra history entry for the iframe, and the Reads gauge adds one camera view. Resizing the viewer keeps its requested locus. +Back follows the map's history. Moving the map creates no extra history entry for the iframe, and the Reads gauge adds one camera view. Resizing the viewer keeps the requested locus. Once you pan, zoom or type a locus inside the viewer, a resize keeps your place instead, until the map moves again. The panel takes the place of the In view table. Press **Esc**, including from inside the viewer, or select **Back to the map** to close it. Focus returns to the map at the same view. Closing the panel clears its previous locus, title, and link to the full viewer. **Open full viewer** opens that locus in a new tab. A closed result has no reads button, and closing an opened result removes its reads panel. When the viewer is off, the panel shows the server's reason. The static demo explains that it has no viewer. diff --git a/tests/test_live_viewer.py b/tests/test_live_viewer.py index 666a8b9f0..2906d191f 100644 --- a/tests/test_live_viewer.py +++ b/tests/test_live_viewer.py @@ -313,6 +313,24 @@ def resized(): out.write(base64.b64decode(png)) b.call("Emulation.setDeviceMetricsOverride", {"width": 1440, "height": 900, "deviceScaleFactor": 1, "mobile": False}, b.session) + # A locus the reader types in igv stays through a resize: only a new map locus moves igv again. + js("var i=savedFrame.contentDocument.getElementById('igv').shadowRoot.querySelector('input.igv-search-input');" + "i.focus();i.select();") + b.call("Input.insertText", {"text": "chr1:2001-3000"}, b.session) + for kind in ("keyDown", "keyUp"): + b.call("Input.dispatchKeyEvent", {"type": kind, "key": "Enter", "code": "Enter", + "windowsVirtualKeyCode": 13, "text": "\r"}, b.session) + typed = b.wait("/^chr1:2,0/.test(savedFrame.contentDocument.getElementById('igv').shadowRoot" + ".querySelector('input.igv-search-input').value)") + js("savedFrame.style.width='50%'") + resized() + time.sleep(0.5) # past viewer.js's 100 ms resize timer + m = re.match(r"chr1:([\d,]+)-([\d,]+)$", viewer_locus() or "") + check("viewer: a locus typed in igv stays through a resize", + typed and m and 1500 <= int(m.group(1).replace(",", "")) <= 3000 + and int(m.group(2).replace(",", "")) <= 3500, viewer_locus()) + js("savedFrame.style.width=''") + resized() b.eval("savedFrame.contentDocument.dispatchEvent(new KeyboardEvent('keydown',{key:'Escape',bubbles:true}))") check("map: Escape closes the reads and returns focus without moving the map", b.eval("document.querySelector('.gm-reads').hidden && document.activeElement===document.querySelector('.gm-canvas') && " From 3b223311d26045e507112d04c10b30a26a5f3221 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:25:32 +0100 Subject: [PATCH 145/255] test(bloodtype): pin split null allele dosage and contradictions Bead: trait-40 Red first: all three new report tests fail on the pre-fix accumulator. --- rust/crates/og-bloodtype/tests/report.rs | 96 ++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/rust/crates/og-bloodtype/tests/report.rs b/rust/crates/og-bloodtype/tests/report.rs index 6cb81053f..165326f5a 100644 --- a/rust/crates/og-bloodtype/tests/report.rs +++ b/rust/crates/og-bloodtype/tests/report.rs @@ -427,3 +427,99 @@ fn a_record_that_spans_a_marker_with_another_base_there_is_no_call() { assert_eq!(status, "absent", "{recs:?}"); } } + +/// The complete table and evidence must agree through both VCF readers. +fn report_both_ways(d: &Path, records: &[String]) -> (String, String) { + let (plain, gz) = both_ways(d, records); + let mut outputs = Vec::new(); + for (vcf, access) in [(plain, Access::Stream), (gz, Access::Index)] { + let files = Files { + vcf: vcf.to_string_lossy().into_owned(), + markers: markers(), + depth: Some(depth(d, 2.0, &HashMap::new())), + cn_loci: Some(cn_loci()), + out: d.join("table.tsv").to_string_lossy().into_owned(), + evidence: d.join("evidence.tsv").to_string_lossy().into_owned(), + }; + assert_eq!(report(&files).unwrap().1, access); + outputs.push(( + std::fs::read_to_string(&files.out).unwrap(), + std::fs::read_to_string(&files.evidence).unwrap(), + )); + } + assert_eq!(outputs[0], outputs[1]); + outputs.remove(0) +} + +fn split_null_marker(name: &str, system: &str, phenotype: &str) { + let d = dir(&format!("split-{name}")); + let m = read_markers(&markers()).unwrap()[name].clone(); + let alts: Vec<&str> = m.alt.split(',').collect(); + let record = |alt: &str, gt: &str| line(&m.chrom, m.pos, &m.ref_, alt, "PASS", gt); + let unsplit = record(&m.alt, "1/2"); + let expected = report_both_ways(&d, std::slice::from_ref(&unsplit)); + let row: Vec<&str> = expected.0.lines().find(|l| l.starts_with(&format!("{system}\t"))) + .unwrap().split('\t').collect(); + assert_eq!(row[..4], [system, phenotype, "null/null", "high"]); + let a = record(alts[0], "1/0"); + let b = record(alts[1], "0/1"); + for records in [ + vec![a.clone(), b.clone()], + vec![b.clone(), a.clone()], + vec![a.clone(), a.clone(), b.clone(), b.clone()], + vec![unsplit.clone(), a.clone(), b.clone()], + vec![b.clone(), a.clone(), unsplit], + vec![record(&format!("{},{}", alts[1], alts[0]), "2/1")], + vec![record(".", "0/0"), a, b], + ] { + assert_eq!(report_both_ways(&d, &records), expected, "{records:?}"); + } + // A repeated call of one ALT is still one copy, even when another + // record lists the same ALT at a different allele index. + let one = record(alts[0], "0/1"); + for gt in ["0/1", "1/1"] { + let single = record(alts[0], gt); + let expected = report_both_ways(&d, std::slice::from_ref(&single)); + let copies = if gt == "0/1" { 1 } else { 2 }; + let reordered = record(&format!("{},{}", alts[1], alts[0]), + if copies == 1 { "0/2" } else { "2/2" }); + let records = [single.clone(), reordered, single]; + assert_eq!(report_both_ways(&d, &records), expected); + assert_eq!(site_both_ways(&d, &records, name), ("called".to_owned(), Some(copies), String::new())); + } + assert_eq!(site_both_ways(&d, &[one], name).1, Some(1)); +} + +#[test] +fn split_kidd_null_alts_match_an_unsplit_call() { + split_null_marker("JK_01N06", "Kidd", "Jk(a-b-)"); +} + +#[test] +fn split_kell_null_alts_match_an_unsplit_call() { + split_null_marker("KEL_02N01", "Kell", "K-k- Kp(a-b-) Js(a-b-)"); +} + +#[test] +fn incompatible_null_alt_dosages_are_no_call_in_both_readers() { + for name in ["JK_01N06", "KEL_02N01"] { + let d = dir(&format!("incompatible-{name}")); + let m = read_markers(&markers()).unwrap()[name].clone(); + let alts: Vec<&str> = m.alt.split(',').collect(); + let record = |alt: &str, gt: &str| line(&m.chrom, m.pos, &m.ref_, alt, "PASS", gt); + // Two distinct ALTs cannot have three copies. Different dosages + // for the same ALT cannot both be true either. + let hom = record(alts[0], "1/1"); + for other in [record(alts[1], "0/1"), record(alts[0], "0/1")] { + for records in [vec![hom.clone(), other.clone()], vec![other.clone(), hom.clone()]] { + let (status, n, note) = site_both_ways(&d, &records, name); + assert_eq!((status.as_str(), n), ("no call", None), "{records:?}"); + assert_eq!(note, "incompatible ALT dosages at this position"); + let (table, _) = report_both_ways(&d, &records); + let row: Vec<&str> = table.lines().find(|l| l.starts_with(&format!("{}\t", m.system))) + .unwrap().split('\t').collect(); + assert_ne!(row[3], "high"); + } + } + } +} From 0a0cfc073de74574954914a2872ce96e280152e9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:31:47 +0100 Subject: [PATCH 146/255] fix(roh): write 0.0, not -0.0, when a sample has no long run An empty f64 sum in Rust is -0.0, so every sample without a run of 5 Mb or more read F_ROH -0.0000 in both reports. Fold from +0.0, check the sign in the scope test, and keep the ROH file's open and read errors named the way every other section names them. --- rust/crates/og-summary/src/collect/group_c/roh.rs | 13 ++++++++++--- rust/crates/og-summary/src/collect/group_c/tests.rs | 9 +++++++++ .../golden/summary-collect/all-bad/all-bad.json | 2 +- .../parity/golden/summary-collect/c-bad1/bad1.json | 2 +- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-summary/src/collect/group_c/roh.rs b/rust/crates/og-summary/src/collect/group_c/roh.rs index 11a538c28..e0790620c 100644 --- a/rust/crates/og-summary/src/collect/group_c/roh.rs +++ b/rust/crates/og-summary/src/collect/group_c/roh.rs @@ -223,11 +223,16 @@ pub fn section(c: &Ctx<'_>) -> Out { // chrom -> the bases of its long runs let mut per_chrom: indexmap::IndexMap = indexmap::IndexMap::new(); let (mut assessed, mut observed) = ([false; 22], [false; 22]); - let mut reader = crate::text_lines(&p.path)?; + let mut reader = crate::text_lines(&p.path).map_err(|e| super::util::os_note(&e))?; let mut line = String::new(); loop { line.clear(); - if !reader.read_line(&mut line).map_err(|e| e.to_string())? { + if !reader.read_line(&mut line).map_err(|e| { + crate::collect::sections::py_error( + "OSError", + &crate::collect::sections::gzip_message(&e), + ) + })? { break; } let c: Vec<&str> = line.split_whitespace().collect(); @@ -285,7 +290,9 @@ pub fn section(c: &Ctx<'_>) -> Out { big.push(row.val()); } } - let long_bp: f64 = per_chrom.values().sum(); + // fold from +0.0: an empty f64 sum is -0.0, which the reports print as + // "-0.0000" for every sample without a long run. + let long_bp: f64 = per_chrom.values().fold(0.0, |a, v| a + v); let has_sites = assessed.iter().any(|yes| *yes); // Old -O r files omit ST rows. Their runs give a lower bound on // chromosome scope, never evidence for a genome-wide interpretation. diff --git a/rust/crates/og-summary/src/collect/group_c/tests.rs b/rust/crates/og-summary/src/collect/group_c/tests.rs index e4cad1c0e..c50a4f5dc 100644 --- a/rust/crates/og-summary/src/collect/group_c/tests.rs +++ b/rust/crates/og-summary/src/collect/group_c/tests.rs @@ -268,6 +268,15 @@ fn roh_scope_counts_sites_without_runs_and_keeps_partial_labels_empty() { let no_runs = roh_scope_summary(&[20], ""); assert_eq!(at(&no_runs, &["f_roh"]), &Value::Float(0.0)); assert_eq!(at(&no_runs, &["relationship"]), &Value::Null); + // Value::Float(-0.0) == Value::Float(0.0), so check the sign apart: a + // sample with no long run must read 0.0, never -0.0, in both reports. + for key in ["f_roh", "long_mb"] { + assert!( + matches!(at(&no_runs, &[key]), Value::Float(f) if f.is_sign_positive()), + "{key} must be +0.0 without long runs, got {:?}", + at(&no_runs, &[key]) + ); + } } #[test] diff --git a/tests/parity/golden/summary-collect/all-bad/all-bad.json b/tests/parity/golden/summary-collect/all-bad/all-bad.json index a20f67e6c..24bc28580 100644 --- a/tests/parity/golden/summary-collect/all-bad/all-bad.json +++ b/tests/parity/golden/summary-collect/all-bad/all-bad.json @@ -525,7 +525,7 @@ "autosomal_over_5mb": [], "masked_mb": 0.0, "sex_chromosome_segments": 0, - "long_mb": -0.0, + "long_mb": 0.0, "f_roh": null, "relationship": null, "upd_flag": null, diff --git a/tests/parity/golden/summary-collect/c-bad1/bad1.json b/tests/parity/golden/summary-collect/c-bad1/bad1.json index ad14386d1..8bf1c4c9e 100644 --- a/tests/parity/golden/summary-collect/c-bad1/bad1.json +++ b/tests/parity/golden/summary-collect/c-bad1/bad1.json @@ -514,7 +514,7 @@ "autosomal_over_5mb": [], "masked_mb": 0.0, "sex_chromosome_segments": 0, - "long_mb": -0.0, + "long_mb": 0.0, "f_roh": null, "relationship": null, "upd_flag": null, From 31694be4a3e94a54fd3ab485f91a62b45d994bf5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:32:07 +0100 Subject: [PATCH 147/255] fix(archaic): name a chromosome the gVCF lacks instead of blaming the order A gVCF with no records on a chromosome the VCF lists was read to its end, and the next chromosome then stopped with an order message. Remember the chromosome not found and name it. The docs now say IBDmix drops every site of one genome at -m 1, and that a sample with no gVCF gets no step 43 output under run-all.sh. --- docs/43-archaic.md | 4 ++-- rust/crates/og-ancestry/src/archaic.rs | 24 +++++++++++++++---- .../tests/i411_archaic_callable.rs | 11 +++++++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/docs/43-archaic.md b/docs/43-archaic.md index 2d25574e5..c90b31222 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -19,7 +19,7 @@ IBDmix needs no unadmixed reference population. It scores each site from the mod ### One genome and the stand-ins -IBDmix takes the modern allele frequency of each site from every modern column in its input, and your column is one of them. With one genome the frequency is that person's own genotype, and IBDmix drops nearly every site at its minimum allele count, so almost nothing scores. The step therefore adds 100 stand-in people. Together they carry round(AF × 200) ALT alleles at each site, where AF is the 1000 Genomes phase 3 frequency of the chosen population (`ARCHAIC_POP`, default `ALL`; `AFR`, `AMR`, `EAS`, `EUR` or `SAS`). The stand-ins' own segments are dropped. +IBDmix takes the modern allele frequency of each site from every modern column in its input, and your column is one of them. With one genome the frequency is that person's own genotype, and IBDmix drops every site at its minimum allele count (`-m 1` needs 2 ALT and 2 REF alleles, and one person has 2 alleles), so nothing scores. The step therefore adds 100 stand-in people. Together they carry round(AF × 200) ALT alleles at each site, where AF is the 1000 Genomes phase 3 frequency of the chosen population (`ARCHAIC_POP`, default `ALL`; `AFR`, `AMR`, `EAS`, `EUR` or `SAS`). The stand-ins' own segments are dropped. So the frequency IBDmix uses at a site is (the stand-ins' ALT count + yours) / 202, or / 200 where your genotype is unknown. That is you inside a sample of about 100 people, about the size of one 1000 Genomes group (they hold 61 to 113 people). Chen et al. ran IBDmix the same way: once per 1000 Genomes group, with the scored person counted in the group's frequency. What follows from it: @@ -97,7 +97,7 @@ they close. It prints how many table sites it read and how many it did not. ./scripts/43-archaic.sh your_name ``` -With `run-all.sh`: step 43 runs once the table is installed (`ARCHAIC_DIR=none` skips it, `ARCHAIC_POP` picks the frequencies). In the Nextflow pipeline: `archaic` in `--tools` with `--archaic_data ` and `--archaic_pop`; `--hla_genes` gives the genes (processes `ARCHAIC_GENOTYPES`, `ARCHAIC_IBDMIX`, `ARCHAIC_REPORT`). +With `run-all.sh`: step 43 runs once the table is installed (`ARCHAIC_DIR=none` skips it, `ARCHAIC_POP` picks the frequencies). It needs the gVCF: a sample that starts from a VCF with no gVCF and index beside it gets no step 43 output, and the Nextflow log names the sample. In the Nextflow pipeline: `archaic` in `--tools` with `--archaic_data ` and `--archaic_pop`; `--hla_genes` gives the genes (processes `ARCHAIC_GENOTYPES`, `ARCHAIC_IBDMIX`, `ARCHAIC_REPORT`). ## Output diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index 89fa026d1..f62b9ee5d 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -915,6 +915,9 @@ struct Callable { chrom: String, /// The last position the confident calls so far cover on `chrom`. covered: i64, + /// The first chromosome of the VCF the gVCF was read to its end for + /// without finding a record on it. + missing: Option, } impl Callable { @@ -951,6 +954,7 @@ impl Callable { seen: Vec::new(), chrom: String::new(), covered: 0, + missing: None, }; callable.pending = callable.read()?; Ok(callable) @@ -1010,15 +1014,25 @@ impl Callable { fn start(&mut self, chrom: &str) -> Out<()> { let here = self.pending.as_ref().is_some_and(|b| b.chrom == chrom); if !here && self.seen.iter().any(|x| x == chrom) { - return usage(format!( - "{} lists {chrom} before a chromosome the VCF lists first; \ - give the gVCF and the VCF in one chromosome order", - self.path - )); + return usage(match &self.missing { + Some(gone) => format!( + "{} has no records on {gone}, which the VCF lists before {chrom}; \ + give the gVCF of the same sample, with every chromosome the VCF lists", + self.path + ), + None => format!( + "{} lists {chrom} before a chromosome the VCF lists first; \ + give the gVCF and the VCF in one chromosome order", + self.path + ), + }); } while self.pending.as_ref().is_some_and(|b| b.chrom != chrom) { self.pending = self.read()?; } + if self.pending.is_none() && self.missing.is_none() { + self.missing = Some(chrom.to_owned()); + } self.chrom = chrom.to_owned(); self.covered = 0; Ok(()) diff --git a/rust/crates/og-ancestry/tests/i411_archaic_callable.rs b/rust/crates/og-ancestry/tests/i411_archaic_callable.rs index 78721e7fe..51236723d 100644 --- a/rust/crates/og-ancestry/tests/i411_archaic_callable.rs +++ b/rust/crates/og-ancestry/tests/i411_archaic_callable.rs @@ -220,8 +220,9 @@ fn a_site_is_covered_up_to_the_furthest_end() { } /// The gVCF is read alongside the VCF: the chromosomes the table lacks are -/// read past, and a gVCF in another chromosome order, or unsorted, stops -/// the step (exit 2) instead of reading as no reference calls. +/// read past, and a gVCF in another chromosome order, unsorted, or missing +/// a chromosome the VCF lists before another, stops the step (exit 2) +/// instead of reading as no reference calls. #[test] fn the_gvcf_is_read_in_the_vcf_order() { let d = dir("order"); @@ -248,6 +249,12 @@ fn the_gvcf_is_read_in_the_vcf_order() { assert_eq!(code, 2, "{err}"); assert!(err.contains("one chromosome order"), "{err}"); + // A gVCF with no records on chr21 is read to its end looking for it; + // the stop then names the missing chromosome, not the order. + let (code, err) = run(&d, vcf, Some(&format!("{other}{chr22}"))); + assert_eq!(code, 2, "{err}"); + assert!(err.contains("has no records on chr21"), "{err}"); + let back = format!( "{chr21}{chr22}{}", block("chr21", 9000, 9100, ".", "GT", "0/0") From 26f7be164db42fa58d37d879f97baa1015910331 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:32:58 +0100 Subject: [PATCH 148/255] fix(bloodtype): combine distinct null ALTs across split records Keep called ALT identities, deduplicate repeated dosages and refuse conflicting or impossible totals. Both VCF readers now preserve split/unsplit tables and evidence. Bead: trait-40 --- CHANGELOG.md | 1 + docs/41-bloodtype.md | 1 + rust/crates/og-bloodtype/src/genotypes.rs | 42 ++++++++++++++++++++++- rust/crates/og-bloodtype/tests/report.rs | 33 +++++++++++++----- 4 files changed, 68 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab677d5bb..f30350c3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Step 41 (blood groups) reads every VCF record that covers a marker site, and their order no longer changes the answer. Before, the first record at a position won. An all-sites VCF writes a reference record and then the indel at one position (`T . 0/0`, then `T TC 0/1` at ABO c.261), so a person with an A or B allele read O. A split multi-allelic record with the SNV first did the same. No column, file or option changes; only the values of the affected sites do. What changes: - A call of the marker's ALT now wins over the other records at the position. Another allele a second record calls there is named in the note, as when one record holds both. + - Distinct ALTs listed by one null marker add their copies across split records. JK_01N06 and KEL_02N01 now give the same evidence and null/null phenotype before and after splitting. Repeated calls of the same ALT count once. Conflicting dosages for one ALT, or totals above two copies, give a no call with an incompatible-dosage note. - A no call of a record that names the marker's ALT (`./.`, or a failed FILTER) now wins over a reference record at the same position, so the site is a no call instead of a confident reference. - A 0/0 record over a REF shorter than the marker's (an all-sites `C . 0/0`, or a gVCF block that starts at the position) now counts as no record, so the reads decide. Before it was a no call: ABO read "A, A1 or A2 not determined", and Vel, HPA-14 and every system with a deletion marker lost confidence. - A record that starts upstream and spans a marker with an allele that changes the marker's base (a deletion over it, or a padded `CG CA`, as DeepVariant writes `CG C,CA` when a deletion and an SNV overlap) now makes the site a no call, with a note that names the record. Before, the site had no record at its own position and 8 reads made it a confident reference: a K/K person read k/k at high confidence. diff --git a/docs/41-bloodtype.md b/docs/41-bloodtype.md index 2456d51b5..9f1941391 100644 --- a/docs/41-bloodtype.md +++ b/docs/41-bloodtype.md @@ -67,6 +67,7 @@ A site the VCF has no record for is the reference allele when the BAM has at lea When several records cover one site, each one is read and their order does not matter: - A call of the table's ALT wins over the other records at the site's position. An all-sites VCF writes `T . 0/0` and then `T TC 0/1` at ABO c.261, so the insertion counts. Another allele that a second record calls there (a split `T G 1/0`) is named in the note. +- When one null marker lists several ALTs, calls of distinct listed ALTs add their copies. Splitting the record keeps the same evidence and phenotype. Repeated calls of the same ALT count once, even if its allele index changes. Different dosages for the same ALT, or a total above two copies, make the site a no call with an incompatible-dosage note. - A no call of a record that names the table's ALT (`./.`, or a failed FILTER) comes next: the ALT may be there, so a reference record beside it does not make the site reference. - Then a reference call, then any other no call. - A record that starts upstream and spans the site with an allele that changes the site's base (a deletion over it, or a padded record such as `CG CA`) makes the site a no call, whatever the site's own records say. A reference call or a no call upstream changes nothing, and neither does an allele that keeps the site's base. diff --git a/rust/crates/og-bloodtype/src/genotypes.rs b/rust/crates/og-bloodtype/src/genotypes.rs index 6eac1aec9..f2506464b 100644 --- a/rust/crates/og-bloodtype/src/genotypes.rs +++ b/rust/crates/og-bloodtype/src/genotypes.rs @@ -22,6 +22,8 @@ //! ALT wins, then a no call of a record that names the marker's ALT, //! then a reference call, then any other no call; the first of equals //! counts, and another allele a losing call names stays in the note; +//! - called copies of distinct ALTs listed by one marker add up; repeated +//! calls of the same ALT count once, and incompatible dosages are no call; //! - a reference-only record whose REF is shorter than the marker's (an //! all-sites record or a gVCF block starting at a deletion marker) says //! nothing past its own bases, so the site reads as if it had no record. @@ -174,6 +176,8 @@ struct Reading { status: Status, n: Option, note: String, + /// Called marker ALTs and their dosages, independent of VCF allele indices. + alts: BTreeMap, } impl Reading { @@ -183,6 +187,7 @@ impl Reading { status: Status::NoCall, n: None, note, + alts: BTreeMap::new(), } } } @@ -196,10 +201,23 @@ struct Evidence { others: Vec, /// The first upstream record that changes the site's base. span: Option, + /// One dosage per called marker ALT, bounded by the marker's ALT list. + alts: BTreeMap, + /// Different calls of one ALT disagree about its dosage. + incompatible: bool, } impl Evidence { fn add(&mut self, r: Reading) { + for (alt, &copies) in &r.alts { + match self.alts.get(alt) { + Some(&previous) if previous != copies => self.incompatible = true, + Some(_) => {} + None => { + self.alts.insert(alt.clone(), copies); + } + } + } let loser = match &mut self.best { Some(b) if r.weight > b.weight => Some(std::mem::replace(b, r)), Some(_) => Some(r), @@ -224,8 +242,19 @@ impl Evidence { site.n = None; site.note = note; } else if let Some(b) = self.best { + let copies: i64 = self.alts.values().sum(); + if self.incompatible || copies > 2 { + site.status = Status::NoCall; + site.n = None; + site.note = "incompatible ALT dosages at this position".to_owned(); + return; + } site.status = b.status; - site.n = b.n; + site.n = if b.weight == Weight::Alt { + Some(copies) + } else { + b.n + }; site.note = if b.status == Status::Called { let others = self.others.into_iter().filter(|o| *o != b.note); std::iter::once(b.note.clone()) @@ -383,6 +412,16 @@ fn read_record( .iter() .filter(|a| idx.iter().any(|i| i == *a)) .count() as i64; + let mut called_alts = BTreeMap::new(); + for (i, alt) in alts.iter().enumerate() { + if wants.contains(alt) { + let index = (i + 1).to_string(); + let copies = alleles.iter().filter(|a| **a == index).count() as i64; + if copies > 0 { + *called_alts.entry((*alt).to_owned()).or_default() += copies; + } + } + } let note = if alleles .iter() .any(|a| *a != "0" && !idx.iter().any(|i| i == a)) @@ -403,6 +442,7 @@ fn read_record( status: Status::Called, n: Some(n), note, + alts: called_alts, }) } diff --git a/rust/crates/og-bloodtype/tests/report.rs b/rust/crates/og-bloodtype/tests/report.rs index 165326f5a..43de618cf 100644 --- a/rust/crates/og-bloodtype/tests/report.rs +++ b/rust/crates/og-bloodtype/tests/report.rs @@ -458,8 +458,13 @@ fn split_null_marker(name: &str, system: &str, phenotype: &str) { let record = |alt: &str, gt: &str| line(&m.chrom, m.pos, &m.ref_, alt, "PASS", gt); let unsplit = record(&m.alt, "1/2"); let expected = report_both_ways(&d, std::slice::from_ref(&unsplit)); - let row: Vec<&str> = expected.0.lines().find(|l| l.starts_with(&format!("{system}\t"))) - .unwrap().split('\t').collect(); + let row: Vec<&str> = expected + .0 + .lines() + .find(|l| l.starts_with(&format!("{system}\t"))) + .unwrap() + .split('\t') + .collect(); assert_eq!(row[..4], [system, phenotype, "null/null", "high"]); let a = record(alts[0], "1/0"); let b = record(alts[1], "0/1"); @@ -481,11 +486,16 @@ fn split_null_marker(name: &str, system: &str, phenotype: &str) { let single = record(alts[0], gt); let expected = report_both_ways(&d, std::slice::from_ref(&single)); let copies = if gt == "0/1" { 1 } else { 2 }; - let reordered = record(&format!("{},{}", alts[1], alts[0]), - if copies == 1 { "0/2" } else { "2/2" }); + let reordered = record( + &format!("{},{}", alts[1], alts[0]), + if copies == 1 { "0/2" } else { "2/2" }, + ); let records = [single.clone(), reordered, single]; assert_eq!(report_both_ways(&d, &records), expected); - assert_eq!(site_both_ways(&d, &records, name), ("called".to_owned(), Some(copies), String::new())); + assert_eq!( + site_both_ways(&d, &records, name), + ("called".to_owned(), Some(copies), String::new()) + ); } assert_eq!(site_both_ways(&d, &[one], name).1, Some(1)); } @@ -511,13 +521,20 @@ fn incompatible_null_alt_dosages_are_no_call_in_both_readers() { // for the same ALT cannot both be true either. let hom = record(alts[0], "1/1"); for other in [record(alts[1], "0/1"), record(alts[0], "0/1")] { - for records in [vec![hom.clone(), other.clone()], vec![other.clone(), hom.clone()]] { + for records in [ + vec![hom.clone(), other.clone()], + vec![other.clone(), hom.clone()], + ] { let (status, n, note) = site_both_ways(&d, &records, name); assert_eq!((status.as_str(), n), ("no call", None), "{records:?}"); assert_eq!(note, "incompatible ALT dosages at this position"); let (table, _) = report_both_ways(&d, &records); - let row: Vec<&str> = table.lines().find(|l| l.starts_with(&format!("{}\t", m.system))) - .unwrap().split('\t').collect(); + let row: Vec<&str> = table + .lines() + .find(|l| l.starts_with(&format!("{}\t", m.system))) + .unwrap() + .split('\t') + .collect(); assert_ne!(row[3], "high"); } } From 6af4386a2dc7e922fc7a94a3d6dbcf0ce5795467 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 10:37:21 +0100 Subject: [PATCH 149/255] fix(recheck): replay only the switches run-all.sh records, and refuse a ~ bash would expand An edited logs/run_status.tsv could name BASH_ENV, PATH or LD_PRELOAD in both 'meta switches' and 'meta command', and the replay then set it for the bash it starts. A switch is now a name in run-all.sh's SWITCHES list too (a unit test keeps the Rust copy equal to it). A ~ after an unquoted = or : is refused, since bash expands it there and printf %q never writes it so. --- docs/recheck.md | 4 +- rust/crates/og-recheck/src/rerun.rs | 79 ++++++++++++++++++++++++++--- rust/crates/og-recheck/src/tests.rs | 33 ++++++++++++ 3 files changed, 106 insertions(+), 10 deletions(-) diff --git a/docs/recheck.md b/docs/recheck.md index bfc471eb2..3f5f93c52 100644 --- a/docs/recheck.md +++ b/docs/recheck.md @@ -68,7 +68,7 @@ bin/og recheck --sample your_sample --rerun bin/og recheck --sample your_sample --rerun --refresh-clinvar # refresh ClinVar first ``` -`run-all.sh` writes its own command into `logs/run_status.tsv` (`meta command`: a `cd` to the directory it was run from, the switches it reads that were set, then its arguments, each shell-quoted; `meta switches`: the names of those switches). `--rerun` reads that command back word by word: the `cd` folder, the switch assignments, the `run-all.sh` path and its arguments. It starts that `run-all.sh` with `bash`, in that folder, with the arguments one by one and those switches only, so a `TOOLS` set in your shell today does not change the run. No shell reads the command. A row of any other shape, for example one edited to add `; rm ...`, is refused with exit 2 and nothing runs; a `cd` folder that no longer exists is refused the same way. Check such a row, then run the command by hand. `NXF_OPTS` and `NXF_JVM_ARGS` are not recorded, because they can hold a proxy password; the replay takes them from your shell. Nextflow's `-resume` then redoes only the tasks whose inputs changed. +`run-all.sh` writes its own command into `logs/run_status.tsv` (`meta command`: a `cd` to the directory it was run from, the switches it reads that were set, then its arguments, each shell-quoted; `meta switches`: the names of those switches). `--rerun` reads that command back word by word: the `cd` folder, the switch assignments, the `run-all.sh` path and its arguments. It starts that `run-all.sh` with `bash`, in that folder, with the arguments one by one and those switches only, so a `TOOLS` set in your shell today does not change the run. A switch is a name in the `SWITCHES` list of `run-all.sh`; an assignment to any other name (`BASH_ENV`, `PATH`, `LD_PRELOAD`) is refused, even when the `meta switches` line names it. No shell reads the command. A row of any other shape, for example one edited to add `; rm ...`, is refused with exit 2 and nothing runs; a `cd` folder that no longer exists is refused the same way. Check such a row, then run the command by hand. `NXF_OPTS` and `NXF_JVM_ARGS` are not recorded, because they can hold a proxy password; the replay takes them from your shell. Nextflow's `-resume` then redoes only the tasks whose inputs changed. It refuses while a run of the sample is going on (`run_status.tsv` has no `finished_utc`), and while another `--rerun` of the sample holds `recheck/rerun.lock`. A run that was killed without its exit trap leaves no `finished_utc` either: run `run-all.sh` by hand once, and the next check can rerun again. @@ -170,4 +170,4 @@ launchd runs an interval missed during sleep when the Mac wakes; one missed whil ## Tests -`cargo test -p og-recheck` checks the classes, the steps each item names, a failed lookup (an offline answer and a server on 127.0.0.1), the files it writes, the refusals during a run and while a lock is held, the lock held while `setup.sh` refreshes ClinVar and let go before the replay, the comparison after a rerun and `--launch-dir`. `tests/test_owngenome_recheck.py` runs `og recheck` from the outside, the deprecated `bin/owngenome_recheck.py`, the live-site plugin and the ClinVar lock block of `run-all.sh` and step 06, each against a planted mistake too. `tests/fake-docker/run-all-recheck.sh` checks that `--rerun` replays the recorded command exactly, with a control where one dropped argument makes it differ. +`cargo test -p og-recheck` checks the classes, the steps each item names, a failed lookup (an offline answer and a server on 127.0.0.1), the files it writes, the refusals during a run and while a lock is held, the lock held while `setup.sh` refreshes ClinVar and let go before the replay, the comparison after a rerun, `--launch-dir`, the last line when lookups fail (connections refused), a recorded command read back word by word and refused when it is not a `run-all.sh` call, and the replay with typed arguments. `tests/test_owngenome_recheck.py` runs `og recheck` from the outside, the deprecated `bin/owngenome_recheck.py`, the live-site plugin and the ClinVar lock block of `run-all.sh` and step 06, each against a planted mistake too. `tests/fake-docker/run-all-recheck.sh` checks that `--rerun` replays the recorded command exactly, with a control where one dropped argument makes it differ. diff --git a/rust/crates/og-recheck/src/rerun.rs b/rust/crates/og-recheck/src/rerun.rs index af72f1474..9a1bd4f34 100644 --- a/rust/crates/og-recheck/src/rerun.rs +++ b/rust/crates/og-recheck/src/rerun.rs @@ -121,6 +121,58 @@ pub enum Rerun { Refused(u8, String), } +/// The switches `scripts/run-all.sh` records (its `SWITCHES` array; a unit +/// test keeps the two equal). A replay sets or drops only these, whatever +/// `meta switches` says, so an edited row can never hand `BASH_ENV`, `PATH`, +/// `LD_PRELOAD` or `DYLD_*` to the `bash` it starts. +pub const SWITCHES: &[&str] = &[ + "GENOME_DIR", + "SKIP_VALIDATION", + "THREADS", + "SKIP_TRIM", + "INTERVALS", + "ALIGN_DIR", + "TOOLS", + "KIR", + "HLA_QC", + "HPO", + "REF_FASTA", + "EH_CATALOG", + "MANTA_CALL_REGIONS", + "PARASCOPY_POPULATION", + "PARAPHASE_REGIONS", + "ANCESTRY_PANEL", + "ANCESTRY_FINE_DIR", + "ARCHAIC_DIR", + "ARCHAIC_POP", + "PGSC_CALC_DIR", + "ANCIENT_PANEL_DIR", + "VIRAL_REF", + "MEGANE_KMERS", + "MOCHA_DATA", + "GRIDSS", + "IMPUTATION", + "SOMATIC", + "EXTRA_CALLERS", + "BENCHMARK", + "FAMILIES", + "OWNGENOME_LIVE_URL", + "LOCITYPER_DATA", + "LOCITYPER_BG_REGION", + "NXF_VER", + "NXF_WORK", + "NXF_HOME", + "NXF_OFFLINE", +]; + +/// The names of `meta switches` that are also in [`SWITCHES`]. +fn known(switches: &str) -> Vec<&str> { + switches + .split_whitespace() + .filter(|n| SWITCHES.contains(n)) + .collect() +} + /// One shell word of a recorded command. #[derive(Debug, Clone, PartialEq, Eq)] enum Token { @@ -131,7 +183,8 @@ enum Token { } /// Bytes `printf %q` leaves unquoted anywhere in a word, and that mean -/// nothing to the shell there. `#` and `~` do too, except first in a word. +/// nothing to the shell there. `#` and `~` do too, except first in a word +/// (and `~` after an unquoted `=` or `:`, see [`words`]). fn plain(b: u8, first: bool) -> bool { b.is_ascii_alphanumeric() || b"%+-./:=@_,".contains(&b) @@ -200,7 +253,8 @@ fn ansi_c(s: &[u8], i: &mut usize, out: &mut Vec) -> Option<()> { /// plain bytes, `\x`, `'...'` and `$'...'`, words split on spaces and tabs, /// and the `&&` operator. `None` for anything else a shell would act on (a /// `;`, a pipe, a redirect, an expansion, a glob, a double quote, a line -/// break) and for text that is not UTF-8. +/// break, a `~` first in a word or after an unquoted `=` or `:`, where bash +/// expands it) and for text that is not UTF-8. fn words(cmd: &str) -> Option> { let s = cmd.as_bytes(); let (mut out, mut i) = (Vec::new(), 0); @@ -215,6 +269,8 @@ fn words(cmd: &str) -> Option> { continue; } let (mut w, mut lead, mut quoted) = (Vec::new(), 0, false); + // The byte before, when it stood unquoted. + let mut prev: Option = None; while i < s.len() && !matches!(s[i], b' ' | b'\t') { let b = s[i]; if b == b'\\' { @@ -226,17 +282,23 @@ fn words(cmd: &str) -> Option> { w.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes()); i += 1 + ch.len_utf8(); quoted = true; + prev = None; } else if b == b'\'' { let end = s[i + 1..].iter().position(|&c| c == b'\'')? + i + 1; w.extend_from_slice(&s[i + 1..end]); i = end + 1; quoted = true; + prev = None; } else if b == b'$' && s.get(i + 1) == Some(&b'\'') { i += 2; ansi_c(s, &mut i, &mut w)?; quoted = true; - } else if plain(b, w.is_empty() && !quoted) { + prev = None; + } else if plain(b, w.is_empty() && !quoted) + && !(b == b'~' && matches!(prev, Some(b'=' | b':'))) + { w.push(b); + prev = Some(b); i += 1; if !quoted { lead = w.len(); @@ -265,10 +327,10 @@ pub struct Call { /// `meta command` as a typed call, or `None` when it is not the shape /// `run-all.sh` writes: `cd DIR && [NAME=VALUE ...] .../scripts/run-all.sh -/// ARG ...`, every `NAME` one of `switches`. Nothing of it reaches a -/// shell. +/// ARG ...`, every `NAME` one of `switches` and of [`SWITCHES`]. Nothing of +/// it reaches a shell. pub fn parse_call(cmd: &str, switches: &str) -> Option { - let names: Vec<&str> = switches.split_whitespace().collect(); + let names = known(switches); let mut t = words(cmd)?.into_iter(); let mut word = || match t.next() { Some(Token::Word(w, lead)) => Some((w, lead)), @@ -314,7 +376,8 @@ pub fn parse_call(cmd: &str, switches: &str) -> Option { /// Replays the last `run-all.sh` command of the sample (`meta command` of /// `logs/run_status.tsv`) with the same switches: the switch variables the -/// run read (`meta switches`) are dropped from the environment, so the +/// run read (`meta switches`, only names in [`SWITCHES`]) are dropped from +/// the environment, so the /// command's own assignments decide. The command is read as a typed call /// ([`parse_call`]) and `run-all.sh` is started by `bash` with its /// arguments one by one; a command of any other shape is refused and @@ -381,7 +444,7 @@ pub fn rerun( append_log(log, &[[&started, "rerun-start", "", "", &cmd]])?; let mut c = Command::new("bash"); c.arg(&call.script).args(&call.args).current_dir(&call.dir); - for name in switches.split_whitespace() { + for name in known(&switches) { c.env_remove(name); } for (name, value) in &call.env { diff --git a/rust/crates/og-recheck/src/tests.rs b/rust/crates/og-recheck/src/tests.rs index a19d8dd4f..c0c4ac869 100644 --- a/rust/crates/og-recheck/src/tests.rs +++ b/rust/crates/og-recheck/src/tests.rs @@ -1262,9 +1262,42 @@ fn a_recorded_command_is_read_as_a_typed_call() { "cd /d /o/scripts/run-all.sh S1 male".to_owned(), "\\cd /d && /o/scripts/run-all.sh S1 male".to_owned(), "cd /d && 'TOOLS=x' /o/scripts/run-all.sh S1".to_owned(), + // bash expands a ~ after an unquoted = or :, printf %q writes \~. + format!("{base} a=~b"), + format!("{base} a:~b"), + format!("{base} --x=a:~/y"), + "cd /d && TOOLS=~/x /o/scripts/run-all.sh S1 male".to_owned(), ] { assert_eq!(parse_call(&bad, "TOOLS"), None, "{bad}"); } + // printf %q's own escape of such a ~ reads back as the ~ itself. + let call = parse_call(&format!("{base} a=\\~b a:\\~b"), "TOOLS").unwrap(); + assert_eq!(call.args[2..], ["a=~b", "a:~b"]); + // A name only an edited 'meta switches' line names is not a switch: + // the replay never hands it to bash. + for name in [ + "BASH_ENV", + "PATH", + "LD_PRELOAD", + "DYLD_INSERT_LIBRARIES", + "ENV", + ] { + let cmd = format!("cd /d && {name}=/tmp/x TOOLS=a /o/scripts/run-all.sh S1 male"); + assert_eq!(parse_call(&cmd, &format!("TOOLS {name}")), None, "{cmd}"); + } +} + +#[test] +fn the_switches_a_replay_may_set_are_the_ones_run_all_records() { + let text = fs::read_to_string( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../scripts/run-all.sh"), + ) + .unwrap(); + let start = text.find("\nSWITCHES=(").expect("SWITCHES in run-all.sh") + 11; + let end = start + text[start..].find(')').unwrap(); + let listed: Vec<&str> = text[start..end].split_whitespace().collect(); + assert!(listed.len() > 10, "{listed:?}"); + assert_eq!(listed, crate::rerun::SWITCHES); } #[test] From c90f127f798cb35f5de21a4d0194b13d8e65e626 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 11:58:57 +0100 Subject: [PATCH 150/255] fix(run-all): skip step 43 for a VCF row with no gVCF beside it --- scripts/run-all.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/run-all.sh b/scripts/run-all.sh index acec2fb02..b8023c3f4 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -238,8 +238,11 @@ else KNOWN+=" chip"; fi # Step 42 reads the gVCF only: DeepVariant writes one for a FASTQ or BAM row, a BAM+VCF row has one when it lies beside the VCF. plan "42 Traits panel" traits "$([ -z "$fv" ] || [ -n "$GROW" ] || echo "no gVCF: ${GV#"$G"/}")" || true # Step 43 against the table setup.sh --archaic builds (DONE is written last); the genes of its segments come from GENCODE. +# It reads the gVCF too (a site with no call is the reference only inside a reference block): with the table installed, +# a row that starts from a VCF without one beside it is skipped, as for step 42. AR=${ARCHAIC_DIR:-${G}/reference/archaic/${ARCHAIC_DATA_VERSION}} -plan "43 Archaic ancestry (IBDmix)" archaic "$(if [ "$AR" = none ]; then echo 'ARCHAIC_DIR=none'; else need "${AR}/DONE"; fi)" \ +plan "43 Archaic ancestry (IBDmix)" archaic "$(if [ "$AR" = none ]; then echo 'ARCHAIC_DIR=none' + elif [ -e "${AR}/DONE" ] && [ -n "$fv" ] && [ -z "$GROW" ]; then echo "no gVCF: ${GV#"$G"/}"; else need "${AR}/DONE"; fi)" \ && NF+=(--archaic_data "$AR" --archaic_pop "${ARCHAIC_POP:-ALL}") && { [[ " ${NF[*]} " == *" --hla_genes "* ]] || [ ! -s "$GT" ] || NF+=(--hla_genes "$GT"); } # Step 44 (ancient populations) is opt-in and listed only when TOOLS names it; it reads the gVCF and the panel # setup.sh --ancient-panel builds (DONE is written last). ANCIENT_PANEL_DIR picks another panel. From 7ff52a054b45653fffb69a4138e6707e17213c07 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:02:50 +0100 Subject: [PATCH 151/255] fix(archaic): give the public-evidence archaic jobs a gVCF, and name step 43 among the gVCF steps --- .github/workflows/public-evidence.yml | 51 +++++++++++++++++++++++++-- docs/43-archaic.md | 4 +-- docs/parameters.md | 2 +- docs/pipeline-overview.md | 4 +-- 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index e40adeb06..8088bb9e9 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -8,7 +8,10 @@ name: Public evidence # downloaded once (its md5 checked) and indexed, for # every chromosome job. # archaic-hg002 setup.sh --archaic and step 43 on HG002's GIAB v4.2.1 -# genotypes, one job per autosome of the chromosomes +# genotypes (with a gVCF of one reference block per +# GIAB benchmark interval, since step 43 reads a site +# with no record as the reference only inside one), +# one job per autosome of the chromosomes # input (all 22 by default: about 150 GB of archaic # downloads in all; each job fetches its three files # side by side first, and setup.sh resumes them and @@ -22,7 +25,9 @@ name: Public evidence # list must make it fail. # archaic-same-person the method check: one 1000 Genomes person with # Chen et al.'s released calls, step 43 on her high- and -# low-coverage genotypes, beside the released figure. +# low-coverage genotypes (each with a gVCF of one +# reference block per chromosome), beside the released +# figure. # ancient-footprint setup.sh --ancient-panel on the full AADR 1240K files, # then step 44 on HG002, each with the peak memory of # its containers (the cgroup's memory.peak), the peak @@ -216,6 +221,28 @@ jobs: mkdir -p "${G}/HG002/vcf" curl -fsSL --retry 5 -o "${G}/HG002/vcf/HG002.vcf.gz" "${GIAB_HG002}.vcf.gz" curl -fsSL --retry 5 -o "${G}/HG002/vcf/HG002.vcf.gz.tbi" "${GIAB_HG002}.vcf.gz.tbi" + # Step 43 reads a site with no record as the reference only inside a + # reference block of the gVCF beside the VCF. GIAB states HG002 + # matches the reference inside its benchmark regions and makes no + # claim outside them, so the gVCF is one block per benchmark + # interval of the part's chromosomes (the VCF gives the variants). + curl -fsSL --retry 5 -o "${EV}/truth.bed" "${GIAB_HG002}_noinconsistent.bed" + FAI="${G}/reference/GRCh38_no_alt_analysis_set.fasta.fai" + { + printf '##fileformat=VCFv4.2\n' + for c in $CHROMS; do awk -F'\t' -v c="chr${c}" '$1 == c {printf "##contig=\n", $1, $2}' "$FAI"; done + printf '##ALT=\n' + printf '##INFO=\n' + printf '##FORMAT=\n' + printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHG002\n' + for c in $CHROMS; do + awk -F'\t' -v c="chr${c}" 'BEGIN {OFS = "\t"} $1 == c {print c, $2 + 1, ".", "N", "<*>", ".", ".", "END=" $3, "GT", "0/0"}' \ + "${EV}/truth.bed" | sort -k2,2n + done + } > "${EV}/HG002.g.vcf" + docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/e" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ + bcftools view -Oz -o /g/HG002/vcf/HG002.g.vcf.gz /e/HG002.g.vcf + echo "gVCF: $(grep -vc '^#' "${EV}/HG002.g.vcf") reference blocks" for pop in ALL EUR; do if ! ARCHAIC_POP=$pop GENOME_DIR="$G" scripts/43-archaic.sh HG002 2>&1 | tee "${OUT}/step43-${pop}.log"; then echo "step 43 (${pop}) failed"; exit 1 @@ -388,7 +415,27 @@ jobs: bcftools index -t -f "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" rm -f "${EV}/${label}".chr*.vcf.gz echo "${label}: $(docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools view -H "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" | wc -l) SNV records" + # Step 43 reads a site with no record as the reference only inside + # a reference block of the gVCF. A panel VCF holds no callable mask, + # so the gVCF is one block over each whole chromosome: every site + # with no ALT reads as the reference, as this check measured before + # step 43 needed a gVCF (the person's records are taken with + # --min-ac 1, so a site with no record is a site with no ALT). + { + printf '##fileformat=VCFv4.2\n' + for c in $LIST; do awk -F'\t' -v c="chr${c}" '$1 == c {printf "##contig=\n", $1, $2}' "$FAI"; done + printf '##ALT=\n' + printf '##INFO=\n' + printf '##FORMAT=\n' + printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t%s\n' "${SAMPLE}${label}" + for c in $LIST; do + awk -F'\t' -v c="chr${c}" 'BEGIN {OFS = "\t"} $1 == c {print c, 1, ".", "N", "<*>", ".", ".", "END=" $2, "GT", "0/0"}' "$FAI" + done + } > "${EV}/${label}.g.vcf" + docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/e" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ + bcftools view -Oz -o "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.g.vcf.gz" "/e/${label}.g.vcf" } + FAI="${G}/reference/GRCh38_no_alt_analysis_set.fasta.fai" pull hc "$HC_URL" '1kGP_high_coverage_Illumina.chr@.filtered.SNV_INDEL_SV_phased_panel.vcf.gz' pull lc "$LC_URL" 'ALL.chr@.shapeit2_integrated_snvindels_v2a_27022019.GRCh38.phased.vcf.gz' args=() diff --git a/docs/43-archaic.md b/docs/43-archaic.md index c90b31222..defd80856 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -97,7 +97,7 @@ they close. It prints how many table sites it read and how many it did not. ./scripts/43-archaic.sh your_name ``` -With `run-all.sh`: step 43 runs once the table is installed (`ARCHAIC_DIR=none` skips it, `ARCHAIC_POP` picks the frequencies). It needs the gVCF: a sample that starts from a VCF with no gVCF and index beside it gets no step 43 output, and the Nextflow log names the sample. In the Nextflow pipeline: `archaic` in `--tools` with `--archaic_data ` and `--archaic_pop`; `--hla_genes` gives the genes (processes `ARCHAIC_GENOTYPES`, `ARCHAIC_IBDMIX`, `ARCHAIC_REPORT`). +With `run-all.sh`: step 43 runs once the table is installed (`ARCHAIC_DIR=none` skips it, `ARCHAIC_POP` picks the frequencies). It needs the gVCF: a sample that starts from a VCF is listed as `skipped (no gVCF: ...)`, as for step 42, unless it has a BAM too and a gVCF with its index beside the VCF (the gVCF `run-all.sh` passes on). In the Nextflow pipeline: `archaic` in `--tools` with `--archaic_data ` and `--archaic_pop`; `--hla_genes` gives the genes (processes `ARCHAIC_GENOTYPES`, `ARCHAIC_IBDMIX`, `ARCHAIC_REPORT`). ## Output @@ -117,7 +117,7 @@ Measured on public HG002 by the `archaic-hg002` jobs of `public-evidence.yml`, o - Install: the archaic files were 150.5 GB in all, from 1.8 GB (chromosome 22) to 13.8 GB (chromosome 2). Downloading them took 2 h 32 min summed over the 22 chromosomes, from 3 to 32 minutes each, at the speed MPI-EVA served that day. Building the table took 3 h 15 min summed, from 2 to 20 minutes per chromosome. The finished table is 0.44 GB with 51,812,908 sites. - Step 43, one population: 19 to 88 seconds per chromosome, 18 minutes summed over the 22. IBDmix runs once per chromosome and archaic genome, `THREADS` chromosomes at a time. -Every figure in this section was measured before step 43 read the gVCF, when each site GIAB's variant-only VCF had no call for was read as the reference, inside GIAB's confident regions or not. The e2e case now gives GIAB's confident regions as the reference blocks and prints its chr22 figure on every run. The whole-genome figure needs a new release run of `public-evidence.yml` with a gVCF. +Every figure in this section was measured before step 43 read the gVCF, when each site GIAB's variant-only VCF had no call for was read as the reference, inside GIAB's confident regions or not. The e2e case now gives GIAB's confident regions as the reference blocks and prints its chr22 figure on every run. The `archaic-hg002` jobs of `public-evidence.yml` give the same blocks, one per GIAB benchmark interval, so the next release run measures the whole-genome figure on this rule. The `archaic-same-person` job has no callable mask for the 1000 Genomes panel data, so it gives one reference block over each whole chromosome: every site with no ALT reads as the reference, as it did when the table below was measured. That run was made before the masks, from a table that held every lifted site. It gave HG002 115.2 Mb of Neanderthal sequence over the 22 autosomes, about twice what Chen et al. report for Europeans (a median of 51.6 Mb per person, and 42.0 to 62.0 Mb across the released per-person calls of their 503 Europeans; see the same-person comparison below for what their method is). The cause was the missing masks: the table carried the positions each archaic genome's own filters exclude, and IBDmix scored them. On chr22, 43% of that run's Neanderthal sequence lay outside the 1000 Genomes GRCh38 strict mask alone. diff --git a/docs/parameters.md b/docs/parameters.md index 021425f70..64205b8b4 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -42,7 +42,7 @@ Control which analysis tools to run. | Parameter | Default | What it does | |---|---|---| -| `--tools` | `pharmcat,cpic,vcfanno,roh,prs,mito_haplogroup,telomere_hunter,mosdepth,mito_variants,html_report,multiqc` | Comma-separated list of tools to run; an unknown name stops the run. DB-heavy tools (vep, slivar, clinical_filter, cpsr, clinvar, expansion_hunter, sample_qc) and extras (stranger, ancestry (needs --ancestry_ref and prs), ancestry_fine (needs --ancestry_fine_panel), ancient (needs --ancient_panel; samples with a gVCF), pypgx, SV callers, cram_archive, cyrius, parascopy, paraphase (PacBio HiFi BAMs only), y_haplogroup (male samples), y_str (Y-STR repeat counts, male samples), consumer_export (samples with a gVCF), bloodtype (blood group antigens; RhD needs the BAM), lpa (the LPA KIV-2 repeat count; needs the BAM), vntyper (MUC1 VNTR frameshifts behind ADTKD-MUC1; needs the BAM), viral (viral DNA in the unmapped reads; needs --viral_ref and a BAM), exomiser (runs when --hpo is set), chip (clonal haematopoiesis from the BAM; needs --chip_gff), archaic (needs --archaic_data), mei (Alu, L1 and SVA insertions with MEGAnE; needs --megane_kmers and a BAM), mocha (mosaic chromosomal alterations; needs --mocha_data), locityper (a second opinion at HLA class II, KIR, CYP2D6 and FCGR; samples with a BAM; needs --locityper_data)) are opt-in. prs and vcfanno are skipped with a warning until --pgs_scoring or a score file is set. With pharmcat, PharmCAT also gets the HLA types of hla_typing and a CYP2D6 call pypgx and cyrius agree on (PGX_CONSENSUS). | +| `--tools` | `pharmcat,cpic,vcfanno,roh,prs,mito_haplogroup,telomere_hunter,mosdepth,mito_variants,html_report,multiqc` | Comma-separated list of tools to run; an unknown name stops the run. DB-heavy tools (vep, slivar, clinical_filter, cpsr, clinvar, expansion_hunter, sample_qc) and extras (stranger, ancestry (needs --ancestry_ref and prs), ancestry_fine (needs --ancestry_fine_panel), ancient (needs --ancient_panel; samples with a gVCF), pypgx, SV callers, cram_archive, cyrius, parascopy, paraphase (PacBio HiFi BAMs only), y_haplogroup (male samples), y_str (Y-STR repeat counts, male samples), consumer_export (samples with a gVCF), bloodtype (blood group antigens; RhD needs the BAM), lpa (the LPA KIV-2 repeat count; needs the BAM), vntyper (MUC1 VNTR frameshifts behind ADTKD-MUC1; needs the BAM), viral (viral DNA in the unmapped reads; needs --viral_ref and a BAM), exomiser (runs when --hpo is set), chip (clonal haematopoiesis from the BAM; needs --chip_gff), archaic (needs --archaic_data; samples with a gVCF), mei (Alu, L1 and SVA insertions with MEGAnE; needs --megane_kmers and a BAM), mocha (mosaic chromosomal alterations; needs --mocha_data), locityper (a second opinion at HLA class II, KIR, CYP2D6 and FCGR; samples with a BAM; needs --locityper_data)) are opt-in. prs and vcfanno are skipped with a warning until --pgs_scoring or a score file is set. With pharmcat, PharmCAT also gets the HLA types of hla_typing and a CYP2D6 call pypgx and cyrius agree on (PGX_CONSENSUS). | ## Annotation databases diff --git a/docs/pipeline-overview.md b/docs/pipeline-overview.md index 4987e5349..8f28913c3 100644 --- a/docs/pipeline-overview.md +++ b/docs/pipeline-overview.md @@ -162,7 +162,7 @@ These run after the core pipeline completes and combine outputs from earlier ste | 40 | [Raw-Data Export](40-export-consumer.md) | Python | `PYTHON_IMAGE` | Opt-in (`TOOLS=...,consumer_export`, after `setup.sh --consumer-sites`; needs the gVCF) | | 41 | [Blood Groups (predicted)](41-bloodtype.md) | samtools + Python | `SAMTOOLS_IMAGE` + `PYTHON_IMAGE` | Opt-in (`TOOLS=...,bloodtype`; RhD needs the BAM) | | 42 | [Traits Panel](42-traits.md) | bcftools + Python + WhatsHap | `BCFTOOLS_IMAGE` + `PYTHON_IMAGE` + `WHATSHAP_IMAGE` | Recommended (reads the gVCF of step 3; WhatsHap phases from the BAM when a rule needs it) | -| 43 | [Archaic Ancestry](43-archaic.md) | IBDmix + Python | `IBDMIX_IMAGE` + `PYTHON_IMAGE` | Opt-in (runs once `setup.sh --archaic` installed its table) | +| 43 | [Archaic Ancestry](43-archaic.md) | IBDmix + Python | `IBDMIX_IMAGE` + `PYTHON_IMAGE` | Opt-in (runs once `setup.sh --archaic` installed its table; needs the gVCF) | | 44 | [Ancient Populations](44-ancient.md) | smartpca (EIGENSOFT) + bcftools + Python | `EIGENSOFT_IMAGE` + `BCFTOOLS_IMAGE` + `PYTHON_IMAGE` | Opt-in (`TOOLS=...,ancient`, after `setup.sh --ancient-panel`; needs the gVCF) | | 45 | [Y-STR Markers](45-y-str.md) | HipSTR | `HIPSTR_IMAGE` | Opt-in (`TOOLS=...,y_str`; male samples) | | 46 | [Lp(a): LPA KIV-2 Repeat Count](46-lpa.md) | samtools + Python | `SAMTOOLS_IMAGE` + `PYTHON_IMAGE` | Opt-in (`TOOLS=...,lpa`; needs the BAM) | @@ -176,7 +176,7 @@ These run after the core pipeline completes and combine outputs from earlier ste ### What a default run covers -A default `./scripts/run-all.sh ` runs **37 numbered steps**: 1b and 2 (only when there is no BAM yet), 3 (only when there is no VCF yet), 4, 5, 6, 7, 8, 9, 9b, 10, 11, 12, 13, 15, 16, 16b, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 26b, 27, 28, 30, 31, 32, 33, 36, 42 and 43. Steps 5, 8, 13, 17, 18, 32 and 33 are reported as skipped when their data is not installed (for 33, somalier's sites and VerifyBamID2's panel, which `setup.sh` installs), 21 when Cyrius is not installed (`setup.sh --cyrius`; non-commercial licence), 25 when no score file is, 26 (inside the PRS run of step 25) when the ancestry panel is not, 26b when its panel (`setup.sh --ancestry-fine-panel`) is not, 43 when its table (`setup.sh --archaic`) is not, 23, 30 and 31 when step 13 did not run, 36 (inside the PharmCAT stage) runs when step 8 or 32 did, and 42 is reported as skipped for a sample that starts from a VCF, unless it has a BAM too and a gVCF beside the VCF (the gVCF `run-all.sh` passes on). It ends with the summary report (`generate-report.sh`). +A default `./scripts/run-all.sh ` runs **37 numbered steps**: 1b and 2 (only when there is no BAM yet), 3 (only when there is no VCF yet), 4, 5, 6, 7, 8, 9, 9b, 10, 11, 12, 13, 15, 16, 16b, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 26b, 27, 28, 30, 31, 32, 33, 36, 42 and 43. Steps 5, 8, 13, 17, 18, 32 and 33 are reported as skipped when their data is not installed (for 33, somalier's sites and VerifyBamID2's panel, which `setup.sh` installs), 21 when Cyrius is not installed (`setup.sh --cyrius`; non-commercial licence), 25 when no score file is, 26 (inside the PRS run of step 25) when the ancestry panel is not, 26b when its panel (`setup.sh --ancestry-fine-panel`) is not, 43 when its table (`setup.sh --archaic`) is not, 23, 30 and 31 when step 13 did not run, 36 (inside the PharmCAT stage) runs when step 8 or 32 did, and 42 and 43 (43 once its table is installed) are reported as skipped for a sample that starts from a VCF, unless it has a BAM too and a gVCF beside the VCF (the gVCF `run-all.sh` passes on). It ends with the summary report (`generate-report.sh`). Off unless you ask for them: 35 (`TOOLS=...,parascopy`, after `setup.sh --parascopy-data`), 35b (`TOOLS=...,paraphase`, on a PacBio HiFi BAM), KIR typing in step 8 (`KIR=true`, after `setup.sh --kir-data`), OptiType beside T1K in step 8 (`HLA_QC=true`), 4b (`GRIDSS=true`), 14 (`IMPUTATION=true`), 29 (`SOMATIC=true`), 37 (`TOOLS=...,y_haplogroup`, after `setup.sh --yleaf-data`, male samples), 40 (`TOOLS=...,consumer_export`, after `setup.sh --consumer-sites`), 41 (`TOOLS=...,bloodtype`), 44 (`TOOLS=...,ancient`, after `setup.sh --ancient-panel`), 45 (`TOOLS=...,y_str`, male samples), 46 (`TOOLS=...,lpa`), 47 (`TOOLS=...,viral`, after `setup.sh --viral-data`), 48 (`HPO=...` with the symptoms as HPO terms, after `setup.sh --exomiser-data`), 49 (`TOOLS=...,chip`, after `setup.sh --chip-data`), 50 (`TOOLS=...,mei`, after `setup.sh --megane-kmers`), 51 (`TOOLS=...,mocha`, after `setup.sh --mocha-data`), 52 (`TOOLS=...,vntyper`), 53 (`TOOLS=...,locityper`, after `setup.sh --locityper-data`), the alternative callers 3a to 3d (`EXTRA_CALLERS=gatk,freebayes,strelka2,octopus`) and the caller comparison (`BENCHMARK=true`). Step 1 (ORA input) and the other alternative scripts (2a, 2b, 3e, 4a, 4c) run only by hand. Step 33 stops the run when somalier's sex from the reads differs from the declared one, as step 16 does (`--sex_check warn` logs it and goes on). Step 34 runs by hand, or in the Nextflow pipeline with `cram_archive` in `--tools`. Steps 38 and 39 work on a family, not one sample. With `FAMILIES=`, `run-all.sh` runs them after the pipeline once the members they need are processed, and records `skipped (waiting: ...)` until then ([Running it](multi-sample.md#running-it)). Without `FAMILIES`, run `38-family.sh` and `39-couple.sh` by hand with a pedigree, or give the Nextflow pipeline `--pedigree` (step 39 then runs for two samples the pedigree marks as partners, when `clinvar` is in `--tools`). From 8e35da2556f9b514876b2dfdcf429144e1600d7a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:03:16 +0100 Subject: [PATCH 152/255] docs(changelog): step 43 in run-all and the public-evidence gVCFs --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35a8087ac..febe0c2ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Step 43 (archaic ancestry) no longer counts a site without reads as a match with the reference genome. It wrote every table site the VCF had no record for as 0/0, so a stretch with no reads, where the archaic genomes carry the common reference allele, could score as a long Neanderthal segment, and one call anywhere on a chromosome was enough to analyse all of it. On a synthetic chromosome with one call far away and no reads over 150 table sites, the pinned IBDmix called a 149 kb segment and a 100% Neanderthal share; it now calls none. `og archaic genotypes` takes the sample's gVCF (`--gvcf`) and reads such a site as the reference only inside a confident reference block: FILTER PASS or `.`, every allele of GT 0, and GQ and depth (MIN_DP, else DP) of 1 or more where given. Any other site with no record goes to IBDmix as unknown (9). The analysed span and `analysed_sites` now run over the sites read, and the command prints how many sites it read and how many it did not. Step 43 now needs the gVCF of step 3, as steps 42 and 44 do: the bash step stops without it and says why, and the Nextflow pipeline skips a samplesheet row without one, with a warning. Without `--gvcf`, a site with no record is unknown; a VCF with a record at every site still needs no gVCF. No option, output file or column is renamed or removed; the values of `_archaic.tsv` and the segments move for a sample with gaps in its coverage. The parity goldens of `archaic-genotypes` record the change. `docs/43-archaic.md` now also says how the 100 stand-ins carry round(AF × 200) ALT alleles with your own genotype counted in the frequency, as Chen et al. counted each person in their group's, what the tests check and what they do not, and the terms of the MPI-EVA archaic genomes, which the pipeline does not redistribute. +- Step 43 (archaic ancestry) no longer counts a site without reads as a match with the reference genome. It wrote every table site the VCF had no record for as 0/0, so a stretch with no reads, where the archaic genomes carry the common reference allele, could score as a long Neanderthal segment, and one call anywhere on a chromosome was enough to analyse all of it. On a synthetic chromosome with one call far away and no reads over 150 table sites, the pinned IBDmix called a 149 kb segment and a 100% Neanderthal share; it now calls none. `og archaic genotypes` takes the sample's gVCF (`--gvcf`) and reads such a site as the reference only inside a confident reference block: FILTER PASS or `.`, every allele of GT 0, and GQ and depth (MIN_DP, else DP) of 1 or more where given. Any other site with no record goes to IBDmix as unknown (9). The analysed span and `analysed_sites` now run over the sites read, and the command prints how many sites it read and how many it did not. Step 43 now needs the gVCF of step 3, as steps 42 and 44 do: the bash step stops without it and says why, the Nextflow pipeline skips a samplesheet row without one, with a warning, and `run-all.sh` lists step 43 as `skipped (no gVCF: ...)` for a sample that starts from a VCF without one beside it. The `archaic-hg002` jobs of `public-evidence.yml` give HG002 a gVCF of one reference block per GIAB benchmark interval, and `archaic-same-person` gives one block over each whole chromosome of the 1000 Genomes panel person, which reads every site with no ALT as the reference, as that check did before. Without `--gvcf`, a site with no record is unknown; a VCF with a record at every site still needs no gVCF. No option, output file or column is renamed or removed; the values of `_archaic.tsv` and the segments move for a sample with gaps in its coverage. The parity goldens of `archaic-genotypes` record the change. `docs/43-archaic.md` now also says how the 100 stand-ins carry round(AF × 200) ALT alleles with your own genotype counted in the frequency, as Chen et al. counted each person in their group's, what the tests check and what they do not, and the terms of the MPI-EVA archaic genomes, which the pipeline does not redistribute. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. From 85e7455b0a6239e458421d0e8cc44bcd9e08b359 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:08:33 +0100 Subject: [PATCH 153/255] fix(ci): fail when the PCGR image or bundle moves without its measured size and digest The PCGR bundle is pinned by size and SHA-256, which the server does not publish. PCGR_DATA_MEASURED_FOR names the image tag and bundle date they were measured for, and check-coupled-versions.sh fails when either moves alone, when the size or digest is missing, or when either is malformed. The Renovate rule now says why it never moves the bundle. --- docs/00-reference-setup.md | 1 + renovate.json | 2 +- scripts/ci/check-coupled-versions.sh | 37 ++++++++++++++++++++++------ versions.env | 7 +++++- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/docs/00-reference-setup.md b/docs/00-reference-setup.md index fa2236f7d..b6baa03d2 100644 --- a/docs/00-reference-setup.md +++ b/docs/00-reference-setup.md @@ -201,6 +201,7 @@ Setup installs it by default. To add the clinical data to an existing setup: ``` The installer checks the archive against `PCGR_DATA_SHA256` in `versions.env`, then extracts it into a temporary folder. +The server publishes no checksum, so the size (`PCGR_DATA_BYTES`) and the digest are measured from a full download. `PCGR_DATA_MEASURED_FOR` names the image tag and bundle date they belong to, and `scripts/ci/check-coupled-versions.sh` fails when `PCGR_IMAGE` or `PCGR_DATA_BUNDLE` moves without it. Only a complete tree is moved to `pcgr_data/20260620/data/`. The archive is deleted after installation. The [PCGR resource list](https://sigven.github.io/pcgr/articles/annotation_resources.html) names the bundle's third-party databases. Their own terms apply. Setup prints the resource link before downloading. diff --git a/renovate.json b/renovate.json index 14f063c4e..09ad6ba29 100644 --- a/renovate.json +++ b/renovate.json @@ -187,7 +187,7 @@ "versioning": "regex:^release_(?\\d+)\\.(?\\d+)$" }, { - "description": "PCGR/CPSR: every update waits for approval on the dependency dashboard, because each release reads its own reference bundle and VEP cache (PCGR_DATA_BUNDLE and PCGR_VEP_CACHE_RELEASE in versions.env). The image comes from ghcr.io/sigven/pcgr (Docker Hub's sigven/pcgr stops at 2.2.5); the image matcher reads ghcr.io like any other registry. allowedVersions keeps the release tags (x.y.z): ghcr also carries development builds (2.3.2.9006) and .singularity tags.", + "description": "PCGR/CPSR: every update waits for approval on the dependency dashboard, because each release reads its own reference bundle and VEP cache (PCGR_DATA_BUNDLE and PCGR_VEP_CACHE_RELEASE in versions.env). Renovate never moves the bundle: no manager matches PCGR_DATA_BUNDLE, PCGR_DATA_BYTES or PCGR_DATA_SHA256, because the server publishes no checksum and the digest has to be measured from a full download. So an approved image PR fails scripts/ci/check-coupled-versions.sh until a person moves the bundle date, the VEP cache release, the size, the digest and PCGR_DATA_MEASURED_FOR in the same PR. The image comes from ghcr.io/sigven/pcgr (Docker Hub's sigven/pcgr stops at 2.2.5); the image matcher reads ghcr.io like any other registry. allowedVersions keeps the release tags (x.y.z): ghcr also carries development builds (2.3.2.9006) and .singularity tags.", "matchDatasources": ["docker"], "matchPackageNames": ["ghcr.io/sigven/pcgr"], "allowedVersions": "/^\\d+\\.\\d+\\.\\d+$/", diff --git a/scripts/ci/check-coupled-versions.sh b/scripts/ci/check-coupled-versions.sh index a00a76208..a50f1a8d9 100755 --- a/scripts/ci/check-coupled-versions.sh +++ b/scripts/ci/check-coupled-versions.sh @@ -7,7 +7,12 @@ # VEP_CACHE_RELEASE must equal the major of VEP_IMAGE (release_116.0 -> 116) # PYPGX_BUNDLE_VERSION must equal the version of PYPGX_IMAGE (0.26.0--pyh... -> 0.26.0) # ANNOTSV_ANNOTATIONS_VERSION must equal major.minor of ANNOTSV_IMAGE (3.5.10--h... -> 3.5) -# PCGR_IMAGE, PCGR_VEP_CACHE_RELEASE and PCGR_DATA_BUNDLE must all be set +# PCGR_IMAGE, PCGR_VEP_CACHE_RELEASE, PCGR_DATA_BUNDLE, PCGR_DATA_BYTES and +# PCGR_DATA_SHA256 must all be set, the size a byte +# count and the digest 64 hex digits +# PCGR_DATA_MEASURED_FOR must be " " +# (2.3.2 20260620): the size and digest were measured +# for that image and bundle, so either moving alone fails # YLEAF_DATA_VERSION must equal the version of YLEAF_IMAGE (3.2.1--pyh... -> 3.2.1) # GENCODE_RELEASE (scripts/lib/common.sh) must be the GENCODE release of the # Ensembl release of VEP_IMAGE, its major minus 66 (116 -> 50) @@ -31,7 +36,7 @@ check() { # shellcheck disable=SC2016 # $1 and ${!v} expand in the inner shell vals=$(env -i bash --noprofile --norc -c '. "$1"; for v in VEP_IMAGE VEP_CACHE_RELEASE PYPGX_IMAGE PYPGX_BUNDLE_VERSION \ ANNOTSV_IMAGE ANNOTSV_ANNOTATIONS_VERSION PCGR_IMAGE PCGR_VEP_CACHE_RELEASE PCGR_DATA_BUNDLE \ - YLEAF_IMAGE YLEAF_DATA_VERSION VERIFYBAMID2_IMAGE; do + PCGR_DATA_BYTES PCGR_DATA_SHA256 PCGR_DATA_MEASURED_FOR YLEAF_IMAGE YLEAF_DATA_VERSION VERIFYBAMID2_IMAGE; do printf "%s=%s\n" "$v" "${!v-}"; done' _ "$1") || { echo "FAIL: cannot read $1"; return 1; } get() { sed -n "s/^$1=//p" <<<"$vals"; } tag() { local i; i=$(get "$1"); i=${i##*/}; [[ "$i" == *:* ]] && echo "${i#*:}"; } @@ -64,15 +69,24 @@ check() { t=$(tag VERIFYBAMID2_IMAGE || true); t=${t%%--*} rule "VB2_PANEL_FILES (scripts/setup.sh) come from the VerifyBamID tag of VERIFYBAMID2_IMAGE" "$tags" "${t:+v$t}" local v missing="" - for v in PCGR_IMAGE PCGR_VEP_CACHE_RELEASE PCGR_DATA_BUNDLE; do + for v in PCGR_IMAGE PCGR_VEP_CACHE_RELEASE PCGR_DATA_BUNDLE PCGR_DATA_BYTES PCGR_DATA_SHA256; do [ -n "$(get "$v")" ] || missing="${missing} ${v}" done if [ -z "$missing" ]; then - echo "OK PCGR_IMAGE, PCGR_VEP_CACHE_RELEASE and PCGR_DATA_BUNDLE are set together" + echo "OK PCGR_IMAGE, PCGR_VEP_CACHE_RELEASE, PCGR_DATA_BUNDLE, PCGR_DATA_BYTES and PCGR_DATA_SHA256 are set together" else echo "FAIL PCGR variables must be set together; unset:${missing}" fail=1 fi + if [[ "$(get PCGR_DATA_BYTES)" =~ ^[1-9][0-9]*$ ]] && [[ "$(get PCGR_DATA_SHA256)" =~ ^[0-9a-f]{64}$ ]]; then + echo "OK PCGR_DATA_BYTES is a byte count and PCGR_DATA_SHA256 a SHA-256 digest" + else + echo "FAIL PCGR_DATA_BYTES must be a byte count and PCGR_DATA_SHA256 64 lowercase hex digits: $(get PCGR_DATA_BYTES), $(get PCGR_DATA_SHA256)" + fail=1 + fi + t=$(tag PCGR_IMAGE || true) + rule "PCGR_DATA_MEASURED_FOR names the PCGR_IMAGE tag and PCGR_DATA_BUNDLE of PCGR_DATA_BYTES and PCGR_DATA_SHA256" \ + "$(get PCGR_DATA_MEASURED_FOR)" "$( [ -n "$t" ] && [ -n "$(get PCGR_DATA_BUNDLE)" ] && echo "$t $(get PCGR_DATA_BUNDLE)")" return "$fail" } @@ -103,20 +117,29 @@ self_test() { [ "$rc" -eq 0 ] || { echo "self-test: versions.env itself failed:"; printf '%s\n' "$out"; fail=1; } # The values the unchanged file holds, so a legitimate bump keeps this # test green; each planted value differs from them. - local vep pypgx annotsv yleaf vb2 + local vep pypgx annotsv yleaf vb2 pcgr vep=$(sed -nE 's/^VEP_IMAGE="[^"]*:release_([0-9]+).*/\1/p' "${ROOT}/versions.env") pypgx=$(sed -nE 's/^PYPGX_IMAGE="[^"]*:([^"]*)--.*/\1/p' "${ROOT}/versions.env") annotsv=$(sed -nE 's/^ANNOTSV_IMAGE="[^"]*:([0-9]+\.[0-9]+).*/\1/p' "${ROOT}/versions.env") yleaf=$(sed -nE 's/^YLEAF_IMAGE="[^"]*:([^"]*)--.*/\1/p' "${ROOT}/versions.env") vb2=$(sed -nE 's/^VERIFYBAMID2_IMAGE="[^"]*:([^"]*)--.*/\1/p' "${ROOT}/versions.env") - if [ -z "$vep" ] || [ -z "$pypgx" ] || [ -z "$annotsv" ] || [ -z "$yleaf" ] || [ -z "$vb2" ]; then - echo "self-test: cannot read VEP_IMAGE, PYPGX_IMAGE, ANNOTSV_IMAGE, YLEAF_IMAGE or VERIFYBAMID2_IMAGE from versions.env"; return 1 + pcgr=$(sed -nE 's/^PCGR_IMAGE="[^"]*:([^"]*)".*/\1/p' "${ROOT}/versions.env") + if [ -z "$vep" ] || [ -z "$pypgx" ] || [ -z "$annotsv" ] || [ -z "$yleaf" ] || [ -z "$vb2" ] || [ -z "$pcgr" ]; then + echo "self-test: cannot read VEP_IMAGE, PYPGX_IMAGE, ANNOTSV_IMAGE, YLEAF_IMAGE, VERIFYBAMID2_IMAGE or PCGR_IMAGE from versions.env"; return 1 fi expect vep-cache "s/^VEP_CACHE_RELEASE=.*/VEP_CACHE_RELEASE=\"$((vep - 1))\"/" "^FAIL VEP_CACHE_RELEASE .*: $((vep - 1)), want ${vep}\$" expect vep-image "s/^(VEP_IMAGE=\"[^\"]*:release_)[0-9]+/\\1$((vep + 1))/" "^FAIL VEP_CACHE_RELEASE .*: ${vep}, want $((vep + 1))\$" expect pypgx 's/^PYPGX_BUNDLE_VERSION=.*/PYPGX_BUNDLE_VERSION="9.9.9"/' "^FAIL PYPGX_BUNDLE_VERSION .*: 9\\.9\\.9, want ${pypgx//./\\.}\$" expect annotsv 's/^ANNOTSV_ANNOTATIONS_VERSION=.*/ANNOTSV_ANNOTATIONS_VERSION="0.1"/' "^FAIL ANNOTSV_ANNOTATIONS_VERSION .*: 0\\.1, want ${annotsv//./\\.}\$" expect pcgr '/^PCGR_DATA_BUNDLE=/d' '^FAIL PCGR variables must be set together; unset: PCGR_DATA_BUNDLE' + expect pcgr-sha256 '/^PCGR_DATA_SHA256=/d' '^FAIL PCGR variables must be set together; unset: PCGR_DATA_SHA256' + expect pcgr-bytes 's/^PCGR_DATA_BYTES=.*/PCGR_DATA_BYTES="7.6 GB"/' '^FAIL PCGR_DATA_BYTES must be a byte count' + expect pcgr-digest 's/^PCGR_DATA_SHA256=.*/PCGR_DATA_SHA256="0123"/' '^FAIL PCGR_DATA_BYTES must be a byte count and PCGR_DATA_SHA256 64' + # A new bundle date, or a new image, without a new size and digest. + expect pcgr-bundle-alone 's/^PCGR_DATA_BUNDLE=.*/PCGR_DATA_BUNDLE="29991231"/' \ + "^FAIL PCGR_DATA_MEASURED_FOR .*: ${pcgr//./\\.} [0-9]+, want ${pcgr//./\\.} 29991231\$" + expect pcgr-image-alone 's/^(PCGR_IMAGE="[^"]*:)[^"]*/\19.9.9/' \ + "^FAIL PCGR_DATA_MEASURED_FOR .*: ${pcgr//./\\.} [0-9]+, want 9\\.9\\.9 [0-9]+\$" expect yleaf 's/^YLEAF_DATA_VERSION=.*/YLEAF_DATA_VERSION="9.9.9"/' "^FAIL YLEAF_DATA_VERSION .*: 9\\.9\\.9, want ${yleaf//./\\.}\$" expect gencode "s/^(GENCODE_RELEASE=)[^ ]*/\\1$((vep - 67))/" "^FAIL GENCODE_RELEASE .*: $((vep - 67)), want $((vep - 66))\$" common expect gencode-vep "s/^(VEP_IMAGE=\"[^\"]*:release_)[0-9]+/\\1$((vep + 1))/" "^FAIL GENCODE_RELEASE .*: $((vep - 66)), want $((vep - 65))\$" diff --git a/versions.env b/versions.env index 5c9595f9d..c305cb838 100644 --- a/versions.env +++ b/versions.env @@ -190,7 +190,12 @@ VEP_CACHE_RELEASE="116" # with PCGR_IMAGE. PCGR_VEP_CACHE_RELEASE="115" PCGR_DATA_BUNDLE="20260620" -# Exact archive size from the server; SHA-256 is measured from the full download. +# Exact archive size from the server; SHA-256 is measured from the full download +# (the server publishes no checksum). PCGR_DATA_MEASURED_FOR names the PCGR_IMAGE +# tag and PCGR_DATA_BUNDLE they were measured for: scripts/ci/check-coupled-versions.sh +# fails when the image or the bundle moves without it, so a bump downloads the +# new bundle and records its size and digest in the same change. +PCGR_DATA_MEASURED_FOR="2.3.2 20260620" PCGR_DATA_BYTES="7664082467" PCGR_DATA_SHA256="f9b8149e37c7e63bd7d877713cdb11a1a609c492b3f1a055fb7e90cd80273e54" # pypgx-bundle git tag (step 32): must equal the version of PYPGX_IMAGE. From da9f925f1bcc369dad85be4d68703f58b1205127 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:40:59 +0100 Subject: [PATCH 154/255] fix(archaic): read an ALT . reference call, and stop on a gVCF that lacks a chromosome or is unsorted in its tail --- rust/crates/og-ancestry/src/archaic.rs | 61 +++++++++--------- .../tests/i411_archaic_callable.rs | 43 ++++++++++++ .../archaic-genotypes/afr-one/gt/chr22.gt.gz | Bin 453 -> 458 bytes .../archaic-genotypes/afr-one/gt/spans.tsv | 2 +- .../golden/archaic-genotypes/afr-one/stderr | 2 +- .../archaic-genotypes/default/gt/chr22.gt.gz | Bin 961 -> 970 bytes .../archaic-genotypes/default/gt/spans.tsv | 2 +- .../golden/archaic-genotypes/default/stderr | 2 +- .../archaic-genotypes/plain/gt/chr22.gt.gz | Bin 514 -> 514 bytes .../archaic-genotypes/plain/gt/spans.tsv | 2 +- .../golden/archaic-genotypes/plain/stderr | 2 +- 11 files changed, 81 insertions(+), 35 deletions(-) diff --git a/rust/crates/og-ancestry/src/archaic.rs b/rust/crates/og-ancestry/src/archaic.rs index f62b9ee5d..75cecf771 100644 --- a/rust/crates/og-ancestry/src/archaic.rs +++ b/rust/crates/og-ancestry/src/archaic.rs @@ -729,9 +729,11 @@ struct VcfRec { } /// The biallelic SNVs (and the other records, as code 9) of a one-sample -/// VCF, in file order. A record whose FILTER is not PASS or `.` gives -/// code 9 (not read), so a DeepVariant RefCall is neither your allele nor -/// the reference. Every record is checked against the last record handed +/// VCF, in file order. A one-base record with ALT `.` (an all-sites VCF's +/// reference call) is the reference (0) when its GT is all 0, and code 9 +/// otherwise. A record whose FILTER is not PASS or `.` gives code 9 (not +/// read), so a DeepVariant RefCall is neither your allele nor the +/// reference. Every record is checked against the last record handed /// out (`previous`), so a record that is skipped cannot hide a chromosome /// that comes back at a lower position. struct Vcf { @@ -782,10 +784,11 @@ impl Vcf { self.text.path.display() )); } - if c[3].chars().count() != 1 || c[4] == "." { + if c[3].chars().count() != 1 { continue; } - if c[4].contains(',') || c[4].chars().count() != 1 { + let reference_only = c[4] == "."; + if !reference_only && (c[4].contains(',') || c[4].chars().count() != 1) { self.previous = Some((c[0].to_owned(), pos)); return Ok(Some(VcfRec { chrom: c[0].to_owned(), @@ -803,7 +806,7 @@ impl Vcf { }; let gt = c[9].split(':').nth(k).unwrap_or("."); let mut code = gt_code(gt).unwrap_or(9); - if c[6] != "PASS" && c[6] != "." { + if (c[6] != "PASS" && c[6] != ".") || (reference_only && code != 0) { code = 9; } self.previous = Some((c[0].to_owned(), pos)); @@ -812,7 +815,7 @@ impl Vcf { pos, call: Sample { r#ref: c[3].to_owned(), - alt: Some(c[4].to_owned()), + alt: (!reference_only).then(|| c[4].to_owned()), code, }, })); @@ -904,8 +907,8 @@ fn confident_reference(c: &[&str]) -> bool { /// VCF is the reference (0) when such a call covers it, and unknown (9) /// otherwise. Only the next record and the furthest end of the calls so /// far are kept; a chromosome passed over is remembered by name, so a gVCF -/// in another chromosome order than the VCF stops the step instead of -/// reading as no calls. +/// in another chromosome order than the VCF, or with no records on a table +/// chromosome of the VCF, stops the step instead of reading as no calls. struct Callable { text: Text, path: String, @@ -915,9 +918,6 @@ struct Callable { chrom: String, /// The last position the confident calls so far cover on `chrom`. covered: i64, - /// The first chromosome of the VCF the gVCF was read to its end for - /// without finding a record on it. - missing: Option, } impl Callable { @@ -954,7 +954,6 @@ impl Callable { seen: Vec::new(), chrom: String::new(), covered: 0, - missing: None, }; callable.pending = callable.read()?; Ok(callable) @@ -1014,24 +1013,21 @@ impl Callable { fn start(&mut self, chrom: &str) -> Out<()> { let here = self.pending.as_ref().is_some_and(|b| b.chrom == chrom); if !here && self.seen.iter().any(|x| x == chrom) { - return usage(match &self.missing { - Some(gone) => format!( - "{} has no records on {gone}, which the VCF lists before {chrom}; \ - give the gVCF of the same sample, with every chromosome the VCF lists", - self.path - ), - None => format!( - "{} lists {chrom} before a chromosome the VCF lists first; \ - give the gVCF and the VCF in one chromosome order", - self.path - ), - }); + return usage(format!( + "{} lists {chrom} before a chromosome the VCF lists first; \ + give the gVCF and the VCF in one chromosome order", + self.path + )); } while self.pending.as_ref().is_some_and(|b| b.chrom != chrom) { self.pending = self.read()?; } - if self.pending.is_none() && self.missing.is_none() { - self.missing = Some(chrom.to_owned()); + if self.pending.is_none() { + return usage(format!( + "{} has no records on {chrom}, which the VCF lists; \ + give the gVCF of the same sample, with every chromosome the VCF lists", + self.path + )); } self.chrom = chrom.to_owned(); self.covered = 0; @@ -1182,6 +1178,13 @@ fn cmd_genotypes(a: &Args, err: &mut dyn Write) -> Out<()> { } } } + // Read the gVCF to its end, so a record out of order after the last + // table site stops the step here too, before any output is published. + if let Some(c) = callable.as_mut() { + while c.pending.is_some() { + c.pending = c.read()?; + } + } if spans.is_empty() { let names: Vec<&str> = tables.iter().map(|(c, _)| c.as_str()).collect(); let mut sorted = names.clone(); @@ -2042,11 +2045,11 @@ mod tests { #[test] fn an_ignored_record_cannot_hide_a_chromosome_return() { - // chr22 comes back after an ignored chrX record (ALT "."), at a + // chr22 comes back after an ignored chrX record (a deletion), at a // lower position: the join would write 0 at chr22:1000 and lose // the call, so it must stop as unsorted. let text = "chr22\t3000\t.\tA\tG\t.\tPASS\t.\tGT\t0/1\n\ - chrX\t1\t.\tA\t.\t.\tPASS\t.\tGT\t0/0\n\ + chrX\t1\t.\tAT\tA\t.\tPASS\t.\tGT\t0/1\n\ chr22\t1000\t.\tC\tT\t.\tPASS\t.\tGT\t1/1\n"; let mut vcf = Vcf::new(Text { path: "sample.vcf".into(), diff --git a/rust/crates/og-ancestry/tests/i411_archaic_callable.rs b/rust/crates/og-ancestry/tests/i411_archaic_callable.rs index 51236723d..fb4526100 100644 --- a/rust/crates/og-ancestry/tests/i411_archaic_callable.rs +++ b/rust/crates/og-ancestry/tests/i411_archaic_callable.rs @@ -255,6 +255,24 @@ fn the_gvcf_is_read_in_the_vcf_order() { assert_eq!(code, 2, "{err}"); assert!(err.contains("has no records on chr21"), "{err}"); + // The same when the gVCF lacks the last table chromosome of the VCF. + let (code, err) = run(&d, vcf, Some(&format!("{chr21}{other}"))); + assert_eq!(code, 2, "{err}"); + assert!(err.contains("has no records on chr22"), "{err}"); + + // A record out of order after the last table site is read too. + let tail = format!( + "{chr21}{chr22}{}{}", + block("chr22", 6000, 6100, ".", "GT", "0/0"), + block("chr22", 100, 200, ".", "GT", "0/0") + ); + let (code, err) = run(&d, vcf, Some(&tail)); + assert_eq!(code, 2, "{err}"); + assert!( + err.contains("is not sorted: chr22:100 comes after 6000"), + "{err}" + ); + let back = format!( "{chr21}{chr22}{}", block("chr21", 9000, 9100, ".", "GT", "0/0") @@ -273,6 +291,31 @@ fn the_gvcf_is_read_in_the_vcf_order() { let _ = std::fs::remove_dir_all(&d); } +/// An all-sites VCF writes a reference call with ALT `.`: with GT all 0 +/// and FILTER PASS or `.` it is the reference, with or without a gVCF, and +/// any other such record is unknown. +#[test] +fn a_reference_record_with_no_alt_is_the_reference() { + let d = dir("alt-dot"); + table(&d, "chr22", [1000, 2000, 3000, 4000]); + let vcf = "chr22\t1000\t.\tA\t.\t50\tPASS\t.\tGT\t0/0\n\ + chr22\t2000\t.\tA\t.\t50\t.\t.\tGT\t0|0\n\ + chr22\t3000\t.\tA\t.\t50\tLowQual\t.\tGT\t0/0\n\ + chr22\t4000\t.\tA\t.\t50\tPASS\t.\tGT\t0/1\n"; + let want: BTreeMap = [(1000, "0"), (2000, "0"), (3000, "9"), (4000, "9")] + .map(|(p, c)| (p, c.to_owned())) + .into(); + let (code, err) = run(&d, vcf, None); + assert_eq!(code, 0, "{err}"); + assert_eq!(codes(&d, "chr22"), want); + assert_eq!(spans(&d), [["chr22", "1000", "2000", "2", "0", "0"]]); + // A gVCF block that does not cover them changes nothing: the record wins. + let (code, err) = run(&d, vcf, Some(&block("chr22", 1, 10, ".", "GT", "0/0"))); + assert_eq!(code, 0, "{err}"); + assert_eq!(codes(&d, "chr22"), want); + let _ = std::fs::remove_dir_all(&d); +} + /// A gVCF of two samples, or one without a header line, is refused. #[test] fn a_gvcf_of_one_sample_only() { diff --git a/tests/parity/golden/archaic-genotypes/afr-one/gt/chr22.gt.gz b/tests/parity/golden/archaic-genotypes/afr-one/gt/chr22.gt.gz index 17e7a3ef2872aac9ffcb14600e9bf26f408be101..46075ee7c0d944c7a0b043103f2981e370275236 100644 GIT binary patch literal 458 zcmV;*0X6;~iwFP!00004|3#9)P8=~1M9m#a{J#74{z2U-VR6mbA0-Dyg%6Y+o$8}>HT28+pp`N zKkV8)E;21W+`3&^n6tuWnof3ONw%@$nqAy(tl75bN>?ZAHAT*}(#yR!*x?|hU)A7r zjHUFO8fr3XN9m>pF~lSb7nvB`L>5>S8P71}o?_9k<6Ri|9le1?#||cDW1q$13cEqY z^z6l=dYQ6MviKt292q3)#jSdICZ}B~7HX)D{p%za4Kk)+*rr=z(IFE9^P?fLC^DYu z#ZW00)hh;mm15E86$5W5#iEa1=d_r{iG_Me&c=y_dW(%7HOw0l**V3ndh7k5S>J5I zq8gZjcXwN`XplK4<@t^mEIMStq}Lj!Em#y;5hU~aPcB$guNcTRi$$Y1YVNZEi%xHn zKroFLEUH&zZm9+=)Jp=H(skZY&@fIMvny`uooq_6sD>B}+dOT>qCqBxbBaZWObjt^ ze#N55oQ7oSORiX`H^|6!8WwHz76UsVgFhHiBPu?>)$`3J zc-TOCduyt)d3gESz1!#R+rFNjZS(eH_3dY~wfpVI?4$|~g1c%Wt zP0vKg&B!~NE+WVwhFQ4G{yPK}Y34f1UfqppPd_2H0p4|H`ZM$E~)EQY!3nz z$z{t9W^p24nPDO>ZOP?3yU;bILWE?jzfPf|V#XE@%V;Z9G|c3{ekT+v(v0tLm*HEe zNUj|C)Giejt{iwmQz}~LI;X`-ph8?G`{F`{xakH*gz<)Cwx80L+o=RINCI2k zSO2Jqii(+YQrW-biHe3icX_KTqc-lI*&IL`7~1zX4=G++n7?3gy;^-Fq^5Un8{&3rJ`XbhZt{urXtN8 vLKY7$72-ypk0}+kbJKx4y<(=K!S!^-Srx}rB$q8_Zsq6?b6wz$DFgrj5`^H0 diff --git a/tests/parity/golden/archaic-genotypes/afr-one/gt/spans.tsv b/tests/parity/golden/archaic-genotypes/afr-one/gt/spans.tsv index dcf5acd63..89b23cbee 100644 --- a/tests/parity/golden/archaic-genotypes/afr-one/gt/spans.tsv +++ b/tests/parity/golden/archaic-genotypes/afr-one/gt/spans.tsv @@ -1,3 +1,3 @@ chrom first last sites sites_with_your_alt sites_other_allele chr21 1010 1290 6 6 0 -chr22 2010 2500 26 21 4 +chr22 2010 2500 30 21 4 diff --git a/tests/parity/golden/archaic-genotypes/afr-one/stderr b/tests/parity/golden/archaic-genotypes/afr-one/stderr index c5b08c8d6..11653b2ee 100644 --- a/tests/parity/golden/archaic-genotypes/afr-one/stderr +++ b/tests/parity/golden/archaic-genotypes/afr-one/stderr @@ -1 +1 @@ -2 chromosomes, 32 sites read, 44 not read (unknown to IBDmix) +2 chromosomes, 36 sites read, 40 not read (unknown to IBDmix) diff --git a/tests/parity/golden/archaic-genotypes/default/gt/chr22.gt.gz b/tests/parity/golden/archaic-genotypes/default/gt/chr22.gt.gz index f95e23eac050e5bfac65d000562295f4e60d529a..4d1b6dd3bab02f6e65ac7ff11004b112c86fb96b 100644 GIT binary patch literal 970 zcmb2|=3oE;mj8EWob{726lgjBEc4F1czO599o|ZctWWoOb!-xBFGO z|K;NQeCDsWtK|tdlnPGDnh~|EZ)LjF47X)#PAUnvI7+~gOOuBHr;?*e2g^hSArObx zh54d`2{PrhML^eu`yzspbz#5gSR#N#Ie!tbb>Y8=pqP&^7&`C>AW=>|0?J1?3=x#1 zl5;L#lN) zl&|xBw}WqT$d7l3;|}C*y&E&{{jCx;|5u)?wps4iv`zb(qNU!sGOuvvx|O@VR?aT+JGJiW#1-9_ z!m@t#`LF(cZGH8w?BKH9_meJe?p(#XfBK9~)mnANZ#z8WuRMR%e%<$`?P~44!oIiu zET2`pW`FfM-lYM?eM2x3vy4r zi?n_De97De+AHh7h@LawZI)^-c|h9m+`q8pH~GO9)%jhTK0Dy~ttodW|7p9dbAz*L z(%;I;BbTo{Te^IQS?c$gKVnNy$NlchXK>e8v^ww_0S=xJPB`&u68MO#hgMPEMLeeL+& zTFZm|!77bm0_4Oh%poZT2f9905Zhg*Lb{=T8 z`p>gVu5G(E8R(}v?@Lon#cl+@P5K(TC+}3=pJ$Q*wP(#=TYc&UJ58*#Z^4Fbx2IL? zyYGGLa5-1luR`rxK`WzIS9ktC`~C9?xrkNQ<97D}qv-NvwNvk7)_wdjQ+;l2J5Y0V z5Ze#SnM*&sd0*O94h}%chRZ(lyq8ye`Z@n0FqBdlt~y=+xc5iZq^(+@5G>n2vrM{l zYWP(x?llj9^1&aafA{TU4O_P!{E0+mO*Wr$Km2=k_x6WsZpmIgqQPgc7;N>C4L(1`G%}^hfTJ0XJOntE99244CMpQIGyyq_ znpP-ib#Nh3SshY~+EyqcQH>c2QXOoI5L8r$*rL`9MI@^Eg@RQF-y#H6)gia2{e>ce zQgTk|;F+jsxzOY)2R zS-+jzdQx_K;)Hb~F?X4?v-#8Gj&419ym{B8t&>DP?QDK{tK?D7(R>TNv-=mHUi08q z$)$^vH(t@vf3R)m-I#grZAsz2spgUgq~E?>2XyU^v*$g36@9gxzvxn&dFpqC{SQsJ%^dwV**|xm z^XTn)dypfKKiaMlTQ={;x&OP&Qn&xkHQ%+O0ch;*`!5#HeR}()IqP|bnD%39`S(v( zx$FHwS~j!j&b83F_x68DS^8#~`bvqP(_UPA%YS{@8@VZrfmJ`=eqHk0Udx01!rOEQZ#!zVC?v0L2gqd)zAiF0U*CNT#HssSd`a}2t^S5;U^v>I$XDCCZTq~Cvi?ue zlh&=-cwk z>b>bvYoH^(ui!b5sdj#;=Da=sjDd#F=2;MR`Oeh;ch0Uc{alcHD)0O3%X#TmIuKalPIKy)~K)3;^vz B@NobD diff --git a/tests/parity/golden/archaic-genotypes/default/gt/spans.tsv b/tests/parity/golden/archaic-genotypes/default/gt/spans.tsv index dcf5acd63..89b23cbee 100644 --- a/tests/parity/golden/archaic-genotypes/default/gt/spans.tsv +++ b/tests/parity/golden/archaic-genotypes/default/gt/spans.tsv @@ -1,3 +1,3 @@ chrom first last sites sites_with_your_alt sites_other_allele chr21 1010 1290 6 6 0 -chr22 2010 2500 26 21 4 +chr22 2010 2500 30 21 4 diff --git a/tests/parity/golden/archaic-genotypes/default/stderr b/tests/parity/golden/archaic-genotypes/default/stderr index c5b08c8d6..11653b2ee 100644 --- a/tests/parity/golden/archaic-genotypes/default/stderr +++ b/tests/parity/golden/archaic-genotypes/default/stderr @@ -1 +1 @@ -2 chromosomes, 32 sites read, 44 not read (unknown to IBDmix) +2 chromosomes, 36 sites read, 40 not read (unknown to IBDmix) diff --git a/tests/parity/golden/archaic-genotypes/plain/gt/chr22.gt.gz b/tests/parity/golden/archaic-genotypes/plain/gt/chr22.gt.gz index a08708fe7f73b2a659751d6eef23092e0e659225..894d7bc0bc5ad54cb3fb71d6304436a63370ea13 100644 GIT binary patch literal 514 zcmV+d0{#6TiwFP!00004|5cO0P8&fGM9#Svfxi0|iB z_wH(a7+ZAB)T?TX-~V3DkM?|iwU_(9c6@lV{_u7@*{{>npVR-NeZ7A=y`JBX_Ot)E z{r=5v-9sNuAJbK9);!#X-B_4&^y%iMG-rNijrokqWoxBOZgZZ)))=cw&#huI+Ua2da?>IqIrPfS|^<7ParkyPW zZl1@N9Ry7WThM*6^1C=RyFZu*K5T{bhi&w>=X}QU7 zztKk1PHNq5xXHEpG@1@lS2n#{-5O1mS~DHyOSVQ+apa;DJB6klj;6tOl9$;UO-FJ( ze}SDs)8YhMNUxPc)8gp%MW5x@uGFIOt~l&K9C-yKmuD&yP35NB+O^Wx4kns*QtPg^ zqwhtb=^%B`#q;=*ZKA1CD_Ijmr#Lhfr?Qn^{|ggMJDlKd;Femwq3M9rn9dw1D~G1y zRO+0s3QddS^puX8ZinU9j&z!#Z3;UmPO|q3P30EMv>je7oC8fesVm!s8%+nP#g1h0 zsuIvtsrj31&Vg47O^Xwdm`W>!rad{DO67GaG#zkiqkH9=rs8O()A01G55>+98tViA E09GUpKL7v# literal 514 zcmV+d0{#6TiwFP!00004|6P;8Zqz^!M9-bC=^k$YA4dk|0#YO*?kfmlQOKes%J=70 z_t^60;I+~1VoX&6QkpZrv&MWz<*>C!Ims>N*nx7YwY<1hIgpl{ z{Pr7dkak+@bi+-q#nT{NXkFcOZ*^;ssjeC{D4wNIWAj##K$^@z0bXq)WUptr}?X=ce zZAb5+K)TR6VDWr>bekYmYc+d!2U6u!x6VS==i6LbpPQmbx|F65}wj6%wq1F3SV zb(#ml-rK6_XVSMdKr Date: Sun, 15 Nov 2026 12:40:59 +0100 Subject: [PATCH 155/255] fix(archaic): keep the same-person panel's no-call sites unknown, and document the gVCF stops --- .github/workflows/public-evidence.yml | 21 ++++++++++++--------- CHANGELOG.md | 2 +- docs/43-archaic.md | 11 ++++++----- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index 8088bb9e9..069401a19 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -387,8 +387,10 @@ jobs: . ./versions.env G="${EV}/genome" OUT="${EV}/out" mkdir -p "$OUT" - # sub LABEL URL-PREFIX NAME-OF-FILE-PATTERN: the person's non-reference - # SNVs on the chromosomes, one VCF. + # pull LABEL URL-PREFIX NAME-OF-FILE-PATTERN: the person's SNV records + # on the chromosomes that are not homozygous reference, one VCF: the + # ALT calls, and the sites with no call, which step 43 reads as + # unknown (a VCF record wins over the gVCF's reference block). pull() { local label=$1 url=$2 pat=$3 c parts="" mkdir -p "${G}/${SAMPLE}${label}/vcf" @@ -397,8 +399,8 @@ jobs: # The container has no CA bundle, so curl fetches the file here. curl -fsSL --retry 5 -o "${EV}/in.vcf.gz" "${url}/${f}" curl -fsSL --retry 5 -o "${EV}/in.vcf.gz.tbi" "${url}/${f}.tbi" - docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/e" "$BCFTOOLS_IMAGE" \ - bcftools view -s "$SAMPLE" -v snps --min-ac 1 -Oz -o "/e/${label}.raw.vcf.gz" /e/in.vcf.gz + docker run --rm -u "$(id -u):$(id -g)" -v "${EV}:/e" "$BCFTOOLS_IMAGE" bash -euo pipefail -c ' + bcftools view -s "$1" -v snps -Ou /e/in.vcf.gz | bcftools view -e "GT=\"ref\"" -Oz -o "/e/$2.raw.vcf.gz"' _ "$SAMPLE" "$label" # The 2019 recall names the contigs 22, not chr22: rename them # (a name already chrN stays as it is). printf '%s\tchr%s\n' "$c" "$c" > "${EV}/rename.txt" @@ -414,13 +416,14 @@ jobs: docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ bcftools index -t -f "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" rm -f "${EV}/${label}".chr*.vcf.gz - echo "${label}: $(docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools view -H "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" | wc -l) SNV records" + echo "${label}: $(docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools view -H "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" | wc -l) SNV records," \ + "$(docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools view -H -i 'GT="mis"' "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" | wc -l) of them with no call" # Step 43 reads a site with no record as the reference only inside # a reference block of the gVCF. A panel VCF holds no callable mask, - # so the gVCF is one block over each whole chromosome: every site - # with no ALT reads as the reference, as this check measured before - # step 43 needed a gVCF (the person's records are taken with - # --min-ac 1, so a site with no record is a site with no ALT). + # so the gVCF is one block over each whole chromosome: a site with + # no record (the person is 0/0 there, or the panel has no variant + # there) reads as the reference, as this check measured before step + # 43 needed a gVCF, and a site with no call stays unknown. { printf '##fileformat=VCFv4.2\n' for c in $LIST; do awk -F'\t' -v c="chr${c}" '$1 == c {printf "##contig=\n", $1, $2}' "$FAI"; done diff --git a/CHANGELOG.md b/CHANGELOG.md index febe0c2ae..a20c40ca3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Step 43 (archaic ancestry) no longer counts a site without reads as a match with the reference genome. It wrote every table site the VCF had no record for as 0/0, so a stretch with no reads, where the archaic genomes carry the common reference allele, could score as a long Neanderthal segment, and one call anywhere on a chromosome was enough to analyse all of it. On a synthetic chromosome with one call far away and no reads over 150 table sites, the pinned IBDmix called a 149 kb segment and a 100% Neanderthal share; it now calls none. `og archaic genotypes` takes the sample's gVCF (`--gvcf`) and reads such a site as the reference only inside a confident reference block: FILTER PASS or `.`, every allele of GT 0, and GQ and depth (MIN_DP, else DP) of 1 or more where given. Any other site with no record goes to IBDmix as unknown (9). The analysed span and `analysed_sites` now run over the sites read, and the command prints how many sites it read and how many it did not. Step 43 now needs the gVCF of step 3, as steps 42 and 44 do: the bash step stops without it and says why, the Nextflow pipeline skips a samplesheet row without one, with a warning, and `run-all.sh` lists step 43 as `skipped (no gVCF: ...)` for a sample that starts from a VCF without one beside it. The `archaic-hg002` jobs of `public-evidence.yml` give HG002 a gVCF of one reference block per GIAB benchmark interval, and `archaic-same-person` gives one block over each whole chromosome of the 1000 Genomes panel person, which reads every site with no ALT as the reference, as that check did before. Without `--gvcf`, a site with no record is unknown; a VCF with a record at every site still needs no gVCF. No option, output file or column is renamed or removed; the values of `_archaic.tsv` and the segments move for a sample with gaps in its coverage. The parity goldens of `archaic-genotypes` record the change. `docs/43-archaic.md` now also says how the 100 stand-ins carry round(AF × 200) ALT alleles with your own genotype counted in the frequency, as Chen et al. counted each person in their group's, what the tests check and what they do not, and the terms of the MPI-EVA archaic genomes, which the pipeline does not redistribute. +- Step 43 (archaic ancestry) no longer counts a site without reads as a match with the reference genome. It wrote every table site the VCF had no record for as 0/0, so a stretch with no reads, where the archaic genomes carry the common reference allele, could score as a long Neanderthal segment, and one call anywhere on a chromosome was enough to analyse all of it. On a synthetic chromosome with one call far away and no reads over 150 table sites, the pinned IBDmix called a 149 kb segment and a 100% Neanderthal share; it now calls none. `og archaic genotypes` takes the sample's gVCF (`--gvcf`) and reads such a site as the reference only inside a confident reference block: FILTER PASS or `.`, every allele of GT 0, and GQ and depth (MIN_DP, else DP) of 1 or more where given. Any other site with no record goes to IBDmix as unknown (9). The analysed span and `analysed_sites` now run over the sites read, and the command prints how many sites it read and how many it did not. Step 43 now needs the gVCF of step 3, as steps 42 and 44 do: the bash step stops without it and says why, the Nextflow pipeline skips a samplesheet row without one, with a warning, and `run-all.sh` lists step 43 as `skipped (no gVCF: ...)` for a sample that starts from a VCF without one beside it. The `archaic-hg002` jobs of `public-evidence.yml` give HG002 a gVCF of one reference block per GIAB benchmark interval, and `archaic-same-person` gives one block over each whole chromosome of the 1000 Genomes panel person, so a site with no record reads as the reference, as that check did before, while the panel's sites with no call for the person are kept as records and stay unknown. Without `--gvcf`, a site with no record is unknown; a VCF with a record at every site still needs no gVCF, and its reference calls with ALT `.` are read as the reference (they were skipped before). A gVCF with no records on a table chromosome of the VCF, or with a record out of order anywhere, stops the step with exit 2. No option, output file or column is renamed or removed; the values of `_archaic.tsv` and the segments move for a sample with gaps in its coverage. The parity goldens of `archaic-genotypes` record the change. `docs/43-archaic.md` now also says how the 100 stand-ins carry round(AF × 200) ALT alleles with your own genotype counted in the frequency, as Chen et al. counted each person in their group's, what the tests check and what they do not, and the terms of the MPI-EVA archaic genomes, which the pipeline does not redistribute. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/43-archaic.md b/docs/43-archaic.md index defd80856..eb2690bc0 100644 --- a/docs/43-archaic.md +++ b/docs/43-archaic.md @@ -79,15 +79,16 @@ The table is built on GRCh37 and lifted once to GRCh38 with Picard `LiftoverVcf` ## Input -- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample. A record whose FILTER is not PASS is read as unknown, and a site where you carry another ALT than the table's is left out. -- `${SAMPLE}/vcf/${SAMPLE}.g.vcf.gz`: the gVCF of step 3, one sample. A variant-only VCF cannot tell a site that matches the reference from a site without reads, so a table site with no record in the VCF is read as the reference only where a confident reference block of the gVCF covers it. A block is confident when its FILTER is PASS or `.`, every allele of its GT is 0, and its GQ and its depth (MIN_DP, else DP) are 1 or more where it gives them. Any other site with no record goes to IBDmix as unknown (9), which IBDmix skips. Without the gVCF the bash step stops and says why, and the Nextflow pipeline skips the sample with a warning (add the `gvcf` and `gvcf_index` columns to its samplesheet row, or start from the BAM or the reads, as for steps 42 and 44). The gVCF must list the chromosomes in the VCF's order. +- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample. A record whose FILTER is not PASS is read as unknown, and a site where you carry another ALT than the table's is left out. A reference call with ALT `.`, as an all-sites VCF writes it, is the reference when its GT is all 0, and unknown otherwise. +- `${SAMPLE}/vcf/${SAMPLE}.g.vcf.gz`: the gVCF of step 3, one sample. A variant-only VCF cannot tell a site that matches the reference from a site without reads, so a table site with no record in the VCF is read as the reference only where a confident reference block of the gVCF covers it. A block is confident when its FILTER is PASS or `.`, every allele of its GT is 0, and its GQ and its depth (MIN_DP, else DP) are 1 or more where it gives them. Any other site with no record goes to IBDmix as unknown (9), which IBDmix skips. Without the gVCF the bash step stops and says why, and the Nextflow pipeline skips the sample with a warning (add the `gvcf` and `gvcf_index` columns to its samplesheet row, or start from the BAM or the reads, as for steps 42 and 44). The gVCF must list the chromosomes in the VCF's order, with records on every table chromosome the VCF lists. - The GENCODE gene lines `setup.sh` installs, for the genes of each segment (without them the segments are listed without genes) `og archaic genotypes` reads the VCF, the gVCF (`--gvcf`) and the table together, keeping one duplicate-position window of the VCF and one record of the gVCF. Positions within each chromosome must be in ascending order in all -three inputs. A position that goes backwards, or a gVCF in another -chromosome order, stops the command with exit 2. Mask runs are written as +three inputs. A position that goes backwards (the gVCF is read to its end +for this), a gVCF in another chromosome order, or a gVCF with no records on +a table chromosome of the VCF, stops the command with exit 2. Mask runs are written as they close. It prints how many table sites it read and how many it did not. ## Command @@ -117,7 +118,7 @@ Measured on public HG002 by the `archaic-hg002` jobs of `public-evidence.yml`, o - Install: the archaic files were 150.5 GB in all, from 1.8 GB (chromosome 22) to 13.8 GB (chromosome 2). Downloading them took 2 h 32 min summed over the 22 chromosomes, from 3 to 32 minutes each, at the speed MPI-EVA served that day. Building the table took 3 h 15 min summed, from 2 to 20 minutes per chromosome. The finished table is 0.44 GB with 51,812,908 sites. - Step 43, one population: 19 to 88 seconds per chromosome, 18 minutes summed over the 22. IBDmix runs once per chromosome and archaic genome, `THREADS` chromosomes at a time. -Every figure in this section was measured before step 43 read the gVCF, when each site GIAB's variant-only VCF had no call for was read as the reference, inside GIAB's confident regions or not. The e2e case now gives GIAB's confident regions as the reference blocks and prints its chr22 figure on every run. The `archaic-hg002` jobs of `public-evidence.yml` give the same blocks, one per GIAB benchmark interval, so the next release run measures the whole-genome figure on this rule. The `archaic-same-person` job has no callable mask for the 1000 Genomes panel data, so it gives one reference block over each whole chromosome: every site with no ALT reads as the reference, as it did when the table below was measured. +Every figure in this section was measured before step 43 read the gVCF, when each site GIAB's variant-only VCF had no call for was read as the reference, inside GIAB's confident regions or not. The e2e case now gives GIAB's confident regions as the reference blocks and prints its chr22 figure on every run. The `archaic-hg002` jobs of `public-evidence.yml` give the same blocks, one per GIAB benchmark interval, so the next release run measures the whole-genome figure on this rule. The `archaic-same-person` job has no callable mask for the 1000 Genomes panel data, so it gives one reference block over each whole chromosome: a site where the panel has no record for the person reads as the reference, as it did when the table below was measured, and a site where the panel has no call for the person stays unknown. That run was made before the masks, from a table that held every lifted site. It gave HG002 115.2 Mb of Neanderthal sequence over the 22 autosomes, about twice what Chen et al. report for Europeans (a median of 51.6 Mb per person, and 42.0 to 62.0 Mb across the released per-person calls of their 503 Europeans; see the same-person comparison below for what their method is). The cause was the missing masks: the table carried the positions each archaic genome's own filters exclude, and IBDmix scored them. On chr22, 43% of that run's Neanderthal sequence lay outside the 1000 Genomes GRCh38 strict mask alone. From 8f49afd5229ec3e2366bf061bdca5aae251384a1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 12:58:13 +0100 Subject: [PATCH 156/255] ci(evidence): count the same-person no-call records at archaic table sites --- .github/workflows/public-evidence.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index 069401a19..9354a9758 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -386,6 +386,7 @@ jobs: set -euo pipefail . ./versions.env G="${EV}/genome" OUT="${EV}/out" + DATA="${G}/reference/archaic/${ARCHAIC_DATA_VERSION}" mkdir -p "$OUT" # pull LABEL URL-PREFIX NAME-OF-FILE-PATTERN: the person's SNV records # on the chromosomes that are not homozygous reference, one VCF: the @@ -416,8 +417,14 @@ jobs: docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/g" "$BCFTOOLS_IMAGE" \ bcftools index -t -f "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" rm -f "${EV}/${label}".chr*.vcf.gz + # How many records, how many of them with no call, and how many of + # those at a table site (the sites step 43 reads as unknown). + docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools query -i 'GT="mis"' -f '%CHROM\t%POS\n' \ + "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" > "${EV}/${label}.nocall.tsv" echo "${label}: $(docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools view -H "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" | wc -l) SNV records," \ - "$(docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools view -H -i 'GT="mis"' "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" | wc -l) of them with no call" + "$(wc -l < "${EV}/${label}.nocall.tsv") of them with no call," \ + "$(for t in "${DATA}"/chr*.tsv.gz; do gzip -dc "$t"; done \ + | awk -F'\t' 'NR == FNR {m[$1 FS $2] = 1; next} ($1 FS $2) in m {n++} END {print n + 0}' "${EV}/${label}.nocall.tsv" -) of those at a table site" # Step 43 reads a site with no record as the reference only inside # a reference block of the gVCF. A panel VCF holds no callable mask, # so the gVCF is one block over each whole chromosome: a site with From c26a02a840663743ce755bc4a61092ea7852e6bb Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:04:31 +0100 Subject: [PATCH 157/255] docs(updating): name the PCGR size, digest and measured-for pins in the bump steps --- docs/updating.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/updating.md b/docs/updating.md index 17f2561f9..a182c8396 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -76,7 +76,7 @@ Each hold is a rule in `renovate.json` with its reason in the rule's `descriptio - **PharmCAT**: diff the diplotype table on the HG002 fixture between the old and the new image, and run the CPIC parser tests on a `report.json` from the new version. - **VEP major**: move `VEP_CACHE_RELEASE` to the new major and `GENCODE_RELEASE` (`scripts/lib/common.sh`) to the new major minus 66 in the same PR, download the new cache (about 26 GB) and rerun step 13. CI cannot run the offline cache. -- **PCGR/CPSR**: move `PCGR_DATA_BUNDLE` and `PCGR_VEP_CACHE_RELEASE` with the image, and rerun step 17 on a sample with an earlier result. CI does not run CPSR on a sample. The image test checks that `cpsr --help` still lists every flag step 17 passes, that `cpsr --version` names the release line of the image tag and the `pcgrr` R package its full release, and runs the bundled VEP with `--database` on 50 variants. +- **PCGR/CPSR**: move `PCGR_DATA_BUNDLE` and `PCGR_VEP_CACHE_RELEASE` with the image. Download the new bundle once, measure its size and sha256, and set `PCGR_DATA_BYTES`, `PCGR_DATA_SHA256` and `PCGR_DATA_MEASURED_FOR` (` `) in the same PR: the server publishes no checksum, and `scripts/ci/check-coupled-versions.sh` fails until all of them move together. Then rerun step 17 on a sample with an earlier result. CI does not run CPSR on a sample. The image test checks that `cpsr --help` still lists every flag step 17 passes, that `cpsr --version` names the release line of the image tag and the `pcgrr` R package its full release, and runs the bundled VEP with `--database` on 50 variants. - **Python**: resolve `scripts/cyrius-constraints.txt` again in the new image, as its header says, and rerun step 21 on a test BAM. - **Nextflow**: the PR runs the stub and E2E jobs on the new version. The prose names `NEXTFLOW_VERSION` rather than a number, and the fake nextflow of the fake-docker suite reads it from `versions.env`, so a patch needs no hand edit. A new release line (26.10 to 27.04, say) also needs the line named in `nextflow.config`, `nextflow.yml` and the prose of [Nextflow Execution](nextflow.md) and [Lessons Learned](lessons-learned.md) moved by hand. - **PGS scoring files and old PRS summaries**: after a scoring file update, rerun step 25 and treat its output as a new baseline ([PRS](25-prs.md#maintenance)). A `_prs_summary.tsv` whose header has `Variants_Used` comes from the scoring path used before 2026-10-01: its `Score` is plink2's average, its `Variants_Used` a dosage sum, and its PGS000738 row (a vitiligo score) is labelled schizophrenia. Rerun step 25 rather than reuse or compare it ([Lessons Learned](lessons-learned.md#a-prs-summary-with-a-variants_used-column-comes-from-the-old-scoring-path)). From 63bca6ba7282d3786fce6962f62eb8538938600f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:07:17 +0100 Subject: [PATCH 158/255] ci(evidence): keep the table scan from filling memory when no record lacks a call --- .github/workflows/public-evidence.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index 9354a9758..4240b63e6 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -424,7 +424,7 @@ jobs: echo "${label}: $(docker run --rm -v "${G}:/g" "$BCFTOOLS_IMAGE" bcftools view -H "/g/${SAMPLE}${label}/vcf/${SAMPLE}${label}.vcf.gz" | wc -l) SNV records," \ "$(wc -l < "${EV}/${label}.nocall.tsv") of them with no call," \ "$(for t in "${DATA}"/chr*.tsv.gz; do gzip -dc "$t"; done \ - | awk -F'\t' 'NR == FNR {m[$1 FS $2] = 1; next} ($1 FS $2) in m {n++} END {print n + 0}' "${EV}/${label}.nocall.tsv" -) of those at a table site" + | awk -F'\t' 'FILENAME == ARGV[1] {m[$1 FS $2] = 1; next} ($1 FS $2) in m {n++} END {print n + 0}' "${EV}/${label}.nocall.tsv" -) of those at a table site" # Step 43 reads a site with no record as the reference only inside # a reference block of the gVCF. A panel VCF holds no callable mask, # so the gVCF is one block over each whole chromosome: a site with From 0651886e0cb18790bad53ff7a7dedd699c533a51 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:21:47 +0100 Subject: [PATCH 159/255] test(run-all): prove step 43 is skipped as 'no gVCF' for a VCF row without one With the archaic table installed, a BAM+VCF row with no gVCF beside the VCF, and a VCF-only row, list and record step 43 as skipped (no gVCF: ...) and nextflow gets no --archaic_data; the same row with its gVCF runs step 43. --- tests/fake-docker/run-all-archaic-gvcf.sh | 60 +++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 tests/fake-docker/run-all-archaic-gvcf.sh diff --git a/tests/fake-docker/run-all-archaic-gvcf.sh b/tests/fake-docker/run-all-archaic-gvcf.sh new file mode 100755 index 000000000..b4a71d8c5 --- /dev/null +++ b/tests/fake-docker/run-all-archaic-gvcf.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Step 43 (archaic ancestry, IBDmix) reads the gVCF: a site with no call is the +# reference only inside a reference block. With the archaic table installed, +# run-all.sh lists step 43 as skipped with 'no gVCF: /vcf/.g.vcf.gz' +# for a row that starts from a VCF with no gVCF and index beside it, as it does +# for step 42, and records the same reason in logs/run_status.tsv: +# 1. BAM+VCF with the gVCF and its index beside the VCF: step 43 runs and is +# 'ok' (the control: the table counts as installed); +# 2. BAM+VCF, no gVCF: skipped, and nextflow gets no --archaic_data; +# 3. a VCF-only row: skipped for the same reason. +# shellcheck source=../../scripts/ci/fake-docker/lib.sh +. "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" + +export GENOME_DIR="${CASE_WORK}/genome" SKIP_VALIDATION=true +G=$GENOME_DIR +seed_reference "$G" +seed_clinvar "$G" +use_output_hook +AV=$(sed -n 's/^ARCHAIC_DATA_VERSION="\(.*\)".*/\1/p' "${REPO_ROOT}/versions.env") +[ -n "$AV" ] || fail "cannot read ARCHAIC_DATA_VERSION from versions.env" +mkdir -p "${G}/reference/archaic/${AV}" +printf 'placeholder\n' > "${G}/reference/archaic/${AV}/DONE" # setup.sh --archaic writes DONE last + +MISSES=0 +miss() { echo "ASSERT FAIL: $*" >&2; MISSES=$((MISSES + 1)); } +# step43 NAME SAMPLE: step 43's result in that run's run_status.tsv (its last row) +step43() { awk -F'\t' '$1 == "step" && $2 == "43" {r = $3} END {print r}' "${G}/$2/logs/run_status.tsv"; } + +# 1. The control: the gVCF and its index beside the VCF. +seed_sample "$G" s1 +printf 'placeholder\n' > "${G}/s1/vcf/s1.g.vcf.gz" +printf 'placeholder\n' > "${G}/s1/vcf/s1.g.vcf.gz.tbi" +run_expect 0 with-gvcf "${SCRIPTS}/run-all.sh" s1 male +output_has with-gvcf '^ 43 Archaic ancestry \(IBDmix\) +runs$' +[ "$(step43 with-gvcf s1)" = ok ] || miss "with-gvcf: step 43 is '$(step43 with-gvcf s1)', want 'ok'" +docker_log_has "^nextflow :: .* --archaic_data ${G}/reference/archaic/${AV} " "with-gvcf: nextflow was not given --archaic_data" + +# 2. BAM+VCF with no gVCF beside the VCF. +seed_sample "$G" s2 +N=$(wc -l < "$FAKE_DOCKER_LOG") # this run's lines of the docker log start after line N +run_expect 0 no-gvcf "${SCRIPTS}/run-all.sh" s2 male +output_has no-gvcf '^ 43 Archaic ancestry \(IBDmix\) +skipped +\(no gVCF: s2/vcf/s2\.g\.vcf\.gz\)$' +output_has no-gvcf '^ 42 Traits panel +skipped +\(no gVCF: s2/vcf/s2\.g\.vcf\.gz\)$' +output_lacks no-gvcf '^ 43 .*data not installed' +[ "$(step43 no-gvcf s2)" = 'skipped (no gVCF: s2/vcf/s2.g.vcf.gz)' ] || miss "no-gvcf: step 43 is '$(step43 no-gvcf s2)'" +# One awk, no pipe: under pipefail, grep -q closing a pipe early would hide the match. +if awk -v n="$N" 'NR > n && / --archaic_data / { hit = 1 } END { exit !hit }' "$FAKE_DOCKER_LOG"; then + miss "no-gvcf: nextflow was given --archaic_data" +fi + +# 3. A VCF-only row. +mkdir -p "${G}/s3/vcf" +printf 'placeholder\n' > "${G}/s3/vcf/s3.vcf.gz" +printf 'placeholder\n' > "${G}/s3/vcf/s3.vcf.gz.tbi" +run_expect 0 vcf-only "${SCRIPTS}/run-all.sh" s3 male +output_has vcf-only '^ 43 Archaic ancestry \(IBDmix\) +skipped +\(no gVCF: s3/vcf/s3\.g\.vcf\.gz\)$' +[ "$(step43 vcf-only s3)" = 'skipped (no gVCF: s3/vcf/s3.g.vcf.gz)' ] || miss "vcf-only: step 43 is '$(step43 vcf-only s3)'" + +[ "$MISSES" -eq 0 ] || fail "${MISSES} result(s) above are wrong" +echo "With the archaic table installed, step 43 runs on a row with its gVCF and is skipped as 'no gVCF' on a BAM+VCF or VCF-only row without one." From 7ac31cdfe8a052a342d2e2459d10a9641bce3911 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 13:58:56 +0100 Subject: [PATCH 160/255] test(pgx): pin that a heteroplasmic MT-RNR1 allele reaches PharmCAT Red on main: og pgx mito-merge holds back every named allele below AF 0.95. Bead: pgx-32 --- .../og-filters/tests/i411_mtrnr1_het.rs | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 rust/crates/og-filters/tests/i411_mtrnr1_het.rs diff --git a/rust/crates/og-filters/tests/i411_mtrnr1_het.rs b/rust/crates/og-filters/tests/i411_mtrnr1_het.rs new file mode 100644 index 000000000..88ba6fba5 --- /dev/null +++ b/rust/crates/og-filters/tests/i411_mtrnr1_het.rs @@ -0,0 +1,119 @@ +//! A heteroplasmic MT-RNR1 allele reaches PharmCAT (issue #411, pgx-04 and +//! pgx-32; floor D10). +//! +//! CPIC defines no heteroplasmy level at which aminoglycosides become safe, +//! and follows the homoplasmic guidance for any detected MT-RNR1 variant. So +//! a PASS named allele at AF 0.05 or more is the call, whatever its +//! heteroplasmy, and the log carries the level. Below 0.05 it is not called, +//! the log says it is below the detection floor, and no `Reference` is +//! passed either: the gene is not called absent. + +use std::collections::HashMap; + +use og_filters::mito::{Alleles, MIN_AF, Rec, decide, parse_alleles}; + +const INC: &str = "Increased risk of aminoglycoside-induced hearing loss"; + +fn alleles() -> Alleles { + let json = r#"{"namedAlleles": [ + {"name": "Reference", "functionValue": "Normal risk of aminoglycoside-induced hearing loss"}, + {"name": "m.827A>G", "functionValue": "Normal risk of aminoglycoside-induced hearing loss"}, + {"name": "m.1494C>T", "functionValue": "Increased risk of aminoglycoside-induced hearing loss"}, + {"name": "m.1555A>G", "functionValue": "Increased risk of aminoglycoside-induced hearing loss"}]}"#; + parse_alleles(&pycompat::json::loads(json).unwrap(), "pharmcat.jar").unwrap() +} + +fn rec(pos: i128, r: &str, alt: &str, af: f64) -> Rec { + Rec { + pos, + ref_: r.into(), + alt: alt.into(), + filter: "PASS".into(), + af: Some(af), + } +} + +/// Two homoplasmic background records and `extra`, with the default floor +/// and depth 500 at every named position. +fn run(extra: &[Rec]) -> (Option, Vec) { + let mut recs = vec![rec(263, "A", "G", 0.998), rec(750, "A", "G", 0.999)]; + recs.extend_from_slice(extra); + let depth: HashMap = (648..=1601).map(|p| (p, 500)).collect(); + let n = recs.len(); + decide(&recs, n, &alleles(), MIN_AF, Some(&depth), 20) +} + +#[test] +fn the_default_floor_is_af_0_05() { + assert!((MIN_AF - 0.05).abs() < f64::EPSILON, "MIN_AF is {MIN_AF}"); +} + +#[test] +fn red_first_m1555_at_af_0_40_is_the_call() { + let (call, lines) = run(&[rec(1555, "A", "G", 0.40)]); + assert_eq!(call.as_deref(), Some("m.1555A>G"), "{lines:?}"); + assert_eq!( + lines[0], + format!( + "MT-RNR1 m.1555A>G passed to PharmCAT as an outside call (heteroplasmic, chrM:1555 A>G AF 0.400; {INC}); CPIC sets no heteroplasmy level at which aminoglycosides are safe, so the homoplasmic guidance applies" + ) + ); +} + +#[test] +fn red_first_m1494_from_0_06_to_0_94_is_the_call() { + for af in [0.05, 0.06, 0.10, 0.50, 0.94, 0.949] { + let (call, lines) = run(&[rec(1494, "C", "T", af)]); + assert_eq!(call.as_deref(), Some("m.1494C>T"), "AF {af}: {lines:?}"); + assert!(lines[0].contains("(heteroplasmic, chrM:1494 C>T AF "), "{lines:?}"); + } +} + +#[test] +fn af_0_95_and_above_still_reads_homoplasmic() { + let (call, lines) = run(&[rec(1555, "A", "G", 0.95)]); + assert_eq!(call.as_deref(), Some("m.1555A>G")); + assert_eq!( + lines[0], + format!( + "MT-RNR1 m.1555A>G passed to PharmCAT as an outside call (homoplasmic, chrM:1555 A>G AF 0.950; {INC})" + ) + ); +} + +#[test] +fn red_first_af_0_03_is_below_the_floor_not_absent() { + let (call, lines) = run(&[rec(1555, "A", "G", 0.03)]); + assert_eq!(call, None, "neither the allele nor Reference: {lines:?}"); + assert_eq!( + lines, + [ + "held back: m.1555A>G (chrM:1555 A>G AF 0.030) is below the detection floor (AF below 0.05): not called, and not absent either", + "no MT-RNR1 call passed to PharmCAT: it stays No Result; read the records above in the chrM call set", + ] + ); +} + +#[test] +fn red_first_a_heteroplasmic_increased_risk_allele_beats_a_homoplasmic_normal_one() { + let (call, lines) = run(&[rec(827, "A", "G", 1.0), rec(1555, "A", "G", 0.20)]); + assert_eq!(call.as_deref(), Some("m.1555A>G"), "{lines:?}"); + assert!(lines[0].contains("(heteroplasmic, chrM:1555 A>G AF 0.200; ")); + assert_eq!( + lines[1], + "also homoplasmic, not passed (PharmCAT takes one MT-RNR1 allele): m.827A>G (chrM:827 A>G AF 1.000; Normal risk of aminoglycoside-induced hearing loss)" + ); +} + +#[test] +fn red_first_a_heteroplasmic_unnamed_change_at_a_named_position_holds_reference_back() { + let (call, lines) = run(&[rec(1555, "A", "C", 0.30)]); + assert_eq!(call, None, "{lines:?}"); + assert_eq!( + lines[0], + "held back: chrM:1555 A>C AF 0.300: a change at an MT-RNR1 position that PharmCAT does not name" + ); + // Below the floor it is noise, as any other record below it. + let (call, _) = run(&[rec(1555, "A", "C", 0.03)]); + assert_eq!(call.as_deref(), Some("Reference")); +} From cbb276ae7456e0c1f8122ef512fb2b6164a59dad Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:00:44 +0100 Subject: [PATCH 161/255] fix(benchmark): count genotype agreement on shared sites and compare Clair3 and any vcf_*/ caller Bead: plat-71 --- scripts/benchmark-variants.sh | 94 ++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 13 deletions(-) diff --git a/scripts/benchmark-variants.sh b/scripts/benchmark-variants.sh index e3cc51c1d..0ff97a8f3 100755 --- a/scripts/benchmark-variants.sh +++ b/scripts/benchmark-variants.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash # benchmark-variants.sh — Compare variant calls across multiple callers -# Supports pairwise concordance (bcftools isec) and truth set benchmarking (hap.py) -# Input: VCFs from vcf/, vcf_gatk/, vcf_freebayes/ directories +# Supports pairwise concordance (bcftools isec, then the genotypes of the shared +# sites) and truth set benchmarking (hap.py) +# Input: .vcf.gz in vcf/ (DeepVariant) and in every vcf_*/ directory +# (vcf_gatk, vcf_freebayes, vcf_octopus, vcf_clair3, or one of your own), and +# Strelka2's vcf_strelka2/results/variants/variants.vcf.gz # Output: comparison tables in $GENOME_DIR//benchmark/ # # Truth sets: --truth VCF (with --regions BED) for any truth set, or @@ -171,6 +174,22 @@ if [ -f "$OCT_VCF" ]; then CALLER_NAMES+=("Octopus") CALLER_VCFS+=("$OCT_VCF") fi +# Every other vcf_/.vcf.gz: Clair3 (step 03e) and any caller's +# VCF placed there by hand. The same glob decides in run-all.sh whether +# BENCHMARK=true queues this step, so what queues it is also what it compares. +for vcf in "${GENOME_DIR}/${SAMPLE}"/vcf_*/"${SAMPLE}.vcf.gz"; do + [ -f "$vcf" ] || continue + dir=${vcf%/*} + dir=${dir##*/vcf_} + case "$dir" in + gatk|freebayes|octopus|strelka2) continue ;; + clair3) name=Clair3 ;; + *[!A-Za-z0-9._-]*|"") echo "WARNING: skipping vcf_${dir}/: the directory name must be letters, digits, '.', '_' or '-'" >&2; continue ;; + *) name=$dir ;; + esac + CALLER_NAMES+=("$name") + CALLER_VCFS+=("$vcf") +done NUM_CALLERS=${#CALLER_VCFS[@]} @@ -189,6 +208,8 @@ if [ "$NUM_CALLERS" -lt 2 ] && [ -z "$TRUTH_VCF" ]; then echo " vcf_freebayes/ (FreeBayes) — run scripts/03b-freebayes.sh" >&2 echo " vcf_strelka2/ (Strelka2) — run scripts/03c-strelka2-germline.sh" >&2 echo " vcf_octopus/ (Octopus) — run scripts/03d-octopus.sh" >&2 + echo " vcf_clair3/ (Clair3) — run scripts/03e-clair3.sh (long reads)" >&2 + echo " vcf_/ () — any other caller's .vcf.gz" >&2 echo "" >&2 echo "Or use --truth --regions for single-caller truth set benchmarking." >&2 exit 1 @@ -382,8 +403,10 @@ else echo "" echo "=== Pairwise Concordance (bcftools isec) ===" - # Build the TSV header - printf "CallerA\tCallerB\tShared\tA_Unique\tB_Unique\tJaccard\n" > "$TSV" + # Build the TSV header. Jaccard counts a site as shared when both callers + # have the same allele there, whatever the genotype; GT_Concordant counts the + # shared sites where the two genotypes are also the same. + printf "CallerA\tCallerB\tShared\tA_Unique\tB_Unique\tJaccard\tGT_Concordant\tGT_Concordance\n" > "$TSV" { echo "================================================================================" @@ -404,6 +427,38 @@ else echo "Restricted to region: ${INTERVALS}" fi + # site_gt VCF: CHROM, POS, REF, ALT and the first sample's genotype of each + # record of a plain VCF, one line each (. when the record has no GT). + site_gt() { + awk -F'\t' '!/^#/ { g = "."; n = split($9, f, ":"); split($10, v, ":") + for (i = 1; i <= n; i++) if (f[i] == "GT") { g = v[i]; break } + print $1 "\t" $2 "\t" $4 "\t" $5 "\t" g }' "$1" + } + # gt_agree A_FILE B_FILE: the number of shared sites whose genotypes agree. + # bcftools isec writes the shared records of A (0002) and of B (0003) in the + # same order, one line per site, so the two files are read side by side and + # never held in memory. Genotypes are compared unphased and with their + # alleles sorted (1|0 = 0/1), and a homozygous call equals its haploid form + # (1/1 = 1), so only a different set of alleles counts as a mismatch. + gt_agree() { + paste <(site_gt "$1") <(site_gt "$2") | awk -F'\t' ' + function canon(g, a, n, i, j, t, same, out) { + gsub(/\|/, "/", g) + n = split(g, a, "/") + for (i = 2; i <= n; i++) for (j = i; j > 1 && (a[j - 1] == "." || (a[j] != "." && a[j - 1] + 0 > a[j] + 0)); j--) { t = a[j]; a[j] = a[j - 1]; a[j - 1] = t } + same = 1 + for (i = 2; i <= n; i++) if (a[i] != a[1]) same = 0 + if (same) return a[1] + out = a[1] + for (i = 2; i <= n; i++) out = out "/" a[i] + return out + } + { if ($1 != $6 || $2 != $7 || $3 != $8 || $4 != $9) bad++ + else if (canon($5) == canon($10)) agree++ } + END { if (bad) print "WARNING: " bad " shared records of the two callers did not line up; counted as genotype mismatches" > "/dev/stderr" + print agree + 0 }' + } + # Normalise each caller's VCF once (split multiallelics, left-align indels) # for a fair comparison; every pair below reads the normalised copies. declare -a NORM_VCFS=() @@ -462,13 +517,22 @@ else JACCARD="0.0000" fi + # Genotype concordance over the shared sites. Without both shared files + # (or with no shared site) there is nothing to compare: N/A. + GT_AGREE="N/A" GT_CONC="N/A" + if [ "$SHARED" -gt 0 ] && [ -f "${ISEC_HOST}/0002.vcf" ] && [ -f "${ISEC_HOST}/0003.vcf" ]; then + GT_AGREE=$(gt_agree "${ISEC_HOST}/0002.vcf" "${ISEC_HOST}/0003.vcf") + GT_CONC=$(awk "BEGIN {printf \"%.4f\", ${GT_AGREE} / ${SHARED}}") + fi + echo " ${CALLER_A} unique: ${A_UNIQUE}" echo " ${CALLER_B} unique: ${B_UNIQUE}" echo " Shared: ${SHARED}" echo " Jaccard index: ${JACCARD}" + echo " Same genotype: ${GT_AGREE} of ${SHARED} shared (${GT_CONC})" - printf "%s\t%s\t%s\t%s\t%s\t%s\n" \ - "$CALLER_A" "$CALLER_B" "$SHARED" "$A_UNIQUE" "$B_UNIQUE" "$JACCARD" >> "$TSV" + printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n" \ + "$CALLER_A" "$CALLER_B" "$SHARED" "$A_UNIQUE" "$B_UNIQUE" "$JACCARD" "$GT_AGREE" "$GT_CONC" >> "$TSV" done done @@ -478,15 +542,19 @@ else # Print results table { echo "" - printf "%-14s %-14s %12s %12s %12s %10s\n" "Caller A" "Caller B" "Shared" "A Unique" "B Unique" "Jaccard" - printf "%-14s %-14s %12s %12s %12s %10s\n" "--------------" "--------------" "------------" "------------" "------------" "----------" - while IFS=$'\t' read -r ca cb shared au bu jaccard; do - printf "%-14s %-14s %12s %12s %12s %10s\n" "$ca" "$cb" "$shared" "$au" "$bu" "$jaccard" + printf "%-14s %-14s %12s %12s %12s %10s %10s\n" "Caller A" "Caller B" "Shared" "A Unique" "B Unique" "Jaccard" "GT Concord" + printf "%-14s %-14s %12s %12s %12s %10s %10s\n" "--------------" "--------------" "------------" "------------" "------------" "----------" "----------" + while IFS=$'\t' read -r ca cb shared au bu jaccard _gtagree gtconc; do + printf "%-14s %-14s %12s %12s %12s %10s %10s\n" "$ca" "$cb" "$shared" "$au" "$bu" "$jaccard" "$gtconc" done < <(tail -n +2 "$TSV") echo "" - echo "NOTE: Jaccard index = shared / (A_unique + B_unique + shared)." - echo "Computed on PASS and unfiltered (.) variants after normalization (bcftools norm -m-both)." - echo "A value of 1.0 means perfect agreement; typical WGS callers agree on ~95%+ of SNPs." + echo "NOTE: Jaccard index = shared / (A_unique + B_unique + shared), where a site is" + echo "shared when both callers have the same allele at the same position, whatever" + echo "the genotype. GT Concord = shared sites with the same genotype / shared sites" + echo "(phase and allele order ignored, 1/1 = 1). Both are computed on PASS and" + echo "unfiltered (.) variants after normalization (bcftools norm -m-both)." + echo "They measure agreement between callers, not accuracy: use --truth or --giab" + echo "for precision and recall." } | tee -a "$SUMMARY" fi From 486e6ae781fe41bda8cc503ec09d8704287a77c4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:02:23 +0100 Subject: [PATCH 162/255] test(e2e): ClinVar tier holds only the classified allele and both parts of a combined hit A synthetic sample with a split T>A,C call where only T>A is a step 6 hit, and a combined hit whose left-aligned position matches neither part. Red on main: the tier holds T>C and misses both parts. Bead: clin-22 Bead: clin-23 --- scripts/ci/e2e-case-paths.tsv | 1 + scripts/ci/e2e-case-times.tsv | 1 + tests/e2e/clinvar-tier-allele-match.sh | 73 ++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100755 tests/e2e/clinvar-tier-allele-match.sh diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index f4dd8e3cd..489f1874a 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -73,6 +73,7 @@ clinical-filter-mane-and-quality-3-modules assets/construct_variants.tsv assets/ clinvar-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh clinvar-combined-allele-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh tests/fixtures/vcf/combined_allele_clinvar.vcf tests/fixtures/vcf/combined_allele_sample.vcf clinvar-cpsr-quality-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv bin/og bin/render_report.py scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh +clinvar-tier-allele-match bin/og scripts/23-clinical-filter.sh scripts/ci/fake-docker/lib.sh cn-engine-1-parity assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og scripts/41-bloodtype.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cn-guard-1-mapq0 bin/cn_loci.tsv bin/og modules/local/parascopy/* scripts/35-paralogs.sh cn-intervals-1-catalog assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og scripts/ci/build-cn-intervals.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv index 5c80940c7..5db7a68b5 100644 --- a/scripts/ci/e2e-case-times.tsv +++ b/scripts/ci/e2e-case-times.tsv @@ -72,6 +72,7 @@ clinical-filter-mane-and-quality-3-modules shard 8 clinical-filter-mane-and-qual clinvar-clip-check-1 shard 8 - clinvar-combined-allele-1 shard 25 - clinvar-cpsr-quality-1 shard 5 clinvar-clip-check-1 +clinvar-tier-allele-match shard 6 - cn-engine-1-parity shard 33 - cn-guard-1-mapq0 shard 5 - cn-intervals-1-catalog shard 0 - diff --git a/tests/e2e/clinvar-tier-allele-match.sh b/tests/e2e/clinvar-tier-allele-match.sh new file mode 100755 index 000000000..fffa34f72 --- /dev/null +++ b/tests/e2e/clinvar-tier-allele-match.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Step 23's ClinVar tier takes a VEP record only when its allele is a step 6 +# hit, not every record at a hit's position. A synthetic sample (no fixture +# file is read) carries, on chr22: +# 30000037 AGC>A and 30000042 T>TGGG the two parts of a combined hit +# 30000100 C>T a plain hit +# 30000200 T>A and T>C one position, two alleles (a split +# T>A,C call); only T>A is a hit +# The step 6 hits are the combined allele (30000038 GCCAT>CATGGG, a position +# neither part has, with PGP_JOINED_FROM naming the parts), 30000100 C>T and +# 30000200 T>A. The tier must hold the two parts, C>T and T>A: four records, +# and not T>C. Every record is common and MODIFIER, so no other tier keeps +# one, and the summary must not list T>C either. +. "$(dirname "$0")/lib.sh" + +T="${SAMPLE}am" +rm -rf "${GENOME_DIR:?}/${T}" +mkdir -p "${GENOME_DIR}/${T}/vep" "${GENOME_DIR}/${T}/clinvar" + +CSQ_FMT='Allele|Consequence|IMPACT|SYMBOL|Gene|Feature_type|Feature|BIOTYPE|CANONICAL|MANE_SELECT|MAX_AF' +csq() { echo "CSQ=${1}|intergenic_variant|MODIFIER||||||||0.3"; } +row() { printf 'chr22\t%s\t.\t%s\t%s\t50\tPASS\t%s\tGT:GQ:DP:AD\t0/1:50:30:15,15\n' "$1" "$2" "$3" "$(csq "$4")"; } +{ + printf '##fileformat=VCFv4.2\n' + printf '##FILTER=\n' + printf '##contig=\n' + printf '##INFO=\n' "$CSQ_FMT" + printf '##FORMAT=\n' + printf '##FORMAT=\n' + printf '##FORMAT=\n' + printf '##FORMAT=\n' + printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t%s\n' "$T" + row 30000037 AGC A - + row 30000042 T TGGG GGG + row 30000100 C T T + row 30000200 T A A + row 30000200 T C C +} > "${GENOME_DIR}/${T}/vep/${T}_vep.vcf" + +{ + printf '##fileformat=VCFv4.2\n' + printf '##contig=\n' + printf '##INFO=\n' + printf '##INFO=\n' + printf '##FORMAT=\n' + printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t%s\n' "$T" + printf 'chr22\t30000038\t9900003\tGCCAT\tCATGGG\t.\tPASS\tCLNSIG=Pathogenic;PGP_JOINED_FROM=chr22:30000037:AGC:A|chr22:30000042:T:TGGG\tGT\t0/1\n' + printf 'chr22\t30000100\t9900009\tC\tT\t.\tPASS\tCLNSIG=Pathogenic\tGT\t0/1\n' + printf 'chr22\t30000200\t9900011\tT\tA\t.\tPASS\tCLNSIG=Likely_pathogenic\tGT\t1/.\n' +} > "${GENOME_DIR}/${T}/clinvar/${T}_clinvar_hits.vcf" + +run_step 23-clinical-filter.sh "$T" +check_step_exit 23-clinical-filter.sh +check "the log names the step 6 hits as the tier's source" has 'ClinVar tier: step 6 hits' "$(cat "$STEP_LOG")" +TIER="${T}/clinical/${T}_clinvar_pathogenic.vcf.gz" +check "the ClinVar tier VCF is readable" vcf_ok "$TIER" +ALLELES=$(bcf query -f '%CHROM:%POS:%REF:%ALT\n' "$TIER" 2>/dev/null) +echo "tier alleles: $(tr '\n' ' ' <<< "$ALLELES")" +has_allele() { grep -cx -- "$1" <<< "$ALLELES"; } +check_eq "the classified allele chr22:30000200 T>A is in the tier" "$(has_allele chr22:30000200:T:A)" 1 +check_eq "the other allele at that position, T>C, is not" "$(has_allele chr22:30000200:T:C)" 0 +check_eq "the plain hit chr22:30000100 C>T is in the tier" "$(has_allele chr22:30000100:C:T)" 1 +check_eq "the combined hit's first part chr22:30000037 AGC>A is in the tier" "$(has_allele chr22:30000037:AGC:A)" 1 +check_eq "the combined hit's second part chr22:30000042 T>TGGG is in the tier" "$(has_allele chr22:30000042:T:TGGG)" 1 +check_eq "the tier holds exactly those four records" "$(vcf_count "$TIER")" 4 +check "the log counts four ClinVar records" has 'ClinVar pathogenic/LP: +4 ' "$(cat "$STEP_LOG")" +SUMMARY="${GENOME_DIR}/${T}/clinical/${T}_clinical_summary.tsv" +check "the summary exists" test -s "$SUMMARY" +check_eq "the summary has no row for chr22:30000200 T>C" \ + "$(awk -F'\t' '$1 == "chr22" && $2 == 30000200 && $3 == "T" && $4 == "C"' "$SUMMARY" | wc -l | tr -d ' ')" 0 +check_eq "the summary lists the four tier records" "$(awk 'NR > 1' "$SUMMARY" | wc -l | tr -d ' ')" 4 + +finish From f21f64bdf56cf6857b4b0cdddcd0263cf01ccc2b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:02:23 +0100 Subject: [PATCH 163/255] fix(clinical): give the ClinVar tier only to the allele ClinVar classified Step 23 picked VEP records by the hits' CHROM and POS, so another allele at a hit's position entered the ClinVar tier, and a combined hit, whose joined allele VEP never saw, dropped out. The tier now keys on CHROM, POS, REF and ALT, and a combined hit gives its PGP_JOINED_FROM parts. Bead: clin-22 Bead: clin-23 --- CHANGELOG.md | 2 ++ docs/23-clinical-filter.md | 6 ++++-- scripts/23-clinical-filter.sh | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85edb5eae..c10765b40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 23's ClinVar tier now holds only the allele ClinVar classified. It took every VEP record at a step 6 hit's position, so when a call of T>A,C split into two records and ClinVar classified only T>A, the T>C record also landed in `clinical/_clinvar_pathogenic.vcf.gz`, in the tier's count and, when no other tier kept it, in `_clinical.vcf.gz` and `_clinical_summary.tsv` with `FREQ_TIER` read as a ClinVar record. The tier now matches CHROM, POS, REF and ALT. A combined hit of step 6 (two neighbouring records that make one allele, `PGP_JOINED_FROM`) was often lost from the tier, because its left-aligned position matches neither part; both parts now enter the tier, and its count counts them as two records. No file, column or option changes; only which records these files hold. The reports' ClinVar section reads the step 6 hits and does not change. The Nextflow `CLINICAL_FILTER` module takes the tier from each record's own ClinVar annotation and is not affected. + - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/23-clinical-filter.md b/docs/23-clinical-filter.md index 4f4eec752..ab3deb059 100644 --- a/docs/23-clinical-filter.md +++ b/docs/23-clinical-filter.md @@ -51,8 +51,10 @@ Missense variants and in-frame insertions/deletions, on the ranked transcript. The same impacts with no frequency value. Their own tier, not rare (see above). ### ClinVar pathogenic/likely pathogenic, at any frequency -- Preferred source: the step 6 hits file, built from the ClinVar file in `clinvar/` that `setup.sh` refreshes. The tier holds the records at those positions. -- Step 6 matches on a split, left-aligned copy of the sample, and this tier selects the VEP records at the same CHROM and POS. An SNV always matches. An indel matches only when the caller already wrote it left-aligned, as DeepVariant does; an indel whose position moves on left-alignment is missing from this tier, though it stays in the step 6 hits and in both reports' ClinVar section. +- Preferred source: the step 6 hits file, built from the ClinVar file in `clinvar/` that `setup.sh` refreshes. The tier holds the VEP records whose allele is a hit: the same CHROM, POS, REF and ALT. +- Another allele at a hit's position is not in the tier. A call of T>A,C splits into two records at one position; when ClinVar classified only T>A, the T>C record stays out. +- Step 6 and step 13 both split and left-align the sample against the same reference, so one allele is written the same way in the hits and in the VEP output. +- A combined hit of step 6 (two neighbouring records that make one allele, `PGP_JOINED_FROM` in the hits) puts both of its parts in the tier. VEP annotated the parts, not the joined allele. The tier's count then counts each part as one record. - Next, when step 6 has not run: `ClinVar_CLNSIG`, the same ClinVar file step 13 annotated with `--custom` (pathogenic or likely pathogenic, not conflicting). It follows a ClinVar refresh once step 13 runs again. - Last: VEP's `CLIN_SIG`, from the VEP cache release, so a ClinVar refresh never reaches it. The step says which source it used. - A common pathogenic allele (for example HFE p.C282Y) stays: this tier has no frequency filter. diff --git a/scripts/23-clinical-filter.sh b/scripts/23-clinical-filter.sh index 434be5fd3..8f42653f8 100755 --- a/scripts/23-clinical-filter.sh +++ b/scripts/23-clinical-filter.sh @@ -255,15 +255,44 @@ rm -f "${CLINVAR_TIER:?}" "${CLINVAR_TIER:?}.tbi" "${CLINVAR_UNRANKED:?}" if [ "$HAS_HITS" -eq 1 ]; then echo "[5] Extracting the step 6 ClinVar hits..." CLINVAR_SOURCE="step 6 ClinVar screen" + # The tier takes a VEP record only when its CHROM, POS, REF and ALT are a + # hit's: another allele at a hit's position (the C of a T>A,C call split + # by step 13) is not what ClinVar classified. Steps 6 and 13 both split and + # left-align against the same reference, so the same allele is written the + # same way. A combined hit of step 6 (PGP_JOINED_FROM=CHROM:POS:REF:ALT|...) + # also gives its parts, the records VEP annotated: VEP never saw the joined + # allele, which is often left-aligned to a position neither part has. + ALLELES="${OUTDIR}/${SAMPLE}_clinvar_alleles.tsv" TARGETS="${OUTDIR}/${SAMPLE}_clinvar_targets.tsv" - awk -F'\t' 'BEGIN {OFS = "\t"} !/^#/ {print $1, $2}' "$CLINVAR_HITS" | sort -u > "$TARGETS" + awk -F'\t' 'BEGIN {OFS = "\t"} !/^#/ { + na = split($5, alt, ",") + for (i = 1; i <= na; i++) print $1, $2, $4, alt[i] + n = split($8, kv, ";") + for (i = 1; i <= n; i++) { + if (kv[i] !~ /^PGP_JOINED_FROM=/) continue + np = split(substr(kv[i], 17), part, "|") + for (j = 1; j <= np; j++) { + # CHROM:POS:REF:ALT, read from the right (a contig name may hold ":") + m = split(part[j], f, ":") + if (m < 4) continue + chrom = f[1] + for (k = 2; k <= m - 3; k++) chrom = chrom ":" f[k] + print chrom, f[m - 2], f[m - 1], f[m] + } + } + }' "$CLINVAR_HITS" | sort -u > "$ALLELES" + cut -f1,2 "$ALLELES" | sort -u > "$TARGETS" if [ -s "$TARGETS" ]; then + # -T narrows the input to the hits' positions; the awk keeps the records + # whose allele is one of them. run_in --cpus 4 --memory 4g "${BCFTOOLS_IMAGE}" \ - bcftools view -f PASS -T "${C}/${SAMPLE}_clinvar_targets.tsv" -Oz -o "${C}/${SAMPLE}_clinvar_unranked.vcf.gz" "${CONTAINER_INPUT}" + bcftools view -f PASS -T "${C}/${SAMPLE}_clinvar_targets.tsv" "${CONTAINER_INPUT}" \ + | awk -F'\t' 'NR == FNR {want[$0] = 1; next} /^#/ || (($1 "\t" $2 "\t" $4 "\t" $5) in want)' "$ALLELES" - \ + | run_in -i --cpus 2 --memory 2g "${BCFTOOLS_IMAGE}" bcftools view -Oz -o "${C}/${SAMPLE}_clinvar_unranked.vcf.gz" else echo " Step 6 found no ClinVar hit." fi - rm -f "${TARGETS:?}" + rm -f "${TARGETS:?}" "${ALLELES:?}" elif [ "$HAS_CUSTOM" -eq 1 ]; then echo "[5] Extracting ClinVar_CLNSIG pathogenic/likely pathogenic (step 13's --custom ClinVar file)..." CLINVAR_SOURCE="ClinVar_CLNSIG (step 13 --custom)" From 0a063bc49a5a89bd3d223fdc124a172c909a6fb5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:02:51 +0100 Subject: [PATCH 164/255] test(traits): low-quality calls at trait sites must read not read (red) Red-first tests for decision D10: a call below GQ 20, or one the caller filtered, at APOE, HFE, SERPINA1 or a traits-table row reads low_gq, and the summary counts low_gq rows. They fail on main, which reads such calls as called. Bead: trait-01 Bead: trait-67 Bead: D10 --- rust/crates/og-steps/src/traits.rs | 130 ++++++++++++++++++ .../og-summary/src/collect/group_c/traits.rs | 53 +++++++ 2 files changed, 183 insertions(+) diff --git a/rust/crates/og-steps/src/traits.rs b/rust/crates/og-steps/src/traits.rs index fa6fef4b1..b714cda42 100644 --- a/rust/crates/og-steps/src/traits.rs +++ b/rust/crates/og-steps/src/traits.rs @@ -1393,6 +1393,136 @@ mod tests { let gt = genotypes(r, &[0, 0, 0]); assert_eq!(rule_line(r, >, None)["phase"], "not_needed"); } + /// The call floor (decision D10): a site whose GQ is below 20, or whose + /// record the caller filtered, is not read. GQ 20, a missing GQ and + /// DeepVariant's RefCall stay readable. + #[test] + fn low_quality_calls_at_rule_sites_are_not_read() { + let (rows, rules) = assets(); + let rule = |id: &str| rules.iter().find(|r| r.id == id).unwrap(); + let sites: Vec<&Site> = rows + .iter() + .chain(rules.iter().flat_map(|r| &r.sites)) + .collect(); + let scratch = Scratch::new(); + let read = |text: &str, id: &str| { + let path = scratch.put("g", text); + rule_line(rule(id), &read_genotypes(&path, &sites).unwrap(), None) + }; + // APOE e2/e4: heterozygous at both sites. + let apoe = |tail: &str| { + format!("chr19\t44908684\tT\tC\tT/C{tail}\nchr19\t44908822\tC\tT\tC/T{tail}\n") + }; + for (tail, status, result) in [ + ("\t0", "low_gq", "."), + ("\t1", "low_gq", "."), + ("\t19", "low_gq", "."), + ("\t20", "called", "e2/e4"), + ("\t99", "called", "e2/e4"), + ("\t.", "called", "e2/e4"), + ("", "called", "e2/e4"), + ("\t50\tPASS", "called", "e2/e4"), + ("\t50\t.", "called", "e2/e4"), + ("\t50\tLowQual", "low_gq", "."), + ] { + let line = read(&apoe(tail), "APOE"); + assert_eq!(line["status"], status, "APOE with '{tail}'"); + assert_eq!(line["result"], result, "APOE with '{tail}'"); + assert_eq!(line["genotypes"], "T/C,C/T", "APOE with '{tail}'"); + if status == "low_gq" { + assert!(line["reading"].contains("not read"), "{}", line["reading"]); + assert!(line["reading"].contains("rs429358"), "{}", line["reading"]); + } + } + // One weak site is enough to stop the rule, and the reading names it. + let line = read( + "chr19\t44908684\tT\tC\tC/C\t1\nchr19\t44908822\tC\tT\tC/C\t50\n", + "APOE", + ); + assert_eq!((&line["status"][..], &line["result"][..]), ("low_gq", ".")); + assert!(line["reading"].contains("rs429358"), "{}", line["reading"]); + assert!(!line["reading"].contains("rs7412"), "{}", line["reading"]); + // A site with no call at all still reads not_covered. + let line = read("chr19\t44908684\tT\tC\tT/C\t1\n", "APOE"); + assert_eq!(line["status"], "not_covered"); + // HFE C282Y/H63D: a LowQual call at C282Y is not read; RefCall and + // a pass are. + let hfe = |filter: &str| { + format!("chr6\t26092913\tG\tA\tG/A\t50\t{filter}\nchr6\t26090951\tC\tG\tC/G\t50\tRefCall\n") + }; + let line = read(&hfe("LowQual"), "HFE"); + assert_eq!((&line["status"][..], &line["result"][..]), ("low_gq", ".")); + assert!(line["reading"].contains("LowQual"), "{}", line["reading"]); + assert_eq!(read(&hfe("PASS"), "HFE")["status"], "called"); + // SERPINA1 PiMZ at GQ 0 is not read; at GQ 30 it is. + let serpina1 = |gq: u32| { + format!("chr14\t94380925\tT\tA\tT/T\t{gq}\nchr14\t94378610\tC\tT\tC/T\t{gq}\n") + }; + assert_eq!(read(&serpina1(0), "SERPINA1")["status"], "low_gq"); + assert_eq!(read(&serpina1(30), "SERPINA1")["status"], "called"); + // A low-quality heterozygous site is not sent to WhatsHap. + let tas2r38 = |gq: u32| { + format!( + "chr7\t141973545\tC\tG\tC/G\t{gq}\nchr7\t141972905\tG\tA\tG/A\t{gq}\nchr7\t141972804\tT\tC\tT/C\t{gq}\n" + ) + }; + for (gq, n) in [(5, 0), (40, 3)] { + let path = scratch.put("g", &tas2r38(gq)); + let gt = read_genotypes(&path, &sites).unwrap(); + let out = scratch.put("phase.vcf", ""); + assert_eq!(phase_vcf(&out, "S", &rules, >).unwrap(), n, "GQ {gq}"); + } + } + + /// The traits table: a weak table row and the APOE line built from weak + /// sites read low_gq with a "not read" reading. + #[test] + fn low_quality_calls_in_the_traits_table_are_not_read() { + let scratch = Scratch::new(); + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../../assets"); + let gt = scratch.put( + "g", + "chr2\t135851076\tG\tA\tG/G\t3\n\ + chr19\t40848628\tA\tT\tA/T\t40\tLowQual\n\ + chr2\t135859184\tC\tT\tC/T\t20\n\ + chr19\t44908684\tT\tC\tC/C\t0\nchr19\t44908822\tC\tT\tC/C\t0\n", + ); + let out = scratch.0.join("traits.tsv").to_string_lossy().into_owned(); + let rules_out = scratch.0.join("rules.tsv").to_string_lossy().into_owned(); + let opts = [ + ("table", root.join("traits.tsv")), + ("rules", root.join("genotype_rules.tsv")), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string_lossy().into_owned())) + .chain([ + ("genotypes".into(), gt), + ("sample".into(), "S".into()), + ("out".into(), out.clone()), + ("rules-out".into(), rules_out), + ]) + .collect(); + assert_eq!(run("table", &opts).unwrap(), 0); + let text = std::fs::read_to_string(&out).unwrap(); + let row = |rsid: &str| -> Vec { + text.lines() + .map(|l| l.split('\t').map(str::to_owned).collect::>()) + .find(|c| c[0] == rsid) + .unwrap_or_else(|| panic!("no row {rsid}")) + }; + // rsid gene chrom pos trait genotype status reading ... + for rsid in ["rs4988235", "rs1801272", "rs429358", "rs7412"] { + let r = row(rsid); + assert_eq!(r[6], "low_gq", "{rsid}: {r:?}"); + assert!(r[7].contains("not read"), "{rsid}: {r:?}"); + } + assert_eq!(row("rs4988235")[5], "G/G"); + assert!(row("rs1801272")[7].contains("LowQual")); + assert_eq!(row("rs182549")[6], "called"); + let apoe = row("rs429358+rs7412"); + assert_eq!((&apoe[5][..], &apoe[6][..]), (".", "low_gq")); + } + #[test] fn unwritable_rules_output_keeps_existing_traits_table() { let scratch = Scratch::new(); diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs index 4e222e974..801489036 100644 --- a/rust/crates/og-summary/src/collect/group_c/traits.rs +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -820,3 +820,56 @@ pub mod apol1 { } } } + +#[cfg(test)] +mod tests { + use crate::collect::{Group, Input, collect}; + use crate::testutil::Dir; + use pycompat::json::Value; + + /// A low_gq row (decision D10: a call below GQ 20, or one the caller + /// filtered) has its own count, so it does not drop out of the totals. + #[test] + fn low_gq_rows_are_counted() { + let d = Dir::new("c-traits-low-gq"); + d.put( + "S/traits/S_traits.tsv", + "rsid\tgene\ttrait\tgenotype\tstatus\treading\tancestry\tnote\tpmids\n\ +rs1\tX\tx\tA/A\tcalled\tr\ta\tn\tp\n\ +rs2\tX\tx\tA/G\tlow_gq\tnot read\ta\tn\tp\n\ +rs3\tX\tx\tC/C\tlow_gq\tnot read\ta\tn\tp\n\ +rs4\tX\tx\t.\tnot_covered\tnot read\ta\tn\tp\n\ +rs429358\tAPOE\tx\tC/C\tlow_gq\tnot read\ta\tn\tp\n", + ); + let input = Input { + sample: "S".into(), + sample_dir: d.path("S"), + declared_sex: None, + assets: d.path("assets"), + group: Group::C, + tables: concat!(env!("CARGO_MANIFEST_DIR"), "/../../../bin").into(), + }; + let v = collect(&input).unwrap().to_json(); + let Value::Object(top) = &v else { + panic!("{v:?}") + }; + let Some(Value::Object(sections)) = top.get("sections") else { + panic!("{v:?}") + }; + let Some(Value::Object(traits)) = sections.get("traits") else { + panic!("{v:?}") + }; + let Some(Value::Object(data)) = traits.get("data") else { + panic!("{traits:?}") + }; + // The APOE rows are left out of the counts, as for the other statuses. + for (k, n) in [ + ("called", 1), + ("not_covered", 1), + ("other_allele", 0), + ("low_gq", 2), + ] { + assert_eq!(data.get(k), Some(&Value::Int(n.into())), "{k}: {data:?}"); + } + } +} From 8709012c7831d2cef3dbb07ab59202635bd8e604 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:04:07 +0100 Subject: [PATCH 165/255] test(dev): og dev licences notices must credit every linked crate and refuse a licence outside the allow list Red first: against the scaffold stub all four tests fail (the verb says "not ported yet"). Bead: lic-02 --- rust/crates/og-dev/tests/licences/main.rs | 208 ++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 rust/crates/og-dev/tests/licences/main.rs diff --git a/rust/crates/og-dev/tests/licences/main.rs b/rust/crates/og-dev/tests/licences/main.rs new file mode 100644 index 000000000..6aadf1f5f --- /dev/null +++ b/rust/crates/og-dev/tests/licences/main.rs @@ -0,0 +1,208 @@ +//! `og dev licences notices` on a planted workspace, through real +//! `cargo metadata`. The workspace links crates the binary must credit +//! (one per kind of licence text) and crates it must leave out (build, dev, +//! procedural macro, another target, a feature that is off). Its Windows +//! build links a crate outside the allow list and one with no licence text, +//! and must fail naming both. +use og_dev::licences; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +static NEXT: AtomicUsize = AtomicUsize::new(0); + +struct Temp(PathBuf); + +impl Temp { + fn new() -> Self { + let p = std::env::temp_dir().join(format!( + "og-licences-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&p).unwrap(); + Self(p) + } + fn write(&self, rel: &str, text: &str) { + let p = self.0.join(rel); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, text).unwrap(); + } + fn path(&self, rel: &str) -> String { + self.0.join(rel).to_str().unwrap().into() + } +} + +impl Drop for Temp { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +const GOOD_TEXT: &str = "Copyright (c) the good authors\nshared permissive text\n"; + +/// A crate at vendor/NAME: its manifest lines after [package], its lib, and +/// any files (path relative to the crate, text). +fn vendor(t: &Temp, name: &str, manifest: &str, files: &[(&str, &str)]) { + t.write( + &format!("vendor/{name}/Cargo.toml"), + &format!("[package]\nname = \"{name}\"\nversion = \"0.1.0\"\nedition = \"2021\"\n{manifest}\n"), + ); + t.write(&format!("vendor/{name}/src/lib.rs"), ""); + for (rel, text) in files { + t.write(&format!("vendor/{name}/{rel}"), text); + } +} + +/// The planted workspace: ws/app (the binary) and its dependencies in +/// vendor/, outside the workspace folder so they are not members. +fn workspace() -> Temp { + let t = Temp::new(); + t.write("ws/Cargo.toml", "[workspace]\nmembers = [\"app\"]\nresolver = \"2\"\n"); + t.write( + "ws/app/Cargo.toml", + r#"[package] +name = "app" +version = "0.1.0" +edition = "2021" +license = "GPL-3.0-or-later" + +[features] +default = ["extra"] +extra = ["dep:extra"] + +[dependencies] +good = { path = "../../vendor/good" } +dual = { path = "../../vendor/dual" } +csys = { path = "../../vendor/csys" } +pm = { path = "../../vendor/pm" } +extra = { path = "../../vendor/extra", optional = true } + +[build-dependencies] +buildonly = { path = "../../vendor/buildonly" } + +[dev-dependencies] +devonly = { path = "../../vendor/devonly" } + +[target.'cfg(windows)'.dependencies] +bad = { path = "../../vendor/bad" } +notext = { path = "../../vendor/notext" } +"#, + ); + t.write("ws/app/src/lib.rs", ""); + vendor(&t, "good", "license = \"MIT\"", &[("LICENSE", GOOD_TEXT)]); + // The same text as good: it must come once, for both. + vendor(&t, "extra", "license = \"MIT OR Apache-2.0\"", &[("LICENSE-MIT", GOOD_TEXT)]); + vendor( + &t, + "dual", + "license = \"MIT/Apache-2.0\"\nauthors = [\"Dual Author \"]", + &[], + ); + vendor( + &t, + "csys", + "license = \"MIT\"\nlinks = \"csys\"", + &[ + ("LICENSE-MIT", "csys crate text\n"), + // cargo wants a build script beside `links`. + ("build.rs", "fn main() {}\n"), + ("vendored/clib/COPYING", "the C library's own notice\n"), + ("vendored/clib/src/clib.c", "int clib(void) { return 0; }\n"), + ], + ); + let sspl = [("LICENSE", "server side text\n")]; + vendor(&t, "pm", "license = \"SSPL-1.0\"\n[lib]\nproc-macro = true\n[dependencies]\npmdep = { path = \"../pmdep\" }", &sspl); + vendor(&t, "pmdep", "license = \"SSPL-1.0\"", &sspl); + vendor(&t, "buildonly", "license = \"SSPL-1.0\"", &sspl); + vendor(&t, "devonly", "license = \"SSPL-1.0\"", &sspl); + vendor(&t, "bad", "license = \"SSPL-1.0\"", &sspl); + vendor(&t, "notext", "license = \"Apache-2.0\"", &[]); + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + let lock = std::process::Command::new(cargo) + .args(["generate-lockfile", "--offline", "--manifest-path"]) + .arg(Path::new(&t.path("ws/Cargo.toml"))) + .output() + .unwrap(); + assert!(lock.status.success(), "{}", String::from_utf8_lossy(&lock.stderr)); + t +} + +fn notices(t: &Temp, targets: &[&str]) -> (u8, String, String) { + let out_file = t.path("THIRD-PARTY-LICENSES.txt"); + let _ = std::fs::remove_file(&out_file); + let mut args: Vec = vec!["notices".into(), "--manifest-path".into(), t.path("ws/Cargo.toml")]; + args.extend(targets.iter().map(|s| s.to_string())); + args.extend(["--package".into(), "app".into(), "--out".into(), out_file.clone()]); + let mut out = Vec::new(); + let mut err = Vec::new(); + let code = licences::cli(&args, &mut out, &mut err); + let text = std::fs::read_to_string(&out_file).unwrap_or_default(); + (code, text, String::from_utf8(err).unwrap()) +} + +const LINUX: &str = "x86_64-unknown-linux-gnu"; + +#[test] +fn every_linked_crate_is_credited_and_nothing_else() { + let t = workspace(); + let (code, text, err) = notices(&t, &["--target", LINUX]); + assert_eq!(code, 0, "{err}"); + let listed: Vec<&str> = text + .lines() + .skip_while(|l| !l.starts_with("Crates (")) + .skip(2) + .take_while(|l| !l.is_empty()) + .collect(); + assert_eq!( + listed, + [ + "csys 0.1.0 MIT", + "dual 0.1.0 MIT/Apache-2.0", + "extra 0.1.0 MIT OR Apache-2.0", + "good 0.1.0 MIT", + ], + "{text}" + ); + // The shared text once, for both crates that ship it. + assert_eq!(text.matches("shared permissive text").count(), 1, "{text}"); + assert!(text.contains("For extra 0.1.0: LICENSE-MIT\nFor good 0.1.0: LICENSE\n"), "{text}"); + // A crate that builds C carries its library's notice from below its root. + assert!(text.contains("For csys 0.1.0: vendored/clib/COPYING\n\nthe C library's own notice"), "{text}"); + // No file, MIT allowed: the MIT text with the crate's authors. + assert!(text.contains("For dual 0.1.0: MIT, no licence file in the crate\n\nCopyright (c) Dual Author \n\nPermission is hereby granted"), "{text}"); + for absent in ["app 0.1.0", "pm 0.1.0", "pmdep", "buildonly", "devonly", "bad 0.1.0", "notext", "server side text"] { + assert!(!text.contains(absent), "{absent} must not be credited:\n{text}"); + } +} + +#[test] +fn a_build_without_default_features_leaves_the_optional_crate_out() { + let t = workspace(); + let (code, text, err) = notices(&t, &["--target", LINUX, "--target-no-default", "aarch64-apple-darwin"]); + assert_eq!(code, 0, "{err}"); + assert!(text.contains("Targets: aarch64-apple-darwin, x86_64-unknown-linux-gnu\n"), "{text}"); + assert!(text.contains("extra 0.1.0 MIT OR Apache-2.0 (only x86_64-unknown-linux-gnu)\n"), "{text}"); + assert!(text.contains("good 0.1.0 MIT\n"), "{text}"); +} + +#[test] +fn a_crate_outside_the_allow_list_or_without_a_text_fails() { + let t = workspace(); + let (code, text, err) = notices(&t, &["--target", LINUX, "--target", "x86_64-pc-windows-gnu"]); + assert_eq!(code, 1, "{err}"); + assert!(text.is_empty(), "no file on an error, got:\n{text}"); + assert!(err.contains("bad 0.1.0 is licensed \"SSPL-1.0\", which the allow list"), "{err}"); + assert!(err.contains("notext 0.1.0 ships no licence file, and \"Apache-2.0\" cannot be met with the MIT text"), "{err}"); + assert_eq!(err.lines().count(), 2, "{err}"); +} + +#[test] +fn usage_errors_exit_2() { + let mut out = Vec::new(); + for args in [&[][..], &["notices"][..], &["notices", "--target"][..], &["assets"][..]] { + let mut err = Vec::new(); + let args: Vec = args.iter().map(|s| s.to_string()).collect(); + assert_eq!(licences::cli(&args, &mut out, &mut err), 2, "{args:?}"); + assert!(String::from_utf8(err).unwrap().contains("usage: og dev licences notices")); + } +} From 3402acc210f5502a8fd01a8e4247e31fc95fc341 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:04:09 +0100 Subject: [PATCH 166/255] fix(pgx): pass a heteroplasmic MT-RNR1 allele to PharmCAT from AF 0.05 CPIC sets no heteroplasmy level at which aminoglycosides are safe, so a named MT-RNR1 allele at AF 0.05 or more is now the call and the log gives its level. Below 0.05 it is held back as below the detection floor, and no Reference is passed. Bead: pgx-04 Bead: D10 --- rust/crates/og-filters/src/mito.rs | 94 +++++++++++++------ .../og-filters/tests/i411_mtrnr1_het.rs | 5 +- 2 files changed, 70 insertions(+), 29 deletions(-) diff --git a/rust/crates/og-filters/src/mito.rs b/rust/crates/og-filters/src/mito.rs index 3ec13f05f..39e2e691d 100644 --- a/rust/crates/og-filters/src/mito.rs +++ b/rust/crates/og-filters/src/mito.rs @@ -1,5 +1,5 @@ //! `og pgx mito-merge --jar pharmcat.jar --mito chrM.vcf.gz [--outside F] -//! [--depth F] [--min-af 0.95] [--min-depth 20] --out F`: an MT-RNR1 +//! [--depth F] [--min-af 0.05] [--min-depth 20] --out F`: an MT-RNR1 //! outside call for PharmCAT from the chrM calls (replaces //! `bin/pgx_mito_merge.py`). //! @@ -8,8 +8,14 @@ //! MT-RNR1 alleles are read from the jar. A chrM record with a named //! single-base allele's position, REF and ALT counts when its FILTER is //! PASS (or `.`) and its allele fraction (FORMAT AF, else INFO AF) is -//! `--min-af` or more. One such allele is the call; of several, the one -//! PharmCAT ranks highest (increased risk, then uncertain, then normal). +//! `--min-af` or more, the detection floor. Heteroplasmic counts: CPIC sets +//! no heteroplasmy level at which aminoglycosides are safe and follows the +//! homoplasmic guidance for any detected MT-RNR1 variant, so the floor is +//! the pipeline's heteroplasmy floor (AF 0.05), not homoplasmy (AF 0.95), +//! and the log line gives the level. A named allele below the floor is not +//! called and not called absent either. One such allele is the call; of +//! several, the one PharmCAT ranks highest (increased risk, then uncertain, +//! then normal). //! With none, `Reference`, but only when `--depth` shows `--min-depth` //! reads or more at every named position. Anything that makes the answer //! unsure holds every call back, with a line saying why ([`Held`]). @@ -25,7 +31,10 @@ //! nothing is passed; //! - with `--min-depth 0` and a named position missing from the depth //! file, the Python crashed (`KeyError`) on the lowest depth; that -//! position now counts as depth 0, as the threshold check already did. +//! position now counts as depth 0, as the threshold check already did; +//! - a named allele at AF 0.05 to 0.95 was held back as heteroplasmic, so +//! MT-RNR1 stayed No Result and the aminoglycoside warning was lost; it +//! is now the call (issue #411). use std::collections::{BTreeSet, HashMap}; use std::fmt; @@ -37,7 +46,10 @@ use pycompat::json::Value; use crate::args::{self, ArgError, Spec}; -pub const MIN_AF: f64 = 0.95; +/// The detection floor: a named allele at this AF or more is the call. +pub const MIN_AF: f64 = 0.05; +/// The AF the log calls homoplasmic; below it, heteroplasmic. +pub const HOMOPLASMIC: f64 = 0.95; pub const MIN_DEPTH: i128 = 20; const GENE: &str = "MT-RNR1"; const RESOURCE: &str = "org/pharmgkb/pharmcat/phenotype/MT_RNR1.json"; @@ -314,7 +326,8 @@ impl Rank { /// Why no call is passed although a record was read. #[derive(Debug, Clone, PartialEq)] pub enum Held { - /// A homoplasmic PASS change at a named position PharmCAT does not name. + /// A PASS change at or above the floor, at a named position PharmCAT + /// does not name. UnnamedChange { shown: String, }, @@ -327,7 +340,8 @@ pub enum Held { name: String, shown: String, }, - Heteroplasmic { + /// A named allele below the detection floor: not called, not absent. + BelowFloor { name: String, shown: String, min_af: f64, @@ -355,13 +369,13 @@ impl fmt::Display for Held { filter, } => write!(f, "{name} ({shown}) is filtered ({filter})"), Held::NoAf { name, shown } => write!(f, "{name} ({shown}) has no allele fraction"), - Held::Heteroplasmic { + Held::BelowFloor { name, shown, min_af, } => write!( f, - "{name} ({shown}) is heteroplasmic (AF below {}); PharmCAT takes one homoplasmic allele", + "{name} ({shown}) is below the detection floor (AF below {}): not called, and not absent either", general(*min_af, 6) ), Held::Near { shown, pos, mnv } => { @@ -379,7 +393,7 @@ impl fmt::Display for Held { } } -/// A homoplasmic named allele. +/// A named allele at or above the floor. #[derive(Debug, Clone, PartialEq, PartialOrd)] struct Called { rank: Rank, @@ -387,6 +401,17 @@ struct Called { name: String, function: String, shown: String, + af: f64, +} + +impl Called { + fn plasmy(&self) -> &'static str { + if self.af >= HOMOPLASMIC { + "homoplasmic" + } else { + "heteroplasmic" + } + } } /// The MT-RNR1 call, if any, and the lines that say why. @@ -429,17 +454,18 @@ pub fn decide( filter: r.filter.clone(), }), None => held.push(Held::NoAf { name, shown }), - Some(af) if af < min_af => held.push(Held::Heteroplasmic { + Some(af) if af < min_af => held.push(Held::BelowFloor { name, shown, min_af, }), - Some(_) => calls.push(Called { + Some(af) => calls.push(Called { rank: Rank::of(function), pos: r.pos, name, function: function.clone(), shown, + af, }), } continue; @@ -482,14 +508,26 @@ pub fn decide( if !calls.is_empty() { calls.sort_by(|x, y| x.partial_cmp(y).unwrap_or(std::cmp::Ordering::Equal)); let best = &calls[0]; - lines.push(format!( - "{GENE} {} passed to PharmCAT as an outside call (homoplasmic, {}; {})", - best.name, best.shown, best.function - )); + let mut line = format!( + "{GENE} {} passed to PharmCAT as an outside call ({}, {}; {})", + best.name, + best.plasmy(), + best.shown, + best.function + ); + if best.af < HOMOPLASMIC { + line.push_str( + "; CPIC sets no heteroplasmy level at which aminoglycosides are safe, so the homoplasmic guidance applies", + ); + } + lines.push(line); for c in &calls[1..] { lines.push(format!( - "also homoplasmic, not passed (PharmCAT takes one {GENE} allele): {} ({}; {})", - c.name, c.shown, c.function + "also {}, not passed (PharmCAT takes one {GENE} allele): {} ({}; {})", + c.plasmy(), + c.name, + c.shown, + c.function )); } return (Some(best.name.clone()), lines); @@ -668,15 +706,15 @@ mod tests { } #[test] - fn red_first_an_af_at_the_floor_is_homoplasmic() { + fn red_first_an_af_at_the_floor_is_called() { let a = alleles(); - let (call, lines) = decide(&[rec(1555, "A", "G", Some(0.95))], 1, &a, 0.95, None, 20); + let (call, lines) = decide(&[rec(1555, "A", "G", Some(0.05))], 1, &a, MIN_AF, None, 20); assert_eq!(call.as_deref(), Some("m.1555A>G"), "{lines:?}"); - let (call, lines) = decide(&[rec(1555, "A", "G", Some(0.949))], 1, &a, 0.95, None, 20); + let (call, lines) = decide(&[rec(1555, "A", "G", Some(0.049))], 1, &a, MIN_AF, None, 20); assert_eq!(call, None); assert_eq!( lines[0], - "held back: m.1555A>G (chrM:1555 A>G AF 0.949) is heteroplasmic (AF below 0.95); PharmCAT takes one homoplasmic allele" + "held back: m.1555A>G (chrM:1555 A>G AF 0.049) is below the detection floor (AF below 0.05): not called, and not absent either" ); } @@ -687,7 +725,7 @@ mod tests { rec(827, "A", "G", Some(1.0)), rec(1555, "A", "G", Some(0.99)), ]; - let (call, lines) = decide(&r, 2, &a, 0.95, None, 20); + let (call, lines) = decide(&r, 2, &a, MIN_AF, None, 20); assert_eq!(call.as_deref(), Some("m.1555A>G")); assert!(lines[1].starts_with("also homoplasmic, not passed")); } @@ -695,13 +733,13 @@ mod tests { #[test] fn an_mnv_is_named_as_one() { let a = alleles(); - let (call, lines) = decide(&[rec(1554, "TA", "CG", Some(1.0))], 1, &a, 0.95, None, 20); + let (call, lines) = decide(&[rec(1554, "TA", "CG", Some(1.0))], 1, &a, MIN_AF, None, 20); assert_eq!(call, None); assert_eq!( lines[0], "held back: a multi-base change (chrM:1554 TA>CG AF 1.000) at or near m.1555, which this script does not map to PharmCAT's names" ); - let (_, lines) = decide(&[rec(955, "AC", "A", Some(1.0))], 1, &a, 0.95, None, 20); + let (_, lines) = decide(&[rec(955, "AC", "A", Some(1.0))], 1, &a, MIN_AF, None, 20); assert!(lines[0].starts_with( "held back: an insertion or deletion (chrM:955 AC>A AF 1.000) at or near m.961" )); @@ -715,13 +753,13 @@ mod tests { .into_iter() .map(|p| (p, 30)) .collect(); - let (call, _) = decide(&r, 1, &a, 0.95, Some(&d), 20); + let (call, _) = decide(&r, 1, &a, MIN_AF, Some(&d), 20); assert_eq!(call.as_deref(), Some("Reference")); d.insert(961, 19); - let (call, lines) = decide(&r, 1, &a, 0.95, Some(&d), 20); + let (call, lines) = decide(&r, 1, &a, MIN_AF, Some(&d), 20); assert_eq!(call, None); assert!(lines[0].contains("1 of its 4 positions have depth below 20 (chrM:961 depth 19)")); - let (call, _) = decide(&r, 1, &a, 0.95, None, 20); + let (call, _) = decide(&r, 1, &a, MIN_AF, None, 20); assert_eq!(call, None); } diff --git a/rust/crates/og-filters/tests/i411_mtrnr1_het.rs b/rust/crates/og-filters/tests/i411_mtrnr1_het.rs index 88ba6fba5..56fe296cf 100644 --- a/rust/crates/og-filters/tests/i411_mtrnr1_het.rs +++ b/rust/crates/og-filters/tests/i411_mtrnr1_het.rs @@ -65,7 +65,10 @@ fn red_first_m1494_from_0_06_to_0_94_is_the_call() { for af in [0.05, 0.06, 0.10, 0.50, 0.94, 0.949] { let (call, lines) = run(&[rec(1494, "C", "T", af)]); assert_eq!(call.as_deref(), Some("m.1494C>T"), "AF {af}: {lines:?}"); - assert!(lines[0].contains("(heteroplasmic, chrM:1494 C>T AF "), "{lines:?}"); + assert!( + lines[0].contains("(heteroplasmic, chrM:1494 C>T AF "), + "{lines:?}" + ); } } From a4eacc69129d1ff80e1c68661bd4add72dc84a8a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:04:11 +0100 Subject: [PATCH 167/255] test(clinvar): one red test per always-report state the reader gets wrong A filtered 0/0, a 0/0 at depth 1, a 0/0 without GQ, a haploid 1, a haploid 0 and a ./. record each get a test of the words they must read. All six fail on the current reader. og clinvar deep-intronic also takes --min-dp, the step's depth floor (default 10), unused until the fix. Bead: clin-14 --- rust/crates/og-clinvar/src/cli.rs | 12 ++- rust/crates/og-clinvar/src/sites.rs | 158 ++++++++++++++++++++++++++-- 2 files changed, 158 insertions(+), 12 deletions(-) diff --git a/rust/crates/og-clinvar/src/cli.rs b/rust/crates/og-clinvar/src/cli.rs index 17c00dc43..1dd988943 100644 --- a/rust/crates/og-clinvar/src/cli.rs +++ b/rust/crates/og-clinvar/src/cli.rs @@ -3,7 +3,7 @@ //! ```text //! og clinvar join-adjacent --mode prepare|finish|tag|build [--fasta REF] [--window N] [VCF] //! og clinvar hits [HITS_VCF] -//! og clinvar deep-intronic [--no-gvcf] [--min-gq N] SITES VCF_CALLS GVCF_CALLS +//! og clinvar deep-intronic [--no-gvcf] [--min-gq N] [--min-dp N] SITES VCF_CALLS GVCF_CALLS //! ``` //! //! A missing file, or `-`, is standard input; plain, gzip and BGZF input are @@ -20,7 +20,7 @@ use crate::{hits, sites}; const USAGE: &str = "usage: og clinvar join-adjacent --mode prepare|finish|tag|build [--fasta REF] [--window N] [VCF] og clinvar hits [HITS_VCF] - og clinvar deep-intronic [--no-gvcf] [--min-gq N] SITES VCF_CALLS GVCF_CALLS"; + og clinvar deep-intronic [--no-gvcf] [--min-gq N] [--min-dp N] SITES VCF_CALLS GVCF_CALLS"; fn refuse(msg: &str) -> u8 { eprintln!("{USAGE}"); @@ -182,7 +182,7 @@ fn join_adjacent(args: &[String], out: &mut impl Write) -> u8 { } fn deep_intronic(args: &[String], out: &mut impl Write) -> u8 { - let l = match Line::parse(args, &["--min-gq"], &["--no-gvcf"]) { + let l = match Line::parse(args, &["--min-gq", "--min-dp"], &["--no-gvcf"]) { Ok(l) => l, Err(e) => return refuse(&e), }; @@ -190,6 +190,10 @@ fn deep_intronic(args: &[String], out: &mut impl Write) -> u8 { Ok(v) => v, Err(e) => return refuse(&e), }; + let min_dp = match l.number("--min-dp", 10) { + Ok(v) => v, + Err(e) => return refuse(&e), + }; let [s, v, g] = l.files.as_slice() else { return refuse("deep-intronic takes SITES VCF_CALLS GVCF_CALLS"); }; @@ -204,6 +208,8 @@ fn deep_intronic(args: &[String], out: &mut impl Write) -> u8 { let o = sites::Options { min_gq: l.get("--min-gq").unwrap_or("30").to_owned(), min_gq_value: min_gq, + min_dp: l.get("--min-dp").unwrap_or("10").to_owned(), + min_dp_value: min_dp, gvcf: !l.flags.contains(&"--no-gvcf"), }; sites::deep_intronic(&texts[0], &texts[1], &texts[2], &o, out) diff --git a/rust/crates/og-clinvar/src/sites.rs b/rust/crates/og-clinvar/src/sites.rs index 87f5d35c1..f24fcffc8 100644 --- a/rust/crates/og-clinvar/src/sites.rs +++ b/rust/crates/og-clinvar/src/sites.rs @@ -28,6 +28,10 @@ pub struct Options { pub min_gq: String, /// The GQ floor. pub min_gq_value: u64, + /// The depth floor as given, for the words. + pub min_dp: String, + /// The depth floor. + pub min_dp_value: u64, pub gvcf: bool, } @@ -196,12 +200,16 @@ pub fn deep_intronic( mod tests { use super::*; - fn run(vcf: &str, gvcf: &str, gvcf_on: bool) -> String { - let sites = "# test\nchrom\tpos\tref\talt\tgene\thgvs\tclinvar_id\n\ - chr1\t100\tA\tG\tG1\tc.1A>G\t1\n"; + /// The call and detail at `site` (CHROM POS REF ALT), joined by `|`. + fn run_at(site: &str, vcf: &str, gvcf: &str, gvcf_on: bool) -> String { + let sites = format!( + "# test\nchrom\tpos\tref\talt\tgene\thgvs\tclinvar_id\n{site}\tG1\tc.1A>G\t1\n" + ); let o = Options { min_gq: "30".into(), min_gq_value: 30, + min_dp: "10".into(), + min_dp_value: 10, gvcf: gvcf_on, }; let mut out = Vec::new(); @@ -223,6 +231,10 @@ mod tests { .join("|") } + fn run(vcf: &str, gvcf: &str, gvcf_on: bool) -> String { + run_at("chr1\t100\tA\tG", vcf, gvcf, gvcf_on) + } + #[test] fn every_kind_of_call() { assert_eq!( @@ -241,21 +253,149 @@ mod tests { run("", "chr1\t90\t120\tA\t<*>\t0/0\t50\t.\n", true), "ref|GQ 50 (gVCF block)" ); + assert_eq!( + run("", "", true), + "no call|no record in the VCF or the gVCF covers the site" + ); + assert_eq!( + run("", "", false), + "no call|no record in the VCF, and no gVCF to tell a reference call from no call" + ); + } + + /// Only PASS, `.` (or no FILTER column) and RefCall read as plain + /// reference; any other FILTER on a `0/0` is a low-confidence reference + /// call that names it. + #[test] + fn a_filtered_reference_call_is_low_confidence() { + for f in ["PASS", ".", "RefCall"] { + assert_eq!( + run("", &format!("chr1\t90\t120\tA\t<*>\t0/0\t99\t.\t{f}\n"), true), + "ref|GQ 99 (gVCF block)", + "FILTER {f}" + ); + } + assert_eq!( + run("", "chr1\t90\t120\tA\t<*>\t0/0\t99\t.\tLowQual\n", true), + "ref (low confidence)|GQ 99 (gVCF block), FILTER LowQual" + ); + assert_eq!( + run("chr1\t100\t100\tA\tG\t0/0\t99\t30\tLowGQ;LowDP\n", "", true), + "ref (low confidence)|GQ 99, DP 30, FILTER LowGQ;LowDP" + ); + } + + /// A reference call on fewer reads than the step's depth floor is + /// shallow, from the record's DP or the block's. + #[test] + fn a_shallow_reference_call_says_so() { + assert_eq!( + run("chr1\t100\t100\tA\tG\t0/0\t99\t1\n", "", true), + "ref (shallow)|GQ 99, DP 1, below the depth floor of DP 10" + ); + assert_eq!( + run("", "chr1\t90\t120\tA\t<*>\t0/0\t99\t1\tPASS\n", true), + "ref (shallow)|GQ 99, DP 1 (gVCF block), below the depth floor of DP 10" + ); + assert_eq!( + run("chr1\t100\t100\tA\tG\t0/0\t99\t10\n", "", true), + "ref|GQ 99, DP 10" + ); + // Below both floors: the GQ makes it low confidence, and both are named. + assert_eq!( + run("chr1\t100\t100\tA\tG\t0/0\t3\t1\n", "", true), + "ref (low confidence)|GQ 3, DP 1, below the call floor of GQ 30, below the depth floor of DP 10" + ); + // A carrier's depth is shown, not floored. + assert_eq!( + run("chr1\t100\t100\tA\tG\t0/1\t45\t4\n", "", true), + "het|GQ 45, DP 4" + ); + } + + /// A reference call with no GQ was never checked against the floor. + #[test] + fn a_reference_call_without_gq_is_low_confidence() { + assert_eq!( + run("chr1\t100\t100\tA\tG\t0/0\t.\t30\n", "", true), + "ref (low confidence)|DP 30, no GQ to check against the call floor of GQ 30" + ); assert_eq!( run("", "chr1\t90\t120\tA\t<*>\t0/0\t.\t.\n", true), - "ref|(gVCF block)" + "ref (low confidence)|(gVCF block), no GQ to check against the call floor of GQ 30" + ); + } + + /// GLA c.640-801G>A, on chrX outside the pseudoautosomal regions. + const GLA: &str = "chrX\t101399747\tC\tT"; + + /// One copy of chrX or chrY outside the pseudoautosomal regions: an + /// alt `1` is hemizygous. + #[test] + fn a_haploid_alt_call_is_hemizygous() { + assert_eq!( + run_at(GLA, "chrX\t101399747\t101399747\tC\tT\t1\t99\t30\n", "", true), + "hemizygous|GQ 99, DP 30" + ); + assert_eq!( + run_at(GLA, "chrX\t101399747\t101399747\tC\tG\t1\t99\t30\n", "", true), + "other allele|C>G hemizygous" + ); + assert_eq!( + run_at("chrY\t3000000\tC\tT", "chrY\t3000000\t3000000\tC\tT\t1\t99\t30\n", "", true), + "hemizygous|GQ 99, DP 30" + ); + } + + /// There a `0` is a reference call, under the same floors. Elsewhere + /// (inside PAR1, on an autosome) a one-copy GT is no call, with why. + #[test] + fn a_haploid_reference_call_is_ref() { + assert_eq!( + run_at(GLA, "chrX\t101399747\t101399747\tC\tT\t0\t99\t30\n", "", true), + "ref|GQ 99, DP 30" + ); + assert_eq!( + run_at(GLA, "chrX\t101399747\t101399747\tC\tT\t0\t20\t30\n", "", true), + "ref (low confidence)|GQ 20, DP 30, below the call floor of GQ 30" + ); + assert_eq!( + run_at(GLA, "", "chrX\t101399700\t101399800\tC\t\t0\t50\t.\n", true), + "ref|GQ 50 (gVCF block)" + ); + // Inside PAR1, and on an autosome, the genome has two copies. + assert_eq!( + run_at("chrX\t100000\tC\tT", "chrX\t100000\t100000\tC\tT\t0\t99\t30\n", "", true), + "no call|the VCF record at the site has one copy (GT 0) where the genome has two" ); + assert_eq!( + run("", "chr1\t90\t120\tA\t<*>\t1\t50\t.\n", true), + "no call|the gVCF record over the site has one copy (GT 1) where the genome has two" + ); + } + + /// A record without a genotype says so, and never that no record exists. + #[test] + fn no_call_says_what_was_there() { assert_eq!( run("", "chr1\t90\t120\tA\t<*>\t./.\t.\t.\n", true), - "no call|the gVCF record over the site is not a PASS reference call (GT ./.)" + "no call|the gVCF record over the site has no genotype (GT ./.)" ); assert_eq!( - run("", "", true), - "no call|no record in the VCF or the gVCF covers the site" + run("chr1\t100\t100\tA\tG\t./.\t.\t.\n", "", false), + "no call|the VCF record at the site has no genotype (GT ./.)" ); assert_eq!( - run("", "", false), - "no call|no record in the VCF, and no gVCF to tell a reference call from no call" + run("chr1\t100\t100\tA\tG\t./.\t.\t.\n", "", true), + "no call|the VCF record at the site has no genotype (GT ./.)" + ); + assert_eq!( + run("", "chr1\t100\t100\tA\tG,<*>\t0/1\t30\t20\tLowQual\n", true), + "no call|the gVCF record over the site is not a reference call (GT 0/1, FILTER LowQual)" + ); + assert_eq!( + run("", "chr1\t90\t120\tA\t<*>\t0/.\t30\t.\n", true), + "no call|the gVCF record over the site is not a reference call (GT 0/.)" ); } } From 41b8128e3c2e321d89ebf59f7095ccabb094f6fd Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:06:23 +0100 Subject: [PATCH 168/255] fix(run): size Nextflow from the container engine's memory, not the host's Docker Desktop and colima run the containers in a VM that is usually smaller than the host, but run-all.sh gave Nextflow the host's CPUs and RAM, so Nextflow booked tasks the VM could not hold and they died with exit 137. When docker info reports less, run-all.sh now passes the engine's figures as the default --max_cpus and --max_memory and writes them to /nextflow/executor.config, passed with -c before the user's options. On native Linux nothing changes. validate-setup.sh adds an INFO line in that case. The launcher e2e case checks it with a docker shim that reports 8 GB and 2 CPUs, with a negative control. Bead: plat-18 --- CHANGELOG.md | 1 + docs/hardware-requirements.md | 6 ++-- docs/troubleshooting.md | 6 ++-- scripts/run-all.sh | 17 +++++++++- scripts/validate-setup.sh | 8 +++++ tests/e2e/run-all-launcher.sh | 62 +++++++++++++++++++++++++++++++++++ 6 files changed, 93 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85edb5eae..28b4388ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- `run-all.sh` now sizes Nextflow from the container engine, not only from the host. Docker Desktop and colima run the containers in a VM that is usually smaller than the Mac, but `run-all.sh` gave Nextflow the Mac's CPUs and RAM. Nextflow then started more tasks than the VM could hold, and the VM killed them (exit 137). When `docker info` reports fewer CPUs or less memory than the host, `run-all.sh` now passes the engine's figures as the default `--max_cpus` and `--max_memory`. It also writes them to `/nextflow/executor.config` and passes that file with `-c`, so Nextflow books no more than the VM has. A `-c` of your own still replaces it. On native Linux the engine reports the host's figures, and the command is unchanged. `validate-setup.sh` adds an `[INFO]` line when Docker has less memory than the host. No option or output file changes. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index 67a7b09ee..4d50ba6f5 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -133,7 +133,7 @@ Every step runs in a Docker container with a hard `--memory` limit. The two entr These are limits, not measured peaks. The two step 44 rows were measured on a 4-CPU GitHub runner on public HG002 ([step 44, Runtime](44-ancient.md#runtime)): the setup's containers held 7.6 GiB of anonymous memory at most (the sampler does not record which container, convertf or smartpca), and the largest container's whole charge, file cache included, reached 11.6 of its 12 GiB; step 44's smartpca used 1.2 GiB. The peaks measured for each task in the end test are in the [runtime table](#runtime-per-step): on 30-40x genomes the alignment task (minimap2 and samtools together) peaked at 18.3-18.4 GB and DeepVariant at 15.9-16.5 GB. For step 25, `PRS_PREPARE` (`og prs format`) peaked at 677 to 698 MB on the 15 default scores, and on the nine scores measured before the four small lipid and blood-pressure scores joined the list pgsc_calc's first task, run on one thread, needed about 14 to 18 GB on a GitHub runner without a genome, and 12.7 to 13.5 GiB on the three end-test genomes ([step 25, Memory](25-prs.md#memory)). Your own run records pgsc_calc's task peaks in `prs/pgsc_calc/pipeline_info/execution_trace.txt`; the `PRS` row of the pipeline's trace is only the launcher. -**Minimum system RAM:** 16 GB. `run-all.sh` passes the machine's RAM as `--max_memory`, so on a 16 GB machine the 32 GB tasks run capped at about 15 GB. A task that needs more is killed (exit 137); its one retry asks for double, and the cap cuts that back to the same 15 GB. With the step scripts, lower DeepVariant's shards and memory with `THREADS` and `DV_MEM`, for example `THREADS=4 DV_MEM=12g ./scripts/03-deepvariant.sh [male|female]`. The other scripts set a fixed `--memory`; edit it in the script to change it. Step 25's pgsc_calc is the exception that does not fit: its first task needs about 14 to 18 GB on the default scores and was killed under a 14.1 GB limit when the host has no swap, so a 16 GB machine needs swap for it ([step 25, Memory](25-prs.md#memory)). +**Minimum system RAM:** 16 GB. `run-all.sh` passes the machine's RAM as `--max_memory` (the container engine's when it has less, as on Docker Desktop and colima), so on a 16 GB machine the 32 GB tasks run capped at about 15 GB. A task that needs more is killed (exit 137); its one retry asks for double, and the cap cuts that back to the same 15 GB. With the step scripts, lower DeepVariant's shards and memory with `THREADS` and `DV_MEM`, for example `THREADS=4 DV_MEM=12g ./scripts/03-deepvariant.sh [male|female]`. The other scripts set a fixed `--memory`; edit it in the script to change it. Step 25's pgsc_calc is the exception that does not fit: its first task needs about 14 to 18 GB on the default scores and was killed under a 14.1 GB limit when the host has no swap, so a 16 GB machine needs swap for it ([step 25, Memory](25-prs.md#memory)). **Recommended:** 32 GB or more, so minimap2 and DeepVariant get close to the 32 GB they ask for. **GRIDSS (step 4b, opt-in):** its container takes 32 GB (a 28 GB Java heap plus overhead), so it needs a machine with 32 GB or more even when nothing else runs. @@ -141,7 +141,7 @@ These are limits, not measured peaks. The two step 44 rows were measured on a 4- ## CPU Requirements -Each pipeline task asks for the CPUs that `conf/base.config` gives its label or its name: 1, 2, 4 or 8. The 8-CPU tasks are minimap2 (the index, and the alignment with its duplicate marking in the same task), fastp, DeepVariant, Manta and VEP. `--max_cpus` and `--max_memory` cap each task; `run-all.sh` passes the machine's CPU count (or `THREADS`) and its RAM. They do not limit how much runs at once: Nextflow starts tasks until their requests fill the machine's CPUs and RAM. On a Mac with Docker Desktop, that is the Mac's CPUs and RAM, not the Docker VM's. `MAX_JOBS` is no longer read. +Each pipeline task asks for the CPUs that `conf/base.config` gives its label or its name: 1, 2, 4 or 8. The 8-CPU tasks are minimap2 (the index, and the alignment with its duplicate marking in the same task), fastp, DeepVariant, Manta and VEP. `--max_cpus` and `--max_memory` cap each task; `run-all.sh` passes the machine's CPU count (or `THREADS`) and its RAM. They do not limit how much runs at once: Nextflow starts tasks until their requests fill the machine's CPUs and RAM. Docker Desktop and colima run the containers in a VM that is usually smaller than the Mac. When `docker info` reports fewer CPUs or less memory than the host, `run-all.sh` passes the VM's figures instead, in whole GB. It also writes them to `/nextflow/executor.config` and passes that file with `-c`. Nextflow then fills the VM, not the Mac, and a task that does not fit waits for room instead of being killed inside the VM. On native Linux the engine reports the host's figures, and nothing changes. `MAX_JOBS` is no longer read. **DeepVariant's shards.** `run-all.sh` runs DeepVariant with at most 8 shards, however many cores the machine has, and fewer when `--max_cpus` (`THREADS`) is lower. More cores help the steps that run beside it. To give DeepVariant more, run `scripts/03-deepvariant.sh` with `THREADS=N` (its `--cpus` and `--num_shards`), or pass `-c` with a `withName: 'DEEPVARIANT'` block: @@ -178,7 +178,7 @@ executor { } ``` -Check the shares on a running task: `docker ps` lists the pipeline's containers as `nxf-...`, and `docker inspect -f '{{.HostConfig.CpuShares}}' ` should print 256. The same `executor` block is the way to keep Docker Desktop's VM from being overbooked: set it to the CPUs and memory the VM has, and pass `THREADS` and `--max_memory` no larger than the VM's (e.g. `THREADS=6 ./scripts/run-all.sh --max_memory 14.GB`). On a Mac, `run-all.sh` otherwise passes the Mac's own CPU count and RAM, and Nextflow refuses a task that asks for more than the `executor` block allows. +Check the shares on a running task: `docker ps` lists the pipeline's containers as `nxf-...`, and `docker inspect -f '{{.HostConfig.CpuShares}}' ` should print 256. On Docker Desktop and colima, `run-all.sh` already writes an `executor` block with the VM's CPUs and memory to `/nextflow/executor.config`. It passes that file before your options, so a `-c` of your own replaces the block. Keep `THREADS` and `--max_memory` no larger than your block, or Nextflow refuses the tasks that ask for more. ### Runtime per step diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 878e85530..34b8a879d 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -40,7 +40,7 @@ dmesg | grep -i "oom\|killed" | tail -10 **Fix:** 1. Increase the `--memory` flag in the script that failed (for step 3, `DV_MEM`). With `run-all.sh` you do not edit a script: the pipeline gives each task a memory request and doubles it on the one retry after an out-of-memory exit, up to `--max_memory` -2. Bound what runs at once. With `run-all.sh`, `--max_memory` (e.g. `./scripts/run-all.sh --max_memory 24.GB`; without it, the machine's RAM) and `THREADS=N` (`--max_cpus`, by default the machine's CPU count) cap each task, not the total: Nextflow starts tasks until their requests fill the machine's CPUs and RAM, and on a Mac with Docker Desktop that is the Mac's, not the VM's. To bound the total, pass a config with an `executor` block ([a shared host](hardware-requirements.md#a-shared-host)); on a Mac, also pass `THREADS` and `--max_memory` no larger than the VM's (e.g. `THREADS=6 ... --max_memory 14.GB`); a rerun reuses the steps that finished +2. Bound what runs at once. With `run-all.sh`, `--max_memory` (e.g. `./scripts/run-all.sh --max_memory 24.GB`; without it, the machine's RAM) and `THREADS=N` (`--max_cpus`, by default the machine's CPU count) cap each task, not the total: Nextflow starts tasks until their requests fill the machine's CPUs and RAM. On Docker Desktop and colima that is the VM's: `run-all.sh` reads its size from `docker info` and books no more (`/nextflow/executor.config`). To bound the total lower, pass a config with an `executor` block ([a shared host](hardware-requirements.md#a-shared-host)); a rerun reuses the steps that finished 3. Increase Docker Desktop memory allocation (see [Docker Desktop not enough memory](#docker-desktop-not-enough-memory-macwindows)) 4. For DeepVariant run as a script, lower `THREADS` (its shard count) and `DV_MEM` together, e.g. `THREADS=4 DV_MEM=16g ./scripts/03-deepvariant.sh ` @@ -207,7 +207,7 @@ grep -E '(%%|!!) executor|^~> ' "$GENOME_DIR//nextflow/.nextflow.log" | `!! executor local > No more task to compute` next to `%% executor local > tasks in the submission queue: 2` (the number is how many wait), with the same tasks listed each time, means tasks are waiting to start while nothing is running. A line `tasks to be completed` means a task is still running. That is the entry above, not this one. -**Cause:** Nextflow before 26.10.0 can lose track of the CPUs and memory it has free ([Lessons learned](lessons-learned.md#the-local-executor-can-lose-track-of-free-memory-and-a-task-that-needs-all-of-it-then-never-starts)). After that it believes less is free than there is, and a task that asks for more than that never starts. The tasks most exposed ask for all, or nearly all, of what the machine has: `run-all.sh` caps a task at the machine's CPU count and RAM, so that is every 8-CPU task on a machine with 8 CPUs or fewer, and DeepVariant and the alignment (32 GB each) on a 32 GB machine. We have seen this in the CI stub runs only, never in a real run. Nextflow 26.10.0, the release `versions.env` pins, fixes the race ([nextflow-io/nextflow#7816](https://github.com/nextflow-io/nextflow/issues/7816)), so this applies when you run an older release. On 26.10.0 or later, a run in this state has another cause: keep the log and open an issue. +**Cause:** Nextflow before 26.10.0 can lose track of the CPUs and memory it has free ([Lessons learned](lessons-learned.md#the-local-executor-can-lose-track-of-free-memory-and-a-task-that-needs-all-of-it-then-never-starts)). After that it believes less is free than there is, and a task that asks for more than that never starts. The tasks most exposed ask for all, or nearly all, of what the machine has: `run-all.sh` caps a task at the machine's CPU count and RAM (the container engine's when it has less), so that is every 8-CPU task on a machine with 8 CPUs or fewer, and DeepVariant and the alignment (32 GB each) on a 32 GB machine. We have seen this in the CI stub runs only, never in a real run. Nextflow 26.10.0, the release `versions.env` pins, fixes the race ([nextflow-io/nextflow#7816](https://github.com/nextflow-io/nextflow/issues/7816)), so this applies when you run an older release. On 26.10.0 or later, a run in this state has another cause: keep the log and open an issue. **Fix:** @@ -931,7 +931,7 @@ HLA typing from WGS data is unreliable in Docker. The two main tools have unreso We have not measured a run on Apple Silicon. Natively on Linux with 8 shards, DeepVariant alone took 8 h 50 min to 13 h 30 min in three measured 30-40x runs ([runtime](hardware-requirements.md#runtime-per-step)); under emulation every step takes longer. **Mitigation strategies:** -1. **Run fewer heavy steps at once.** With `run-all.sh`, bound the total with an `executor` block set to Docker Desktop's CPUs and memory ([a shared host](hardware-requirements.md#a-shared-host)), and pass `THREADS` and `--max_memory` no larger than the VM's (e.g. `THREADS=6 ... --max_memory 14.GB`); `--max_cpus` and `--max_memory` alone cap each task, not the total. +1. **Run fewer heavy steps at once.** `run-all.sh` already books no more than Docker Desktop's CPUs and memory, which it reads from `docker info`. To run fewer at once, bound the total lower with an `executor` block ([a shared host](hardware-requirements.md#a-shared-host)), and pass `THREADS` and `--max_memory` no larger than the block (e.g. `THREADS=6 ... --max_memory 14.GB`); `--max_cpus` and `--max_memory` alone cap each task, not the total. 2. **Lower the step scripts' limits.** For step 3, `THREADS=2 DV_MEM=12g` instead of the default 8 CPUs and 32 GB ([DeepVariant crashes on Mac](#step-3-deepvariant-crashes-on-mac-amd64-emulation)). 3. **Offload the heaviest steps.** Run steps 2, 3, 18, 19 on a remote Linux machine and bring back the outputs. Everything after step 3 needs only the VCF or BAM. 4. **Use a cloud instance.** A Hetzner CCX33 has 8 vCPUs and 32 GB at ~$0.18/hr; plan for more than a day of run time on 8 vCPUs. diff --git a/scripts/run-all.sh b/scripts/run-all.sh index acec2fb02..d52c88a70 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -3,6 +3,8 @@ # Usage: GENOME_DIR=/data ./scripts/run-all.sh [nextflow options] # Options after the sex go to `nextflow run` as given (--sex_check warn), except -bg. --max_cpus (THREADS) and --max_memory # cap each task (default: the host's); they do not limit how much runs at once: Nextflow fills the machine's CPUs and RAM. +# When the container engine has less (Docker Desktop and colima run the containers in a VM), the defaults are the +# engine's, and /nextflow/executor.config makes Nextflow fill only that. # A full run on 8 CPUs can take more than a day: start it inside tmux or screen, or with nohup. A closed terminal stops # it, and -resume then restarts the task that was running (DeepVariant is one task). # Needs Docker, bash 4.4+, Java 17+ and Nextflow (NEXTFLOW_VERSION in versions.env). Results: GENOME_DIR//. @@ -283,9 +285,22 @@ for t in ${T//,/ }; do [[ "${KNOWN} " == *" ${t} "* ]] || { echo "ERROR: unknown arg --cytoband "$(data_file cytoband || true)"; ! GEX=$(data_file gencode_exons) || NF+=(--gencode_exons "$GEX"); arg --delly_exclude "$(data_file delly_exclude || true)"; arg --manta_call_regions "${MANTA_CALL_REGIONS:-}" # Step 20's annotation of the chrM calls (MITO_VARIANTS_ANNOTATE): ClinVar with its index, and gnomAD's chrM sites. MC="${G}/clinvar/clinvar_chr.vcf.gz"; if [ -s "$MC" ] && [ -s "${MC}.tbi" ]; then NF+=(--mito_clinvar "$MC"); fi; arg --gnomad_mito "${G}/gnomad_mito/${GNOMAD_MITO_URL##*/}" -[ -z "${INTERVALS:-}" ] || NF+=(--intervals "$INTERVALS"); [[ " $* " == *" --max_cpus"* ]] || NF+=(--max_cpus "${USER_THREADS:-$(getconf _NPROCESSORS_ONLN)}") +[ -z "${INTERVALS:-}" ] || NF+=(--intervals "$INTERVALS") +# The CPUs and RAM the tasks get: the host's, or the container engine's when docker info reports less. Docker Desktop +# and colima run the containers in a VM, often smaller than the host; Nextflow books the host's RAM by default, and +# a task the VM cannot hold is killed (exit 137). Then the default caps are the engine's, and executor.config sets the +# pool Nextflow fills to it, so a task waits for room instead. A later -c of the user's replaces that pool. +NC=$(getconf _NPROCESSORS_ONLN) EM="" EC="" EXC="${S}/nextflow/executor.config" EX="" M=$(awk '/^MemTotal:/ {print int($2 / 1048576)}' /proc/meminfo 2>/dev/null || echo $(( $(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824 ))) +read -r EM EC _ < <(docker info --format '{{.MemTotal}} {{.NCPU}}' 2>/dev/null) || true +if [[ "$EM" =~ ^[0-9]+$ ]] && [ $((EM / 1073741824)) -ge 1 ] && { [ "${M:-0}" -lt 1 ] || [ $((EM / 1073741824)) -lt "$M" ]; }; then M=$((EM / 1073741824)) EX=1; fi +if [[ "$EC" =~ ^[0-9]+$ ]] && [ "$EC" -ge 1 ] && [ "$EC" -lt "$NC" ]; then NC=$EC EX=1; fi +[[ " $* " == *" --max_cpus"* ]] || NF+=(--max_cpus "${USER_THREADS:-$NC}") [[ " $* " == *" --max_memory"* ]] || [ "${M:-0}" -lt 1 ] || NF+=(--max_memory "${M}.GB") +if [ -n "$EX" ]; then + { printf 'executor {\n cpus = %s\n' "$NC"; [ "${M:-0}" -lt 1 ] || printf " memory = '%s GB'\n" "$M"; printf '}\n'; } > "$EXC" + NF+=(-c "$EXC"); echo "[Resources] the container engine has less than this host: Nextflow books at most ${NC} CPUs and ${M} GB (${EXC})" +else rm -f "$EXC"; fi # Publish by hard link when it is sure to work: the BAM and every other output are then stored once, not twice. # Nextflow does not fall back to a copy when a link fails; the run stops. The tasks run as root (the docker profile) # and their files can stay root's, and Linux lets another user link a file it does not own only when diff --git a/scripts/validate-setup.sh b/scripts/validate-setup.sh index e8c19b6e4..a89a2db76 100755 --- a/scripts/validate-setup.sh +++ b/scripts/validate-setup.sh @@ -362,6 +362,14 @@ if [ "$RAM_KB" -gt 0 ] 2>/dev/null; then else info "Could not detect system RAM" fi +# --- Container engine smaller than the host --- +# Docker Desktop and colima run the containers in a VM. run-all.sh then sizes the run to the VM, by the same rule (whole GB). +ENGINE_GB=0 HOST_GB=0 +[[ "${DOCKER_MEM_BYTES:-}" =~ ^[0-9]+$ ]] && ENGINE_GB=$((DOCKER_MEM_BYTES / 1073741824)) +[[ "$RAM_KB" =~ ^[0-9]+$ ]] && HOST_GB=$((RAM_KB / 1048576)) +if [ "$ENGINE_GB" -ge 1 ] && [ "$HOST_GB" -ge 1 ] && [ "$ENGINE_GB" -lt "$HOST_GB" ]; then + info "Docker has ${ENGINE_GB} GB of this host's ${HOST_GB} GB: run-all.sh sizes the tasks to Docker's ${ENGINE_GB} GB, not the host's" +fi # --- Disk space --- # Check free space in GENOME_DIR if set, otherwise CWD diff --git a/tests/e2e/run-all-launcher.sh b/tests/e2e/run-all-launcher.sh index d59818c0d..d9915896a 100644 --- a/tests/e2e/run-all-launcher.sh +++ b/tests/e2e/run-all-launcher.sh @@ -21,6 +21,11 @@ # the BAM is one file with two names (work/ and aligned/), and no published # file is a symbolic link the direct (copy) run lacks. The second run has # the runner's own setting back (1): run-all.sh passes no link mode. +# 6. A container engine with less memory and fewer CPUs than the host, as +# Docker Desktop and colima have (a docker shim that reports 8 GB and +# 2 CPUs, and a stand-in nextflow that records its arguments): run-all.sh +# passes the engine's as --max_memory and --max_cpus, and the pool +# Nextflow fills, executor.config, is the engine's. # Each comparison gets a negative control in the same run: a planted missing # file must be reported, the first run's trace must show the alignment and # calling tasks, and the first run's trace (tasks COMPLETED) must fail the @@ -77,6 +82,63 @@ published() { | grep -vE "^(fastq|nextflow|logs|vep)/|^run_manifest\.tsv$|^summary\.json$|^${P}_report\.(html|txt)$|^${P}_delta\.(html|txt|json)$|^history/" } +# --- 6. a container engine smaller than the host -------------------------------------------- +# Its own GENOME_DIR with an empty VCF: the stand-in nextflow runs nothing and exits 1, so +# run-all.sh then writes the reports of nothing. The negative control is an engine as big as +# the host, as on native Linux: the host's caps, and no executor.config passed or left behind. +ENG="${CASE_TMP}/engine" G3="${E2E_WORK}/genome-engine" E=ENGINE1 +rm -rf "$ENG" "$G3"; mkdir -p "${ENG}/bin" "${G3}/reference" "${G3}/${E}/vcf" +: > "${G3}/${E}/vcf/${E}.vcf.gz"; : > "${G3}/${E}/vcf/${E}.vcf.gz.tbi" +cat > "${ENG}/bin/docker" <<'SH' +#!/usr/bin/env bash +# docker info prints ENGINE_INFO; anything else goes to the next docker on PATH. +if [ "${1:-}" = info ]; then echo "$ENGINE_INFO"; exit 0; fi +self=$(cd "$(dirname "$0")" && pwd) +IFS=: read -r -a dirs <<< "$PATH" +for d in "${dirs[@]}"; do if [ "$d" != "$self" ] && [ -x "${d}/docker" ]; then exec "${d}/docker" "$@"; fi; done +echo "docker shim: no real docker on PATH" >&2; exit 127 +SH +cat > "${ENG}/bin/nextflow" <<'SH' +#!/usr/bin/env bash +# Records its arguments on one line and a copy of the executor.config it was given; runs nothing. +echo " $* " > "${ENGINE_REC}.args" +prev=""; for a in "$@"; do if [ "$prev" = -c ] && [ "${a##*/}" = executor.config ]; then cp "$a" "${ENGINE_REC}.config"; fi; prev=$a; done +exit 1 +SH +chmod +x "${ENG}/bin/docker" "${ENG}/bin/nextflow" +# The host's CPUs and RAM in whole GB, as run-all.sh reads them; the small engine has 8 GB (less on a +# host with 8 GB or less) and 2 CPUs (1 on a host with 2 or fewer). +HC=$(getconf _NPROCESSORS_ONLN) +HM=$(awk '/^MemTotal:/ {print int($2 / 1048576)}' /proc/meminfo 2>/dev/null || echo $(( $(sysctl -n hw.memsize) / 1073741824 ))) +EG=$(( HM > 8 ? 8 : HM / 2 )) ECPU=$(( HC > 2 ? 2 : 1 )) +engine_run() { # engine_run NAME 'MEMTOTAL NCPU': run-all.sh with docker info answering that; its log is logs/.engine-NAME.log + rm -f "${ENG}/${1}".* + echo "+ scripts/run-all.sh ${E} female, docker info reporting '${2}' (${1})" + env -u THREADS PATH="${ENG}/bin:${PATH}" ENGINE_INFO="$2" ENGINE_REC="${ENG}/${1}" GENOME_DIR="$G3" SKIP_VALIDATION=true \ + TOOLS=roh EH_CATALOG="${G3}/none.json" "${REPO}/scripts/run-all.sh" "$E" female > "${E2E_WORK}/logs/${CASE_NAME}.engine-${1}.log" 2>&1 + echo "+ exit $? (the stand-in nextflow exits 1)" +} +EXC="${G3}/${E}/nextflow/executor.config" +engine_run small "$((EG * 1073741824)) ${ECPU}" +SMALL=$(cat "${ENG}/small.args" 2>/dev/null) +echo "+ nextflow${SMALL}" +check "the stand-in nextflow ran (small engine)" test -n "$SMALL" +check "nextflow got --max_memory with the engine's ${EG} GB, not the host's ${HM} GB" has " --max_memory ${EG}\.GB " "$SMALL" +check "nextflow got --max_cpus with the engine's ${ECPU} CPUs, not the host's ${HC}" has " --max_cpus ${ECPU} " "$SMALL" +check "nextflow got -c ${EXC}" has " -c ${EXC} " "$SMALL" +check "executor.config sets the pool's memory to ${EG} GB" has "^ *memory = '${EG} GB'\$" "$(cat "${ENG}/small.config" 2>/dev/null)" +check "executor.config sets the pool's CPUs to ${ECPU}" has "^ *cpus += ${ECPU}\$" "$(cat "${ENG}/small.config" 2>/dev/null)" +check "run-all.sh says it books the engine's size" has '^\[Resources\] the container engine has less than this host' \ + "$(cat "${E2E_WORK}/logs/${CASE_NAME}.engine-small.log" 2>/dev/null)" +engine_run big "$(( (HM + 4) * 1073741824 )) $((HC + 4))" +BIG=$(cat "${ENG}/big.args" 2>/dev/null) +echo "+ nextflow${BIG}" +check "negative control: an engine as big as the host gives the host's --max_memory ${HM}.GB" has " --max_memory ${HM}\.GB " "$BIG" +check "negative control: and the host's --max_cpus ${HC}" has " --max_cpus ${HC} " "$BIG" +check "negative control: and no executor.config" lacks "executor\.config" "$BIG" +check "negative control: the small engine's executor.config is not left behind" test ! -e "$EXC" +rm -rf "$G3" "$ENG" + # --- first run, with fs.protected_hardlinks 0 --------------------------------------------- PH=$(cat /proc/sys/fs/protected_hardlinks 2>/dev/null) restore_ph() { [ -z "$PH" ] || sudo -n sysctl -q -w "fs.protected_hardlinks=${PH}" > /dev/null 2>&1; } From b876980458c70bc94c7c3c0ce45c4ad60860a4ea Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:07:00 +0100 Subject: [PATCH 169/255] test(benchmark): a 0/1 against 1/1 site is a genotype mismatch, and Clair3 is compared Bead: plat-71 --- tests/fake-docker/run-all-switches.sh | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/fake-docker/run-all-switches.sh b/tests/fake-docker/run-all-switches.sh index a10340cd9..baea851cb 100644 --- a/tests/fake-docker/run-all-switches.sh +++ b/tests/fake-docker/run-all-switches.sh @@ -6,6 +6,9 @@ # EXTRA_CALLERS stop it with exit 2 before nextflow. # plain run the host's CPU count and RAM as --max_cpus and --max_memory; # BENCHMARK=true with one caller VCF is skipped, not failed +# benchmark BENCHMARK=true counts the shared sites whose genotypes +# agree (0/1 against 1/1 is a mismatch), and compares Clair3 +# and any other vcf_*/ VCF run-all.sh counts # ALIGN_DIR run its BAM replaces the plain run's samplesheet row # plain again back to aligned/: a kept row names this call's BAM only # publish mode --publish_dir_mode link for root (a fake `id`) when a @@ -99,6 +102,48 @@ if compgen -G "${G}/.pgp-link-probe*" > /dev/null || compgen -G "${G}/sample1/.p fail "the link probe left files behind" fi +# BENCHMARK=true compares genotypes, and finds every caller run-all.sh counts. +# The isec hook writes two callers that agree on both alleles but not on one +# genotype (0/1 against 1/1): the Jaccard index is 1, the genotypes agree on +# one site of two. +cat > "${CASE_WORK}/isec-hook" <<'HOOK' +#!/usr/bin/env bash +set -euo pipefail +. "${CASE_WORK:?}/host-path.sh" +w=("$@") +for ((i = 0; i < ${#w[@]} - 1; i++)); do [ "${w[i]}" != -p ] || d=$(host_path "${w[i + 1]}"); done +[ -n "${d:-}" ] && [[ " $* " == *" bcftools isec "* ]] || exec "${CASE_WORK}/hook-outputs" "$@" +mkdir -p "$d" +h=$'##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS' +printf '%s\n' "$h" > "${d}/0000.vcf" +printf '%s\n' "$h" > "${d}/0001.vcf" +printf '%s\nchr1\t10\t.\tA\tG\t50\tPASS\t.\tGT:DP\t0/1:30\nchr1\t20\t.\tC\tT\t50\tPASS\t.\tGT\t1|0\n' "$h" > "${d}/0002.vcf" +printf '%s\nchr1\t10\t.\tA\tG\t50\tPASS\t.\tDP:GT\t30:1/1\nchr1\t20\t.\tC\tT\t50\tPASS\t.\tGT\t0/1\n' "$h" > "${d}/0003.vcf" +HOOK +chmod +x "${CASE_WORK}/isec-hook" +callers() { + rm -rf "${G}/sample1"/vcf_* "${G}/sample1/benchmark" + local d + for d in "$@"; do + mkdir -p "${G}/sample1/${d}" + printf 'placeholder\n' > "${G}/sample1/${d}/sample1.vcf.gz" + printf 'placeholder\n' > "${G}/sample1/${d}/sample1.vcf.gz.tbi" + done +} +cmp_row() { sed -n 2p "${G}/sample1/benchmark/comparison.tsv" 2>/dev/null; } +callers vcf vcf_gatk +run_expect 0 bench-gt env TOOLS=pharmcat BENCHMARK=true FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/isec-hook" "${SCRIPTS}/run-all.sh" sample1 male +[ "$(cmp_row)" = $'DeepVariant\tGATK\t2\t0\t0\t1.0000\t1\t0.5000' ] \ + || fail "0/1 against 1/1 is not a genotype mismatch in comparison.tsv: $(cmp_row)" +# Clair3 (step 03e) and a VCF in a directory of one's own: run-all.sh queues +# the step for them, and the step compares them instead of exiting 1. +callers vcf vcf_clair3 vcf_mine +run_expect 0 bench-clair3 env TOOLS=pharmcat BENCHMARK=true FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/isec-hook" "${SCRIPTS}/run-all.sh" sample1 male +grep -qE $'^DeepVariant\tClair3\t2\t0\t0\t1.0000\t1\t0.5000$' "${G}/sample1/benchmark/comparison.tsv" \ + || fail "DeepVariant and Clair3 were not compared: $(cat "${G}/sample1/benchmark/comparison.tsv" 2>/dev/null)" +grep -qE $'^Clair3\tmine\t' "${G}/sample1/benchmark/comparison.tsv" || fail "vcf_mine/ was not compared" +rm -rf "${G}/sample1"/vcf_* "${G}/sample1/benchmark" "${G}/sample1/vcf/sample1.vcf.gz" "${G}/sample1/vcf/sample1.vcf.gz.tbi" + n=$(grep -c '^nextflow :: ' "$FAKE_DOCKER_LOG") run_expect 2 bg env TOOLS=pharmcat "${SCRIPTS}/run-all.sh" sample1 male -bg output_has bg "ERROR: run-all.sh does not take -bg" From 8e31cdcd4a74cacbdcfb6df197cfe7c5302ede6b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:07:02 +0100 Subject: [PATCH 170/255] docs(results): move each report chapter of the results guide to its own page docs/interpreting-results.md held every chapter in one 883-line file. Each chapter now has a page under docs/results/, moved word for word. The overview keeps the Before You Look bullets, the Before You Panic table, the example outputs and the general advice, and lists the chapter pages. Every old #anchor on the overview lands on a link to the section's new place. The root .gitignore drops every results/ folder (Nextflow output), so the seven pages are added with -f; once tracked, git follows their edits. The repeat table moved with its chapter, so tests/test_str_ranges.py reads it from docs/results/structural-and-repeats.md. --- bin/str_intermediate_ranges.tsv | 2 +- docs/interpreting-results.md | 698 ++----------------------- docs/results/ancestry-and-lineage.md | 104 ++++ docs/results/clinvar.md | 234 +++++++++ docs/results/pharmacogenomics.md | 40 ++ docs/results/quality.md | 17 + docs/results/secondary-findings.md | 60 +++ docs/results/structural-and-repeats.md | 180 +++++++ docs/results/traits-and-somatic.md | 52 ++ tests/test_str_ranges.py | 15 +- zensical.toml | 11 +- 11 files changed, 747 insertions(+), 666 deletions(-) create mode 100644 docs/results/ancestry-and-lineage.md create mode 100644 docs/results/clinvar.md create mode 100644 docs/results/pharmacogenomics.md create mode 100644 docs/results/quality.md create mode 100644 docs/results/secondary-findings.md create mode 100644 docs/results/structural-and-repeats.md create mode 100644 docs/results/traits-and-somatic.md diff --git a/bin/str_intermediate_ranges.tsv b/bin/str_intermediate_ranges.tsv index f89587b18..fa834b5f0 100644 --- a/bin/str_intermediate_ranges.tsv +++ b/bin/str_intermediate_ranges.tsv @@ -5,7 +5,7 @@ # next_min: the first count of the next class, named in next_class; the range ends right below it. # label: what the report prints beside a count in the range. citation: the GeneReviews chapter that defines the range. # Only loci whose GeneReviews chapter defines such a range are listed. tests/test_str_ranges.py checks the rows -# and that docs/interpreting-results.md prints the same ranges. +# and that docs/results/structural-and-repeats.md prints the same ranges. locus intermediate_min intermediate_max next_min next_class label citation HTT 27 35 36 reduced penetrance intermediate (not disease-causing; may expand in offspring) GeneReviews: Huntington Disease, https://www.ncbi.nlm.nih.gov/books/NBK1305/ FMR1 45 54 55 premutation intermediate (gray zone; not disease-causing; may expand in offspring) GeneReviews: FMR1 Disorders, https://www.ncbi.nlm.nih.gov/books/NBK1384/ diff --git a/docs/interpreting-results.md b/docs/interpreting-results.md index 576e963c2..b236ba59a 100644 --- a/docs/interpreting-results.md +++ b/docs/interpreting-results.md @@ -28,669 +28,53 @@ If this is your first time looking at your own genomic data, the numbers can be | Heteroplasmic mitochondrial variants | 20-40 | Low-level heteroplasmy (<5%) is universal and age-related | | VUS (Variants of Uncertain Significance) | 20-200+ | "Uncertain" means **not enough data yet**, not "probably bad" | -### The VUS Trap - -The single biggest source of unnecessary anxiety in personal genomics is **VUS — Variants of Uncertain Significance**. These are variants where: - -- There is not enough scientific evidence to classify them as either pathogenic or benign -- The vast majority will eventually be reclassified as **benign** as more data accumulates -- They are **not actionable** — no clinical decision should be made based on a VUS -- CPSR may report dozens or hundreds of VUS. This is normal and expected. - -**Rule of thumb:** If a variant is classified as VUS, treat it the same as if it were not tested. Do not change screening or management based on a VUS. Check back in 1-2 years when ClinVar may have reclassified it. - -**One nuance:** If you have a strong family history of a condition AND a VUS appears in the relevant high-penetrance gene (e.g., BRCA1/2, TP53, MLH1/MSH2), it may be worth mentioning to a genetic counselor — not to act on the VUS, but because the family history itself may warrant enhanced screening regardless of the variant's classification. - -### ClinVar Star Ratings - -Not all ClinVar classifications are equally reliable. Each entry has a **review status** indicated by stars: - -| Stars | Review Status | Reliability | -|---|---|---| -| 0 | No assertion criteria | Low — submitter did not explain their reasoning | -| 1 | Single submitter, criteria provided | Moderate — one lab's interpretation | -| 2 | Two or more submitters, no conflict | Good — multiple labs agree | -| 3 | Expert panel reviewed | High — reviewed by specialists | -| 4 | Practice guideline | Highest — established clinical standard | - -**Focus on 2+ star entries.** Single-submitter (1-star) pathogenic calls are sometimes reclassified. If you find a scary-looking pathogenic variant with 0-1 stars, check the ClinVar entry directly at [ncbi.nlm.nih.gov/clinvar](https://www.ncbi.nlm.nih.gov/clinvar/) — look at the "Review status" and "Last evaluated" date. - -### Carrier Status Is Not Disease - -The most common "pathogenic" finding in any genome is **heterozygous carrier status for recessive conditions**. This means: - -- You have ONE copy of a variant that causes disease when BOTH copies are affected -- For most recessive conditions, heterozygous carriers are **not clinically affected** (though some carrier states confer subtle phenotypic effects — e.g., sickle cell trait, HFE carriers and iron loading) -- The primary relevance is for **family planning**: if your partner carries the same gene, each child has a 25% chance of being affected -- **Note — MUTYH**: Biallelic (homozygous or compound het) MUTYH carriers have a well-established high colorectal cancer risk. For **monoallelic** (single-copy) carriers, the evidence is more nuanced: some meta-analyses show a modest risk increase, but a counseling framework for moderate-penetrance CRC genes (Genetics in Medicine) notes that risk estimates for monoallelic MUTYH are conflicting and that screening recommendations (e.g., earlier colonoscopy) were historically tied to carriers with a CRC family history. Discuss with a genetic counselor, especially if you have a family history of CRC -- Examples: GJB2 (hearing loss), CFTR (cystic fibrosis), HFE (hemochromatosis) - -**Your own carrier list (step 24 report).** The ClinVar card of both reports is followed by a carrier list for this one sample, with no partner needed: every step 6 ClinVar hit in the 3,292 recessive and X-linked genes of `assets/carrier_genes.tsv` (the list of the [couple screen](39-couple.md#the-carrier-list)). An allele that list marks as a hypomorph (for example BTD p.Asp444His or the GALT Duarte variants) or a pseudodeficiency allele (HEXA c.739C>T and c.745C>T, which lower the enzyme test result but cause no Tay-Sachs disease) is listed as set aside, never as a carrier finding on its own. - -**Secondary findings follow each gene's ACMG rule.** The secondary-findings card (step 23's section of the report) applies the reporting rule of each of the 84 ACMG SF v3.3 genes (Lee et al., Genet Med 2025;27(8):101454, Table 1), written gene by gene in `assets/acmg_sf.tsv`: - -- a dominant gene (BRCA1, LDLR, MYH7 and most others): one pathogenic or likely pathogenic variant is reported; -- a recessive gene (MUTYH, ATP7B, BTD, CYP27A1, GAA, RPE65, CASQ2, TRDN): two variants are needed, a homozygote or two different ones even when their phase is unknown. One heterozygous MUTYH variant is listed apart as not reported, with the reason; -- HFE: only homozygous p.Cys282Tyr. A heterozygous p.Cys282Tyr or a homozygous p.His63Asp is not reported; -- X-linked: GLA (Fabry disease) and OTC are reported in a hemizygous male and a heterozygous female alike. ABCD1 (X-linked adrenoleukodystrophy) is reported hemizygous, homozygous or with two heterozygous variants, never for one heterozygous variant; -- BTD with p.Asp444His: reported as partial, not profound, biotinidase deficiency; -- AnnotSV class 4 and 5 structural variants (step 5) in BRCA1, BRCA2, MLH1, MSH2 and LDLR reach the same card, and so does a deletion of the 3' end of EPCAM (exons 8 and 9), which silences MSH2 (Kovacs et al., Hum Mutat 2009;30(2):197-203). - -**Two hits across callers: a deletion plus a variant on the other copy.** Small-variant callers see one hit when the other copy of the gene is deleted. So the carrier list also joins the deletions the other engines call with the ClinVar hits, gene by gene (`og summary collect`, `two_hits`). The deletions come from AnnotSV's class 4 and 5 deletions (step 5), step 35's interval table (the recurrent regions of `bin/cn_loci.tsv`, such as 1q21.1), its HBA1/HBA2 count and Parascopy's SMN1 count. A recessive gene that a deletion and a hit both reach gets one line: - -- **biallelic** when the reads show the two hits on different copies. Either the hit is called homozygous inside the deletion, so it sits on the one copy left (the line and the carrier list call it hemizygous), or both calls carry read-backed phase (a `|` genotype with the same phase set, PS) on different copies; -- **possibly biallelic** otherwise: the phase is unknown. Most short-read calls carry no phase, so this is the usual line. Step 05's checked table keeps no genotype column yet, and step 35's counts have no phase, so in a pipeline run a deletion pairs as possibly biallelic unless the hit is called homozygous inside it. A parent's genome (step 38, [family analysis](multi-sample.md#family-analysis-step-38-opt-in)) or long reads can settle it; -- no line when read-backed phase puts both on the same copy (cis): the other copy is intact. - -Examples: an exon deletion of MUTYH plus a pathogenic MUTYH variant; three alpha-globin genes plus an HBA2 point variant; one SMN1 copy plus an SMN1 variant (short reads place few SMN1 variants reliably, so check with a targeted test); and the 1q21.1 deletion plus one of the two RBM8A low-expression alleles (c.-21G>A, rs139428292, or c.67+32G>C, rs201779890), the cause of TAR syndrome (thrombocytopenia with absent radius; Albers et al., Nat Genet 2012;44(4):435-439, PMID 22366785). On its own an RBM8A low-expression allele stays set aside. The [couple screen](39-couple.md) (step 39) applies the same pairing between partners: a hypomorph allele of the list in one partner and a full loss of function in the same gene in the other (a ClinVar pathogenic variant, a Delly deletion) is a shared gene with its risk for a child; the allele in both partners, or in one alone, stays set aside. In the secondary-findings card a called deletion counts as one of the two variants a recessive gene needs, unless read-backed phase puts it on the variant's copy. The line under the carrier list names the deletion callers it read and the ones it could not read. A deletion AnnotSV ranks below class 4 is not counted. - -The TTN rule (truncating variants only) and the LMNA rule (missense only with cardiac evidence) are printed beside the finding, not applied. It is still a list to review with a clinician. - --- ## Start Here: The Three Most Important Outputs -### 1. ClinVar Screen (Step 6) - -**What it tells you:** Known pathogenic variants in your genome, as classified by ClinVar (NCBI's public database of clinically significant variants). - -**Where to look:** `${SAMPLE}/clinvar/` - -**How to read it:** -- Each line in the output VCF is a variant in your genome that matches a known ClinVar entry -- Step 6 intersects against the **pathogenic-only** ClinVar subset (Pathogenic + Likely_pathogenic). Every hit in this output is at a position ClinVar classifies as disease-associated — benign/VUS entries are excluded at the database level -- The `CLNSIG` field confirms the classification - -**What to expect:** -- 0-10 pathogenic/likely pathogenic hits is typical for a 30X WGS -- Most pathogenic hits are **carrier status** (heterozygous) for recessive conditions — you carry one copy but aren't affected -- A heterozygous pathogenic variant in a recessive gene (like GJB2 for hearing loss) means you're a **carrier**, not affected -- A homozygous pathogenic variant, or a heterozygous variant in a dominant gene, needs attention - -**When to worry:** -- Pathogenic variant in a **dominant** gene (one copy is enough to cause disease) -- **Two** pathogenic variants in the same recessive gene (one from each parent) -- Any variant in cancer predisposition genes (BRCA1, BRCA2, MLH1, MSH2, etc.) - -### 2. PharmCAT Report (Step 7) - -**What it tells you:** How your genes affect drug metabolism. These results are clinically relevant and should be shared with your prescribing physician. - -**Where to look:** `${SAMPLE}/pharmcat/` (Nextflow) or `${SAMPLE}/vcf/` (bash scripts) — PharmCAT writes its reports there. Open the HTML report in a browser. - -**Key genes to check:** - -| Gene | Affects | Common Impact | -|---|---|---| -| CYP2C19 | PPIs, clopidogrel, SSRIs, voriconazole | Rapid metabolizers burn through drugs too fast | -| CYP2D6 | Codeine, tramadol, tamoxifen, many psych meds | Poor metabolizers get toxic buildup | -| CYP2C9 | Warfarin, NSAIDs, phenytoin | Dose adjustment needed | -| DPYD | 5-fluorouracil (cancer drug) | Poor metabolizers can die from standard doses | -| SLCO1B1 | Statins (simvastatin, atorvastatin) | Increased myopathy risk | -| NAT2 | Isoniazid (TB), hydralazine | Slow acetylators have more side effects | -| UGT1A1 | Irinotecan, atazanavir | Two \*28 alleles are associated with Gilbert syndrome (elevated bilirubin) | - -**What to do:** Share the PharmCAT report with your prescribing physician or pharmacist. PharmCAT is a research tool — its authors explicitly note that missing positions, unphased input, and undetected structural variation (especially CYP2D6) can affect genotype and phenotype calls. The report is a valuable starting point for pharmacogenomic-guided prescribing, but clinical confirmation may be warranted before making medication changes, especially for high-risk drugs (DPYD, CYP2D6-dependent opioids). - -### 3. CPSR Report (Step 17) - -**What it tells you:** Cancer predisposition screening using CPSR's curated cancer gene panels (panel 0 covers 500+ genes). Step 17 also turns on CPSR's secondary findings: the ACMG SF list, which includes cardiac and metabolic genes outside cancer (see [Before you look](#before-you-look-what-the-pipeline-can-tell-you)). - -**Where to look:** `${SAMPLE}/cpsr/` — open the HTML report in a browser. - -**How to read it:** -- CPSR puts every variant in its genes into one of the five ACMG/AMP classes: **Pathogenic**, **Likely pathogenic**, **VUS** (uncertain significance), **Likely benign** and **Benign**. A variant that already has a ClinVar entry can carry ClinVar's class instead of the one CPSR computed. -- Only **Pathogenic** and **Likely pathogenic** call for clinical attention. The report lists them first, and the secondary findings in their own section. - ---- - -## Structural Variants (Steps 4, 5, 15, 18, 19, 22) - -### What Are Structural Variants? - -Unlike SNPs (single letter changes), structural variants are large rearrangements: -- **Deletions (DEL):** A chunk of DNA is missing -- **Duplications (DUP):** A chunk is copied extra times -- **Inversions (INV):** A chunk is flipped backwards -- **Translocations (BND):** A chunk moved to a different chromosome -- **Insertions (INS):** New DNA inserted - -### A Note About BND (Breakend) Calls - -If you run Manta or Delly, you will see many **BND** calls — often hundreds or thousands. BND indicates a "breakend" where one end of a read pair maps to a different chromosome or a distant location. This sounds alarming ("translocation!") but: - -- **Most BND calls are artifacts** of repetitive regions, segmental duplications, or mobile elements -- A typical genome has 1,000-3,000 BND calls from Manta and 5,000+ from Delly -- **Fewer than 5 are likely real** inter-chromosomal translocations in a healthy genome -- BND calls require **multi-caller support** (called by both Manta and Delly at overlapping breakpoints) to be considered credible -- Unless a BND disrupts a known disease gene AND is confirmed by a second caller, it can be safely ignored - -### How Many Is Normal? - -A typical human genome has: -- ~5,000-10,000 structural variants total -- Most are in non-coding regions and harmless -- ~5-20 may affect genes -- 0-2 may be clinically significant - -### Which Callers to Trust? - -If you ran multiple SV callers: -- **Called by 2+ callers (Manta + Delly, or Manta + CNVpytor):** Lower false-positive rate -- **Called by 1 caller only:** Lower confidence, may be false positive -- **duphold DHFFC < 0.7 for deletions:** High confidence (depth drops as expected) -- **duphold DHBFC > 1.3 for duplications:** High confidence (depth rises as expected) - -### SV Consensus (Step 22) - -`${SAMPLE}/sv_merged/${SAMPLE}_sv_consensus.vcf.gz` keeps the SVs that two or more callers found (Manta, Delly, CNVpytor and, when they ran, Sniffles2 and TIDDIT): SURVIVOR pairs two calls when both breakpoints lie within 1 kb of each other and their type and strands agree. `SUPP` says how many callers found each one and `SUPP_VEC` which. GRIDSS is left out (its breakend records never match). It is the short list to read first, but it drops real SVs that only one caller found; see [step 22](22-survivor-merge.md#limitations). - -### AnnotSV Output - -The AnnotSV TSV (step 5) adds clinical annotations to each SV: -- `ACMG_class`: 1 (benign) to 5 (pathogenic) -- `Overlapped_CDS_percent`: How much of a gene is affected -- Focus on SVs with `ACMG_class` 4 or 5 that overlap known disease genes - ---- - -## STR Expansions (Step 9) - -### What Are Repeat Expansions? - -Some regions of DNA have short sequences repeated many times (e.g., CAG CAG CAG...). When the number of repeats exceeds a threshold, it can cause disease. - -### How to Read ExpansionHunter Output - -The output VCF lists each tested locus with the number of repeats found. Key loci: - -| Locus | Gene | Normal | Intermediate / Premutation | Pathogenic | Disease | -|---|---|---|---|---|---| -| HTT | HTT | <=26 | 27-35 (mutable normal); 36-39 (reduced penetrance) | >=40 (full penetrance) | Huntington's disease | -| FMR1 | FMR1 | <45 | 45-54 (intermediate); 55-200 (premutation) | >200 | Fragile X syndrome | -| ATXN1 | ATXN1 | <33 | — | >39 | Spinocerebellar ataxia 1 | -| C9orf72 | C9orf72 | <24 | 24-30 (gray zone, lab cutoffs vary) | Typically hundreds-thousands | ALS / Frontotemporal dementia | -| DMPK | DMPK | <35 | 35-49 (premutation) | >=50 | Myotonic dystrophy type 1 | - -**HTT 36-39 repeats (reduced penetrance):** Individuals in this range may or may not develop Huntington's disease. The risk increases with repeat length but is not certain. GeneReviews classifies >=40 as full penetrance and 36-39 as reduced penetrance. Alleles of 27-35 ("mutable normal") do not cause disease but may expand in offspring. - -**C9orf72 gray zone:** The exact pathogenic threshold for C9orf72 is not established. Laboratory cutoffs are discordant (JNNP 2021 review). Clearly pathogenic expansions are typically hundreds to thousands of repeats. Short-read WGS has limited ability to size very large expansions accurately. - -**FMR1 intermediate zone (45-54 repeats):** Not affected, but repeats may expand in offspring. Carriers should receive genetic counseling. Premutation (55-200) carries risk of FXTAS (males >50) and FXPOI. - -The report prints no all-clear verdict for these loci. It lists every locus in the ExpansionHunter output, the five above first, with its repeat count, Stranger's status and the catalog's normal maximum (`STR_NORMAL_MAX`), and, when step 9b (Stranger) ran, every locus Stranger marks `pre_mutation` or `full_mutation`; without step 9b, compare the counts with the table yourself. It also says: - -- **Not sized.** A locus whose call rests on flanking reads only prints "size not determined (flanking reads only)"; one with in-repeat reads only (longer than a read) prints its count as at least the lower end of ExpansionHunter's interval; one with fewer than 2 spanning reads, or an interval that reaches the catalog's pathogenic minimum, is marked low confidence. Such a locus is listed apart and never shown as normal, because an unsized repeat can hide an expansion. -- **Intermediate range.** A sized count inside the HTT, FMR1 or DMPK intermediate range of the table above prints the range, a note that it does not cause disease but may expand in offspring, and the GeneReviews chapter. The report reads these ranges from `bin/str_intermediate_ranges.tsv`, and `tests/test_str_ranges.py` checks that the table above prints the same ones. -- **RFC1.** A `full_mutation` flag at RFC1 carries the motif verdict of step 9b ([RFC1 note](09b-stranger.md#notes)). - -A count under every threshold is not a guarantee: repeat-expansion calls from short reads can be wrong at some loci, and a long expansion cannot be sized. Intermediate-range results should be discussed with a genetic counselor. - -### Stranger Annotation (Step 9b) - -Step 9b adds a `STR_STATUS` field to each locus in the VCF so you do not need to look up thresholds manually: - -| STR_STATUS | Meaning | -|---|---| -| `normal` | Repeat count is within the established normal range | -| `pre_mutation` | Elevated repeat count; not currently disease-causing but carries risk of expansion in offspring or late-onset carrier effects (e.g. FXTAS, FXPOI for FMR1) | -| `full_mutation` | Repeat count exceeds the pathogenic threshold for this locus | - -The `Disease`, `InheritanceMode`, `SourceDisplay`, `STR_NORMAL_MAX`, and `STR_PATHOLOGIC_MIN` INFO fields give the clinical context for each locus (a field the catalog has no value for prints as `.`). Use a VCF viewer or `bcftools query` to extract them: - -```bash -bcftools query -f '%INFO/STR_STATUS\t%INFO/Disease\t%INFO/InheritanceMode\t%INFO/SourceDisplay\t%INFO/STR_NORMAL_MAX\t%INFO/STR_PATHOLOGIC_MIN\n' \ - expansion_hunter/_eh_stranger.vcf -``` - -`full_mutation` at any locus warrants follow-up with a clinical geneticist. Short-read WGS has limited sizing accuracy for very large expansions (>150 repeats), so confirmatory testing may be recommended. - ---- - -## Telomere Length (Step 10) - -### What It Means - -Telomeres are protective caps at the ends of chromosomes that shorten with cell division. TelomereHunter measures `tel_content`: intratelomeric reads per million reads with 48-52% GC. It is a relative telomere content, not a telomere length, and the report labels it "Telomere content (relative)". - -### How to Interpret - -- **Higher `tel_content` = more telomeric reads** (generally correlates with longer telomeres) -- **Lower `tel_content` = fewer telomeric reads** (generally correlates with shorter telomeres) -- There is no universal "normal" range — compare between samples of similar age, sequenced on the same platform -- Typical `tel_content` for 30X WGS: 300-800 (varies by sequencing platform and coverage) - -### Limitations - -- **Not a clinical telomere length measurement.** TelomereHunter was developed for cancer genome analysis, not as a validated healthy-population aging assay -- **Not a "biological age" readout.** While telomere length correlates with aging at a population level, it is a rough estimate of aging rate and is not established as a clinically important standalone risk marker for individuals (see Vaiserman & Krasnienkov, "Telomere Length as a Marker of Biological Age," 2021) -- Short-read WGS systematically underestimates telomere length compared to dedicated assays (TRF, FlowFISH) -- Useful only for **relative comparisons** between samples run on the same platform — not absolute measurements and not individual health predictions +1. [ClinVar Screen (Step 6)](results/clinvar.md#clinvar-screen-step-6){ #1-clinvar-screen-step-6 } +2. [PharmCAT Report (Step 7)](results/pharmacogenomics.md#pharmcat-report-step-7){ #2-pharmcat-report-step-7 } +3. [CPSR Report (Step 17)](results/secondary-findings.md#cpsr-report-step-17){ #3-cpsr-report-step-17 } --- -## ROH Analysis (Step 11) - -### What It Means - -Runs of Homozygosity (ROH) are long stretches where both copies of your DNA are identical. Everyone has some ROH, but extensive ROH can indicate: -- Parental relatedness (consanguinity) -- Uniparental disomy -- Population bottleneck effects - -### How to Read - -Add up the autosomal segments of 5 Mb or more and compare the total with the table in [step 11](11-roh-analysis.md#total-roh-and-parental-relationship), which gives the expected total for each parental relationship (about 45 Mb for second cousins, 180 Mb for first cousins). In short: - -- **No segments of 5 Mb or more, or only a few:** no sign that your parents are related -- **Many small segments (1-5 Mb):** population-level background, typical of population isolates (Ashkenazi, Finnish, etc.) -- **One chromosome carrying most of the long runs:** possible uniparental disomy rather than related parents. The reports flag a chromosome when its segments of 5 Mb or more add up to at least 20 Mb and to more than half of all such autosomal segments ([step 11](11-roh-analysis.md#one-chromosome-carrying-most-of-the-runs-possible-uniparental-disomy)) - -### Centromeric Artifacts - -Some apparent ROH near centromeres are artifacts of low-coverage sequencing in repetitive regions. These can be ignored. - ---- - -## Mitochondrial Haplogroup (Step 12) - -### What It Means - -Your mitochondrial haplogroup traces your maternal ancestry lineage. It's determined by the specific set of variants in your mitochondrial DNA (inherited only from your mother). - -### Common European Haplogroups - -| Haplogroup | Origin | Notes | -|---|---|---| -| H | Western Europe | Most common in Europe (~40%) | -| U | Northern/Eastern Europe | Second most common (~15%) | -| T | Near East / Mediterranean | ~10% of Europeans | -| K | Near East | ~6% of Europeans, Ashkenazi ~30% | -| J | Near East | ~8% of Europeans | -| V | Iberian Peninsula / Scandinavia | ~4% | -| I | Near East / Europe | ~3% | - -### Medical Relevance - -Mitochondrial haplogroups have weak associations with some diseases (Parkinson's, diabetes, longevity), but these are population-level statistics, not individual predictions. The main clinical value is in step 20 (GATK Mutect2 mitochondrial mode), which detects disease-causing mitochondrial variants and heteroplasmy. - -### Exploring Your Branch - -Both reports link your haplogroup to FamilyTreeDNA Discover (free, no upload) and to YFull's tree. Discover shows the branch's age, where its testers live and the ancient DNA samples on it. For the Y haplogroup of step 37 the reports add the same two links. The free and paid routes, what each one takes and what an upload shares: [step 12](12-mito-haplogroup.md#explore-your-branch-further) and [step 37](37-y-haplogroup.md#explore-your-branch-further). The pipeline never uploads anything. - ---- - -## VEP Annotation (Step 13) - -### What It Adds - -VEP annotates every variant with: -- **Consequence type:** missense, nonsense, synonymous, splice site, etc. -- **SIFT score:** Predicts if amino acid change is tolerated (>0.05) or damaging (<0.05) -- **PolyPhen score:** Predicts if change is benign (<0.15), possibly damaging (0.15-0.85), or probably damaging (>0.85) -- **gnomAD frequencies:** How common this variant is in gnomAD's exomes (`gnomADe_AF`) and genomes (`gnomADg_AF`) - -### gnomAD Frequency: Your Best Sanity Check - -The single most useful annotation VEP adds is the **gnomAD allele frequency** — how common a variant is in the general population. Step 13 runs VEP with `--everything`, which turns on both `--af_gnomade` and `--af_gnomadg`, so each variant gets two frequencies: `gnomADe_AF` from gnomAD's exomes and `gnomADg_AF` from its genomes. A non-coding variant outside exome capture regions has no exome frequency but can still have a genome frequency, so check both before calling a variant absent from gnomAD. The recipes below do that. - -**Key principle:** A variant that is common in healthy people is almost certainly benign, regardless of what any prediction tool says. - -| gnomAD AF | Interpretation | Action | -|---|---|---| -| > 5% (0.05) | Common polymorphism | Benign. Ignore. | -| 1-5% | Low-frequency variant | Almost certainly benign | -| 0.1-1% | Uncommon | Probably benign, but check ClinVar | -| 0.01-0.1% | Rare | Worth investigating if in a disease gene | -| < 0.01% | Very rare | Potentially significant. Check ClinVar + literature | -| Absent | Novel or ultra-rare | Could be significant OR a sequencing artifact. Verify with a second method | - -**If a variant is "pathogenic" in ClinVar but has gnomAD AF > 1%:** The ClinVar entry may be outdated or wrong. Truly pathogenic variants for severe diseases are almost always rare (< 0.1%) because natural selection removes them from the population. - -### Filtering Strategy - -Step 23 (the [clinical filter](#clinical-filter-step-23)) already applies the filters most people want. For your own queries, use the [Quick Variant Filtering Recipes](#quick-variant-filtering-recipes) below. Do not `grep` the VEP VCF: VEP writes its values by position inside the pipe-separated `CSQ` field, never as `name=value`, so a grep for `gnomAD_AF` or `SYMBOL=` matches nothing, and a grep for `HIGH` or `1/1` can match text in other fields. `bcftools +split-vep` reads the fields by name. - -### What "HIGH Impact" Means - -VEP classifies variant impact as: - -| Impact | Types | Interpretation | -|---|---|---| -| HIGH | Stop gained, frameshift, splice donor/acceptor | Likely breaks the protein | -| MODERATE | Missense, in-frame insertion/deletion | Changes the protein, may or may not matter | -| LOW | Synonymous, splice region | Probably no functional effect | -| MODIFIER | Intronic, intergenic, UTR | Usually non-functional | - -**Everyone has ~100-150 HIGH impact variants.** Most are in one copy (heterozygous) of non-essential genes. Don't panic at the number. - ---- - -## Pathogenicity Scores (Step 30 — vcfanno) - -If you ran step 30, your VCF now includes quantitative pathogenicity scores beyond VEP's qualitative SIFT/PolyPhen predictions. These scores are widely used as computational evidence in variant interpretation (see ClinGen's PP3/BP4 calibration framework). - -### CADD (Combined Annotation Dependent Depletion) - -Scores **all** variant types (coding, non-coding, splice, regulatory). Uses a PHRED-like scale where higher = more deleterious. - -| CADD PHRED | Interpretation | Context | -|---|---|---| -| < 10 | Likely benign | Bottom 90% of genome variation | -| 10-20 | Uncertain | Top 10%, but most are still benign | -| 20-25 | Potentially deleterious | Top 1% — investigate if in a disease gene | -| 25-30 | Likely deleterious | Top 0.3% — strong candidate for pathogenicity | -| > 30 | Highly deleterious | Top 0.1% — likely damaging if in a constrained gene | - -**When to use CADD:** Best for non-coding and splice-region variants where SIFT/PolyPhen don't apply. For missense variants, REVEL and AlphaMissense are more specific. - -### REVEL (Rare Exome Variant Ensemble Learner) - -Scores **missense variants only**. Combines 13 individual tools into a single 0-1 score. Recommended by ClinGen for ACMG PP3/BP4 evidence. - -The levels below are the REVEL row of Table 2 in [Pejaver et al. 2022](https://doi.org/10.1016/j.ajhg.2022.10.013), the ClinGen calibration of PP3/BP4: - -| REVEL Score | ClinGen Evidence Level | Interpretation | -|---|---|---| -| <= 0.003 | BP4_Very Strong | Very strong evidence of benign | -| > 0.003 to <= 0.016 | BP4_Strong | Strong evidence of benign | -| > 0.016 to <= 0.183 | BP4_Moderate | Moderate evidence of benign | -| > 0.183 to <= 0.290 | BP4_Supporting | Supporting evidence of benign | -| > 0.290 to < 0.644 | No evidence | Uncertain significance | -| >= 0.644 to < 0.773 | PP3_Supporting | Supporting evidence of pathogenicity | -| >= 0.773 to < 0.932 | PP3_Moderate | Moderate evidence of pathogenicity | -| >= 0.932 | PP3_Strong | Strong evidence of pathogenicity | - -There is no PP3 Very Strong level for REVEL: no score reached it in the calibration. - -**When to use REVEL:** First-line score for evaluating missense variants. If REVEL >= 0.644, investigate the variant seriously. - -### AlphaMissense - -DeepMind's protein-structure-informed **missense** classifier. Uses AlphaFold2 protein structure predictions to assess amino acid substitution impact. - -| am_pathogenicity | am_class | Interpretation | -|---|---|---| -| < 0.34 | likely_benign | Predicted benign by protein structure analysis | -| 0.34-0.564 | ambiguous | Uncertain — use other evidence | -| > 0.564 | likely_pathogenic | Predicted damaging based on protein structure | - -These cut-offs are AlphaMissense's own class boundaries, set by its authors. They are not ACMG/ClinGen evidence levels. - -**When to use AlphaMissense:** Complements REVEL as a second opinion. If they disagree, investigate further. Agreement does not add up to stronger PP3 evidence: ClinGen's calibration recommends one tool, used genome-wide, for PP3/BP4, so picking whichever tool scores a variant highest would bias the result. - -### SpliceAI - -Deep learning model predicting **splice-altering** variants. Scores four types of splice disruption: acceptor gain (AG), acceptor loss (AL), donor gain (DG), donor loss (DL). - -| Max Delta Score | Interpretation | -|---|---| -| < 0.2 | Unlikely to affect splicing | -| 0.2-0.5 | May affect splicing — investigate | -| 0.5-0.8 | Likely affects splicing | -| > 0.8 | Strong evidence of splice disruption | - -**When to use SpliceAI:** VEP already flags canonical splice site variants (GT/AG dinucleotides). SpliceAI catches **cryptic** splice variants — intronic or exonic variants that create new splice sites or disrupt existing ones through more subtle mechanisms. - -### gnomAD Gene Constraint (Step 23 summary) - -These are per-gene metrics (not per-variant) added to the clinical filter summary TSV. They measure how intolerant a gene is to different types of mutations. - -| Metric | Threshold | Meaning | -|---|---|---| -| LOEUF < 0.35 | Constrained for loss-of-function | Gene is intolerant to LoF mutations — a HIGH impact variant here is more likely to cause disease | -| pLI >= 0.9 | Loss-of-function intolerant | Same as LOEUF but older metric. LOEUF is preferred. | -| mis_Z > 3.09 | Constrained for missense | Gene is intolerant to missense mutations — a REVEL-high missense here is more concerning | - -**Combining scores:** A rare variant (gnomAD AF < 0.01%) with CADD > 25, in a constrained gene (LOEUF < 0.35), with ClinVar pathogenic classification, is a high-confidence pathogenic finding. Any one of these alone is insufficient. - ---- - -### Quick Variant Filtering Recipes - -Copy-paste these commands to extract the most clinically relevant variants. They read step 13's output, `${GENOME_DIR}/${SAMPLE}/vep/${SAMPLE}_vep.vcf.gz`, with `bcftools +split-vep` from the pipeline's pinned bcftools image, which picks each value out of the `CSQ` field by name (the same way step 23 does). `-s worst` keeps the most severe consequence of each variant; `-d` keeps every transcript. - -```bash -# Run from the repository root, with GENOME_DIR and SAMPLE set -source versions.env -bcf() { docker run --rm -i -v "${GENOME_DIR}:/genome" -w /genome "${BCFTOOLS_IMAGE}" bcftools "$@"; } -VEP_VCF="${SAMPLE}/vep/${SAMPLE}_vep.vcf.gz" # relative to GENOME_DIR - -# Rare: below 0.1% in gnomAD exomes and genomes, or absent from them -RARE='(gnomADe_AF="." || gnomADe_AF<0.001) && (gnomADg_AF="." || gnomADg_AF<0.001)' -FIELDS='%CHROM\t%POS\t%REF\t%ALT\t%SYMBOL\t%Consequence\t%gnomADe_AF\t%gnomADg_AF\t[%GT]\n' - -# 1. Homozygous high-impact variants (stop gained, frameshift, splice donor/acceptor) -bcf view -i 'GT="AA"' "$VEP_VCF" | - bcf +split-vep - -s worst -i 'IMPACT="HIGH"' -f "$FIELDS" - -# 2. Rare high-impact variants -bcf +split-vep "$VEP_VCF" -s worst -i "IMPACT=\"HIGH\" && ${RARE}" -f "$FIELDS" - -# 3. Compound heterozygous candidates: genes with two or more rare heterozygous -# HIGH or MODERATE variants. The phase is unknown: both can sit on the same -# copy of the gene, so this is a list to curate, not a finding. -bcf view -i 'GT="het"' "$VEP_VCF" | - bcf +split-vep - -s worst -i "(IMPACT=\"HIGH\" || IMPACT=\"MODERATE\") && ${RARE}" -f '%SYMBOL\n' | - sort | uniq -c | awk '$1 >= 2' | sort -rn - -# 4. HIGH or MODERATE variants in genes you name, matched on the exact symbol. -# The list here is a few cancer genes for illustration; CPSR (step 17) already -# reports the full ACMG secondary-findings list. For pharmacogenes, use -# GENES="CYP2D6 CYP2C19 CYP2C9 DPYD UGT1A1 SLCO1B1 TPMT NUDT15". -GENES="BRCA1 BRCA2 MLH1 MSH2 MSH6 PMS2 APC MUTYH TP53" -bcf +split-vep "$VEP_VCF" -d -i 'IMPACT="HIGH" || IMPACT="MODERATE"' -f "$FIELDS" | - awk -F'\t' -v genes="$GENES" 'BEGIN { n = split(genes, g, " "); for (i = 1; i <= n; i++) want[g[i]] = 1 } $5 in want' | - sort -u - -# 5. Rare missense variants that SIFT and PolyPhen both call damaging -bcf +split-vep "$VEP_VCF" -s worst \ - -i "Consequence~\"missense_variant\" && SIFT~\"^deleterious[(]\" && PolyPhen~\"^probably_damaging\" && ${RARE}" \ - -f "$FIELDS" - -# 6. Missense variants absent from both gnomAD sets (novel or ultra-rare) -bcf +split-vep "$VEP_VCF" -s worst \ - -i 'Consequence~"missense_variant" && gnomADe_AF="." && gnomADg_AF="."' -f "$FIELDS" -``` - -If the VCF came from a VEP run without gnomAD frequencies, `+split-vep` stops with `the tag "gnomADe_AF" is not defined`: drop the `RARE` term and the two gnomAD columns from `FIELDS`. PharmCAT (step 7) misses some alleles, and CYP2D6 needs the BAM-based callers (steps 21 and 32), so recipe 4 with the pharmacogene list is a cross-check, not a replacement. - -**Important:** These are starting points, not definitive screens. Any interesting finding should be cross-referenced with ClinVar and ideally confirmed by a second method (Sanger sequencing or a clinical lab). - ---- - -## CNVpytor Results (Step 18) - -CNVpytor detects **copy number variants** using read depth analysis — complementary to Manta's paired-end/split-read approach. - -**Where to look:** `${SAMPLE}/cnvpytor/${SAMPLE}_cnvs.txt` - -**Format:** Each line has: type, region, size, normalized_RD, e-value1, e-value2, e-value3, e-value4, q0 - -**What to expect:** -- 3,000-4,000 total CNVs (mostly deletions) -- 1,500-2,000 significant (e-value < 0.01) -- Calls at chromosome starts (chr1:1-10000) are telomeric artifacts — ignore them - -**Filtering:** -```bash -# Significant CNVs only (e-value < 0.01) -awk '$5 < 0.01' ${SAMPLE}_cnvs.txt - -# Large deletions (>100kb, potentially clinically relevant) -awk '$1 == "deletion" && $3 > 100000 && $5 < 0.01' ${SAMPLE}_cnvs.txt - -# Large duplications -awk '$1 == "duplication" && $3 > 100000 && $5 < 0.01' ${SAMPLE}_cnvs.txt -``` - -**Multi-caller overlap:** CNVs found by both Manta AND CNVpytor have lower false-positive rates. Cross-reference by checking if the same genomic region appears in both output files. - ---- - -## Delly Results (Step 19) - -Delly is a third structural variant caller, detecting deletions, duplications, inversions, and translocations. - -**Where to look:** `${SAMPLE}/delly/${SAMPLE}_sv.vcf.gz` - -**Quick summary:** -```bash -# Count SVs by type -bcftools query -f '%INFO/SVTYPE\n' ${SAMPLE}_sv.vcf.gz | sort | uniq -c - -# Filter PASS variants only -bcftools view -f PASS ${SAMPLE}_sv.vcf.gz | grep -cv '^#' -``` - -**What to expect:** -- 5,000-15,000 total SV calls -- Most are small deletions (<1kb) -- PASS filter reduces count significantly - -**Multi-caller overlap:** SVs detected by Manta + Delly + CNVpytor (or any 2 of 3) have substantially lower false-positive rates. Single-caller calls, especially large ones, should be viewed with caution. - ---- - -## Mitochondrial Variants (Step 20) - -GATK Mutect2 in mitochondrial mode detects variants with heteroplasmy fractions — the proportion of your mitochondria carrying each variant. - -**Where to look:** `${SAMPLE}/mito/${SAMPLE}_chrM_filtered.vcf.gz` - -**Key field:** `AF` (allele fraction) indicates heteroplasmy level: - -| AF Level | Meaning | -|---|---| -| >0.95 | Homoplasmic — fixed in all mitochondria (haplogroup-defining) | -| 0.10-0.95 | Heteroplasmic — clinically significant range | -| 0.03-0.10 | Low-level heteroplasmy — often age-related somatic | -| <0.03 | Near detection limit | - -**What to expect:** -- 50-70 PASS variants total -- 25-35 homoplasmic (haplogroup variants) -- 25-35 low-level heteroplasmic (mostly <5%) -- Length variants in the D-loop poly-C tracts (m.303-315 and m.16184-16193) in most samples: the reports list them apart and do not count them as heteroplasmy - -**When to investigate further:** -- Heteroplasmic variant at a known disease position: the report lists each heteroplasmic call, and each call ClinVar classes as anything but benign, with its ClinVar class and gnomAD v3.1 frequency (`mito/_chrM_annotated.tsv`). [MITOMAP](https://www.mitomap.org/) has the curated status; look it up by hand, as it refuses scripted requests -- m.3243A>G (MELAS) or m.8344A>G (MERRF) at detectable heteroplasmy levels -- Any position in MT-ATP6, MT-ND genes with AF >0.10 - -**Important caveats about heteroplasmy thresholds:** -- There is **no single absolute heteroplasmy threshold** that determines clinical significance. Thresholds vary by variant and by tissue (ClinGen/MSeqDR mtDNA interpretation specifications) -- **Blood and saliva underrepresent heteroplasmy** for many mitochondrial diseases. WGS from saliva, a cheek swab or blood may show lower heteroplasmy levels than affected tissues (muscle, nerve). m.3243A>G in particular shows different clinical phenotypes at very different heteroplasmy levels across tissues -- The AF values from this pipeline reflect the tissue your sample came from. A low or absent heteroplasmy level there does **not** rule out clinically significant heteroplasmy in other tissues -- For any detected pathogenic mtDNA variant, discuss with a specialist who can order tissue-specific testing if warranted - -**Cross-reference:** Compare with step 12 (haplogrep3) — your homoplasmic variants should match your assigned haplogroup. - ---- - -## Somatic Variants (Step 29) [EXPERIMENTAL] - -Mutect2 in tumor-only mode looks for somatic mutations -- variants acquired during your lifetime rather than inherited. Consumer WGS is usually made from saliva or a cheek swab, a mix of cheek cells and white blood cells; the main category of interest is **clonal hematopoiesis (CHIP)**, which lives in the blood cells. - -**Where to look:** `${SAMPLE}/somatic/${SAMPLE}_somatic_filtered.vcf.gz` - -**Key field:** `AF` (allele fraction) indicates the clone size: - -| AF Range | Likely Source | -|---|---| -| 0.45-0.55 | Heterozygous germline (false positive) | -| ~1.0 | Homozygous germline (false positive) | -| 0.10-0.40 | Could be a large somatic clone, mosaic, or noisy germline | -| below 0.10 | At 30X this is one to three reads: mostly noise, see below | - -**What to expect:** -- Thousands of PASS calls in a healthy individual -- the vast majority are germline false positives -- At 30X a variant in 2% of the reads has less than one supporting read on average, and one in 10% about three. Only variants at roughly 10% allele fraction or more can be told apart from noise. That is an approximate rule about reads, not a clone size: a heterozygous variant at 10% allele fraction sits in about 20% of the sampled cells, and copy number and the mix of cell types in the sample shift it. Most CHIP (defined from 2% allele fraction) is invisible here, and a clean result does not rule it out -- Without a matched normal sample, germline variants that are rare in gnomAD will often pass all filters - -**CHIP genes to check:** DNMT3A, TET2, ASXL1, TP53, JAK2, SF3B1, SRSF2, PPM1D, CBL. CHIP prevalence increases with age and is associated with elevated cardiovascular risk and risk of hematologic malignancies. - -**Important caveats:** -- This step has a much higher false positive rate than any other step in the pipeline -- Do not interpret PASS variants as confirmed somatic without cross-referencing with the germline VCF (step 3) and gnomAD frequencies (step 13) -- If a variant is also called at ~50% AF by DeepVariant in step 3, it is almost certainly germline - ---- - -## Reads, Alignment, Variant Calling and Coverage (Steps 1b, 2, 3, 16, 16b) - -These steps check that the data is good enough for everything else. Look at them first if a later result looks strange. - -- **fastp (step 1b):** `${SAMPLE}/fastq_trimmed/${SAMPLE}_fastp.html`. A few percent of reads trimmed or dropped is normal; a large share points at a library or upload problem. -- **Alignment and variant calling (steps 2 and 3):** the [example output](#variant-calling-step-3) below shows what a 30X genome looks like: about 4.5 to 5.5 million variants and a Ti/Tv ratio of 2.0 to 2.1. -- **Coverage (step 16b, mosdepth):** `${SAMPLE}/mosdepth/${SAMPLE}.mosdepth.summary.txt`. The `total` row should show a mean near the depth you paid for (about 30). Below 15, small-variant calls lose accuracy. See [step 16b](16b-mosdepth.md#interpreting-results). -- **Sex check (step 16, indexcov):** `${SAMPLE}/indexcov/` estimates the copy number of chrX and chrY from the BAM index. About 1 and 1 is XY, about 2 and 0 is XX. A result that does not match the sex you gave the pipeline means a sample swap, a mislabelled file or a real sex-chromosome difference; [step 16](16-indexcov.md#interpretation) lists the patterns. Re-check the input before reading anything else. - -## HLA Typing (Step 8) - -**Where to look:** `${SAMPLE}/hla_t1k/${SAMPLE}_hla_genotype.tsv`, two alleles per HLA gene. - -The main use is drug safety: a few HLA alleles predict severe reactions to specific drugs, for example HLA-B\*57:01 with abacavir and HLA-B\*58:01 with allopurinol. The report marks each allele of `bin/hla_alleles.tsv` (the drug alleles, and disease-associated ones such as HLA-B\*27 and HLA-C\*06:02) present or absent, and derives the coeliac DQ types (DQ2.5, DQ2.2, DQ8) and the number of DQB1\*02 copies from the DQA1 and DQB1 rows; [step 8](08-hla-typing.md#what-the-report-derives) gives the rules and the sources. The genotypes are unphased, so DQ2.5 reads "likely cis" or "possible trans". These lines are information about alleles, not a diagnosis: carrying one does not mean the condition or reaction will happen, and not carrying one does not rule it out. A quality-0 gene with many tied candidates reads "no confident call", and the HLA-HFE row is a sequence allele, not the HFE C282Y/H63D genotype, which comes from the variant calls: the ClinVar hits list each change, and the HFE rule of the traits step reads the two together ([step 42](42-traits.md)). Typing from short-read WGS is approximate. HLA-A and HLA-B also reach PharmCAT (step 36), so the PharmCAT report and the CPIC recommendations (step 27) give the drug guidance for them; the CPIC file marks them `[outside call]`. If one of those alleles appears, or is missing and you are about to take the drug, ask for a clinical HLA test; do not rely on this output for transplant matching. With `HLA_QC=true` (opt-in) the HLA section also says, for HLA-A, HLA-B and HLA-C, whether OptiType agrees with T1K at two fields, with both calls; a gene they disagree on, or did not compare, is not passed to PharmCAT when it is HLA-A or HLA-B. With `KIR=true` (opt-in) the KIR genes are typed too, in `${SAMPLE}/kir_t1k/`, and the HLA section lists the genes T1K typed; no step interprets them. - -## Clinical Filter (Step 23) - -**Where to look:** `${SAMPLE}/clinical/${SAMPLE}_clinical_summary.tsv`, one row per variant with gene, impact, scores and gnomAD constraint. - -Step 23 keeps the HIGH-impact variants, the MODERATE ones below 1% in gnomAD exomes, the ClinVar pathogenic ones and, when step 30 ran, the variants that CADD, SpliceAI, REVEL or AlphaMissense flag. Expect a few hundred rows. Most are heterozygous variants in genes that tolerate one broken copy. Read the rows in constrained genes (low LOEUF, see [gene constraint](#gnomad-gene-constraint-step-23-summary)) first, then the homozygous ones. The HIGH-impact rows have no frequency filter, so check gnomAD for each before you worry. [Step 23](23-clinical-filter.md#what-gets-filtered) gives the exact rules. - -## Variant Prioritization (Step 31) - -**Where to look:** `${SAMPLE}/slivar/${SAMPLE}_slivar_summary.tsv`, `${SAMPLE}/slivar/${SAMPLE}_compound_hets.tsv` and `${SAMPLE}/slivar/${SAMPLE}_compound_het_pairs.tsv`. - -slivar sorts the damaging variants into four groups (rare HIGH, rare MODERATE with damaging scores, ClinVar pathogenic, and frequency unknown: HIGH or MODERATE with no population frequency, so not called rare) and lists genes where you carry two such variants. The genotypes alone cannot tell whether the two variants sit on different copies of the gene (in trans, which can cause recessive disease) or on the same copy (in cis, which usually does not), so step 31 reads the phase from the reads: `${SAMPLE}_compound_het_pairs.tsv` says `trans`, `cis` or `unknown` for each pair, with the reads that say so. Only variants close enough for one read pair to cover both (a few hundred bases) can be phased this way, so most pairs read `unknown`. Expect a thousand or more candidate pairs; nearly all are noise. A `trans` pair matters only in a gene that fits your health history, and an `unknown` one needs a parent's DNA or long reads to phase. See [step 31](31-slivar.md#interpretation). - -## More Pharmacogenomics: Cyrius, CPIC, pypgx and the Consensus (Steps 21, 27, 32, 36) - -- **CPIC lookup (step 27):** `${SAMPLE}/cpic/${SAMPLE}_cpic_recommendations.txt` turns PharmCAT's calls into the drugs with CPIC guidance. Only genes whose phenotype names a changed function or a positive test get drug entries. Genes PharmCAT called with no such phenotype (`n/a`, the reference VKORC1 genotype `-1639 GG`, `Uncertain Susceptibility`) are listed in their own section without drugs; PharmCAT's HTML report has any guidance for them. Genes PharmCAT could not call are listed separately at the end; their absence from the drug list does not mean normal function. Its section "Calls From Other Tools" says which of HLA-A, HLA-B and CYP2D6 reached PharmCAT from the BAM-based callers, and why the others did not. -- **PGx consensus (step 36):** `${SAMPLE}/pgx_consensus/${SAMPLE}_pgx_consensus.tsv` shows, for HLA-A, HLA-B and CYP2D6, what each caller said and what was passed to PharmCAT. CYP2D6 is passed only when pypgx and Cyrius give the same diplotype and the depth at CYP2D6 passed its check; otherwise it reads `indeterminate`. -- **pypgx (step 32):** `${SAMPLE}/pypgx/${SAMPLE}_pypgx_summary.tsv` calls the 29 genes of `assets/pgx_genes.tsv`, 13 of them (the `bam` rows) with copy number from the BAM (GSTT1, which GRCh38 has only on the ALT contig `chr22_KI270879v1_alt`, is reported as not callable on the default no-ALT reference; a BAM aligned to a reference with ALT or HLA contigs stops the step with a `REFUSED` line, because its depth splits between a gene and its ALT copy), so it sees gene deletions and duplications PharmCAT cannot. `${SAMPLE}_pharmcat_comparison.tsv`, written into the same folder by step 27 (CPIC lookup), shows where the two tools agree. -- **Cyrius (step 21, opt-in, non-commercial licence):** `${SAMPLE}/cyrius/${SAMPLE}_cyp2d6.tsv` gives a second CYP2D6 call from the BAM. Without it, CYP2D6 stays `indeterminate`. - -CYP2D6 is the hard gene: a nearby pseudogene and frequent copy-number changes confuse short reads. The pipeline gives CYP2D6 drug guidance only when two callers agree; even then, take any result that would change a prescription to a pharmacist or a certified pharmacogenomics test first. - -## SMN1 and SMN2 Copy Number (Step 35, opt-in) - -**Where to look:** `${SAMPLE}/paralogs/${SAMPLE}_smn_copy_number.tsv`, one row per stretch of the SMN1/SMN2 locus with the copy number of the two genes together (agCN) and of each (psCN), each with a quality. - -One SMN1 copy suggests SMA carrier status; two do not rule it out (two copies on one chromosome, none on the other). Trust a value only with quality 20 or more and filter `PASS`, and confirm anything that matters with a clinical SMN1 test. [Step 35](35-paralogs.md#interpreting-results) explains the columns. - -**The silent-carrier marker:** `${SAMPLE}/paralogs/${SAMPLE}_smn1_marker.tsv` reads SMN1 g.27134T>G (rs143838139), and both reports and the live person page print it right under the SMN1 copies: `present`, `uncertain`, `not seen`, `too little depth` or `not assessed`. With two SMN1 copies, `present` raises the chance that both sit on one chromosome (2+0, a silent carrier) and `not seen` lowers it; it proves neither, and the figures depend on ancestry. A carrier test in a clinical laboratory confirms it. [The SMN1 silent-carrier marker](35-paralogs.md#the-smn1-silent-carrier-marker) gives the thresholds. - -## Lp(a) and the LPA KIV-2 Count (Step 46, opt-in) - -**Where to look:** `${SAMPLE}/lpa/${SAMPLE}_lpa_kiv2.tsv`, the KIV-2 repeat count over both chromosomes with its 95% interval and class, and `${SAMPLE}/lpa/${SAMPLE}_lpa_kiv2_4925.tsv`, the KIV-2 4925G>A splice variant. Both reports put the count beside the two LPA risk variants of step 42 (rs10455872, rs3798220), and print the 4925G>A line right under the count: `present` or `uncertain` with the reads that carry it and the copies, `not seen`, `too little depth` or `not assessed`. - -Fewer repeats or a risk allele point to a higher Lp(a); a 4925G>A copy lowers the Lp(a) made from the chromosome that carries it. Neither tells which chromosome carries what, and `not seen` is not proof of absence at low depth. The count is a research estimate: the Lp(a) blood test is the measure, once in every adult. [Step 46](46-lpa.md) explains the columns. - -## Mobile Element Insertions (Step 50, opt-in) - -**Where to look:** `${SAMPLE}/mei/${SAMPLE}_mei_sites.tsv` for the three known disease insertions (SVA insertions in FKTN, TAF1 and BBS1), and `${SAMPLE}/mei/${SAMPLE}_mei.tsv` for every Alu, L1 and SVA insertion MEGAnE called, with one or two copies. - -Expect many calls: a genome carries about 1,200 such insertions, nearly all common and harmless. Read the catalog first. **Seen** means an insertion of the same element was called inside the site's window; one copy of the FKTN or BBS1 insertion is carrier status for a recessive condition, and the TAF1 insertion on a male's single X is what causes X-linked dystonia-parkinsonism. **Not seen** is not proof of absence: short reads find an SVA's edges, not its body. **Not callable** means too few reads there (under 10x) to tell. Confirm anything that matters in a clinical laboratory. [Step 50](50-mei.md) explains the columns. - -## Ancient Populations (Step 44, opt-in) - -**Where to look:** `${SAMPLE}/ancient/${SAMPLE}_ancient.tsv`, the 20 ancient groups nearest to you on a PCA of present-day West Eurasians, with their dates, places and number of people (`n`), and `${SAMPLE}/ancient/${SAMPLE}_ancient_pca.svg`, the plot with you marked. - -Closeness on the PCA is genetic similarity, not descent: being near an ancient group does not mean you descend from those people. A group with `n` of 1 or 2 has a noisy position. The axes describe West Eurasia only, so a genome with most of its ancestry from elsewhere lands at the edge and its nearest groups say little. `sites_used` in `${SAMPLE}_ancient_info.tsv` tells how many panel sites your gVCF covered; a whole genome covers nearly all of them (1,133,951 of the 1,150,106 that lift to GRCh38, for a gVCF built from the public GIAB HG002 genotypes). [Step 44](44-ancient.md) explains the columns. - -## Polygenic Risk Scores (Step 25) - -**Where to look:** `${SAMPLE}/prs/${SAMPLE}_prs_summary.tsv`, one row per condition, and the PRS section of the report. - -With the ancestry panel installed (`scripts/setup.sh --ancestry-panel`), each score has a **Percentile** and an **Ancestry_Group**: the 80th percentile means your score is higher than 80% of the reference samples whose genetic ancestry is most similar to yours (step 26 names that group). A percentile is still not a probability: most people at a high percentile never get the condition, and lifestyle and family history often weigh more. Percentiles of different conditions are not comparable either. - -Without the panel the report says "Raw score only": the sum cannot be compared with anyone, so read nothing into its size. A score with `NA` as its sum matched under 75% of its variants and was not scored; `Matched_Pct` says how many. When `Input` is `vcf` (no gVCF), the sums leave out the sites where you match the reference ([step 25](25-prs.md#interpreting-results) explains why). When `ChrX` says `left out`, the sample's sex was not given and the score's chrX rows are not in its sum ([chrX and the sex](25-prs.md#chrx-and-the-sex)). The breast cancer score is for women and the prostate cancer score for men: when you declared the other sex, `Sex_Check` says `not applicable to the declared sex` and the score has no percentile. The blood pressure and lipid scores rank a genetic tendency only; a measured blood pressure or lipid panel says far more. The Alzheimer's score includes APOE: read [Before you look](#before-you-look-what-the-pipeline-can-tell-you) first. - -## Reports (Steps 24 and 28) - -- **HTML report (step 24):** `${SAMPLE}/${SAMPLE}_report.html` collects the headline results of the other steps on one page, including the ClinVar hits table. It contains health findings: share it only as you would share a medical record. -- **MultiQC (step 28):** `${SAMPLE}/multiqc/multiqc_report.html` puts the QC of fastp, samtools, mosdepth and the other tools in one page. It is about data quality, not about your health. +## The Report Chapters + +Each chapter of the report has its own page in this guide. The links below keep the old addresses of the sections on this page working. + +- [Secondary Findings and Carrier Status](results/secondary-findings.md) + - [Carrier Status Is Not Disease](results/secondary-findings.md#carrier-status-is-not-disease){ #carrier-status-is-not-disease } + - [Clinical Filter (Step 23)](results/secondary-findings.md#clinical-filter-step-23){ #clinical-filter-step-23 } + - [Variant Prioritization (Step 31)](results/secondary-findings.md#variant-prioritization-step-31){ #variant-prioritization-step-31 } +- [ClinVar and Variant Annotation](results/clinvar.md) + - [The VUS Trap](results/clinvar.md#the-vus-trap){ #the-vus-trap } + - [ClinVar Star Ratings](results/clinvar.md#clinvar-star-ratings){ #clinvar-star-ratings } + - [VEP Annotation (Step 13)](results/clinvar.md#vep-annotation-step-13){ #vep-annotation-step-13 }: [What It Adds](results/clinvar.md#what-it-adds){ #what-it-adds }, [gnomAD Frequency: Your Best Sanity Check](results/clinvar.md#gnomad-frequency-your-best-sanity-check){ #gnomad-frequency-your-best-sanity-check }, [Filtering Strategy](results/clinvar.md#filtering-strategy){ #filtering-strategy }, [What "HIGH Impact" Means](results/clinvar.md#what-high-impact-means){ #what-high-impact-means } + - [Pathogenicity Scores (Step 30 — vcfanno)](results/clinvar.md#pathogenicity-scores-step-30-vcfanno){ #pathogenicity-scores-step-30-vcfanno }: [CADD](results/clinvar.md#cadd-combined-annotation-dependent-depletion){ #cadd-combined-annotation-dependent-depletion }, [REVEL](results/clinvar.md#revel-rare-exome-variant-ensemble-learner){ #revel-rare-exome-variant-ensemble-learner }, [AlphaMissense](results/clinvar.md#alphamissense){ #alphamissense }, [SpliceAI](results/clinvar.md#spliceai){ #spliceai }, [gnomAD Gene Constraint (Step 23 summary)](results/clinvar.md#gnomad-gene-constraint-step-23-summary){ #gnomad-gene-constraint-step-23-summary } + - [Quick Variant Filtering Recipes](results/clinvar.md#quick-variant-filtering-recipes){ #quick-variant-filtering-recipes } +- [Pharmacogenomics and HLA](results/pharmacogenomics.md) + - [HLA Typing (Step 8)](results/pharmacogenomics.md#hla-typing-step-8){ #hla-typing-step-8 } + - [More Pharmacogenomics: Cyrius, CPIC, pypgx and the Consensus (Steps 21, 27, 32, 36)](results/pharmacogenomics.md#more-pharmacogenomics-cyrius-cpic-pypgx-and-the-consensus-steps-21-27-32-36){ #more-pharmacogenomics-cyrius-cpic-pypgx-and-the-consensus-steps-21-27-32-36 } +- [Structural Variants, Copy Number and Repeats](results/structural-and-repeats.md) + - [Structural Variants (Steps 4, 5, 15, 18, 19, 22)](results/structural-and-repeats.md#structural-variants-steps-4-5-15-18-19-22){ #structural-variants-steps-4-5-15-18-19-22 }: [What Are Structural Variants?](results/structural-and-repeats.md#what-are-structural-variants){ #what-are-structural-variants }, [A Note About BND (Breakend) Calls](results/structural-and-repeats.md#a-note-about-bnd-breakend-calls){ #a-note-about-bnd-breakend-calls }, [How Many Is Normal?](results/structural-and-repeats.md#how-many-is-normal){ #how-many-is-normal }, [Which Callers to Trust?](results/structural-and-repeats.md#which-callers-to-trust){ #which-callers-to-trust }, [SV Consensus (Step 22)](results/structural-and-repeats.md#sv-consensus-step-22){ #sv-consensus-step-22 }, [AnnotSV Output](results/structural-and-repeats.md#annotsv-output){ #annotsv-output } + - [STR Expansions (Step 9)](results/structural-and-repeats.md#str-expansions-step-9){ #str-expansions-step-9 }: [What Are Repeat Expansions?](results/structural-and-repeats.md#what-are-repeat-expansions){ #what-are-repeat-expansions }, [How to Read ExpansionHunter Output](results/structural-and-repeats.md#how-to-read-expansionhunter-output){ #how-to-read-expansionhunter-output }, [Stranger Annotation (Step 9b)](results/structural-and-repeats.md#stranger-annotation-step-9b){ #stranger-annotation-step-9b } + - [CNVpytor Results (Step 18)](results/structural-and-repeats.md#cnvpytor-results-step-18){ #cnvpytor-results-step-18 } + - [Delly Results (Step 19)](results/structural-and-repeats.md#delly-results-step-19){ #delly-results-step-19 } + - [SMN1 and SMN2 Copy Number (Step 35, opt-in)](results/structural-and-repeats.md#smn1-and-smn2-copy-number-step-35-opt-in){ #smn1-and-smn2-copy-number-step-35-opt-in } + - [Lp(a) and the LPA KIV-2 Count (Step 46, opt-in)](results/structural-and-repeats.md#lpa-and-the-lpa-kiv-2-count-step-46-opt-in){ #lpa-and-the-lpa-kiv-2-count-step-46-opt-in } + - [Mobile Element Insertions (Step 50, opt-in)](results/structural-and-repeats.md#mobile-element-insertions-step-50-opt-in){ #mobile-element-insertions-step-50-opt-in } +- [Ancestry, Lineage and Polygenic Scores](results/ancestry-and-lineage.md) + - [ROH Analysis (Step 11)](results/ancestry-and-lineage.md#roh-analysis-step-11){ #roh-analysis-step-11 }: [What It Means](results/ancestry-and-lineage.md#what-it-means){ #what-it-means_1 }, [How to Read](results/ancestry-and-lineage.md#how-to-read){ #how-to-read }, [Centromeric Artifacts](results/ancestry-and-lineage.md#centromeric-artifacts){ #centromeric-artifacts } + - [Mitochondrial Haplogroup (Step 12)](results/ancestry-and-lineage.md#mitochondrial-haplogroup-step-12){ #mitochondrial-haplogroup-step-12 }: [What It Means](results/ancestry-and-lineage.md#what-it-means_1){ #what-it-means_2 }, [Common European Haplogroups](results/ancestry-and-lineage.md#common-european-haplogroups){ #common-european-haplogroups }, [Medical Relevance](results/ancestry-and-lineage.md#medical-relevance){ #medical-relevance }, [Exploring Your Branch](results/ancestry-and-lineage.md#exploring-your-branch){ #exploring-your-branch } + - [Mitochondrial Variants (Step 20)](results/ancestry-and-lineage.md#mitochondrial-variants-step-20){ #mitochondrial-variants-step-20 } + - [Ancient Populations (Step 44, opt-in)](results/ancestry-and-lineage.md#ancient-populations-step-44-opt-in){ #ancient-populations-step-44-opt-in } + - [Polygenic Risk Scores (Step 25)](results/ancestry-and-lineage.md#polygenic-risk-scores-step-25){ #polygenic-risk-scores-step-25 } +- [Telomeres and Somatic Variants](results/traits-and-somatic.md) + - [Telomere Length (Step 10)](results/traits-and-somatic.md#telomere-length-step-10){ #telomere-length-step-10 }: [What It Means](results/traits-and-somatic.md#what-it-means){ #what-it-means }, [How to Interpret](results/traits-and-somatic.md#how-to-interpret){ #how-to-interpret }, [Limitations](results/traits-and-somatic.md#limitations){ #limitations } + - [Somatic Variants (Step 29) [EXPERIMENTAL]](results/traits-and-somatic.md#somatic-variants-step-29-experimental){ #somatic-variants-step-29-experimental } +- [Data Quality and Reports](results/quality.md) + - [Reads, Alignment, Variant Calling and Coverage (Steps 1b, 2, 3, 16, 16b)](results/quality.md#reads-alignment-variant-calling-and-coverage-steps-1b-2-3-16-16b){ #reads-alignment-variant-calling-and-coverage-steps-1b-2-3-16-16b } + - [Reports (Steps 24 and 28)](results/quality.md#reports-steps-24-and-28){ #reports-steps-24-and-28 } --- @@ -819,7 +203,7 @@ NOTE: Centromeric ROH (chr1:125-143MB, chr9:42-60MB, chr18:15-20MB) are technica No line under the heading means no segment of 5 Mb or more: no evidence of related parents. If there are some, add them up and compare the total with the table in [step 11](11-roh-analysis.md#total-roh-and-parental-relationship). -### Telomere Length (Step 10) +### Telomere Length (Step 10) { #telomere-length-step-10_1 } ``` tel_content: diff --git a/docs/results/ancestry-and-lineage.md b/docs/results/ancestry-and-lineage.md new file mode 100644 index 000000000..b4786ef5d --- /dev/null +++ b/docs/results/ancestry-and-lineage.md @@ -0,0 +1,104 @@ +# Ancestry, Lineage and Polygenic Scores + +Part of [Interpreting Your Results](../interpreting-results.md). Read [Before You Look](../interpreting-results.md#before-you-look-what-the-pipeline-can-tell-you) first. + +## ROH Analysis (Step 11) + +### What It Means + +Runs of Homozygosity (ROH) are long stretches where both copies of your DNA are identical. Everyone has some ROH, but extensive ROH can indicate: +- Parental relatedness (consanguinity) +- Uniparental disomy +- Population bottleneck effects + +### How to Read + +Add up the autosomal segments of 5 Mb or more and compare the total with the table in [step 11](../11-roh-analysis.md#total-roh-and-parental-relationship), which gives the expected total for each parental relationship (about 45 Mb for second cousins, 180 Mb for first cousins). In short: + +- **No segments of 5 Mb or more, or only a few:** no sign that your parents are related +- **Many small segments (1-5 Mb):** population-level background, typical of population isolates (Ashkenazi, Finnish, etc.) +- **One chromosome carrying most of the long runs:** possible uniparental disomy rather than related parents. The reports flag a chromosome when its segments of 5 Mb or more add up to at least 20 Mb and to more than half of all such autosomal segments ([step 11](../11-roh-analysis.md#one-chromosome-carrying-most-of-the-runs-possible-uniparental-disomy)) + +### Centromeric Artifacts + +Some apparent ROH near centromeres are artifacts of low-coverage sequencing in repetitive regions. These can be ignored. + +--- + +## Mitochondrial Haplogroup (Step 12) + +### What It Means + +Your mitochondrial haplogroup traces your maternal ancestry lineage. It's determined by the specific set of variants in your mitochondrial DNA (inherited only from your mother). + +### Common European Haplogroups + +| Haplogroup | Origin | Notes | +|---|---|---| +| H | Western Europe | Most common in Europe (~40%) | +| U | Northern/Eastern Europe | Second most common (~15%) | +| T | Near East / Mediterranean | ~10% of Europeans | +| K | Near East | ~6% of Europeans, Ashkenazi ~30% | +| J | Near East | ~8% of Europeans | +| V | Iberian Peninsula / Scandinavia | ~4% | +| I | Near East / Europe | ~3% | + +### Medical Relevance + +Mitochondrial haplogroups have weak associations with some diseases (Parkinson's, diabetes, longevity), but these are population-level statistics, not individual predictions. The main clinical value is in step 20 (GATK Mutect2 mitochondrial mode), which detects disease-causing mitochondrial variants and heteroplasmy. + +### Exploring Your Branch + +Both reports link your haplogroup to FamilyTreeDNA Discover (free, no upload) and to YFull's tree. Discover shows the branch's age, where its testers live and the ancient DNA samples on it. For the Y haplogroup of step 37 the reports add the same two links. The free and paid routes, what each one takes and what an upload shares: [step 12](../12-mito-haplogroup.md#explore-your-branch-further) and [step 37](../37-y-haplogroup.md#explore-your-branch-further). The pipeline never uploads anything. + +--- + +## Mitochondrial Variants (Step 20) + +GATK Mutect2 in mitochondrial mode detects variants with heteroplasmy fractions — the proportion of your mitochondria carrying each variant. + +**Where to look:** `${SAMPLE}/mito/${SAMPLE}_chrM_filtered.vcf.gz` + +**Key field:** `AF` (allele fraction) indicates heteroplasmy level: + +| AF Level | Meaning | +|---|---| +| >0.95 | Homoplasmic — fixed in all mitochondria (haplogroup-defining) | +| 0.10-0.95 | Heteroplasmic — clinically significant range | +| 0.03-0.10 | Low-level heteroplasmy — often age-related somatic | +| <0.03 | Near detection limit | + +**What to expect:** +- 50-70 PASS variants total +- 25-35 homoplasmic (haplogroup variants) +- 25-35 low-level heteroplasmic (mostly <5%) +- Length variants in the D-loop poly-C tracts (m.303-315 and m.16184-16193) in most samples: the reports list them apart and do not count them as heteroplasmy + +**When to investigate further:** +- Heteroplasmic variant at a known disease position: the report lists each heteroplasmic call, and each call ClinVar classes as anything but benign, with its ClinVar class and gnomAD v3.1 frequency (`mito/_chrM_annotated.tsv`). [MITOMAP](https://www.mitomap.org/) has the curated status; look it up by hand, as it refuses scripted requests +- m.3243A>G (MELAS) or m.8344A>G (MERRF) at detectable heteroplasmy levels +- Any position in MT-ATP6, MT-ND genes with AF >0.10 + +**Important caveats about heteroplasmy thresholds:** +- There is **no single absolute heteroplasmy threshold** that determines clinical significance. Thresholds vary by variant and by tissue (ClinGen/MSeqDR mtDNA interpretation specifications) +- **Blood and saliva underrepresent heteroplasmy** for many mitochondrial diseases. WGS from saliva, a cheek swab or blood may show lower heteroplasmy levels than affected tissues (muscle, nerve). m.3243A>G in particular shows different clinical phenotypes at very different heteroplasmy levels across tissues +- The AF values from this pipeline reflect the tissue your sample came from. A low or absent heteroplasmy level there does **not** rule out clinically significant heteroplasmy in other tissues +- For any detected pathogenic mtDNA variant, discuss with a specialist who can order tissue-specific testing if warranted + +**Cross-reference:** Compare with step 12 (haplogrep3) — your homoplasmic variants should match your assigned haplogroup. + +--- + +## Ancient Populations (Step 44, opt-in) + +**Where to look:** `${SAMPLE}/ancient/${SAMPLE}_ancient.tsv`, the 20 ancient groups nearest to you on a PCA of present-day West Eurasians, with their dates, places and number of people (`n`), and `${SAMPLE}/ancient/${SAMPLE}_ancient_pca.svg`, the plot with you marked. + +Closeness on the PCA is genetic similarity, not descent: being near an ancient group does not mean you descend from those people. A group with `n` of 1 or 2 has a noisy position. The axes describe West Eurasia only, so a genome with most of its ancestry from elsewhere lands at the edge and its nearest groups say little. `sites_used` in `${SAMPLE}_ancient_info.tsv` tells how many panel sites your gVCF covered; a whole genome covers nearly all of them (1,133,951 of the 1,150,106 that lift to GRCh38, for a gVCF built from the public GIAB HG002 genotypes). [Step 44](../44-ancient.md) explains the columns. + +## Polygenic Risk Scores (Step 25) + +**Where to look:** `${SAMPLE}/prs/${SAMPLE}_prs_summary.tsv`, one row per condition, and the PRS section of the report. + +With the ancestry panel installed (`scripts/setup.sh --ancestry-panel`), each score has a **Percentile** and an **Ancestry_Group**: the 80th percentile means your score is higher than 80% of the reference samples whose genetic ancestry is most similar to yours (step 26 names that group). A percentile is still not a probability: most people at a high percentile never get the condition, and lifestyle and family history often weigh more. Percentiles of different conditions are not comparable either. + +Without the panel the report says "Raw score only": the sum cannot be compared with anyone, so read nothing into its size. A score with `NA` as its sum matched under 75% of its variants and was not scored; `Matched_Pct` says how many. When `Input` is `vcf` (no gVCF), the sums leave out the sites where you match the reference ([step 25](../25-prs.md#interpreting-results) explains why). When `ChrX` says `left out`, the sample's sex was not given and the score's chrX rows are not in its sum ([chrX and the sex](../25-prs.md#chrx-and-the-sex)). The breast cancer score is for women and the prostate cancer score for men: when you declared the other sex, `Sex_Check` says `not applicable to the declared sex` and the score has no percentile. The blood pressure and lipid scores rank a genetic tendency only; a measured blood pressure or lipid panel says far more. The Alzheimer's score includes APOE: read [Before you look](../interpreting-results.md#before-you-look-what-the-pipeline-can-tell-you) first. diff --git a/docs/results/clinvar.md b/docs/results/clinvar.md new file mode 100644 index 000000000..1c32c2ca6 --- /dev/null +++ b/docs/results/clinvar.md @@ -0,0 +1,234 @@ +# ClinVar and Variant Annotation + +Part of [Interpreting Your Results](../interpreting-results.md). Read [Before You Look](../interpreting-results.md#before-you-look-what-the-pipeline-can-tell-you) first. + +## The VUS Trap + +The single biggest source of unnecessary anxiety in personal genomics is **VUS — Variants of Uncertain Significance**. These are variants where: + +- There is not enough scientific evidence to classify them as either pathogenic or benign +- The vast majority will eventually be reclassified as **benign** as more data accumulates +- They are **not actionable** — no clinical decision should be made based on a VUS +- CPSR may report dozens or hundreds of VUS. This is normal and expected. + +**Rule of thumb:** If a variant is classified as VUS, treat it the same as if it were not tested. Do not change screening or management based on a VUS. Check back in 1-2 years when ClinVar may have reclassified it. + +**One nuance:** If you have a strong family history of a condition AND a VUS appears in the relevant high-penetrance gene (e.g., BRCA1/2, TP53, MLH1/MSH2), it may be worth mentioning to a genetic counselor — not to act on the VUS, but because the family history itself may warrant enhanced screening regardless of the variant's classification. + +## ClinVar Star Ratings + +Not all ClinVar classifications are equally reliable. Each entry has a **review status** indicated by stars: + +| Stars | Review Status | Reliability | +|---|---|---| +| 0 | No assertion criteria | Low — submitter did not explain their reasoning | +| 1 | Single submitter, criteria provided | Moderate — one lab's interpretation | +| 2 | Two or more submitters, no conflict | Good — multiple labs agree | +| 3 | Expert panel reviewed | High — reviewed by specialists | +| 4 | Practice guideline | Highest — established clinical standard | + +**Focus on 2+ star entries.** Single-submitter (1-star) pathogenic calls are sometimes reclassified. If you find a scary-looking pathogenic variant with 0-1 stars, check the ClinVar entry directly at [ncbi.nlm.nih.gov/clinvar](https://www.ncbi.nlm.nih.gov/clinvar/) — look at the "Review status" and "Last evaluated" date. + +--- + +## ClinVar Screen (Step 6) + +**What it tells you:** Known pathogenic variants in your genome, as classified by ClinVar (NCBI's public database of clinically significant variants). + +**Where to look:** `${SAMPLE}/clinvar/` + +**How to read it:** +- Each line in the output VCF is a variant in your genome that matches a known ClinVar entry +- Step 6 intersects against the **pathogenic-only** ClinVar subset (Pathogenic + Likely_pathogenic). Every hit in this output is at a position ClinVar classifies as disease-associated — benign/VUS entries are excluded at the database level +- The `CLNSIG` field confirms the classification + +**What to expect:** +- 0-10 pathogenic/likely pathogenic hits is typical for a 30X WGS +- Most pathogenic hits are **carrier status** (heterozygous) for recessive conditions — you carry one copy but aren't affected +- A heterozygous pathogenic variant in a recessive gene (like GJB2 for hearing loss) means you're a **carrier**, not affected +- A homozygous pathogenic variant, or a heterozygous variant in a dominant gene, needs attention + +**When to worry:** +- Pathogenic variant in a **dominant** gene (one copy is enough to cause disease) +- **Two** pathogenic variants in the same recessive gene (one from each parent) +- Any variant in cancer predisposition genes (BRCA1, BRCA2, MLH1, MSH2, etc.) + + +--- + +## VEP Annotation (Step 13) + +### What It Adds + +VEP annotates every variant with: +- **Consequence type:** missense, nonsense, synonymous, splice site, etc. +- **SIFT score:** Predicts if amino acid change is tolerated (>0.05) or damaging (<0.05) +- **PolyPhen score:** Predicts if change is benign (<0.15), possibly damaging (0.15-0.85), or probably damaging (>0.85) +- **gnomAD frequencies:** How common this variant is in gnomAD's exomes (`gnomADe_AF`) and genomes (`gnomADg_AF`) + +### gnomAD Frequency: Your Best Sanity Check + +The single most useful annotation VEP adds is the **gnomAD allele frequency** — how common a variant is in the general population. Step 13 runs VEP with `--everything`, which turns on both `--af_gnomade` and `--af_gnomadg`, so each variant gets two frequencies: `gnomADe_AF` from gnomAD's exomes and `gnomADg_AF` from its genomes. A non-coding variant outside exome capture regions has no exome frequency but can still have a genome frequency, so check both before calling a variant absent from gnomAD. The recipes below do that. + +**Key principle:** A variant that is common in healthy people is almost certainly benign, regardless of what any prediction tool says. + +| gnomAD AF | Interpretation | Action | +|---|---|---| +| > 5% (0.05) | Common polymorphism | Benign. Ignore. | +| 1-5% | Low-frequency variant | Almost certainly benign | +| 0.1-1% | Uncommon | Probably benign, but check ClinVar | +| 0.01-0.1% | Rare | Worth investigating if in a disease gene | +| < 0.01% | Very rare | Potentially significant. Check ClinVar + literature | +| Absent | Novel or ultra-rare | Could be significant OR a sequencing artifact. Verify with a second method | + +**If a variant is "pathogenic" in ClinVar but has gnomAD AF > 1%:** The ClinVar entry may be outdated or wrong. Truly pathogenic variants for severe diseases are almost always rare (< 0.1%) because natural selection removes them from the population. + +### Filtering Strategy + +Step 23 (the [clinical filter](secondary-findings.md#clinical-filter-step-23)) already applies the filters most people want. For your own queries, use the [Quick Variant Filtering Recipes](#quick-variant-filtering-recipes) below. Do not `grep` the VEP VCF: VEP writes its values by position inside the pipe-separated `CSQ` field, never as `name=value`, so a grep for `gnomAD_AF` or `SYMBOL=` matches nothing, and a grep for `HIGH` or `1/1` can match text in other fields. `bcftools +split-vep` reads the fields by name. + +### What "HIGH Impact" Means + +VEP classifies variant impact as: + +| Impact | Types | Interpretation | +|---|---|---| +| HIGH | Stop gained, frameshift, splice donor/acceptor | Likely breaks the protein | +| MODERATE | Missense, in-frame insertion/deletion | Changes the protein, may or may not matter | +| LOW | Synonymous, splice region | Probably no functional effect | +| MODIFIER | Intronic, intergenic, UTR | Usually non-functional | + +**Everyone has ~100-150 HIGH impact variants.** Most are in one copy (heterozygous) of non-essential genes. Don't panic at the number. + +--- + +## Pathogenicity Scores (Step 30 — vcfanno) + +If you ran step 30, your VCF now includes quantitative pathogenicity scores beyond VEP's qualitative SIFT/PolyPhen predictions. These scores are widely used as computational evidence in variant interpretation (see ClinGen's PP3/BP4 calibration framework). + +### CADD (Combined Annotation Dependent Depletion) + +Scores **all** variant types (coding, non-coding, splice, regulatory). Uses a PHRED-like scale where higher = more deleterious. + +| CADD PHRED | Interpretation | Context | +|---|---|---| +| < 10 | Likely benign | Bottom 90% of genome variation | +| 10-20 | Uncertain | Top 10%, but most are still benign | +| 20-25 | Potentially deleterious | Top 1% — investigate if in a disease gene | +| 25-30 | Likely deleterious | Top 0.3% — strong candidate for pathogenicity | +| > 30 | Highly deleterious | Top 0.1% — likely damaging if in a constrained gene | + +**When to use CADD:** Best for non-coding and splice-region variants where SIFT/PolyPhen don't apply. For missense variants, REVEL and AlphaMissense are more specific. + +### REVEL (Rare Exome Variant Ensemble Learner) + +Scores **missense variants only**. Combines 13 individual tools into a single 0-1 score. Recommended by ClinGen for ACMG PP3/BP4 evidence. + +The levels below are the REVEL row of Table 2 in [Pejaver et al. 2022](https://doi.org/10.1016/j.ajhg.2022.10.013), the ClinGen calibration of PP3/BP4: + +| REVEL Score | ClinGen Evidence Level | Interpretation | +|---|---|---| +| <= 0.003 | BP4_Very Strong | Very strong evidence of benign | +| > 0.003 to <= 0.016 | BP4_Strong | Strong evidence of benign | +| > 0.016 to <= 0.183 | BP4_Moderate | Moderate evidence of benign | +| > 0.183 to <= 0.290 | BP4_Supporting | Supporting evidence of benign | +| > 0.290 to < 0.644 | No evidence | Uncertain significance | +| >= 0.644 to < 0.773 | PP3_Supporting | Supporting evidence of pathogenicity | +| >= 0.773 to < 0.932 | PP3_Moderate | Moderate evidence of pathogenicity | +| >= 0.932 | PP3_Strong | Strong evidence of pathogenicity | + +There is no PP3 Very Strong level for REVEL: no score reached it in the calibration. + +**When to use REVEL:** First-line score for evaluating missense variants. If REVEL >= 0.644, investigate the variant seriously. + +### AlphaMissense + +DeepMind's protein-structure-informed **missense** classifier. Uses AlphaFold2 protein structure predictions to assess amino acid substitution impact. + +| am_pathogenicity | am_class | Interpretation | +|---|---|---| +| < 0.34 | likely_benign | Predicted benign by protein structure analysis | +| 0.34-0.564 | ambiguous | Uncertain — use other evidence | +| > 0.564 | likely_pathogenic | Predicted damaging based on protein structure | + +These cut-offs are AlphaMissense's own class boundaries, set by its authors. They are not ACMG/ClinGen evidence levels. + +**When to use AlphaMissense:** Complements REVEL as a second opinion. If they disagree, investigate further. Agreement does not add up to stronger PP3 evidence: ClinGen's calibration recommends one tool, used genome-wide, for PP3/BP4, so picking whichever tool scores a variant highest would bias the result. + +### SpliceAI + +Deep learning model predicting **splice-altering** variants. Scores four types of splice disruption: acceptor gain (AG), acceptor loss (AL), donor gain (DG), donor loss (DL). + +| Max Delta Score | Interpretation | +|---|---| +| < 0.2 | Unlikely to affect splicing | +| 0.2-0.5 | May affect splicing — investigate | +| 0.5-0.8 | Likely affects splicing | +| > 0.8 | Strong evidence of splice disruption | + +**When to use SpliceAI:** VEP already flags canonical splice site variants (GT/AG dinucleotides). SpliceAI catches **cryptic** splice variants — intronic or exonic variants that create new splice sites or disrupt existing ones through more subtle mechanisms. + +### gnomAD Gene Constraint (Step 23 summary) + +These are per-gene metrics (not per-variant) added to the clinical filter summary TSV. They measure how intolerant a gene is to different types of mutations. + +| Metric | Threshold | Meaning | +|---|---|---| +| LOEUF < 0.35 | Constrained for loss-of-function | Gene is intolerant to LoF mutations — a HIGH impact variant here is more likely to cause disease | +| pLI >= 0.9 | Loss-of-function intolerant | Same as LOEUF but older metric. LOEUF is preferred. | +| mis_Z > 3.09 | Constrained for missense | Gene is intolerant to missense mutations — a REVEL-high missense here is more concerning | + +**Combining scores:** A rare variant (gnomAD AF < 0.01%) with CADD > 25, in a constrained gene (LOEUF < 0.35), with ClinVar pathogenic classification, is a high-confidence pathogenic finding. Any one of these alone is insufficient. + +--- + +### Quick Variant Filtering Recipes + +Copy-paste these commands to extract the most clinically relevant variants. They read step 13's output, `${GENOME_DIR}/${SAMPLE}/vep/${SAMPLE}_vep.vcf.gz`, with `bcftools +split-vep` from the pipeline's pinned bcftools image, which picks each value out of the `CSQ` field by name (the same way step 23 does). `-s worst` keeps the most severe consequence of each variant; `-d` keeps every transcript. + +```bash +# Run from the repository root, with GENOME_DIR and SAMPLE set +source versions.env +bcf() { docker run --rm -i -v "${GENOME_DIR}:/genome" -w /genome "${BCFTOOLS_IMAGE}" bcftools "$@"; } +VEP_VCF="${SAMPLE}/vep/${SAMPLE}_vep.vcf.gz" # relative to GENOME_DIR + +# Rare: below 0.1% in gnomAD exomes and genomes, or absent from them +RARE='(gnomADe_AF="." || gnomADe_AF<0.001) && (gnomADg_AF="." || gnomADg_AF<0.001)' +FIELDS='%CHROM\t%POS\t%REF\t%ALT\t%SYMBOL\t%Consequence\t%gnomADe_AF\t%gnomADg_AF\t[%GT]\n' + +# 1. Homozygous high-impact variants (stop gained, frameshift, splice donor/acceptor) +bcf view -i 'GT="AA"' "$VEP_VCF" | + bcf +split-vep - -s worst -i 'IMPACT="HIGH"' -f "$FIELDS" + +# 2. Rare high-impact variants +bcf +split-vep "$VEP_VCF" -s worst -i "IMPACT=\"HIGH\" && ${RARE}" -f "$FIELDS" + +# 3. Compound heterozygous candidates: genes with two or more rare heterozygous +# HIGH or MODERATE variants. The phase is unknown: both can sit on the same +# copy of the gene, so this is a list to curate, not a finding. +bcf view -i 'GT="het"' "$VEP_VCF" | + bcf +split-vep - -s worst -i "(IMPACT=\"HIGH\" || IMPACT=\"MODERATE\") && ${RARE}" -f '%SYMBOL\n' | + sort | uniq -c | awk '$1 >= 2' | sort -rn + +# 4. HIGH or MODERATE variants in genes you name, matched on the exact symbol. +# The list here is a few cancer genes for illustration; CPSR (step 17) already +# reports the full ACMG secondary-findings list. For pharmacogenes, use +# GENES="CYP2D6 CYP2C19 CYP2C9 DPYD UGT1A1 SLCO1B1 TPMT NUDT15". +GENES="BRCA1 BRCA2 MLH1 MSH2 MSH6 PMS2 APC MUTYH TP53" +bcf +split-vep "$VEP_VCF" -d -i 'IMPACT="HIGH" || IMPACT="MODERATE"' -f "$FIELDS" | + awk -F'\t' -v genes="$GENES" 'BEGIN { n = split(genes, g, " "); for (i = 1; i <= n; i++) want[g[i]] = 1 } $5 in want' | + sort -u + +# 5. Rare missense variants that SIFT and PolyPhen both call damaging +bcf +split-vep "$VEP_VCF" -s worst \ + -i "Consequence~\"missense_variant\" && SIFT~\"^deleterious[(]\" && PolyPhen~\"^probably_damaging\" && ${RARE}" \ + -f "$FIELDS" + +# 6. Missense variants absent from both gnomAD sets (novel or ultra-rare) +bcf +split-vep "$VEP_VCF" -s worst \ + -i 'Consequence~"missense_variant" && gnomADe_AF="." && gnomADg_AF="."' -f "$FIELDS" +``` + +If the VCF came from a VEP run without gnomAD frequencies, `+split-vep` stops with `the tag "gnomADe_AF" is not defined`: drop the `RARE` term and the two gnomAD columns from `FIELDS`. PharmCAT (step 7) misses some alleles, and CYP2D6 needs the BAM-based callers (steps 21 and 32), so recipe 4 with the pharmacogene list is a cross-check, not a replacement. + +**Important:** These are starting points, not definitive screens. Any interesting finding should be cross-referenced with ClinVar and ideally confirmed by a second method (Sanger sequencing or a clinical lab). diff --git a/docs/results/pharmacogenomics.md b/docs/results/pharmacogenomics.md new file mode 100644 index 000000000..b0f0d15f4 --- /dev/null +++ b/docs/results/pharmacogenomics.md @@ -0,0 +1,40 @@ +# Pharmacogenomics and HLA + +Part of [Interpreting Your Results](../interpreting-results.md). Read [Before You Look](../interpreting-results.md#before-you-look-what-the-pipeline-can-tell-you) first. + +## PharmCAT Report (Step 7) + +**What it tells you:** How your genes affect drug metabolism. These results are clinically relevant and should be shared with your prescribing physician. + +**Where to look:** `${SAMPLE}/pharmcat/` (Nextflow) or `${SAMPLE}/vcf/` (bash scripts) — PharmCAT writes its reports there. Open the HTML report in a browser. + +**Key genes to check:** + +| Gene | Affects | Common Impact | +|---|---|---| +| CYP2C19 | PPIs, clopidogrel, SSRIs, voriconazole | Rapid metabolizers burn through drugs too fast | +| CYP2D6 | Codeine, tramadol, tamoxifen, many psych meds | Poor metabolizers get toxic buildup | +| CYP2C9 | Warfarin, NSAIDs, phenytoin | Dose adjustment needed | +| DPYD | 5-fluorouracil (cancer drug) | Poor metabolizers can die from standard doses | +| SLCO1B1 | Statins (simvastatin, atorvastatin) | Increased myopathy risk | +| NAT2 | Isoniazid (TB), hydralazine | Slow acetylators have more side effects | +| UGT1A1 | Irinotecan, atazanavir | Two \*28 alleles are associated with Gilbert syndrome (elevated bilirubin) | + +**What to do:** Share the PharmCAT report with your prescribing physician or pharmacist. PharmCAT is a research tool — its authors explicitly note that missing positions, unphased input, and undetected structural variation (especially CYP2D6) can affect genotype and phenotype calls. The report is a valuable starting point for pharmacogenomic-guided prescribing, but clinical confirmation may be warranted before making medication changes, especially for high-risk drugs (DPYD, CYP2D6-dependent opioids). + +--- + +## HLA Typing (Step 8) + +**Where to look:** `${SAMPLE}/hla_t1k/${SAMPLE}_hla_genotype.tsv`, two alleles per HLA gene. + +The main use is drug safety: a few HLA alleles predict severe reactions to specific drugs, for example HLA-B\*57:01 with abacavir and HLA-B\*58:01 with allopurinol. The report marks each allele of `bin/hla_alleles.tsv` (the drug alleles, and disease-associated ones such as HLA-B\*27 and HLA-C\*06:02) present or absent, and derives the coeliac DQ types (DQ2.5, DQ2.2, DQ8) and the number of DQB1\*02 copies from the DQA1 and DQB1 rows; [step 8](../08-hla-typing.md#what-the-report-derives) gives the rules and the sources. The genotypes are unphased, so DQ2.5 reads "likely cis" or "possible trans". These lines are information about alleles, not a diagnosis: carrying one does not mean the condition or reaction will happen, and not carrying one does not rule it out. A quality-0 gene with many tied candidates reads "no confident call", and the HLA-HFE row is a sequence allele, not the HFE C282Y/H63D genotype, which comes from the variant calls: the ClinVar hits list each change, and the HFE rule of the traits step reads the two together ([step 42](../42-traits.md)). Typing from short-read WGS is approximate. HLA-A and HLA-B also reach PharmCAT (step 36), so the PharmCAT report and the CPIC recommendations (step 27) give the drug guidance for them; the CPIC file marks them `[outside call]`. If one of those alleles appears, or is missing and you are about to take the drug, ask for a clinical HLA test; do not rely on this output for transplant matching. With `HLA_QC=true` (opt-in) the HLA section also says, for HLA-A, HLA-B and HLA-C, whether OptiType agrees with T1K at two fields, with both calls; a gene they disagree on, or did not compare, is not passed to PharmCAT when it is HLA-A or HLA-B. With `KIR=true` (opt-in) the KIR genes are typed too, in `${SAMPLE}/kir_t1k/`, and the HLA section lists the genes T1K typed; no step interprets them. + +## More Pharmacogenomics: Cyrius, CPIC, pypgx and the Consensus (Steps 21, 27, 32, 36) + +- **CPIC lookup (step 27):** `${SAMPLE}/cpic/${SAMPLE}_cpic_recommendations.txt` turns PharmCAT's calls into the drugs with CPIC guidance. Only genes whose phenotype names a changed function or a positive test get drug entries. Genes PharmCAT called with no such phenotype (`n/a`, the reference VKORC1 genotype `-1639 GG`, `Uncertain Susceptibility`) are listed in their own section without drugs; PharmCAT's HTML report has any guidance for them. Genes PharmCAT could not call are listed separately at the end; their absence from the drug list does not mean normal function. Its section "Calls From Other Tools" says which of HLA-A, HLA-B and CYP2D6 reached PharmCAT from the BAM-based callers, and why the others did not. +- **PGx consensus (step 36):** `${SAMPLE}/pgx_consensus/${SAMPLE}_pgx_consensus.tsv` shows, for HLA-A, HLA-B and CYP2D6, what each caller said and what was passed to PharmCAT. CYP2D6 is passed only when pypgx and Cyrius give the same diplotype and the depth at CYP2D6 passed its check; otherwise it reads `indeterminate`. +- **pypgx (step 32):** `${SAMPLE}/pypgx/${SAMPLE}_pypgx_summary.tsv` calls the 29 genes of `assets/pgx_genes.tsv`, 13 of them (the `bam` rows) with copy number from the BAM (GSTT1, which GRCh38 has only on the ALT contig `chr22_KI270879v1_alt`, is reported as not callable on the default no-ALT reference; a BAM aligned to a reference with ALT or HLA contigs stops the step with a `REFUSED` line, because its depth splits between a gene and its ALT copy), so it sees gene deletions and duplications PharmCAT cannot. `${SAMPLE}_pharmcat_comparison.tsv`, written into the same folder by step 27 (CPIC lookup), shows where the two tools agree. +- **Cyrius (step 21, opt-in, non-commercial licence):** `${SAMPLE}/cyrius/${SAMPLE}_cyp2d6.tsv` gives a second CYP2D6 call from the BAM. Without it, CYP2D6 stays `indeterminate`. + +CYP2D6 is the hard gene: a nearby pseudogene and frequent copy-number changes confuse short reads. The pipeline gives CYP2D6 drug guidance only when two callers agree; even then, take any result that would change a prescription to a pharmacist or a certified pharmacogenomics test first. diff --git a/docs/results/quality.md b/docs/results/quality.md new file mode 100644 index 000000000..08ddc776f --- /dev/null +++ b/docs/results/quality.md @@ -0,0 +1,17 @@ +# Data Quality and Reports + +Part of [Interpreting Your Results](../interpreting-results.md). Read [Before You Look](../interpreting-results.md#before-you-look-what-the-pipeline-can-tell-you) first. + +## Reads, Alignment, Variant Calling and Coverage (Steps 1b, 2, 3, 16, 16b) + +These steps check that the data is good enough for everything else. Look at them first if a later result looks strange. + +- **fastp (step 1b):** `${SAMPLE}/fastq_trimmed/${SAMPLE}_fastp.html`. A few percent of reads trimmed or dropped is normal; a large share points at a library or upload problem. +- **Alignment and variant calling (steps 2 and 3):** the [example output](../interpreting-results.md#variant-calling-step-3) shows what a 30X genome looks like: about 4.5 to 5.5 million variants and a Ti/Tv ratio of 2.0 to 2.1. +- **Coverage (step 16b, mosdepth):** `${SAMPLE}/mosdepth/${SAMPLE}.mosdepth.summary.txt`. The `total` row should show a mean near the depth you paid for (about 30). Below 15, small-variant calls lose accuracy. See [step 16b](../16b-mosdepth.md#interpreting-results). +- **Sex check (step 16, indexcov):** `${SAMPLE}/indexcov/` estimates the copy number of chrX and chrY from the BAM index. About 1 and 1 is XY, about 2 and 0 is XX. A result that does not match the sex you gave the pipeline means a sample swap, a mislabelled file or a real sex-chromosome difference; [step 16](../16-indexcov.md#interpretation) lists the patterns. Re-check the input before reading anything else. + +## Reports (Steps 24 and 28) + +- **HTML report (step 24):** `${SAMPLE}/${SAMPLE}_report.html` collects the headline results of the other steps on one page, including the ClinVar hits table. It contains health findings: share it only as you would share a medical record. +- **MultiQC (step 28):** `${SAMPLE}/multiqc/multiqc_report.html` puts the QC of fastp, samtools, mosdepth and the other tools in one page. It is about data quality, not about your health. diff --git a/docs/results/secondary-findings.md b/docs/results/secondary-findings.md new file mode 100644 index 000000000..5f355c585 --- /dev/null +++ b/docs/results/secondary-findings.md @@ -0,0 +1,60 @@ +# Secondary Findings and Carrier Status + +Part of [Interpreting Your Results](../interpreting-results.md). Read [Before You Look](../interpreting-results.md#before-you-look-what-the-pipeline-can-tell-you) first. + +## Carrier Status Is Not Disease + +The most common "pathogenic" finding in any genome is **heterozygous carrier status for recessive conditions**. This means: + +- You have ONE copy of a variant that causes disease when BOTH copies are affected +- For most recessive conditions, heterozygous carriers are **not clinically affected** (though some carrier states confer subtle phenotypic effects — e.g., sickle cell trait, HFE carriers and iron loading) +- The primary relevance is for **family planning**: if your partner carries the same gene, each child has a 25% chance of being affected +- **Note — MUTYH**: Biallelic (homozygous or compound het) MUTYH carriers have a well-established high colorectal cancer risk. For **monoallelic** (single-copy) carriers, the evidence is more nuanced: some meta-analyses show a modest risk increase, but a counseling framework for moderate-penetrance CRC genes (Genetics in Medicine) notes that risk estimates for monoallelic MUTYH are conflicting and that screening recommendations (e.g., earlier colonoscopy) were historically tied to carriers with a CRC family history. Discuss with a genetic counselor, especially if you have a family history of CRC +- Examples: GJB2 (hearing loss), CFTR (cystic fibrosis), HFE (hemochromatosis) + +**Your own carrier list (step 24 report).** The ClinVar card of both reports is followed by a carrier list for this one sample, with no partner needed: every step 6 ClinVar hit in the 3,292 recessive and X-linked genes of `assets/carrier_genes.tsv` (the list of the [couple screen](../39-couple.md#the-carrier-list)). An allele that list marks as a hypomorph (for example BTD p.Asp444His or the GALT Duarte variants) or a pseudodeficiency allele (HEXA c.739C>T and c.745C>T, which lower the enzyme test result but cause no Tay-Sachs disease) is listed as set aside, never as a carrier finding on its own. + +**Secondary findings follow each gene's ACMG rule.** The secondary-findings card (step 23's section of the report) applies the reporting rule of each of the 84 ACMG SF v3.3 genes (Lee et al., Genet Med 2025;27(8):101454, Table 1), written gene by gene in `assets/acmg_sf.tsv`: + +- a dominant gene (BRCA1, LDLR, MYH7 and most others): one pathogenic or likely pathogenic variant is reported; +- a recessive gene (MUTYH, ATP7B, BTD, CYP27A1, GAA, RPE65, CASQ2, TRDN): two variants are needed, a homozygote or two different ones even when their phase is unknown. One heterozygous MUTYH variant is listed apart as not reported, with the reason; +- HFE: only homozygous p.Cys282Tyr. A heterozygous p.Cys282Tyr or a homozygous p.His63Asp is not reported; +- X-linked: GLA (Fabry disease) and OTC are reported in a hemizygous male and a heterozygous female alike. ABCD1 (X-linked adrenoleukodystrophy) is reported hemizygous, homozygous or with two heterozygous variants, never for one heterozygous variant; +- BTD with p.Asp444His: reported as partial, not profound, biotinidase deficiency; +- AnnotSV class 4 and 5 structural variants (step 5) in BRCA1, BRCA2, MLH1, MSH2 and LDLR reach the same card, and so does a deletion of the 3' end of EPCAM (exons 8 and 9), which silences MSH2 (Kovacs et al., Hum Mutat 2009;30(2):197-203). + +**Two hits across callers: a deletion plus a variant on the other copy.** Small-variant callers see one hit when the other copy of the gene is deleted. So the carrier list also joins the deletions the other engines call with the ClinVar hits, gene by gene (`og summary collect`, `two_hits`). The deletions come from AnnotSV's class 4 and 5 deletions (step 5), step 35's interval table (the recurrent regions of `bin/cn_loci.tsv`, such as 1q21.1), its HBA1/HBA2 count and Parascopy's SMN1 count. A recessive gene that a deletion and a hit both reach gets one line: + +- **biallelic** when the reads show the two hits on different copies. Either the hit is called homozygous inside the deletion, so it sits on the one copy left (the line and the carrier list call it hemizygous), or both calls carry read-backed phase (a `|` genotype with the same phase set, PS) on different copies; +- **possibly biallelic** otherwise: the phase is unknown. Most short-read calls carry no phase, so this is the usual line. Step 05's checked table keeps no genotype column yet, and step 35's counts have no phase, so in a pipeline run a deletion pairs as possibly biallelic unless the hit is called homozygous inside it. A parent's genome (step 38, [family analysis](../multi-sample.md#family-analysis-step-38-opt-in)) or long reads can settle it; +- no line when read-backed phase puts both on the same copy (cis): the other copy is intact. + +Examples: an exon deletion of MUTYH plus a pathogenic MUTYH variant; three alpha-globin genes plus an HBA2 point variant; one SMN1 copy plus an SMN1 variant (short reads place few SMN1 variants reliably, so check with a targeted test); and the 1q21.1 deletion plus one of the two RBM8A low-expression alleles (c.-21G>A, rs139428292, or c.67+32G>C, rs201779890), the cause of TAR syndrome (thrombocytopenia with absent radius; Albers et al., Nat Genet 2012;44(4):435-439, PMID 22366785). On its own an RBM8A low-expression allele stays set aside. The [couple screen](../39-couple.md) (step 39) applies the same pairing between partners: a hypomorph allele of the list in one partner and a full loss of function in the same gene in the other (a ClinVar pathogenic variant, a Delly deletion) is a shared gene with its risk for a child; the allele in both partners, or in one alone, stays set aside. In the secondary-findings card a called deletion counts as one of the two variants a recessive gene needs, unless read-backed phase puts it on the variant's copy. The line under the carrier list names the deletion callers it read and the ones it could not read. A deletion AnnotSV ranks below class 4 is not counted. + +The TTN rule (truncating variants only) and the LMNA rule (missense only with cardiac evidence) are printed beside the finding, not applied. It is still a list to review with a clinician. + +--- + +## CPSR Report (Step 17) + +**What it tells you:** Cancer predisposition screening using CPSR's curated cancer gene panels (panel 0 covers 500+ genes). Step 17 also turns on CPSR's secondary findings: the ACMG SF list, which includes cardiac and metabolic genes outside cancer (see [Before you look](../interpreting-results.md#before-you-look-what-the-pipeline-can-tell-you)). + +**Where to look:** `${SAMPLE}/cpsr/` — open the HTML report in a browser. + +**How to read it:** +- CPSR puts every variant in its genes into one of the five ACMG/AMP classes: **Pathogenic**, **Likely pathogenic**, **VUS** (uncertain significance), **Likely benign** and **Benign**. A variant that already has a ClinVar entry can carry ClinVar's class instead of the one CPSR computed. +- Only **Pathogenic** and **Likely pathogenic** call for clinical attention. The report lists them first, and the secondary findings in their own section. + +--- + +## Clinical Filter (Step 23) + +**Where to look:** `${SAMPLE}/clinical/${SAMPLE}_clinical_summary.tsv`, one row per variant with gene, impact, scores and gnomAD constraint. + +Step 23 keeps the HIGH-impact variants, the MODERATE ones below 1% in gnomAD exomes, the ClinVar pathogenic ones and, when step 30 ran, the variants that CADD, SpliceAI, REVEL or AlphaMissense flag. Expect a few hundred rows. Most are heterozygous variants in genes that tolerate one broken copy. Read the rows in constrained genes (low LOEUF, see [gene constraint](clinvar.md#gnomad-gene-constraint-step-23-summary)) first, then the homozygous ones. The HIGH-impact rows have no frequency filter, so check gnomAD for each before you worry. [Step 23](../23-clinical-filter.md#what-gets-filtered) gives the exact rules. + +## Variant Prioritization (Step 31) + +**Where to look:** `${SAMPLE}/slivar/${SAMPLE}_slivar_summary.tsv`, `${SAMPLE}/slivar/${SAMPLE}_compound_hets.tsv` and `${SAMPLE}/slivar/${SAMPLE}_compound_het_pairs.tsv`. + +slivar sorts the damaging variants into four groups (rare HIGH, rare MODERATE with damaging scores, ClinVar pathogenic, and frequency unknown: HIGH or MODERATE with no population frequency, so not called rare) and lists genes where you carry two such variants. The genotypes alone cannot tell whether the two variants sit on different copies of the gene (in trans, which can cause recessive disease) or on the same copy (in cis, which usually does not), so step 31 reads the phase from the reads: `${SAMPLE}_compound_het_pairs.tsv` says `trans`, `cis` or `unknown` for each pair, with the reads that say so. Only variants close enough for one read pair to cover both (a few hundred bases) can be phased this way, so most pairs read `unknown`. Expect a thousand or more candidate pairs; nearly all are noise. A `trans` pair matters only in a gene that fits your health history, and an `unknown` one needs a parent's DNA or long reads to phase. See [step 31](../31-slivar.md#interpretation). diff --git a/docs/results/structural-and-repeats.md b/docs/results/structural-and-repeats.md new file mode 100644 index 000000000..7afbb24aa --- /dev/null +++ b/docs/results/structural-and-repeats.md @@ -0,0 +1,180 @@ +# Structural Variants, Copy Number and Repeats + +Part of [Interpreting Your Results](../interpreting-results.md). Read [Before You Look](../interpreting-results.md#before-you-look-what-the-pipeline-can-tell-you) first. + +## Structural Variants (Steps 4, 5, 15, 18, 19, 22) + +### What Are Structural Variants? + +Unlike SNPs (single letter changes), structural variants are large rearrangements: +- **Deletions (DEL):** A chunk of DNA is missing +- **Duplications (DUP):** A chunk is copied extra times +- **Inversions (INV):** A chunk is flipped backwards +- **Translocations (BND):** A chunk moved to a different chromosome +- **Insertions (INS):** New DNA inserted + +### A Note About BND (Breakend) Calls + +If you run Manta or Delly, you will see many **BND** calls — often hundreds or thousands. BND indicates a "breakend" where one end of a read pair maps to a different chromosome or a distant location. This sounds alarming ("translocation!") but: + +- **Most BND calls are artifacts** of repetitive regions, segmental duplications, or mobile elements +- A typical genome has 1,000-3,000 BND calls from Manta and 5,000+ from Delly +- **Fewer than 5 are likely real** inter-chromosomal translocations in a healthy genome +- BND calls require **multi-caller support** (called by both Manta and Delly at overlapping breakpoints) to be considered credible +- Unless a BND disrupts a known disease gene AND is confirmed by a second caller, it can be safely ignored + +### How Many Is Normal? + +A typical human genome has: +- ~5,000-10,000 structural variants total +- Most are in non-coding regions and harmless +- ~5-20 may affect genes +- 0-2 may be clinically significant + +### Which Callers to Trust? + +If you ran multiple SV callers: +- **Called by 2+ callers (Manta + Delly, or Manta + CNVpytor):** Lower false-positive rate +- **Called by 1 caller only:** Lower confidence, may be false positive +- **duphold DHFFC < 0.7 for deletions:** High confidence (depth drops as expected) +- **duphold DHBFC > 1.3 for duplications:** High confidence (depth rises as expected) + +### SV Consensus (Step 22) + +`${SAMPLE}/sv_merged/${SAMPLE}_sv_consensus.vcf.gz` keeps the SVs that two or more callers found (Manta, Delly, CNVpytor and, when they ran, Sniffles2 and TIDDIT): SURVIVOR pairs two calls when both breakpoints lie within 1 kb of each other and their type and strands agree. `SUPP` says how many callers found each one and `SUPP_VEC` which. GRIDSS is left out (its breakend records never match). It is the short list to read first, but it drops real SVs that only one caller found; see [step 22](../22-survivor-merge.md#limitations). + +### AnnotSV Output + +The AnnotSV TSV (step 5) adds clinical annotations to each SV: +- `ACMG_class`: 1 (benign) to 5 (pathogenic) +- `Overlapped_CDS_percent`: How much of a gene is affected +- Focus on SVs with `ACMG_class` 4 or 5 that overlap known disease genes + + +--- + +## STR Expansions (Step 9) + +### What Are Repeat Expansions? + +Some regions of DNA have short sequences repeated many times (e.g., CAG CAG CAG...). When the number of repeats exceeds a threshold, it can cause disease. + +### How to Read ExpansionHunter Output + +The output VCF lists each tested locus with the number of repeats found. Key loci: + +| Locus | Gene | Normal | Intermediate / Premutation | Pathogenic | Disease | +|---|---|---|---|---|---| +| HTT | HTT | <=26 | 27-35 (mutable normal); 36-39 (reduced penetrance) | >=40 (full penetrance) | Huntington's disease | +| FMR1 | FMR1 | <45 | 45-54 (intermediate); 55-200 (premutation) | >200 | Fragile X syndrome | +| ATXN1 | ATXN1 | <33 | — | >39 | Spinocerebellar ataxia 1 | +| C9orf72 | C9orf72 | <24 | 24-30 (gray zone, lab cutoffs vary) | Typically hundreds-thousands | ALS / Frontotemporal dementia | +| DMPK | DMPK | <35 | 35-49 (premutation) | >=50 | Myotonic dystrophy type 1 | + +**HTT 36-39 repeats (reduced penetrance):** Individuals in this range may or may not develop Huntington's disease. The risk increases with repeat length but is not certain. GeneReviews classifies >=40 as full penetrance and 36-39 as reduced penetrance. Alleles of 27-35 ("mutable normal") do not cause disease but may expand in offspring. + +**C9orf72 gray zone:** The exact pathogenic threshold for C9orf72 is not established. Laboratory cutoffs are discordant (JNNP 2021 review). Clearly pathogenic expansions are typically hundreds to thousands of repeats. Short-read WGS has limited ability to size very large expansions accurately. + +**FMR1 intermediate zone (45-54 repeats):** Not affected, but repeats may expand in offspring. Carriers should receive genetic counseling. Premutation (55-200) carries risk of FXTAS (males >50) and FXPOI. + +The report prints no all-clear verdict for these loci. It lists every locus in the ExpansionHunter output, the five above first, with its repeat count, Stranger's status and the catalog's normal maximum (`STR_NORMAL_MAX`), and, when step 9b (Stranger) ran, every locus Stranger marks `pre_mutation` or `full_mutation`; without step 9b, compare the counts with the table yourself. It also says: + +- **Not sized.** A locus whose call rests on flanking reads only prints "size not determined (flanking reads only)"; one with in-repeat reads only (longer than a read) prints its count as at least the lower end of ExpansionHunter's interval; one with fewer than 2 spanning reads, or an interval that reaches the catalog's pathogenic minimum, is marked low confidence. Such a locus is listed apart and never shown as normal, because an unsized repeat can hide an expansion. +- **Intermediate range.** A sized count inside the HTT, FMR1 or DMPK intermediate range of the table above prints the range, a note that it does not cause disease but may expand in offspring, and the GeneReviews chapter. The report reads these ranges from `bin/str_intermediate_ranges.tsv`, and `tests/test_str_ranges.py` checks that the table above prints the same ones. +- **RFC1.** A `full_mutation` flag at RFC1 carries the motif verdict of step 9b ([RFC1 note](../09b-stranger.md#notes)). + +A count under every threshold is not a guarantee: repeat-expansion calls from short reads can be wrong at some loci, and a long expansion cannot be sized. Intermediate-range results should be discussed with a genetic counselor. + +### Stranger Annotation (Step 9b) + +Step 9b adds a `STR_STATUS` field to each locus in the VCF so you do not need to look up thresholds manually: + +| STR_STATUS | Meaning | +|---|---| +| `normal` | Repeat count is within the established normal range | +| `pre_mutation` | Elevated repeat count; not currently disease-causing but carries risk of expansion in offspring or late-onset carrier effects (e.g. FXTAS, FXPOI for FMR1) | +| `full_mutation` | Repeat count exceeds the pathogenic threshold for this locus | + +The `Disease`, `InheritanceMode`, `SourceDisplay`, `STR_NORMAL_MAX`, and `STR_PATHOLOGIC_MIN` INFO fields give the clinical context for each locus (a field the catalog has no value for prints as `.`). Use a VCF viewer or `bcftools query` to extract them: + +```bash +bcftools query -f '%INFO/STR_STATUS\t%INFO/Disease\t%INFO/InheritanceMode\t%INFO/SourceDisplay\t%INFO/STR_NORMAL_MAX\t%INFO/STR_PATHOLOGIC_MIN\n' \ + expansion_hunter/_eh_stranger.vcf +``` + +`full_mutation` at any locus warrants follow-up with a clinical geneticist. Short-read WGS has limited sizing accuracy for very large expansions (>150 repeats), so confirmatory testing may be recommended. + +--- + +## CNVpytor Results (Step 18) + +CNVpytor detects **copy number variants** using read depth analysis — complementary to Manta's paired-end/split-read approach. + +**Where to look:** `${SAMPLE}/cnvpytor/${SAMPLE}_cnvs.txt` + +**Format:** Each line has: type, region, size, normalized_RD, e-value1, e-value2, e-value3, e-value4, q0 + +**What to expect:** +- 3,000-4,000 total CNVs (mostly deletions) +- 1,500-2,000 significant (e-value < 0.01) +- Calls at chromosome starts (chr1:1-10000) are telomeric artifacts — ignore them + +**Filtering:** +```bash +# Significant CNVs only (e-value < 0.01) +awk '$5 < 0.01' ${SAMPLE}_cnvs.txt + +# Large deletions (>100kb, potentially clinically relevant) +awk '$1 == "deletion" && $3 > 100000 && $5 < 0.01' ${SAMPLE}_cnvs.txt + +# Large duplications +awk '$1 == "duplication" && $3 > 100000 && $5 < 0.01' ${SAMPLE}_cnvs.txt +``` + +**Multi-caller overlap:** CNVs found by both Manta AND CNVpytor have lower false-positive rates. Cross-reference by checking if the same genomic region appears in both output files. + +--- + +## Delly Results (Step 19) + +Delly is a third structural variant caller, detecting deletions, duplications, inversions, and translocations. + +**Where to look:** `${SAMPLE}/delly/${SAMPLE}_sv.vcf.gz` + +**Quick summary:** +```bash +# Count SVs by type +bcftools query -f '%INFO/SVTYPE\n' ${SAMPLE}_sv.vcf.gz | sort | uniq -c + +# Filter PASS variants only +bcftools view -f PASS ${SAMPLE}_sv.vcf.gz | grep -cv '^#' +``` + +**What to expect:** +- 5,000-15,000 total SV calls +- Most are small deletions (<1kb) +- PASS filter reduces count significantly + +**Multi-caller overlap:** SVs detected by Manta + Delly + CNVpytor (or any 2 of 3) have substantially lower false-positive rates. Single-caller calls, especially large ones, should be viewed with caution. + +--- + +## SMN1 and SMN2 Copy Number (Step 35, opt-in) + +**Where to look:** `${SAMPLE}/paralogs/${SAMPLE}_smn_copy_number.tsv`, one row per stretch of the SMN1/SMN2 locus with the copy number of the two genes together (agCN) and of each (psCN), each with a quality. + +One SMN1 copy suggests SMA carrier status; two do not rule it out (two copies on one chromosome, none on the other). Trust a value only with quality 20 or more and filter `PASS`, and confirm anything that matters with a clinical SMN1 test. [Step 35](../35-paralogs.md#interpreting-results) explains the columns. + +**The silent-carrier marker:** `${SAMPLE}/paralogs/${SAMPLE}_smn1_marker.tsv` reads SMN1 g.27134T>G (rs143838139), and both reports and the live person page print it right under the SMN1 copies: `present`, `uncertain`, `not seen`, `too little depth` or `not assessed`. With two SMN1 copies, `present` raises the chance that both sit on one chromosome (2+0, a silent carrier) and `not seen` lowers it; it proves neither, and the figures depend on ancestry. A carrier test in a clinical laboratory confirms it. [The SMN1 silent-carrier marker](../35-paralogs.md#the-smn1-silent-carrier-marker) gives the thresholds. + +## Lp(a) and the LPA KIV-2 Count (Step 46, opt-in) + +**Where to look:** `${SAMPLE}/lpa/${SAMPLE}_lpa_kiv2.tsv`, the KIV-2 repeat count over both chromosomes with its 95% interval and class, and `${SAMPLE}/lpa/${SAMPLE}_lpa_kiv2_4925.tsv`, the KIV-2 4925G>A splice variant. Both reports put the count beside the two LPA risk variants of step 42 (rs10455872, rs3798220), and print the 4925G>A line right under the count: `present` or `uncertain` with the reads that carry it and the copies, `not seen`, `too little depth` or `not assessed`. + +Fewer repeats or a risk allele point to a higher Lp(a); a 4925G>A copy lowers the Lp(a) made from the chromosome that carries it. Neither tells which chromosome carries what, and `not seen` is not proof of absence at low depth. The count is a research estimate: the Lp(a) blood test is the measure, once in every adult. [Step 46](../46-lpa.md) explains the columns. + +## Mobile Element Insertions (Step 50, opt-in) + +**Where to look:** `${SAMPLE}/mei/${SAMPLE}_mei_sites.tsv` for the three known disease insertions (SVA insertions in FKTN, TAF1 and BBS1), and `${SAMPLE}/mei/${SAMPLE}_mei.tsv` for every Alu, L1 and SVA insertion MEGAnE called, with one or two copies. + +Expect many calls: a genome carries about 1,200 such insertions, nearly all common and harmless. Read the catalog first. **Seen** means an insertion of the same element was called inside the site's window; one copy of the FKTN or BBS1 insertion is carrier status for a recessive condition, and the TAF1 insertion on a male's single X is what causes X-linked dystonia-parkinsonism. **Not seen** is not proof of absence: short reads find an SVA's edges, not its body. **Not callable** means too few reads there (under 10x) to tell. Confirm anything that matters in a clinical laboratory. [Step 50](../50-mei.md) explains the columns. diff --git a/docs/results/traits-and-somatic.md b/docs/results/traits-and-somatic.md new file mode 100644 index 000000000..648d7851e --- /dev/null +++ b/docs/results/traits-and-somatic.md @@ -0,0 +1,52 @@ +# Telomeres and Somatic Variants + +Part of [Interpreting Your Results](../interpreting-results.md). Read [Before You Look](../interpreting-results.md#before-you-look-what-the-pipeline-can-tell-you) first. + +## Telomere Length (Step 10) + +### What It Means + +Telomeres are protective caps at the ends of chromosomes that shorten with cell division. TelomereHunter measures `tel_content`: intratelomeric reads per million reads with 48-52% GC. It is a relative telomere content, not a telomere length, and the report labels it "Telomere content (relative)". + +### How to Interpret + +- **Higher `tel_content` = more telomeric reads** (generally correlates with longer telomeres) +- **Lower `tel_content` = fewer telomeric reads** (generally correlates with shorter telomeres) +- There is no universal "normal" range — compare between samples of similar age, sequenced on the same platform +- Typical `tel_content` for 30X WGS: 300-800 (varies by sequencing platform and coverage) + +### Limitations + +- **Not a clinical telomere length measurement.** TelomereHunter was developed for cancer genome analysis, not as a validated healthy-population aging assay +- **Not a "biological age" readout.** While telomere length correlates with aging at a population level, it is a rough estimate of aging rate and is not established as a clinically important standalone risk marker for individuals (see Vaiserman & Krasnienkov, "Telomere Length as a Marker of Biological Age," 2021) +- Short-read WGS systematically underestimates telomere length compared to dedicated assays (TRF, FlowFISH) +- Useful only for **relative comparisons** between samples run on the same platform — not absolute measurements and not individual health predictions + +--- + +## Somatic Variants (Step 29) [EXPERIMENTAL] + +Mutect2 in tumor-only mode looks for somatic mutations -- variants acquired during your lifetime rather than inherited. Consumer WGS is usually made from saliva or a cheek swab, a mix of cheek cells and white blood cells; the main category of interest is **clonal hematopoiesis (CHIP)**, which lives in the blood cells. + +**Where to look:** `${SAMPLE}/somatic/${SAMPLE}_somatic_filtered.vcf.gz` + +**Key field:** `AF` (allele fraction) indicates the clone size: + +| AF Range | Likely Source | +|---|---| +| 0.45-0.55 | Heterozygous germline (false positive) | +| ~1.0 | Homozygous germline (false positive) | +| 0.10-0.40 | Could be a large somatic clone, mosaic, or noisy germline | +| below 0.10 | At 30X this is one to three reads: mostly noise, see below | + +**What to expect:** +- Thousands of PASS calls in a healthy individual -- the vast majority are germline false positives +- At 30X a variant in 2% of the reads has less than one supporting read on average, and one in 10% about three. Only variants at roughly 10% allele fraction or more can be told apart from noise. That is an approximate rule about reads, not a clone size: a heterozygous variant at 10% allele fraction sits in about 20% of the sampled cells, and copy number and the mix of cell types in the sample shift it. Most CHIP (defined from 2% allele fraction) is invisible here, and a clean result does not rule it out +- Without a matched normal sample, germline variants that are rare in gnomAD will often pass all filters + +**CHIP genes to check:** DNMT3A, TET2, ASXL1, TP53, JAK2, SF3B1, SRSF2, PPM1D, CBL. CHIP prevalence increases with age and is associated with elevated cardiovascular risk and risk of hematologic malignancies. + +**Important caveats:** +- This step has a much higher false positive rate than any other step in the pipeline +- Do not interpret PASS variants as confirmed somatic without cross-referencing with the germline VCF (step 3) and gnomAD frequencies (step 13) +- If a variant is also called at ~50% AF by DeepVariant in step 3, it is almost certainly germline diff --git a/tests/test_str_ranges.py b/tests/test_str_ranges.py index b842e9633..38b345fbe 100644 --- a/tests/test_str_ranges.py +++ b/tests/test_str_ranges.py @@ -12,10 +12,11 @@ 2. bin/str_intermediate_ranges.tsv, the GeneReviews ranges the report names (HTT, FMR1, DMPK): the columns og summary collect reads, one row per locus, whole numbers with min <= max ending right below the next class, a - label and a GeneReviews citation; docs/interpreting-results.md prints the - same range in the locus's row; every locus has a row in str_ranges.tsv, - and the range lies above its normal_max and below its pathogenic_min, so a - count the report calls intermediate is one Stranger calls pre_mutation + label and a GeneReviews citation; docs/results/structural-and-repeats.md + prints the same range in the locus's row; every locus has a row in + str_ranges.tsv, and the range lies above its normal_max and below its + pathogenic_min, so a count the report calls intermediate is one Stranger + calls pre_mutation (control: a planted HTT range of 20-35 is caught). That og reads every row is og-summary's own test (collect::group_c::tests). 3. The "Key Disease Thresholds" table of docs/09-str-expansions.md: each @@ -35,7 +36,7 @@ import test_str_catalog # noqa: E402 TSV = os.path.join(REPO, "bin", "str_intermediate_ranges.tsv") -DOC = os.path.join(REPO, "docs", "interpreting-results.md") +DOC = os.path.join(REPO, "docs", "results", "structural-and-repeats.md") STEP_DOC = os.path.join(REPO, "docs", "09-str-expansions.md") COLUMNS = ["locus", "intermediate_min", "intermediate_max", "next_min", "next_class", "label", "citation"] RANGE_COLUMNS = ["locus", "source_id", "normal_max", "intermediate_min", "intermediate_max", "pathogenic_min", @@ -65,7 +66,7 @@ def table(path): def doc_rows(): - """Locus -> its row of the repeat table in docs/interpreting-results.md.""" + """Locus -> its row of the repeat table in docs/results/structural-and-repeats.md.""" out = {} for line in open(DOC): c = [x.strip() for x in line.strip().strip("|").split("|")] @@ -155,7 +156,7 @@ def inside(locus, lo, hi): check(f"{d['locus']}: a label and a next class", bool(d["label"].strip() and d["next_class"].strip()), d) check(f"{d['locus']}: cited to a GeneReviews chapter", bool(CITATION.match(d["citation"])), d["citation"]) row = docs.get(d["locus"].upper(), "") - check(f"{d['locus']}: docs/interpreting-results.md prints {lo}-{hi} in its row", + check(f"{d['locus']}: docs/results/structural-and-repeats.md prints {lo}-{hi} in its row", bool(re.search(rf"(? Date: Sun, 15 Nov 2026 14:07:02 +0100 Subject: [PATCH 171/255] docs(changelog): note the results guide split into one page per chapter --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf6c9e6d..371624eee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The variant calling page names the Genome Reference Consortium as the source of the PAR intervals in `assets/par_grch38.bed`. - Every verb the 1.0.0 release still has to port now exists in the `og` binary, each in its own module, as a stub that prints `og : not ported yet` and exits 2. The new crates are og-render, og-live, og-couple and og-admix. Each port then fills one module and never edits `rust/og/src/main.rs` or a crate's module list, so the ports can land in parallel. No step calls these verbs yet, and no output changes. - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. +- The guide to interpreting results now has one page per report chapter, under `docs/results/`: secondary findings and carrier status, ClinVar and variant annotation, pharmacogenomics and HLA, structural variants and repeats, ancestry and lineage, telomeres and somatic variants, and data quality. The text moved word for word. `docs/interpreting-results.md` keeps the "Before You Look" bullets, the "Before You Panic" table, the example outputs and the general advice, and lists every chapter page. Each old link to a section of that page still works: it lands on a link to the section's new place. + - The repeat table moved with its chapter, so `tests/test_str_ranges.py` now reads it from `docs/results/structural-and-repeats.md`. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. From 9509cc6a7344160ee8b901d02176b85be24adbcf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:07:54 +0100 Subject: [PATCH 172/255] fix(clinvar): read FILTER, depth and one-copy calls at always-report sites One state table decides each site's words. Only FILTER PASS, '.' and RefCall leave a reference call plain; any other FILTER makes it "ref (low confidence)" and names the FILTER. A reference call with no GQ is low confidence too, since nothing checked it against the floor. A depth below the step's depth floor reads "ref (shallow)". On chrX and chrY outside the pseudoautosomal regions a one-copy 1 is "hemizygous" and a one-copy 0 a reference call; elsewhere a one-copy GT is no call, with why. A no-call record says what it holds (no genotype, not a reference call) and never claims a PASS check or that no record exists. Bead: clin-09 Bead: clin-10 Bead: clin-11 Bead: clin-12 Bead: clin-13 Bead: clin-14 --- rust/crates/og-clinvar/src/sites.rs | 317 ++++++++++++++++++++-------- 1 file changed, 233 insertions(+), 84 deletions(-) diff --git a/rust/crates/og-clinvar/src/sites.rs b/rust/crates/og-clinvar/src/sites.rs index f24fcffc8..5c290bf83 100644 --- a/rust/crates/og-clinvar/src/sites.rs +++ b/rust/crates/og-clinvar/src/sites.rs @@ -4,25 +4,32 @@ //! The list (`assets/deep_intronic_sites.tsv`) has `#` comment lines, a //! header row and the columns `chrom pos ref alt gene hgvs clinvar_id`. The //! calls are `bcftools query -//! '%CHROM\t%POS\t%END\t%REF\t%ALT\t[%GT]\t[%GQ]\t[%DP]\n'` of the split PASS -//! VCF and of the gVCF at those positions. One row per site comes out, with -//! two columns more: +//! '%CHROM\t%POS\t%END\t%REF\t%ALT\t[%GT]\t[%GQ]\t[%DP]\t%FILTER\n'` of the +//! split PASS VCF and of the gVCF at those positions (an input without the +//! FILTER column reads as FILTER `.`). One row per site comes out, with two +//! columns more: //! //! - `call`: the genotype in words when the sample carries the site's -//! allele; `ref` for a reference call (a `0/0` record, or a gVCF reference -//! block over the site), `ref (low confidence)` when its GQ is below the +//! allele (`hemizygous` for one copy on chrX or chrY outside the +//! pseudoautosomal regions); `ref` for a reference call (a record whose +//! alleles are all `0`, or a gVCF reference block over the site) with +//! FILTER `PASS`, `.` or `RefCall`, a GQ at or above the call floor and a +//! depth at or above the depth floor; `ref (low confidence)` for a +//! reference call with another FILTER, a GQ below the call floor or no GQ; +//! `ref (shallow)` for one whose only fault is a depth below the depth //! floor; `other allele` when the sample carries another allele there; //! `no call` otherwise. -//! - `detail`: the call's `GQ n, DP n` (`GQ n (gVCF block)` for a block), the -//! floor a low-confidence reference call is below, the other allele, or -//! why there is no call. +//! - `detail`: the call's `GQ n, DP n` (`GQ n (gVCF block)` for a block, +//! with its DP when that is below the depth floor), each reason a +//! reference call is not plain `ref`, the other allele, or what the record +//! at the site holds when there is no call. use std::io::{self, Write}; use crate::text::{is_digits, nth, same, split}; use crate::zygosity; -/// The floor and whether there is a gVCF to read reference blocks from. +/// The floors and whether there is a gVCF to read reference blocks from. pub struct Options { /// The GQ floor as given, for the words. pub min_gq: String, @@ -45,36 +52,72 @@ fn leading_number(s: &[u8]) -> f64 { .unwrap_or(0.0) } -fn carries(gt: &[u8]) -> bool { - gt.iter().any(|c| (b'1'..=b'9').contains(c)) -} +/// GRCh38's pseudoautosomal regions, 1-based and inclusive, as in +/// `assets/par_grch38.bed`. +const PAR_X: [(u64, u64); 2] = [(10_001, 2_781_479), (155_701_383, 156_030_895)]; +const PAR_Y: [(u64, u64); 2] = [(10_001, 2_781_479), (56_887_903, 57_217_415)]; -fn is_ref(gt: &[u8]) -> bool { - matches!(gt, [b'0', b'/' | b'|', b'0']) +/// True where a caller may write one copy: chrX and chrY outside the +/// pseudoautosomal regions. +fn one_copy(chrom: &[u8], pos: u64) -> bool { + let par = match chrom.strip_prefix(b"chr").unwrap_or(chrom) { + b"X" => &PAR_X, + b"Y" => &PAR_Y, + _ => return false, + }; + !par.iter().any(|&(a, b)| (a..=b).contains(&pos)) } -impl Options { - fn below(&self, gq: &[u8]) -> bool { - is_digits(gq) && leading_number(gq) < self.min_gq_value as f64 - } +/// What a record's GT says. +#[derive(Clone, Copy, PartialEq)] +enum Gt { + /// Every allele `0`. + Ref, + /// An alt allele. + Carries, + /// No allele called. + NoGenotype, + /// One allele where the genome has two. + OneCopyWhereTwo, + /// Anything else, such as `0/.`. + Other, +} - fn refcall(&self, gq: &[u8]) -> &'static [u8] { - if self.below(gq) { - b"ref (low confidence)" +impl Gt { + fn of(gt: &[u8], one_copy_site: bool) -> Gt { + let alleles: Vec<&[u8]> = gt + .split(|&c| c == b'/' || c == b'|') + .filter(|a| !a.is_empty()) + .collect(); + if alleles.iter().all(|&a| a == b".") { + Gt::NoGenotype + } else if alleles.len() == 1 && !one_copy_site { + Gt::OneCopyWhereTwo + } else if alleles.iter().all(|&a| a == b"0") { + Gt::Ref + } else if gt.iter().any(|c| (b'1'..=b'9').contains(c)) { + Gt::Carries } else { - b"ref" + Gt::Other } } +} - fn floor_note(&self, gq: &[u8]) -> Vec { - if self.below(gq) { - [b", below the call floor of GQ ", self.min_gq.as_bytes()].concat() - } else { - Vec::new() - } +/// A carried GT in words: `hemizygous` for one copy, else as the hits word it. +fn words(gt: &[u8]) -> Vec { + if gt.contains(&b'/') || gt.contains(&b'|') { + zygosity(gt) + } else { + b"hemizygous".to_vec() } } +/// True for a FILTER that leaves a reference call plain: `PASS`, `.`, none, +/// or DeepVariant's `RefCall` (the model reads the site as reference). +fn clean(filter: &[u8]) -> bool { + matches!(filter, b"" | b"." | b"PASS" | b"RefCall") +} + /// `GQ n, DP n`, each when it is a whole number. fn quality(gq: &[u8], dp: &[u8]) -> Vec { let mut out = Vec::new(); @@ -90,6 +133,81 @@ fn quality(gq: &[u8], dp: &[u8]) -> Vec { out } +impl Options { + /// A reference call: its words and detail from its GQ, depth and FILTER. + fn reference(&self, gq: &[u8], dp: &[u8], filter: &[u8], block: bool) -> (Vec, Vec) { + let filtered = !clean(filter); + let no_gq = !is_digits(gq); + let low_gq = !no_gq && leading_number(gq) < self.min_gq_value as f64; + let shallow = is_digits(dp) && leading_number(dp) < self.min_dp_value as f64; + let call: &[u8] = if filtered || no_gq || low_gq { + b"ref (low confidence)" + } else if shallow { + b"ref (shallow)" + } else { + b"ref" + }; + let mut detail = quality(gq, if block && !shallow { b"" } else { dp }); + if block { + if !detail.is_empty() { + detail.push(b' '); + } + detail.extend_from_slice(b"(gVCF block)"); + } + let mut notes: Vec> = Vec::new(); + if filtered { + notes.push([b"FILTER ", filter].concat()); + } + if low_gq { + notes.push([b"below the call floor of GQ ", self.min_gq.as_bytes()].concat()); + } + if no_gq { + notes.push( + [ + b"no GQ to check against the call floor of GQ ", + self.min_gq.as_bytes(), + ] + .concat(), + ); + } + if shallow { + notes.push([b"below the depth floor of DP ", self.min_dp.as_bytes()].concat()); + } + for n in notes { + if !detail.is_empty() { + detail.extend_from_slice(b", "); + } + detail.extend(n); + } + (call.to_vec(), detail) + } +} + +/// Why a record (`what`) gives no call. +fn unread(what: &[u8], gt: &[u8], filter: &[u8], state: Gt) -> Vec { + let gt: &[u8] = if gt.is_empty() { b"." } else { gt }; + match state { + Gt::NoGenotype => [what, b" has no genotype (GT ", gt, b")"].concat(), + Gt::OneCopyWhereTwo => [ + what, + b" has one copy (GT ", + gt, + b") where the genome has two", + ] + .concat(), + _ if !clean(filter) => [ + what, + b" is not a reference call (GT ", + gt, + b", FILTER ", + filter, + b")", + ] + .concat(), + _ => [what, b" is not a reference call (GT ", gt, b")"].concat(), + } +} + /// The lines of a text, without line endings. fn lines(text: &[u8]) -> Vec<&[u8]> { let mut v: Vec<&[u8]> = text.split(|&c| c == b'\n').collect(); @@ -101,67 +219,59 @@ fn lines(text: &[u8]) -> Vec<&[u8]> { /// The call at one site. fn call(st: &[&[u8]], vcf: &[Vec<&[u8]>], gvcf: &[Vec<&[u8]>], o: &Options) -> (Vec, Vec) { - let (mut call, mut detail, mut other) = (Vec::new(), Vec::new(), Vec::new()); + let pos = leading_number(nth(st, 2)); + let haploid = one_copy(nth(st, 1), pos as u64); + let (mut found, mut other, mut why) = (None, Vec::new(), None); for vr in vcf { if !same(nth(vr, 1), nth(st, 1)) || !same(nth(vr, 2), nth(st, 2)) { continue; } - let gt = nth(vr, 6); + let (gt, gq, dp, filter) = (nth(vr, 6), nth(vr, 7), nth(vr, 8), nth(vr, 9)); + let state = Gt::of(gt, haploid); if same(nth(vr, 4), nth(st, 3)) && same(nth(vr, 5), nth(st, 4)) { - if carries(gt) { - call = zygosity(gt); - detail = quality(nth(vr, 7), nth(vr, 8)); - } else if is_ref(gt) { - call = o.refcall(nth(vr, 7)).to_vec(); - detail = [quality(nth(vr, 7), nth(vr, 8)), o.floor_note(nth(vr, 7))].concat(); + match state { + Gt::Carries => found = Some((words(gt), quality(gq, dp))), + Gt::Ref => found = Some(o.reference(gq, dp, filter, false)), + _ => { + why.get_or_insert_with(|| { + unread(b"the VCF record at the site", gt, filter, state) + }); + } } - } else if carries(gt) && other.is_empty() { - other = [nth(vr, 4), b">", nth(vr, 5), b" ", &zygosity(gt)].concat(); + } else if state == Gt::Carries && other.is_empty() { + other = [nth(vr, 4), b">", nth(vr, 5), b" ", &words(gt)].concat(); } } - if call.is_empty() && !other.is_empty() { + if let Some(f) = found { + return f; + } + if !other.is_empty() { return (b"other allele".to_vec(), other); } - if call.is_empty() { - let pos = leading_number(nth(st, 2)); - let over = gvcf.iter().find(|gr| { - same(nth(gr, 1), nth(st, 1)) - && leading_number(nth(gr, 2)) <= pos - && leading_number(nth(gr, 3)) >= pos - }); - if let Some(gr) = over { - let (gt, gq) = (nth(gr, 6), nth(gr, 7)); - if is_ref(gt) { - let sep: &[u8] = if is_digits(gq) { b" " } else { b"" }; - return ( - o.refcall(gq).to_vec(), - [ - quality(gq, b""), - sep.to_vec(), - b"(gVCF block)".to_vec(), - o.floor_note(gq), - ] - .concat(), - ); - } - return ( + let over = gvcf.iter().find(|gr| { + same(nth(gr, 1), nth(st, 1)) + && leading_number(nth(gr, 2)) <= pos + && leading_number(nth(gr, 3)) >= pos + }); + if let Some(gr) = over { + let (gt, filter) = (nth(gr, 6), nth(gr, 9)); + return match Gt::of(gt, haploid) { + Gt::Ref => o.reference(nth(gr, 7), nth(gr, 8), filter, true), + state => ( b"no call".to_vec(), - [ - &b"the gVCF record over the site is not a PASS reference call (GT "[..], - gt, - b")", - ] - .concat(), - ); - } - let why: &[u8] = if o.gvcf { - b"no record in the VCF or the gVCF covers the site" - } else { - b"no record in the VCF, and no gVCF to tell a reference call from no call" + unread(b"the gVCF record over the site", gt, filter, state), + ), }; - return (b"no call".to_vec(), why.to_vec()); } - (call, detail) + if let Some(w) = why { + return (b"no call".to_vec(), w); + } + let why: &[u8] = if o.gvcf { + b"no record in the VCF or the gVCF covers the site" + } else { + b"no record in the VCF, and no gVCF to tell a reference call from no call" + }; + (b"no call".to_vec(), why.to_vec()) } /// Writes the header and one row per site of `sites`, in its order. @@ -270,7 +380,11 @@ mod tests { fn a_filtered_reference_call_is_low_confidence() { for f in ["PASS", ".", "RefCall"] { assert_eq!( - run("", &format!("chr1\t90\t120\tA\t<*>\t0/0\t99\t.\t{f}\n"), true), + run( + "", + &format!("chr1\t90\t120\tA\t<*>\t0/0\t99\t.\t{f}\n"), + true + ), "ref|GQ 99 (gVCF block)", "FILTER {f}" ); @@ -334,15 +448,30 @@ mod tests { #[test] fn a_haploid_alt_call_is_hemizygous() { assert_eq!( - run_at(GLA, "chrX\t101399747\t101399747\tC\tT\t1\t99\t30\n", "", true), + run_at( + GLA, + "chrX\t101399747\t101399747\tC\tT\t1\t99\t30\n", + "", + true + ), "hemizygous|GQ 99, DP 30" ); assert_eq!( - run_at(GLA, "chrX\t101399747\t101399747\tC\tG\t1\t99\t30\n", "", true), + run_at( + GLA, + "chrX\t101399747\t101399747\tC\tG\t1\t99\t30\n", + "", + true + ), "other allele|C>G hemizygous" ); assert_eq!( - run_at("chrY\t3000000\tC\tT", "chrY\t3000000\t3000000\tC\tT\t1\t99\t30\n", "", true), + run_at( + "chrY\t3000000\tC\tT", + "chrY\t3000000\t3000000\tC\tT\t1\t99\t30\n", + "", + true + ), "hemizygous|GQ 99, DP 30" ); } @@ -352,20 +481,40 @@ mod tests { #[test] fn a_haploid_reference_call_is_ref() { assert_eq!( - run_at(GLA, "chrX\t101399747\t101399747\tC\tT\t0\t99\t30\n", "", true), + run_at( + GLA, + "chrX\t101399747\t101399747\tC\tT\t0\t99\t30\n", + "", + true + ), "ref|GQ 99, DP 30" ); assert_eq!( - run_at(GLA, "chrX\t101399747\t101399747\tC\tT\t0\t20\t30\n", "", true), + run_at( + GLA, + "chrX\t101399747\t101399747\tC\tT\t0\t20\t30\n", + "", + true + ), "ref (low confidence)|GQ 20, DP 30, below the call floor of GQ 30" ); assert_eq!( - run_at(GLA, "", "chrX\t101399700\t101399800\tC\t\t0\t50\t.\n", true), + run_at( + GLA, + "", + "chrX\t101399700\t101399800\tC\t\t0\t50\t.\n", + true + ), "ref|GQ 50 (gVCF block)" ); // Inside PAR1, and on an autosome, the genome has two copies. assert_eq!( - run_at("chrX\t100000\tC\tT", "chrX\t100000\t100000\tC\tT\t0\t99\t30\n", "", true), + run_at( + "chrX\t100000\tC\tT", + "chrX\t100000\t100000\tC\tT\t0\t99\t30\n", + "", + true + ), "no call|the VCF record at the site has one copy (GT 0) where the genome has two" ); assert_eq!( From 31d5adfae0b872c3e09067345e1f79ac07e066bf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:08:51 +0100 Subject: [PATCH 173/255] docs(benchmark): say what the concordance columns measure and that step 03a runs without BQSR Bead: plat-55 Bead: plat-71 --- CHANGELOG.md | 4 ++++ CLAUDE.md | 3 ++- docs/benchmarking.md | 25 +++++++++++++++++++++---- docs/tool-rationale.md | 6 +++--- scripts/03e-clair3.sh | 2 +- 5 files changed, 31 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85edb5eae..7fd7d868e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- `benchmark-variants.sh` now compares genotypes, not just alleles, between callers. Its pairwise mode counted a site as shared when both callers had the same allele there, so a site one caller called 0/1 and the other 1/1 still scored as full agreement, and the summary said a Jaccard index of 1.0 meant perfect agreement. `comparison.tsv` gains two columns at the end, `GT_Concordant` and `GT_Concordance`: the shared sites whose genotypes also agree, ignoring phase and allele order, with a homozygous call equal to its haploid form. The existing columns are unchanged. + - The step now also compares Clair3 (`vcf_clair3/`) and `.vcf.gz` in any other `vcf_*/` directory. `run-all.sh` with `BENCHMARK=true` already queued the step for those directories, and the step then found one caller and exited 1, so the run reported a failure; step 03e also told long-read users to run it. + - The docs stop calling step 03a the clinical-lab standard without a caveat: it runs HaplotypeCaller without base quality score recalibration (BQSR) and with `-ERC NONE`, so it is not the full GATK Best Practices workflow. Part of #411 (plat-55, plat-71). + - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/CLAUDE.md b/CLAUDE.md index 1b8788d1a..3855512d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -185,7 +185,8 @@ Change its line in `versions.env`, plus the coupled data variable its comment na - **BWA-MEM2 index files**: Created alongside FASTA. Check for `.bwt.2bit.64`. - **ALIGN_DIR env var**: All alternative scripts accept `ALIGN_DIR=aligned_bwamem2`. - **TIDDIT --skip_assembly auto-detected**: Checks for BWA index files. -- **benchmark-variants.sh**: Pairwise mode (auto-discovers vcf*/ dirs) and truth set mode (hap.py). +- **benchmark-variants.sh**: Pairwise mode and truth set mode (hap.py). It compares `vcf/`, Strelka2's `vcf_strelka2/results/variants/variants.vcf.gz` and `.vcf.gz` in every `vcf_*/` dir (Clair3 included), the same glob `run-all.sh` uses for `BENCHMARK=true`. `comparison.tsv` has the allele-level Jaccard and `GT_Concordant`/`GT_Concordance`, the shared sites whose genotypes also agree. +- **03a runs without BQSR** and with `-ERC NONE`: not the full GATK Best Practices workflow, so its calls can differ from a clinical lab's GATK calls. ### bcftools - **`bcftools sort` requires `##contig` headers** — fails on VCFs without them. Inject from reference `.fai`. diff --git a/docs/benchmarking.md b/docs/benchmarking.md index 5d0eb63a7..ce72bf95c 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -19,18 +19,20 @@ For background on caller performance differences, see: ## Available Alternative Caller Scripts -The pipeline ships with four variant callers. Each writes output to a separate directory so they can run in parallel without conflicts. +The pipeline calls with DeepVariant by default and ships five alternative caller scripts: four for short reads and Clair3 for long reads. Each writes output to a separate directory so they can run in parallel without conflicts. The image each script runs is on [Image versions](versions.md). | Script | Caller | Output Directory | Description | |---|---|---|---| | `scripts/03-deepvariant.sh` | DeepVariant (`DEEPVARIANT_IMAGE`) | `vcf/` | Default caller. Deep learning model, highest precision and F1. | -| `scripts/03a-gatk-haplotypecaller.sh` | GATK HaplotypeCaller (`GATK_IMAGE`) | `vcf_gatk/` | Gold standard in clinical labs. Good precision/recall balance, GVCF support. | +| `scripts/03a-gatk-haplotypecaller.sh` | GATK HaplotypeCaller (`GATK_IMAGE`) | `vcf_gatk/` | Widely used in clinical labs, but this script is not their full GATK Best Practices workflow: it calls the duplicate-marked step-02 BAM without base quality score recalibration (BQSR) and writes a plain VCF (`-ERC NONE`), not a GVCF. Good precision/recall balance. | | `scripts/03b-freebayes.sh` | FreeBayes (image set in the script) | `vcf_freebayes/` | Bayesian caller. Highest sensitivity, most false positives, single-threaded. | | `scripts/03c-strelka2-germline.sh` | Strelka2 (image set in the script) | `vcf_strelka2/` | Fast heuristic caller (SNVs + indels). Best with BWA-MEM2 alignments. | +| `scripts/03d-octopus.sh` | Octopus (`OCTOPUS_IMAGE`) | `vcf_octopus/` | Haplotype-based caller. | +| `scripts/03e-clair3.sh` | Clair3 (`CLAIR3_IMAGE`) | `vcf_clair3/` | Long-read caller (ONT, PacBio HiFi) for a BAM from `scripts/02b-alignment-longread.sh`. | -All four scripts accept the same arguments: +DeepVariant, GATK, FreeBayes and Strelka2 take the same arguments: ```bash export GENOME_DIR=/path/to/your/data @@ -59,6 +61,19 @@ There are two ways to evaluate caller performance: Compare two or more caller VCFs against each other to measure agreement. This tells you how much the callers agree, but not which one is "right." +`./scripts/benchmark-variants.sh ` does this for every caller VCF it finds: `vcf/` (DeepVariant), `vcf_strelka2/results/variants/variants.vcf.gz`, and `.vcf.gz` in every `vcf_*/` directory (`vcf_gatk/`, `vcf_freebayes/`, `vcf_octopus/`, `vcf_clair3/`, or a directory of your own, named in the results by what follows `vcf_`). That is the same set that makes `run-all.sh` with `BENCHMARK=true` queue the step. It normalises each VCF once (`bcftools norm -m-both`), runs `bcftools isec` on PASS and unfiltered records for each pair, and writes `benchmark/comparison.tsv`: + +| Column | Meaning | +|---|---| +| `Shared`, `A_Unique`, `B_Unique` | sites both callers have, and sites only one has. A site is shared when both have the same allele at the same position, whatever the genotype | +| `Jaccard` | `Shared / (Shared + A_Unique + B_Unique)`: agreement on which alleles are present | +| `GT_Concordant` | shared sites where the two genotypes are also the same | +| `GT_Concordance` | `GT_Concordant / Shared`; `N/A` when nothing is shared | + +A site one caller calls 0/1 and the other 1/1 counts in `Shared` and in the Jaccard index, but not in `GT_Concordant`. Genotypes are compared without phase and with their alleles sorted (`1|0` equals `0/1`), and a homozygous call equals its haploid form (`1/1` equals `1`), so a caller that writes male chrX haploid still agrees with one that writes it diploid. + +The commands below do the allele part by hand: + Use `bcftools isec` to compute the intersection and unique calls: ```bash @@ -252,7 +267,8 @@ Each hap.py run produces several output files: | **Precision** | How many called variants are real (low = too many false positives) | TP / (TP + FP) | | **Recall** | How many real variants were found (low = missing real variants) | TP / (TP + FN) | | **F1** | Harmonic mean of precision and recall (balanced accuracy) | 2 * (Precision * Recall) / (Precision + Recall) | -| **Jaccard** | Overlap between two call sets (pairwise mode) | Shared / (Shared + Unique_A + Unique_B) | +| **Jaccard** | Overlap between two call sets (pairwise mode): the same allele at the same position, whatever the genotype | Shared / (Shared + Unique_A + Unique_B) | +| **GT concordance** | Shared sites where the two callers also give the same genotype (pairwise mode) | GT_Concordant / Shared | Where: - **TP** (True Positive) = variant called correctly @@ -267,6 +283,7 @@ Where: - **Jaccard > 0.95**: Two callers agree on the vast majority of calls. - **Jaccard 0.85-0.95**: Moderate agreement. Differences are mostly in low-confidence regions or indels. - **Jaccard < 0.85**: Significant disagreement. Worth investigating the unique calls from each caller. +- **GT concordance well below 1 with a high Jaccard**: the callers find the same variants but disagree on zygosity. Look at the depth and allele balance at those sites. ### Typical Results (30X WGS, GRCh38) diff --git a/docs/tool-rationale.md b/docs/tool-rationale.md index 7ef551912..2307e2a19 100644 --- a/docs/tool-rationale.md +++ b/docs/tool-rationale.md @@ -83,7 +83,7 @@ The cost is time. With 8 shards DeepVariant 1.10 took 8 h 50 min to 13 h 30 min ### GATK HaplotypeCaller: when to use it -- **Clinical reproducibility.** GATK HaplotypeCaller is the gold standard in clinical genetics labs and CLIA-certified pipelines. If you need to compare your calls with a clinical lab report, running GATK alongside DeepVariant helps identify discrepancies. +- **Closer to a clinical lab's calls.** Many clinical genetics labs and CLIA-certified pipelines call with GATK HaplotypeCaller. If you need to compare your calls with a clinical lab report, running GATK alongside DeepVariant helps identify discrepancies. `scripts/03a-gatk-haplotypecaller.sh` is not the full GATK Best Practices germline workflow, though: it calls the duplicate-marked step-02 BAM directly, without base quality score recalibration (BQSR), and writes a plain VCF (`-ERC NONE`). Its calls can differ from a lab's GATK calls for that reason alone. - **Cohort calling.** GATK's GVCF mode (`-ERC GVCF`) produces per-sample genomic VCFs that can be joint-genotyped across a cohort with `GenomicsDBImport` + `GenotypeGVCFs`. This is the standard workflow for family studies and population-scale projects. - **Balanced performance.** GATK sits between DeepVariant (highest precision) and FreeBayes (highest sensitivity), offering a different precision/recall tradeoff that can complement either tool. @@ -104,7 +104,7 @@ The cost is time. With 8 shards DeepVariant 1.10 took 8 h 50 min to 13 h 30 min | Priority | Recommended Caller | |---|---| | Highest accuracy (single sample) | DeepVariant | -| Clinical lab compatibility | GATK HaplotypeCaller | +| Closest to a clinical lab's calls | GATK HaplotypeCaller (without BQSR here, so not identical to a lab's GATK Best Practices calls) | | Maximum sensitivity (research) | FreeBayes (with quality filtering) | | Fastest runtime | Strelka2 (72 min on 8 threads in one historical run; align with BWA-MEM2 for it) | | Cohort / family joint calling | GATK HaplotypeCaller (GVCF mode) | @@ -213,7 +213,7 @@ Stick with the defaults. The default tools were chosen for the best single-tool | Question | Answer | Action | |---|---|---| -| Do I need to match a clinical lab? | Yes | Use BWA-MEM2 + GATK HC | +| Do I need to match a clinical lab? | Yes | Use BWA-MEM2 + GATK HC; it still runs without BQSR, so expect some differences | | Do I want maximum accuracy for a specific variant? | Yes | Run DeepVariant + GATK, keep shared calls | | Am I exploring broadly and can tolerate false positives? | Yes | Run all three SNP callers | | Do I plan to use Strelka2? | Yes | Align with BWA-MEM2 (XS tag required) | diff --git a/scripts/03e-clair3.sh b/scripts/03e-clair3.sh index b4fedfe2c..66a6af801 100755 --- a/scripts/03e-clair3.sh +++ b/scripts/03e-clair3.sh @@ -131,4 +131,4 @@ echo " PASS variants: ${PASS_COUNT}" echo "" echo "Next steps:" echo " - Run downstream VCF steps (ClinVar, PharmCAT, VEP) pointing at vcf_clair3/" -echo " - Compare with DeepVariant: ./scripts/benchmark-variants.sh ${SAMPLE}" +echo " - Compare with DeepVariant (vcf/) or any other caller's vcf_*/ VCF: ./scripts/benchmark-variants.sh ${SAMPLE}" From 69dc5cfefce13d42b3be44fbc14f8dd3cf7f4163 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:09:18 +0100 Subject: [PATCH 174/255] refactor(live): ops card and /metrics text in og-live on og-runlog og-live's ops module gives each run its hours left, memory and disk headroom, last activity and five alerts, and writes the Prometheus text, from og-runlog's eta and headroom instead of the Python run_eta and run_headroom modules. og-runlog gains a fault-inject feature: named planted mistakes the red controls turn on per thread; without the feature none is compiled in. Bead: D1 --- rust/Cargo.lock | 6 + rust/crates/og-live/Cargo.toml | 8 + rust/crates/og-live/src/ops.rs | 955 +++++++++++++++++++++++++- rust/crates/og-runlog/Cargo.toml | 4 + rust/crates/og-runlog/src/eta.rs | 140 +++- rust/crates/og-runlog/src/headroom.rs | 79 ++- 6 files changed, 1161 insertions(+), 31 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 6bf5d012d..4b446571b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -752,6 +752,12 @@ dependencies = [ [[package]] name = "og-live" version = "0.1.0" +dependencies = [ + "jiff", + "og-nf", + "og-runlog", + "pycompat", +] [[package]] name = "og-lookup" diff --git a/rust/crates/og-live/Cargo.toml b/rust/crates/og-live/Cargo.toml index 1e1e7ad65..85508abff 100644 --- a/rust/crates/og-live/Cargo.toml +++ b/rust/crates/og-live/Cargo.toml @@ -6,7 +6,15 @@ edition.workspace = true license.workspace = true publish.workspace = true +[features] +# Planted mistakes for the red controls of tests/ops*.rs (og_runlog::eta::fault). +fault-inject = ["og-runlog/fault-inject"] + [dependencies] +pycompat.workspace = true +og-nf.workspace = true +og-runlog.workspace = true +jiff.workspace = true [lints] workspace = true diff --git a/rust/crates/og-live/src/ops.rs b/rust/crates/og-live/src/ops.rs index 0b4d3ea7b..58474de8c 100644 --- a/rust/crates/og-live/src/ops.rs +++ b/rust/crates/og-live/src/ops.rs @@ -1,2 +1,953 @@ -//! The live site's run progress and estimates. Empty until its unit of the -//! 1.0.0 plan lands. +//! How a run is doing, on the page and in Prometheus: the live site's ops +//! card and `GET /metrics`. +//! +//! Per run, [`run_ops`] gives the hours left (a range, from +//! [`og_runlog::eta`]), the memory and disk headroom (from +//! [`og_runlog::headroom`]), when the run last showed activity, and five +//! alerts: failed, stalled, late, memory and disk. Two outputs, one source: +//! [`RunOps::to_json`] is `plugins.ops` of `state.json`, drawn by +//! `assets/live/plugins/ops.js`, and [`metrics_text`] writes the same numbers +//! in the Prometheus text format for the example rules in +//! `assets/monitoring/pgp-alerts.yml`. +//! +//! Nothing here writes: the step history is read, never updated (`og run +//! eta` adds a run to it), so the genome folder can stay read-only. The paths +//! it measures are the run folder, the run's work folder (`nextflow/work`, or +//! an absolute `workDir` from the weblog) and an absolute outdir from the +//! weblog, as this machine sees them. No path, task name or parameter reaches +//! the card or a metric; the labels are `run` (the run's folder name) and +//! `fs` (`work`, `results` or `work and results`). + +use jiff::tz::TimeZone; +use og_nf::time::{ISO_Z, iso, parse_utc}; +use og_runlog::eta::{self, fault}; +use og_runlog::headroom::{self, BadPercent, Disk, TaskMemory}; +use og_runlog::host::{Host, cpu_count}; +use og_runlog::state::{RunState, State, StepState, trace_files}; +use og_runlog::steps::Step; +use pycompat::json::Value; +use pycompat::value::py_str; +use std::fmt::Write as _; +use std::io; +use std::path::{Path, PathBuf}; + +/// A run quiet this long, with no running step inside its usual time, is +/// stalled (`--stall-minutes`). +pub const DEFAULT_STALL_MINUTES: f64 = 60.0; +/// The content type of `GET /metrics`. +pub const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8"; + +/// Every metric `/metrics` can write, in the order it writes them: name, +/// type, help. `docs/alerts.md` lists the same names. +pub const METRICS: [(&str, &str, &str); 21] = [ + ("pgp_runs", "gauge", "Runs the site knows."), + ( + "pgp_disk_warn_ratio", + "gauge", + "The disk warn share (--disk-warn-percent / 100).", + ), + ( + "pgp_host_memory_bytes", + "gauge", + "RAM of the machine the site runs on.", + ), + ( + "pgp_metrics_errors", + "gauge", + "Runs whose numbers could not be read in this scrape.", + ), + ("pgp_run_running", "gauge", "1 while the run is going."), + ( + "pgp_run_failed", + "gauge", + "1 when the run ended badly: a failed step, a non-zero exit or a Nextflow error.", + ), + ( + "pgp_run_failed_steps", + "gauge", + "Steps of the run in state failed.", + ), + ( + "pgp_run_stalled", + "gauge", + "1 when the run is going but has shown no activity for --stall-minutes.", + ), + ( + "pgp_run_seconds_since_activity", + "gauge", + "Seconds since the run's last message or file change.", + ), + ( + "pgp_run_late_steps", + "gauge", + "Running steps already past the high end of their usual time.", + ), + ( + "pgp_run_unknown_steps", + "gauge", + "Steps left with no usual time.", + ), + ("pgp_run_seconds_left_low", "gauge", "Low end of the time left."), + ( + "pgp_run_seconds_left_high", + "gauge", + "High end of the time left (absent when a step is late).", + ), + ( + "pgp_run_tasks_near_memory_request", + "gauge", + "Finished tasks whose peak_rss reached the memory warn share of their request.", + ), + ( + "pgp_run_memory_retries", + "gauge", + "Task attempts retried after a memory or time kill.", + ), + ("pgp_run_max_memory_bytes", "gauge", "The run's max_memory."), + ( + "pgp_run_max_memory_above_ram", + "gauge", + "1 when the run's max_memory is above this machine's RAM.", + ), + ( + "pgp_disk_used_ratio", + "gauge", + "Used share of the filesystem.", + ), + ( + "pgp_disk_inodes_used_ratio", + "gauge", + "Used share of the filesystem's inodes.", + ), + ("pgp_disk_free_bytes", "gauge", "Bytes free on the filesystem."), + ("pgp_disk_size_bytes", "gauge", "Size of the filesystem."), +]; + +/// The site's ops options (also `KEY=VALUE` lines in `--config`). +#[derive(Debug, Clone, Default)] +pub struct OpsConfig { + /// The genome folder; else the run folder's parent. + pub root: Option, + /// The step history to read (`--eta-box`); else `PGP_BOX` or the short + /// host name. + pub eta_box: Option, + /// The cores of the box that runs the pipeline (`--eta-cores`); else + /// this machine's. + pub eta_cores: Option, + /// `--disk-warn-percent`, as given; else `OWNGENOME_DISK_WARN_PERCENT` + /// or 90. + pub disk_warn_percent: Option, + /// `--mem-warn-percent`, as given; else `OWNGENOME_MEM_WARN_PERCENT` or + /// 90. + pub mem_warn_percent: Option, + /// `--stall-minutes`; none or 0 is [`DEFAULT_STALL_MINUTES`]. + pub stall_minutes: Option, +} + +/// What [`run_ops`] reads besides the run. +pub struct Ctx<'a> { + pub now: f64, + pub steps: &'a [Step], + pub host: &'a dyn Host, + /// The zone the trace files' times are in. + pub tz: &'a TimeZone, +} + +/// How bad an alert is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Level { + Info, + Warning, + Critical, +} + +impl Level { + pub fn as_str(self) -> &'static str { + match self { + Level::Info => "info", + Level::Warning => "warning", + Level::Critical => "critical", + } + } +} + +/// What an alert is about. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AlertKind { + Failed, + Stalled, + Late, + Memory, + Disk, +} + +impl AlertKind { + pub fn as_str(self) -> &'static str { + match self { + AlertKind::Failed => "failed", + AlertKind::Stalled => "stalled", + AlertKind::Late => "late", + AlertKind::Memory => "memory", + AlertKind::Disk => "disk", + } + } +} + +/// One alert of the card. +#[derive(Debug, Clone, PartialEq)] +pub struct Alert { + pub kind: AlertKind, + pub level: Level, + pub text: String, +} + +/// Whether the run is going, and how it went. +#[derive(Debug, Clone, PartialEq)] +pub struct RunHealth { + pub state: RunState, + pub failed: bool, + pub failed_steps: Vec, + /// The exit of the last `pgp_run_finished` message, as it came (null + /// when there is none). + pub exit: Value, + pub stalled: bool, + pub seconds_since_activity: Option, + pub stall_seconds: f64, +} + +/// The hours left of a run going. +#[derive(Debug, Clone, PartialEq)] +pub struct HoursLeft { + pub low_s: f64, + /// `None` when a running step is past its usual high end. + pub high_s: Option, + pub late_steps: Vec, + pub unknown_steps: Vec, + pub basis: String, + pub cores: usize, +} + +/// The run's memory: the machine against `max_memory`, and the tasks near +/// their request. +#[derive(Debug, Clone, PartialEq)] +pub struct Memory { + pub host_bytes: Option, + pub max_memory_bytes: Option, + pub max_memory_above_ram: bool, + pub warn_ratio: f64, + pub near_request: usize, + pub retries: usize, + /// The tasks at or above the warn share of their request. + pub tasks: Vec, +} + +/// The ops card of one run. +#[derive(Debug, Clone, PartialEq)] +pub struct RunOps { + pub run: RunHealth, + /// Only while the run is going. + pub hours_left: Option, + pub memory: Memory, + pub disk: Vec, + pub disk_warn_ratio: f64, + pub alerts: Vec, +} + +/// The card, or the one line it shows instead. +#[derive(Debug, Clone, PartialEq)] +pub enum OpsState { + Ok(Box), + Error(String), +} + +/// The messages of the newest run: after the last `pgp_run_finished` that +/// is not the newest message, and from the last weblog `started`. +pub fn current_run(events: &[Value]) -> &[Value] { + let mut start = 0; + for (i, ev) in events.iter().enumerate().take(events.len().saturating_sub(1)) { + if is_event(ev, "pgp_run_finished") { + start = i + 1; + } + } + if let Some(i) = (start..events.len()) + .rev() + .find(|&i| is_event(&events[i], "started")) + { + start = i; + } + &events[start..] +} + +fn field<'v>(ev: &'v Value, k: &str) -> Option<&'v Value> { + match ev { + Value::Object(o) => o.get(k), + _ => None, + } +} + +fn is_event(ev: &Value, kind: &str) -> bool { + matches!(field(ev, "event"), Some(Value::Str(e)) if e == kind) +} + +/// An ISO time such as `2026-01-02T10:00:00Z`, in seconds. +fn epoch_of(v: Option<&Value>) -> Option { + match v { + Some(Value::Str(s)) => parse_utc(s, &[ISO_Z]), + _ => None, + } +} + +fn is_dir(p: Option<&Path>) -> Option<&Path> { + p.filter(|p| p.is_dir()) +} + +/// An absolute path from the weblog, else nothing. +fn abs_dir(v: Option) -> Option { + match v { + Some(Value::Str(s)) if s.starts_with('/') && !fault::on("ops-bare-run-no-work") => Some(s), + _ => None, + } +} + +/// The newest sign of life: a message's `utcTime`, the run's +/// `logs/run_status.tsv`, or a trace file of the run. +pub fn last_activity(run_dir: Option<&Path>, events: &[Value], started: Option) -> Option { + let mut times: Vec = events + .iter() + .filter_map(|ev| epoch_of(field(ev, "utcTime"))) + .collect(); + if let Some(dir) = is_dir(run_dir) { + times.extend(pycompat::mtime::mtime(dir.join("logs").join("run_status.tsv")).ok()); + times.extend( + trace_files(dir, started) + .iter() + .filter_map(|p| pycompat::mtime::mtime(p).ok()), + ); + } + times.into_iter().reduce(f64::max) +} + +/// `(work, results)` to measure: the run folder's `nextflow/work`, else an +/// absolute `workDir` from the weblog; the run folder, else an absolute +/// outdir from the weblog (a relative one is under the weblog's +/// `launchDir`). +pub fn folders(run_dir: Option<&Path>, events: &[Value]) -> (Option, Option) { + let (m_work, m_out, _) = headroom::metadata(events); + let have = is_dir(run_dir); + let text = |p: &Path| p.to_string_lossy().into_owned(); + let work = match have.map(|d| d.join("nextflow").join("work")) { + Some(w) if w.is_dir() => Some(text(&w)), + _ => abs_dir(m_work), + }; + let results = match have { + Some(d) => Some(text(d)), + None => match m_out { + Some(Value::Str(s)) if s.starts_with('/') => Some(s), + _ => None, + }, + }; + (work, results) +} + +/// The run's started time as the history reads it (whole seconds of its ISO +/// form). +fn started_of(state: &State<'_>) -> Option { + state + .started + .and_then(iso) + .and_then(|v| parse_utc(&v, &[ISO_Z])) +} + +/// This run's own tasks from its trace files. +fn trace_rows(run_dir: Option<&Path>, sample: &str, started: Option, tz: &TimeZone) -> Vec { + let Some(dir) = is_dir(run_dir) else { + return Vec::new(); + }; + trace_files(dir, started) + .iter() + .flat_map(|p| eta::read_trace(p, tz)) + .filter(|r| og_runlog::tasks::is_sample(r.tag.as_deref(), sample)) + .collect() +} + +/// A weblog value as text (`max_memory` may come as a number). +fn value_text(v: &Value) -> Option { + match v { + Value::Null => None, + Value::Str(s) => Some(s.clone()), + v => py_str(v).ok(), + } +} + +/// An exit that means the run ended well, or none: null, 0 (or 0.0, false). +fn exit_ok(v: &Value) -> bool { + match v { + Value::Null => true, + Value::Int(i) => *i == 0, + Value::Float(x) => *x == 0.0, + Value::Bool(b) => !b, + _ => false, + } +} + +fn plural(n: usize, one: &str, many: &str) -> String { + if n > 1 { many.to_owned() } else { one.to_owned() } +} + +/// The ops card of the run in `run_dir` (none or missing for a run known +/// from its messages only), whose state is `state` and whose stored +/// messages are `events`. A warn percent outside 1 to 100 is the error. +pub fn run_ops( + run_dir: Option<&Path>, + state: &State<'_>, + events: &[Value], + cfg: &OpsConfig, + ctx: &Ctx<'_>, +) -> Result { + let now = ctx.now; + let root = cfg + .root + .clone() + .or_else(|| run_dir.and_then(Path::parent).map(Path::to_path_buf)); + let cur = current_run(events); + let sample = if state.sample.is_empty() { + run_dir + .and_then(Path::file_name) + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default() + } else { + state.sample.clone() + }; + // run-all.sh posts pgp_run_finished as its last message; the site marks + // the run finished on it too. + let ended = cur.last().is_some_and(|ev| is_event(ev, "pgp_run_finished")); + let run_st = if ended && state.run == RunState::Running { + RunState::Finished + } else { + state.run + }; + let running = run_st == RunState::Running; + let started = started_of(state); + + // Hours left: the ranges from the history as it is, without adding to it. + let cores = cfg.eta_cores.filter(|c| *c != 0).unwrap_or_else(cpu_count); + let history = root + .as_deref() + .map(|r| eta::read_history(&eta::history_path(r, &eta::box_name(cfg.eta_box.as_deref())))) + .unwrap_or_default(); + let mut exclude = root.as_deref().map(eta::live_traces).unwrap_or_default(); + if let Some(dir) = is_dir(run_dir) + && state.run == RunState::Running + { + exclude.extend(trace_files(dir, started).iter().filter_map(|p| { + p.file_name().map(|n| n.to_string_lossy().into_owned()) + })); + } + let ranges = eta::step_ranges(&history, ctx.steps, cores, &exclude); + let rem = running.then(|| eta::remaining(state, &ranges, now, cores)); + + // Headroom on the folders found here. The weblog metadata is taken out + // of the messages first, so the snapshot cannot fall back to a weblog + // path that folders() left out. + let (work, results) = folders(run_dir, cur); + let (_, _, cap) = headroom::metadata(cur); + let plain: Vec = cur + .iter() + .map(|ev| match ev { + Value::Object(o) => Value::Object( + o.iter() + .filter(|(k, _)| *k != "metadata") + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ), + v => v.clone(), + }) + .collect(); + let rows = trace_rows(run_dir, &sample, started, ctx.tz); + let cap = cap.as_ref().and_then(value_text); + let snap = headroom::snapshot( + ctx.host, + ctx.steps, + &headroom::Request { + events: &plain, + trace_rows: &rows, + work_dir: work.as_deref(), + outdir: results.as_deref(), + max_memory: cap.as_deref(), + disk_warn_percent: cfg.disk_warn_percent.as_deref(), + mem_warn_percent: cfg.mem_warn_percent.as_deref(), + }, + )?; + + // What went wrong or is going wrong. + let failed_steps: Vec = state + .steps + .iter() + .filter(|s| s.state == StepState::Failed) + .map(|s| s.step.step.clone()) + .collect(); + let exit = cur + .iter() + .rfind(|ev| is_event(ev, "pgp_run_finished")) + .map_or(Value::Null, |ev| field(ev, "exit").cloned().unwrap_or(Value::Null)); + let error = cur.iter().any(|ev| is_event(ev, "error")); + let finished = run_st == RunState::Finished; + let bad_exit = !exit_ok(&exit) && !fault::on("ops-exit-ignored"); + let failed = !failed_steps.is_empty() || (finished && (bad_exit || error)); + let quiet = last_activity(run_dir, cur, started).map(|a| (now - a).max(0.0)); + let stall_s = cfg.stall_minutes.filter(|m| *m != 0.0).unwrap_or(DEFAULT_STALL_MINUTES) * 60.0; + // Running steps still inside their usual high end: a long task posts + // nothing meanwhile. + let covered = state.steps.iter().any(|s| { + let high = ranges.get(&s.step.step).and_then(|r| r.high_s); + let st = s.started_utc.as_deref().and_then(|v| parse_utc(v, &[ISO_Z])); + s.state == StepState::Running + && matches!((high, st), (Some(h), Some(st)) if now - st < h) + && !fault::on("ops-stall-ignores-steps") + }); + // A run with no end record that the state calls stopped has been quiet + // even longer: still stalled, so the alert holds for a run that died + // without an end instead of resolving. + let alive = run_st == RunState::Running + || (run_st == RunState::Stopped && !fault::on("ops-stopped-not-stalled")); + let stalled = alive + && quiet.is_some_and(|q| q >= stall_s || fault::on("ops-stall-ignores-quiet")) + && !covered; + let late: Vec = match &rem { + Some(r) if !fault::on("ops-late-dropped") => r.late_steps.clone(), + _ => Vec::new(), + }; + let mem_hints: Vec<&TaskMemory> = snap.tasks.iter().filter(|t| t.warn).collect(); + let retries: usize = if fault::on("ops-retries-uncounted") { + 0 + } else { + snap.tasks.iter().map(|t| t.oom_retry).sum() + }; + let disk_warn: Vec<&Disk> = snap + .disk + .iter() + .filter(|d| d.warn && !fault::on("ops-disk-warn-ignored")) + .collect(); + + let mut alerts = Vec::new(); + if failed { + let text = if !failed_steps.is_empty() { + format!( + "{} {} failed", + plural(failed_steps.len(), "step", "steps"), + failed_steps.join(", ") + ) + } else if !exit_ok(&exit) { + format!("the run ended with exit {}", py_str(&exit).unwrap_or_default()) + } else { + "Nextflow reported an error".to_owned() + }; + alerts.push(Alert { + kind: AlertKind::Failed, + level: Level::Critical, + text, + }); + } + if stalled { + let minutes = pycompat::fmt::fixed(quiet.unwrap_or(0.0) / 60.0, 0); + alerts.push(Alert { + kind: AlertKind::Stalled, + level: Level::Warning, + text: format!( + "no activity for {minutes} minutes, and no running step is inside its usual time" + ), + }); + } + if !late.is_empty() { + let text = late + .iter() + .map(|k| format!("step {k} is taking longer than usual")) + .collect::>() + .join("; "); + alerts.push(Alert { + kind: AlertKind::Late, + level: Level::Warning, + text, + }); + } + if retries > 0 || snap.host.warn { + let mut parts = Vec::new(); + if retries > 0 { + parts.push(format!( + "{retries} task {} retried after a memory or time kill", + plural(retries, "attempt", "attempts") + )); + } + if snap.host.warn { + parts.push("max_memory is above this machine's memory".to_owned()); + } + alerts.push(Alert { + kind: AlertKind::Memory, + level: Level::Warning, + text: parts.join("; "), + }); + } else if !mem_hints.is_empty() { + alerts.push(Alert { + kind: AlertKind::Memory, + level: Level::Info, + text: format!( + "{} {} reached {}% of the memory asked for (peak_rss counts shared pages, so this may be fine)", + mem_hints.len(), + plural(mem_hints.len(), "task", "tasks"), + pycompat::fmt::fixed(snap.mem_warn_ratio * 100.0, 0) + ), + }); + } + for d in &disk_warn { + alerts.push(Alert { + kind: AlertKind::Disk, + level: if d.used_ratio.unwrap_or(0.0) >= 0.98 { + Level::Critical + } else { + Level::Warning + }, + text: format!( + "disk {} is at or above {}% used", + d.fs, + pycompat::fmt::fixed(snap.warn_used_ratio * 100.0, 0) + ), + }); + } + + Ok(RunOps { + run: RunHealth { + state: run_st, + failed, + failed_steps, + exit, + stalled, + seconds_since_activity: quiet, + stall_seconds: stall_s, + }, + hours_left: rem.map(|r| HoursLeft { + low_s: r.low_s, + high_s: r.high_s, + late_steps: late, + unknown_steps: r.unknown_steps, + basis: r.basis, + cores, + }), + memory: Memory { + host_bytes: snap.host.memory_bytes, + max_memory_bytes: snap.host.max_memory_bytes, + max_memory_above_ram: snap.host.warn, + warn_ratio: snap.mem_warn_ratio, + near_request: mem_hints.len(), + retries, + tasks: mem_hints.into_iter().cloned().collect(), + }, + disk: snap.disk, + disk_warn_ratio: snap.warn_used_ratio, + alerts, + }) +} + +/// The state hook: the card, or the line it shows instead. Only a warn +/// percent outside 1 to 100 fails it (the readers take a missing or +/// half-written file as no value), and the rest of the run's state stays. +pub fn hook(run_dir: Option<&Path>, state: &State<'_>, events: &[Value], cfg: &OpsConfig, ctx: &Ctx<'_>) -> OpsState { + match run_ops(run_dir, state, events, cfg, ctx) { + Ok(o) => OpsState::Ok(Box::new(o)), + Err(BadPercent(m)) => OpsState::Error(m.chars().take(200).collect()), + } +} + +fn int(v: impl TryInto) -> Value { + v.try_into().map_or(Value::Null, Value::Int) +} + +fn opt_int(v: Option>) -> Value { + v.map_or(Value::Null, int) +} + +fn opt_float(v: Option) -> Value { + v.map_or(Value::Null, Value::Float) +} + +fn strs(v: &[String]) -> Value { + Value::Array(v.iter().map(|s| Value::Str(s.clone())).collect()) +} + +fn obj(pairs: [(&str, Value); N]) -> Value { + Value::Object(pairs.into_iter().map(|(k, v)| (k.to_owned(), v)).collect()) +} + +impl RunOps { + /// `plugins.ops` of `state.json`, key for key. + pub fn to_json(&self) -> Value { + let r = &self.run; + let m = &self.memory; + obj([ + ( + "run", + obj([ + ("state", Value::Str(r.state.as_str().to_owned())), + ("failed", Value::Bool(r.failed)), + ("failed_steps", strs(&r.failed_steps)), + ("exit", r.exit.clone()), + ("stalled", Value::Bool(r.stalled)), + ("seconds_since_activity", opt_float(r.seconds_since_activity)), + ("stall_seconds", Value::Float(r.stall_seconds)), + ]), + ), + ( + "hours_left", + self.hours_left.as_ref().map_or(Value::Null, |h| { + obj([ + ("low_s", Value::Float(h.low_s)), + ("high_s", opt_float(h.high_s)), + ("late_steps", strs(&h.late_steps)), + ("unknown_steps", strs(&h.unknown_steps)), + ("basis", Value::Str(h.basis.clone())), + ("cores", int(h.cores)), + ]) + }), + ), + ( + "memory", + obj([ + ("host_bytes", opt_int(m.host_bytes)), + ("max_memory_bytes", opt_int(m.max_memory_bytes)), + ("max_memory_above_ram", Value::Bool(m.max_memory_above_ram)), + ("warn_ratio", Value::Float(m.warn_ratio)), + ("near_request", int(m.near_request)), + ("retries", int(m.retries)), + ( + "tasks", + Value::Array( + m.tasks + .iter() + .map(|t| { + obj([ + ("step", t.step.clone().map_or(Value::Null, Value::Str)), + ("process", Value::Str(t.process.clone())), + ("state", Value::Str(t.state.clone())), + ("request_bytes", opt_int(t.request_bytes)), + ("peak_rss_bytes", opt_int(t.peak_rss_bytes)), + ("ratio", opt_float(t.ratio)), + ("warn", Value::Bool(t.warn)), + ("oom_retry", int(t.oom_retry)), + ]) + }) + .collect(), + ), + ), + ]), + ), + ( + "disk", + Value::Array( + self.disk + .iter() + .map(|d| { + obj([ + ("fs", Value::Str(d.fs.clone())), + ("used_ratio", opt_float(d.used_ratio)), + ("free_bytes", int(d.free_bytes)), + ("size_bytes", int(d.size_bytes)), + ("inodes_used_ratio", opt_float(d.inodes_used_ratio)), + ("missing", Value::Bool(d.missing)), + ("warn", Value::Bool(d.warn)), + ]) + }) + .collect(), + ), + ), + ("disk_warn_ratio", Value::Float(self.disk_warn_ratio)), + ( + "alerts", + Value::Array( + self.alerts + .iter() + .map(|a| { + obj([ + ("kind", Value::Str(a.kind.as_str().to_owned())), + ("level", Value::Str(a.level.as_str().to_owned())), + ("text", Value::Str(a.text.clone())), + ]) + }) + .collect(), + ), + ), + ]) + } +} + +impl OpsState { + /// `plugins.ops` of `state.json`: the card, or `{"error": ...}`. + pub fn to_json(&self) -> Value { + match self { + OpsState::Ok(o) => o.to_json(), + OpsState::Error(e) => obj([("error", Value::Str(e.clone()))]), + } + } +} + +/// A number as `/metrics` writes it: a flag as 1 or 0, a whole number as +/// itself, any other as Python's `repr` (`3600.0`, `1e-05`). +#[derive(Debug, Clone, Copy)] +enum Num { + Flag(bool), + Whole(i128), + Real(f64), +} + +impl std::fmt::Display for Num { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Num::Flag(b) => f.write_str(if *b { "1" } else { "0" }), + Num::Whole(i) => write!(f, "{i}"), + Num::Real(x) => f.write_str(&pycompat::fmt::repr(*x)), + } + } +} + +fn whole(v: impl TryInto) -> Option { + v.try_into().ok().map(Num::Whole) +} + +/// A label value, escaped as the text format asks. +fn label(v: &str) -> String { + v.replace('\\', "\\\\") + .replace('\n', "\\n") + .replace('"', "\\\"") +} + +/// The disk warn share `/metrics` reports: `--disk-warn-percent` (else +/// `OWNGENOME_DISK_WARN_PERCENT`, else 90) over 100; none when it is not a +/// number from 1 to 100. +pub fn disk_warn_ratio(cfg: &OpsConfig) -> Option { + headroom::warn_percent(cfg.disk_warn_percent.as_deref(), Some("OWNGENOME_DISK_WARN_PERCENT")) + .ok() + .map(|p| p / 100.0) +} + +/// The Prometheus text of `runs` (run name, card). A value that is not +/// known is left out, never written as 0. +pub fn metrics_text( + runs: &[(&str, &RunOps)], + disk_warn_ratio: Option, + host_bytes: Option, + errors: usize, +) -> String { + let mut samples: Vec> = vec![Vec::new(); METRICS.len()]; + let mut put = |name: &str, value: Option, labels: &[(&str, &str)]| { + let Some(value) = value else { return }; + let i = METRICS + .iter() + .position(|(n, _, _)| *n == name) + .expect("every metric written is in METRICS"); + let lab = labels + .iter() + .map(|(k, v)| format!("{k}=\"{}\"", label(v))) + .collect::>() + .join(","); + samples[i].push(if lab.is_empty() { + format!("{name} {value}") + } else { + format!("{name}{{{lab}}} {value}") + }); + }; + put("pgp_runs", whole(runs.len()), &[]); + put("pgp_disk_warn_ratio", disk_warn_ratio.map(Num::Real), &[]); + put("pgp_host_memory_bytes", host_bytes.and_then(whole), &[]); + put("pgp_metrics_errors", whole(errors), &[]); + for (name, o) in runs { + let run = [("run", *name)]; + let (r, m) = (&o.run, &o.memory); + put("pgp_run_running", Some(Num::Flag(r.state == RunState::Running)), &run); + put("pgp_run_failed", Some(Num::Flag(r.failed)), &run); + put("pgp_run_failed_steps", whole(r.failed_steps.len()), &run); + put("pgp_run_stalled", Some(Num::Flag(r.stalled)), &run); + put("pgp_run_seconds_since_activity", r.seconds_since_activity.map(Num::Real), &run); + if let Some(h) = &o.hours_left { + put("pgp_run_late_steps", whole(h.late_steps.len()), &run); + put("pgp_run_unknown_steps", whole(h.unknown_steps.len()), &run); + put("pgp_run_seconds_left_low", Some(Num::Real(h.low_s)), &run); + put("pgp_run_seconds_left_high", h.high_s.map(Num::Real), &run); + } + put("pgp_run_tasks_near_memory_request", whole(m.near_request), &run); + put("pgp_run_memory_retries", whole(m.retries), &run); + put("pgp_run_max_memory_bytes", m.max_memory_bytes.and_then(whole), &run); + if m.max_memory_bytes.is_some() { + put("pgp_run_max_memory_above_ram", Some(Num::Flag(m.max_memory_above_ram)), &run); + } + for d in &o.disk { + let at = [("run", *name), ("fs", d.fs.as_str())]; + put("pgp_disk_used_ratio", d.used_ratio.map(Num::Real), &at); + put("pgp_disk_inodes_used_ratio", d.inodes_used_ratio.map(Num::Real), &at); + put("pgp_disk_free_bytes", whole(d.free_bytes), &at); + put("pgp_disk_size_bytes", whole(d.size_bytes), &at); + } + } + let mut out = String::new(); + for ((name, typ, help), lines) in METRICS.iter().zip(&samples) { + if lines.is_empty() { + continue; + } + let _ = writeln!(out, "# HELP {name} {help}"); + let _ = writeln!(out, "# TYPE {name} {typ}"); + for l in lines { + out.push_str(l); + out.push('\n'); + } + } + if out.is_empty() { + out.push('\n'); + } + out +} + +/// The token in `path` (`--metrics-token-file`), surrounding white space +/// cut; none for no file given or an empty one. +pub fn read_token(path: Option<&Path>) -> io::Result> { + let Some(path) = path else { return Ok(None) }; + let text = std::fs::read_to_string(path)?; + let tok = pycompat::text::strip(&text, None); + Ok((!tok.is_empty()).then(|| tok.to_owned())) +} + +/// Who asks for `/metrics`. +#[derive(Debug, Clone, Copy, Default)] +pub struct Asker<'a> { + /// The site has a password. + pub password_set: bool, + /// The request carries a valid session. + pub logged_in: bool, + /// Its `Authorization` header. + pub authorization: Option<&'a str>, +} + +/// Equal bytes, in time that does not depend on where they differ. +fn same_secret(a: &[u8], b: &[u8]) -> bool { + a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 +} + +/// Whether `/metrics` answers `who`. Without a password the site is +/// loopback only and `/metrics` is open, like every page. With one: a +/// logged-in browser, or `Authorization: Bearer `. `token` reads the +/// token, and is called only when one is needed (an error: the token file +/// cannot be read). +pub fn metrics_allowed( + who: &Asker<'_>, + token: impl FnOnce() -> io::Result>, +) -> io::Result { + if !who.password_set || who.logged_in { + return Ok(true); + } + let auth = who.authorization.unwrap_or(""); + let (scheme, given) = auth.split_once(' ').unwrap_or((auth, "")); + let Some(token) = token()? else { + return Ok(false); + }; + Ok(pycompat::text::lower(scheme) == "bearer" + && same_secret( + pycompat::text::strip(given, None).as_bytes(), + token.as_bytes(), + )) +} diff --git a/rust/crates/og-runlog/Cargo.toml b/rust/crates/og-runlog/Cargo.toml index 45da21400..cfa1bec6f 100644 --- a/rust/crates/og-runlog/Cargo.toml +++ b/rust/crates/og-runlog/Cargo.toml @@ -6,6 +6,10 @@ edition.workspace = true license.workspace = true publish.workspace = true +[features] +# Planted mistakes for the red controls of og-live's ops tests (eta::fault). +fault-inject = [] + [dependencies] pycompat.workspace = true og-io.workspace = true diff --git a/rust/crates/og-runlog/src/eta.rs b/rust/crates/og-runlog/src/eta.rs index 3ecd862e8..57579bdd3 100644 --- a/rust/crates/og-runlog/src/eta.rs +++ b/rust/crates/og-runlog/src/eta.rs @@ -45,6 +45,40 @@ pub const LAST_RUNS: usize = 5; /// The threads the catalogue's hours were measured on. pub const DOCS_THREADS: f64 = 16.0; +/// Planted mistakes for the red controls of the live site's tests +/// (`og-live/tests/ops*.rs`, docs/rust.md). With the cargo feature +/// `fault-inject` a test turns one on by name for its own thread, and the +/// check it plants against must then fail. Without the feature [`on`] is +/// always false and the release binary holds none of them. +/// +/// [`on`]: fault::on +pub mod fault { + #[cfg(feature = "fault-inject")] + thread_local! { + static ON: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + + /// Whether the planted mistake `name` is on in this thread. + #[inline] + pub fn on(name: &str) -> bool { + #[cfg(feature = "fault-inject")] + { + ON.with(|c| c.get() == Some(name)) + } + #[cfg(not(feature = "fault-inject"))] + { + let _ = name; + false + } + } + + /// Turns the planted mistake `name` on for this thread. + #[cfg(feature = "fault-inject")] + pub fn plant(name: &'static str) { + ON.with(|c| c.set(Some(name))); + } +} + /// One trace row, as the history and the headroom read it. #[derive(Debug, Clone, PartialEq)] pub struct TraceTask { @@ -94,6 +128,9 @@ pub fn read_trace(path: &Path, tz: &TimeZone) -> Vec { let real = duration(r.get("realtime")); let complete = time(r.get("complete"), tz).or_else(|| submit.zip(dur).map(|(s, d)| s + d)); let start = time(r.get("start"), tz).or_else(|| { + if fault::on("eta-old-start-is-submit") { + return submit; + } complete.and_then(|c| { if let Some(x) = real { Some(c - x) @@ -107,6 +144,11 @@ pub fn read_trace(path: &Path, tz: &TimeZone) -> Vec { } else { r.get("cpus").and_then(|v| pycompat::num::parse_int(v).ok()) }; + let forced = fault::on("eta-forced-ok"); + let mut memory_bytes = memory(r.get("memory")); + if fault::on("eta-memory-decimal-gb") && r.get("memory").is_some_and(|m| m.ends_with("GB")) { + memory_bytes = memory_bytes.map(|m| m / 1024 / 1024 / 1024 * 1_000_000_000); + } out.push(TraceTask { trace_file: file.clone(), hash: r.get("hash").unwrap_or("").replace('/', ""), @@ -118,10 +160,18 @@ pub fn read_trace(path: &Path, tz: &TimeZone) -> Vec { .to_owned(), process: proc, tag, - status: pycompat::text::upper(r.get("status").unwrap_or("")), - exit: pycompat::text::strip(r.get("exit").unwrap_or(""), None).to_owned(), + status: if forced { + "COMPLETED".to_owned() + } else { + pycompat::text::upper(r.get("status").unwrap_or("")) + }, + exit: if forced { + "0".to_owned() + } else { + pycompat::text::strip(r.get("exit").unwrap_or(""), None).to_owned() + }, cpus, - memory_bytes: memory(r.get("memory")), + memory_bytes, peak_rss_bytes: memory(r.get("peak_rss")), realtime_s: real, start, @@ -144,6 +194,9 @@ fn run_level(t: &TraceTask) -> bool { /// True when every task of a step's group ended well: per process and tag, /// the attempt that ended last is COMPLETED with exit 0. fn finished(ts: &[&TraceTask]) -> bool { + if fault::on("eta-finished-always") { + return true; + } type Last<'t> = (f64, usize, &'t TraceTask); let mut last: IndexMap<(Option<&str>, Option<&str>), Last<'_>> = IndexMap::new(); for (i, t) in ts.iter().enumerate() { @@ -223,12 +276,20 @@ pub fn history_rows( if !seen.insert((t.hash.as_str(), t.attempt.as_str())) { continue; } + let mut file = t.trace_file.clone(); + let mut process = t.process.clone().unwrap_or_default(); + if fault::on("eta-history-path-in-name") { + file.insert_str(0, "pipeline_info/"); + } + if fault::on("eta-history-sample-in-process") { + process.push_str("_demo1"); + } rows.push([ - t.trace_file.clone(), + file, t.hash.clone(), t.attempt.clone(), key.0.to_owned(), - t.process.clone().unwrap_or_default(), + process, if key.1.is_empty() { "yes" } else { "no" }.to_owned(), fmt_opt_i(t.cpus), fmt_opt_i(t.memory_bytes), @@ -253,6 +314,12 @@ pub fn box_name(name: Option<&str>) -> String { .map(str::to_owned) .or(env.filter(|e| !e.is_empty())) .unwrap_or_else(crate::host::short_hostname); + if fault::on("eta-box-same") { + return "same".to_owned(); + } + if fault::on("eta-box-raw") { + return raw; + } let clean: String = raw .chars() .map(|c| { @@ -341,7 +408,7 @@ pub fn ingest( let name = basename(&tp); let others: IndexSet<(String, String)> = by_file .iter() - .filter(|(f, _)| **f != name) + .filter(|(f, _)| **f != name && !fault::on("eta-ingest-adds-again")) .flat_map(|(_, rs)| rs.iter().map(|r| (r[1].clone(), r[2].clone()))) .collect(); let new: Vec = history_rows(&read_trace(&tp, tz), &proc_step, box_cpus) @@ -384,7 +451,15 @@ fn num(v: &str) -> Option { /// The catalogue's hours were measured on 16 threads: fewer cores scale /// them up, more never scale them below that. pub fn docs_scale(cores: usize) -> f64 { - let x = DOCS_THREADS / cores.max(1) as f64; + let threads = if fault::on("eta-docs-threads-8") { + 8.0 + } else { + DOCS_THREADS + }; + let x = threads / cores.max(1) as f64; + if fault::on("eta-docs-scale-down") { + return x; + } if x > 1.0 { x } else { 1.0 } } @@ -457,7 +532,14 @@ pub fn step_ranges( .map(|m| m.values().filter(|v| !v.spans.is_empty()).collect()) .unwrap_or_default(); mine.sort_by(|a, b| a.end.cmp(&b.end)); - let mine = &mine[mine.len().saturating_sub(LAST_RUNS)..]; + let last = if fault::on("eta-last-runs-2") { + 2 + } else if fault::on("eta-last-runs-6") { + 6 + } else { + LAST_RUNS + }; + let mine = &mine[mine.len().saturating_sub(last)..]; let range = if !mine.is_empty() { let spans: Vec = mine .iter() @@ -478,9 +560,16 @@ pub fn step_ranges( .unwrap_or(1), ), } - } else if let (Some(h), "measured") = (s.hours, s.hours_from.as_str()) { + } else if let (Some(h), "measured") = ( + s.hours, + if fault::on("eta-estimates-count") { + "measured" + } else { + s.hours_from.as_str() + }, + ) { let h = h * 3600.0; - let lo = num(&s.hours_low); + let lo = num(&s.hours_low).filter(|_| !fault::on("eta-no-hours-low")); Range { low_s: Some(lo.map_or(h, |l| l * 3600.0)), high_s: Some(h * scale), @@ -497,9 +586,10 @@ pub fn step_ranges( cpus: Some(1), } } else { + let zero = fault::on("eta-none-is-zero").then_some(0.0); Range { - low_s: None, - high_s: None, + low_s: zero, + high_s: zero, n: 0, source: Source::None, label: "not measured".to_owned(), @@ -535,7 +625,7 @@ pub fn remaining( for s in &state.steps { steps.insert(s.step.step.as_str(), s); } - let left: Vec<&str> = if state.run == RunState::Finished { + let left: Vec<&str> = if state.run == RunState::Finished && !fault::on("eta-finished-ignored") { Vec::new() } else { steps @@ -561,13 +651,18 @@ pub fn remaining( if steps[k].state == StepState::Running && let Some(st) = started { - let gone = if now - st > 0.0 { now - st } else { 0.0 }; + let mut gone = if now - st > 0.0 { now - st } else { 0.0 }; + if fault::on("eta-gone-ignored") { + gone = 0.0; + } let l = lo - gone; lo = if l > 0.0 { l } else { 0.0 }; hi -= gone; if hi < 0.0 { hi = 0.0; - late.push(k.to_owned()); + if !fault::on("eta-late-not-flagged") { + late.push(k.to_owned()); + } } } dur.insert(k, (lo, hi)); @@ -589,6 +684,9 @@ pub fn remaining( path.push(k); let mut up = Vec::new(); for u in &steps[k].step.upstream { + if fault::on("eta-chain-ignores-upstream") { + break; + } if let Some((u, _)) = dur.get_key_value(u.as_str()) && let Some((&u, _)) = steps.get_key_value(*u) { @@ -621,6 +719,11 @@ pub fn remaining( .unwrap_or(1); acc + dur[k].1 * cpus as f64 }); + let work = if fault::on("eta-no-cpu-work") { + 0.0 + } else { + work + }; let cores_f = cores.max(1) as f64; let bound = work / cores_f; let high = if late.is_empty() { @@ -656,13 +759,18 @@ pub fn remaining( /// finished_utc`. Their partial spans are not history yet. pub fn live_traces(outdir: &Path) -> IndexSet { let mut out = IndexSet::new(); + if fault::on("eta-live-traces-none") { + return out; + } let pattern = outdir.join("*").join("logs").join("run_status.tsv"); for p in pycompat::mtime::glob(pattern.as_os_str(), false) { let sample_dir = p.parent().and_then(Path::parent).unwrap_or(Path::new("/")); let meta = match crate::files::read_run_status(&p) { Ok(st) => st.unwrap_or_default().meta, Err(_) => { - out.extend(trace_files(sample_dir, None).iter().map(|t| basename(t))); + if !fault::on("eta-unreadable-status-ignored") { + out.extend(trace_files(sample_dir, None).iter().map(|t| basename(t))); + } continue; } }; diff --git a/rust/crates/og-runlog/src/headroom.rs b/rust/crates/og-runlog/src/headroom.rs index 4ac2785c7..615b49b10 100644 --- a/rust/crates/og-runlog/src/headroom.rs +++ b/rust/crates/og-runlog/src/headroom.rs @@ -10,7 +10,7 @@ //! on its nearest existing parent, and two folders on one file system are //! reported once. Labels are `work` and `results`, never a path. -use crate::eta::TraceTask; +use crate::eta::{TraceTask, fault}; use crate::host::Host; use crate::steps::{Step, process_steps}; use crate::util::abspath; @@ -46,8 +46,13 @@ pub fn warn_percent(value: Option<&str>, env: Option<&str>) -> Result) -> i128 { } fn row(r: &TaskRow<'_>, order: usize) -> Row { + let mut t = read_row(r, order); + if fault::on("headroom-request-default-8g") { + t.memory = t.memory.filter(|m| *m != 0).or(Some(8 << 30)); + } + if fault::on("headroom-weblog-decimal-peak") && matches!(r, TaskRow::Weblog(_)) { + t.peak = t.peak.map(|p| p * 1_000_000_000 / (1 << 30)); + } + t +} + +fn read_row(r: &TaskRow<'_>, order: usize) -> Row { match r { TaskRow::Trace(t) => Row { key: Key::Pair(format!("s{}", t.hash), format!("s{}", t.attempt)), @@ -208,9 +224,10 @@ pub fn task_memory(rows: &[TaskRow<'_>], warn: f64, steps: &[Step]) -> Vec= (old.attempt, finished(&old.status)) - }); + let replaces = fault::on("headroom-replaces-always") + || latest.get(&t.key).is_none_or(|old| { + (t.attempt, finished(&t.status)) >= (old.attempt, finished(&old.status)) + }); if replaces { latest.insert(t.key.clone(), t); } @@ -226,11 +243,19 @@ pub fn task_memory(rows: &[TaskRow<'_>], warn: f64, steps: &[Step]) -> Vec = ts .iter() - .filter(|t| t.attempt < last.attempt && MEMORY_EXITS.contains(&t.exit.as_str())) + .filter(|t| { + let exits: &[&str] = if fault::on("headroom-exits-143-only") { + &MEMORY_EXITS[..1] + } else { + &MEMORY_EXITS + }; + t.attempt < last.attempt && exits.contains(&t.exit.as_str()) + }) .map(|t| t.attempt) .collect(); retried.sort_unstable(); @@ -250,7 +275,13 @@ pub fn task_memory(rows: &[TaskRow<'_>], warn: f64, steps: &[Step]) -> Vec= warn), + warn: ratio.is_some_and(|r| { + if fault::on("headroom-warn-strict") { + r > 0.99 + } else { + r * 100.0 >= warn + } + }), ratio, oom_retry: retried.len(), process: proc, @@ -285,6 +316,9 @@ pub struct Disk { /// `path`, or its nearest parent that exists. fn nearest_existing(host: &dyn Host, path: &Path) -> PathBuf { let mut p = abspath(path); + if fault::on("headroom-missing-not-parent") { + return p; + } while !host.exists(&p) { match p.parent() { Some(up) if up != p => p = up.to_path_buf(), @@ -306,7 +340,7 @@ pub fn disk(host: &dyn Host, paths: &[(&str, Option<&str>)], warn: f64) -> Vec)], warn: f64) -> Vec 0).then(|| (s.files - s.ffree.min(s.files)) as f64 / s.files as f64); + let inodes = inodes.or(fault::on("headroom-inodes-always").then_some(0.0)); let used_ratio = (used + free > 0).then(|| used as f64 / (used + free) as f64); + let used_ratio = if fault::on("headroom-used-above-size") { + Some(1.5) + } else { + used_ratio + }; by_dev.insert(dev, out.len()); out.push(Disk { fs: (*label).to_owned(), @@ -359,9 +399,15 @@ pub fn metadata(events: &[Value]) -> (Option, Option, Option) -> Result text(m_cap.as_ref()), }; let phys = host.physical_memory(); - let cap = cap_text.and_then(|c| og_nf::units::memory(Some(&c))); + let cap = cap_text + .and_then(|c| og_nf::units::memory(Some(&c))) + .filter(|_| !fault::on("headroom-host-never-warns")); + let labels = if fault::on("headroom-label-is-path") { + (work.unwrap_or("work"), out.unwrap_or("results")) + } else { + ("work", "results") + }; Ok(Snapshot { - disk: disk(host, &[("work", work), ("results", out)], dw), + disk: disk(host, &[(labels.0, work), (labels.1, out)], dw), tasks: task_memory(&rows, mw, steps), host: HostMemory { memory_bytes: phys, From 3c578c7915056a96eb88f2518184d7101fef75b6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:10:08 +0100 Subject: [PATCH 175/255] fix(clinvar): hand FILTER and the depth floor to the always-report reader, and count REF mismatches Both mirrors of step 06 query each site's FILTER as a ninth column and pass the step's depth floor (CLINVAR_MIN_DP, ext.min_dp) to og clinvar deep-intronic. The sample's bcftools norm -c w keeps a record whose REF is not the reference's and only warns REF_MISMATCH; the step now counts those records in its summary and prints a notice when there are any, since such a record cannot match a ClinVar allele. Bead: clin-09 Bead: clin-10 Bead: clin-13 --- modules/local/clinvar_screen/main.nf | 31 +++++++++++++++++++++------ scripts/06-clinvar-screen.sh | 32 ++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/modules/local/clinvar_screen/main.nf b/modules/local/clinvar_screen/main.nf index f8241cedb..29af0325d 100644 --- a/modules/local/clinvar_screen/main.nf +++ b/modules/local/clinvar_screen/main.nf @@ -36,8 +36,13 @@ _clinvar_deep_intronic.tsv gives the sample's call at every site of assets/deep_intronic_sites.tsv (known pathogenic deep intronic alleles), carrier or not: a genotype, "ref" with the GQ of the record or of the - gVCF block over the site, or why there is no call. gvcf and gvcf_index - are [] for a sample without a gVCF. + gVCF block over the site ("ref (low confidence)" for a filtered call or + one below ext.min_gq or without a GQ, "ref (shallow)" below ext.min_dp), + or why there is no call. gvcf and gvcf_index are [] for a sample without + a gVCF. + + The log counts the sample records whose REF differs from the reference + (bcftools norm -c w warns REF_MISMATCH and keeps them as written). CLINVAR_CDNA_CHECK, below, runs scripts/06's cDNA read check (og cdna check) on these hits: _clinvar_cdna_check.tsv says for @@ -141,8 +146,15 @@ process CLINVAR_SCREEN { echo "NOTICE: \${LEFT_OUT} '\${FILTER}' records left out: they lie on \$(grep -c . sample_dropped.txt) contigs that ClinVar or the reference lacks (first ones: \$(awk 'NR <= 5' sample_dropped.txt | paste -sd, -))" fi # --multi-overlaps .: a 1/2 record splits into 1/. and ./1, not 1/0 and 0/1. - bcftools view -f "\${FILTER}" -T sample_targets.tsv -Ou ${vcf} | \\ - bcftools norm -m -any --multi-overlaps . -c w -f ${reference} -Oz -o ${meta.id}_pass.vcf.gz - + # -c w warns REF_MISMATCH for a record whose REF is not the reference's and + # keeps it as written; such a record matches no ClinVar allele, so the log counts them. + if ! bcftools view -f "\${FILTER}" -T sample_targets.tsv -Ou ${vcf} | \\ + bcftools norm -m -any --multi-overlaps . -c w -f ${reference} -Oz -o ${meta.id}_pass.vcf.gz - 2> norm.log; then + cat norm.log >&2 + exit 1 + fi + cat norm.log >&2 + REF_MISMATCH=\$(grep -c '^REF_MISMATCH' norm.log || true) bcftools index -t ${meta.id}_pass.vcf.gz if [ "\$(bcftools index -n ${meta.id}_pass.vcf.gz)" -eq 0 ]; then echo "ERROR: no records left after the '\${FILTER}' filter on the shared contigs" >&2 @@ -204,14 +216,15 @@ process CLINVAR_SCREEN { # Step 3b: the always-report list, as scripts/06 builds it: each site's # record among the split PASS records, else the gVCF record over it (a - # reference block gives "ref" with its GQ), else no call, with why. + # reference block gives "ref" with its GQ), else no call, with why; the + # FILTER, GQ and DP of a reference call are read against the floors. # query_format FILE: '.' for a GQ or DP its header does not define. query_format() { gq=.; dp=. bcftools view -h "\$1" > query_header.txt if grep -q '^##FORMAT== 7 { print \$1, \$2 }' ${sites} | sort -k1,1 -k2,2n -u > sites_pos.tsv bcftools query -T sites_pos.tsv -f "\$(query_format ${meta.id}_pass.vcf.gz)" -o sites_vcf.txt ${meta.id}_pass.vcf.gz @@ -219,7 +232,7 @@ process CLINVAR_SCREEN { if [ "${has_gvcf}" = yes ]; then bcftools query -R sites_pos.tsv -f "\$(query_format sample.g.vcf.gz)" -o sites_gvcf.txt sample.g.vcf.gz fi - og clinvar deep-intronic ${has_gvcf == 'yes' ? '' : '--no-gvcf '}--min-gq ${min_gq} ${sites} sites_vcf.txt sites_gvcf.txt \\ + og clinvar deep-intronic ${has_gvcf == 'yes' ? '' : '--no-gvcf '}--min-gq ${min_gq} --min-dp ${min_dp} ${sites} sites_vcf.txt sites_gvcf.txt \\ > ${meta.id}_clinvar_deep_intronic.tsv echo "Known deep intronic sites, always reported:" awk -F'\\t' 'NR > 1 { printf " %s %s %s:%s %s>%s: %s%s\\n", \$5, \$6, \$1, \$2, \$3, \$4, \$8, (\$9 != "" ? ", " \$9 : "") }' \\ @@ -227,6 +240,10 @@ process CLINVAR_SCREEN { HITS=\$(grep -c -v '^#' ${meta.id}_clinvar_hits.vcf || true) echo "ClinVar pathogenic hits: \${HITS}" + echo "Records whose REF differs from the reference (REF_MISMATCH): \${REF_MISMATCH}" + if [ "\${REF_MISMATCH}" -gt 0 ]; then + echo "NOTICE: \${REF_MISMATCH} sample records have a REF that is not the reference's (bcftools norm REF_MISMATCH above): they stay as written and cannot match a ClinVar allele there. Check that the VCF was called on this reference." + fi # Grouped by ClinVar review stars, as scripts/06 prints them # (og clinvar hits, bin/og on the task PATH). if [ "\${HITS}" -gt 0 ]; then diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index c55155635..fc0ec8333 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -23,7 +23,9 @@ # clinvar/${SAMPLE}_clinvar_deep_intronic.tsv gives the sample's call at every # site of assets/deep_intronic_sites.tsv (known pathogenic deep intronic # alleles), carrier or not: a genotype, "ref" with the GQ of the record or the -# gVCF block, or why there is no call (CLINVAR_ALWAYS_REPORT names another list). +# gVCF block ("ref (low confidence)" for a filtered call or one below the GQ +# floor or without a GQ, "ref (shallow)" below the depth floor), or why there +# is no call (CLINVAR_ALWAYS_REPORT names another list). # A multiallelic record is split with the other alt allele written as missing # (1/2 gives 1/. and ./1), so a hit from it never reads as het. # A hit whose call has a genotype quality below CLINVAR_MIN_GQ (default 30) or @@ -32,6 +34,8 @@ # other than the default is written into the hits VCF as # ##PGP_CALL_FLOOR=GQ=..,DP=.., so og clinvar hits and both reports read # the floor this run used; without the line they use the default. +# The summary counts the sample records whose REF differs from the reference +# (bcftools norm -c w warns REF_MISMATCH and keeps them as written). set -euo pipefail SAMPLE=${1:?Usage: $0 } @@ -160,11 +164,22 @@ fi # --multi-overlaps .: the split of a 1/2 record writes 1/. and ./1, not 1/0 # and 0/1. The other copy holds the other alt allele, not the reference, so a # hit from such a record (one length of a poly-T tract) must not read as het. +# -c w warns REF_MISMATCH for a record whose REF is not the reference's and +# keeps it as written; such a record matches no ClinVar allele, so the +# summary counts them. PASS_VCF="${OUTPUT_DIR}/${SAMPLE}_pass.vcf.gz" -run_in --cpus 2 --memory 2g "$BCFTOOLS_IMAGE" \ +NORM_LOG="${OUTPUT_DIR}/${SAMPLE}_norm.log" +if ! run_in --cpus 2 --memory 2g "$BCFTOOLS_IMAGE" \ sh -c "set -e; bcftools view -f '${FILTER}' -T '$(cpath "$SAMPLE_TARGETS")' -Ou '$(cpath "$VCF")' \ | bcftools norm -m -any --multi-overlaps . -c w -f '$(cpath "$REF")' -Oz -o '$(cpath "$PASS_VCF")' - - bcftools index -f -t '$(cpath "$PASS_VCF")'" + bcftools index -f -t '$(cpath "$PASS_VCF")'" 2> "$NORM_LOG"; then + cat "$NORM_LOG" >&2 + rm -f "$NORM_LOG" + exit 1 +fi +cat "$NORM_LOG" >&2 +REF_MISMATCH=$(grep -c '^REF_MISMATCH' "$NORM_LOG" || true) +rm -f "$NORM_LOG" PASS_COUNT=$(bcftools_run bcftools index -n "$(cpath "$PASS_VCF")") if [ "$PASS_COUNT" -eq 0 ]; then @@ -248,7 +263,8 @@ rm -f "$SHARED" "${SHARED}.tbi" "${HITS_TSV}.body" "$CLINVAR_TARGETS" "$SAMPLE_T # Step 3b: the always-report list. Every site of the list gets the sample's # call: its record in the split PASS records above, else the gVCF record over # it (a reference block gives "ref" with the block's GQ), else no call, with -# why. og clinvar deep-intronic words the genotype as the hits are. +# why. og clinvar deep-intronic words the genotype as the hits are, and reads +# the FILTER, GQ and DP of a reference call against the floors above. # query_format FILE: the bcftools query format og clinvar deep-intronic reads, # with '.' for a GQ or DP the file's header does not define. query_format() { @@ -256,7 +272,7 @@ query_format() { h=$(bcftools_run bcftools view -h "$(cpath "$1")") grep -q '^##FORMAT== 7 {print $1, $2}' "$SITES" | sort -k1,1 -k2,2n -u > "$SITES_POS" @@ -272,7 +288,7 @@ fi NO_GVCF=(); [ "$HAS_GVCF" = yes ] || NO_GVCF=(--no-gvcf) # The list can live outside GENOME_DIR (CLINVAR_ALWAYS_REPORT), so it goes in on stdin. run_in -i -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$BCFTOOLS_IMAGE" /pgp-bin/og clinvar deep-intronic ${NO_GVCF[@]+"${NO_GVCF[@]}"} \ - --min-gq "$MIN_GQ" - "$(cpath "${DEEP_TSV}.vcf")" "$(cpath "${DEEP_TSV}.gvcf")" < "$SITES" > "${DEEP_TSV}.part" + --min-gq "$MIN_GQ" --min-dp "$MIN_DP" - "$(cpath "${DEEP_TSV}.vcf")" "$(cpath "${DEEP_TSV}.gvcf")" < "$SITES" > "${DEEP_TSV}.part" mv "${DEEP_TSV}.part" "$DEEP_TSV" rm -f "$SITES_POS" "${DEEP_TSV}.vcf" "${DEEP_TSV}.gvcf" @@ -315,6 +331,10 @@ echo "cDNA scan of every exon: ${SCAN_TSV}" echo "Known deep intronic sites, always reported: ${DEEP_TSV}" awk -F'\t' 'NR > 1 {printf " %s %s %s:%s %s>%s: %s%s\n", $5, $6, $1, $2, $3, $4, $8, ($9 != "" ? ", " $9 : "")}' "$DEEP_TSV" echo "Count: ${HIT_COUNT} pathogenic hits" +echo "Records whose REF differs from the reference (REF_MISMATCH): ${REF_MISMATCH}" +if [ "$REF_MISMATCH" -gt 0 ]; then + echo "NOTICE: ${REF_MISMATCH} sample records have a REF that is not the reference's (bcftools norm REF_MISMATCH above): they stay as written and cannot match a ClinVar allele there. Check that the VCF was called on this reference." +fi if [ "$HIT_COUNT" -gt 0 ]; then HIT_ROWS=$(run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$BCFTOOLS_IMAGE" /pgp-bin/og clinvar hits "$(cpath "$HITS")") LOW_COUNT=$(awk -F'\t' '$13 == "yes"' <<< "$HIT_ROWS" | wc -l | tr -d ' ') From 9e6a06c0bd6923ae22b8d923277cb2e8d4244feb Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:10:13 +0100 Subject: [PATCH 176/255] fix(pgx): hold back a named MT-RNR1 allele whose AF reads nan; goldens for the new calls The parity goldens record the deliberate changes against the old Python: AF 0.4 is now the call, --min-af 0.3 logs it as heteroplasmic, and a nan AF is held back as having no allele fraction. The unit test of step 07 expects the call down to AF 0.05. Bead: pgx-32 --- rust/crates/og-filters/src/mito.rs | 6 +++++- .../crates/og-filters/tests/i411_mtrnr1_het.rs | 10 ++++++++++ tests/parity/golden/pgx-mito-merge/het/out.tsv | 1 + tests/parity/golden/pgx-mito-merge/het/stdout | 1 + tests/parity/golden/pgx-mito-merge/nan/out.tsv | 0 tests/parity/golden/pgx-mito-merge/nan/stdout | 2 ++ .../golden/pgx-mito-merge/stored-min-af/stdout | 1 + tests/test_pgx_mito_merge.py | 18 +++++++++++++----- 8 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 tests/parity/golden/pgx-mito-merge/het/out.tsv create mode 100644 tests/parity/golden/pgx-mito-merge/het/stdout create mode 100644 tests/parity/golden/pgx-mito-merge/nan/out.tsv create mode 100644 tests/parity/golden/pgx-mito-merge/nan/stdout create mode 100644 tests/parity/golden/pgx-mito-merge/stored-min-af/stdout diff --git a/rust/crates/og-filters/src/mito.rs b/rust/crates/og-filters/src/mito.rs index 39e2e691d..f4512d5b0 100644 --- a/rust/crates/og-filters/src/mito.rs +++ b/rust/crates/og-filters/src/mito.rs @@ -34,7 +34,9 @@ //! position now counts as depth 0, as the threshold check already did; //! - a named allele at AF 0.05 to 0.95 was held back as heteroplasmic, so //! MT-RNR1 stayed No Result and the aminoglycoside warning was lost; it -//! is now the call (issue #411). +//! is now the call (issue #411); +//! - a named allele whose AF reads `nan` was passed as homoplasmic; it is +//! now held back as having no allele fraction. use std::collections::{BTreeSet, HashMap}; use std::fmt; @@ -453,7 +455,9 @@ pub fn decide( shown, filter: r.filter.clone(), }), + // A NaN AF says nothing about detection: no AF. None => held.push(Held::NoAf { name, shown }), + Some(af) if af.is_nan() => held.push(Held::NoAf { name, shown }), Some(af) if af < min_af => held.push(Held::BelowFloor { name, shown, diff --git a/rust/crates/og-filters/tests/i411_mtrnr1_het.rs b/rust/crates/og-filters/tests/i411_mtrnr1_het.rs index 56fe296cf..ba1bd88a5 100644 --- a/rust/crates/og-filters/tests/i411_mtrnr1_het.rs +++ b/rust/crates/og-filters/tests/i411_mtrnr1_het.rs @@ -120,3 +120,13 @@ fn red_first_a_heteroplasmic_unnamed_change_at_a_named_position_holds_reference_ let (call, _) = run(&[rec(1555, "A", "C", 0.03)]); assert_eq!(call.as_deref(), Some("Reference")); } + +#[test] +fn red_first_a_nan_af_is_no_af_not_a_call() { + let (call, lines) = run(&[rec(1555, "A", "G", f64::NAN)]); + assert_eq!(call, None, "{lines:?}"); + assert_eq!( + lines[0], + "held back: m.1555A>G (chrM:1555 A>G AF nan) has no allele fraction" + ); +} diff --git a/tests/parity/golden/pgx-mito-merge/het/out.tsv b/tests/parity/golden/pgx-mito-merge/het/out.tsv new file mode 100644 index 000000000..7625289f3 --- /dev/null +++ b/tests/parity/golden/pgx-mito-merge/het/out.tsv @@ -0,0 +1 @@ +MT-RNR1 m.1555A>G diff --git a/tests/parity/golden/pgx-mito-merge/het/stdout b/tests/parity/golden/pgx-mito-merge/het/stdout new file mode 100644 index 000000000..2dff47160 --- /dev/null +++ b/tests/parity/golden/pgx-mito-merge/het/stdout @@ -0,0 +1 @@ +pgx_mito_merge: MT-RNR1 m.1555A>G passed to PharmCAT as an outside call (heteroplasmic, chrM:1555 A>G AF 0.400; Increased risk of aminoglycoside-induced hearing loss); CPIC sets no heteroplasmy level at which aminoglycosides are safe, so the homoplasmic guidance applies diff --git a/tests/parity/golden/pgx-mito-merge/nan/out.tsv b/tests/parity/golden/pgx-mito-merge/nan/out.tsv new file mode 100644 index 000000000..e69de29bb diff --git a/tests/parity/golden/pgx-mito-merge/nan/stdout b/tests/parity/golden/pgx-mito-merge/nan/stdout new file mode 100644 index 000000000..b3c34caae --- /dev/null +++ b/tests/parity/golden/pgx-mito-merge/nan/stdout @@ -0,0 +1,2 @@ +pgx_mito_merge: held back: m.1555A>G (chrM:1555 A>G AF nan) has no allele fraction +pgx_mito_merge: no MT-RNR1 call passed to PharmCAT: it stays No Result; read the records above in the chrM call set diff --git a/tests/parity/golden/pgx-mito-merge/stored-min-af/stdout b/tests/parity/golden/pgx-mito-merge/stored-min-af/stdout new file mode 100644 index 000000000..2dff47160 --- /dev/null +++ b/tests/parity/golden/pgx-mito-merge/stored-min-af/stdout @@ -0,0 +1 @@ +pgx_mito_merge: MT-RNR1 m.1555A>G passed to PharmCAT as an outside call (heteroplasmic, chrM:1555 A>G AF 0.400; Increased risk of aminoglycoside-induced hearing loss); CPIC sets no heteroplasmy level at which aminoglycosides are safe, so the homoplasmic guidance applies diff --git a/tests/test_pgx_mito_merge.py b/tests/test_pgx_mito_merge.py index d2e64e324..b2ebd5aca 100644 --- a/tests/test_pgx_mito_merge.py +++ b/tests/test_pgx_mito_merge.py @@ -8,8 +8,10 @@ VCFs (GT 0/1 at every AF, as Mutect2 writes them), and checks: 1. a homoplasmic PASS m.1555A>G (AF 0.99) is the call; step 36's lines are copied before it and an MT-RNR1 line there is dropped; - 2. AF 0.95 counts as homoplasmic, 0.949 does not; --min-af moves the bound; - 3. a heteroplasmic, a filtered or an AF-less m.1555A>G passes nothing and + 2. AF 0.95 counts as homoplasmic; a heteroplasmic allele down to the + detection floor, AF 0.05, is the call too (CPIC: no safe heteroplasmy + level), 0.049 is not; --min-af moves the floor; + 3. one below the floor, a filtered or an AF-less m.1555A>G passes nothing and says why; so do a homoplasmic PASS change at a named position that PharmCAT does not name and a PASS deletion near m.961; a filtered one, or a low-AF unnamed change, does not; @@ -132,14 +134,20 @@ def main(): # 2. the AF bound rc, log, lines = c.run(BACKGROUND + rec(1555, "A", "G", 0.95)) check("2: AF 0.95 is homoplasmic", c.mt(lines) == ["m.1555A>G"], log) - rc, log, lines = c.run(BACKGROUND + rec(1555, "A", "G", 0.949)) - check("2: AF 0.949 passes nothing", c.mt(lines) == [], log) + rc, log, lines = c.run(BACKGROUND + rec(1555, "A", "G", 0.4)) + check("2: AF 0.4 is the call", c.mt(lines) == ["m.1555A>G"], log) + check("2: AF 0.4: the log says heteroplasmic and why it still counts", + "(heteroplasmic, chrM:1555 A>G AF 0.400; " in log and "homoplasmic guidance applies" in log, log) + rc, log, lines = c.run(BACKGROUND + rec(1555, "A", "G", 0.05)) + check("2: AF 0.05 is the call", c.mt(lines) == ["m.1555A>G"], log) + rc, log, lines = c.run(BACKGROUND + rec(1555, "A", "G", 0.049)) + check("2: AF 0.049 passes nothing", c.mt(lines) == [], log) rc, log, lines = c.run(BACKGROUND + rec(1555, "A", "G", 0.9), extra=("--min-af", "0.85")) check("2: --min-af 0.85 takes AF 0.9", c.mt(lines) == ["m.1555A>G"], log) # 3. what passes nothing for desc, records, want in [ - ("heteroplasmic", rec(1555, "A", "G", 0.4), "is heteroplasmic (AF below 0.95)"), + ("below the floor", rec(1555, "A", "G", 0.03), "is below the detection floor (AF below 0.05)"), ("filtered", rec(1555, "A", "G", 0.99, filt="possible_numt"), "is filtered (possible_numt)"), ("without an allele fraction", rec(1555, "A", "G", None), "has no allele fraction"), ("a change PharmCAT does not name", rec(1555, "A", "C", 0.99), From 3a840f57393a480ab7a460557c6711aef4719f1f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:12:02 +0100 Subject: [PATCH 177/255] test(clinvar): fake-docker case for the REF_MISMATCH count and the site query in both mirrors Step 06 runs to its summary against a fake bcftools that warns REF_MISMATCH twice, then not at all: the count line reads 2 and then 0, and the notice shows only for 2. The always-report call gets the depth floor, and the query_format of the script and of the module, run with the Groovy escapes undone, both end with %FILTER. Red on main: the count line is missing, and the module's format has no FILTER. Bead: clin-13 --- tests/fake-docker/i411-site-states.sh | 105 ++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100755 tests/fake-docker/i411-site-states.sh diff --git a/tests/fake-docker/i411-site-states.sh b/tests/fake-docker/i411-site-states.sh new file mode 100755 index 000000000..5980c5c6d --- /dev/null +++ b/tests/fake-docker/i411-site-states.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Step 06's always-report list reads FILTER and the depth floor, and step 06 +# counts the sample records whose REF is not the reference's, in both mirrors: +# - scripts/06-clinvar-screen.sh: bcftools norm -c w keeps a record whose REF +# differs from the reference and only warns REF_MISMATCH on stderr. The +# summary counts those warnings and prints a notice when there are any, +# so a mismatched record no longer leaves only a log line; +# - it hands og clinvar deep-intronic the depth floor (CLINVAR_MIN_DP), and +# its query format ends with %FILTER, so a filtered or shallow reference +# call no longer reads as plain "ref"; +# - modules/local/clinvar_screen/main.nf builds the same query format (its +# query_format runs here, Groovy escapes undone), passes ext.min_dp, and +# counts REF_MISMATCH from its own bcftools norm. +# shellcheck source=../../scripts/ci/fake-docker/lib.sh +. "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" + +export GENOME_DIR="${CASE_WORK}/genome" +seed_reference "$GENOME_DIR" +seed_clinvar "$GENOME_DIR" +seed_sample "$GENOME_DIR" sample1 +use_output_hook + +# The answers step 06 reads from bcftools, every call's outputs, the two +# REF_MISMATCH warnings bcftools norm writes for the sample when +# ${CASE_WORK}/mismatch exists, and the two cDNA tables og writes relative to +# its -w directory, so the step runs to its summary. +cat > "${CASE_WORK}/hook-step06" <<'HOOK' +#!/usr/bin/env bash +"${CASE_WORK:?}/hook-outputs" "$@" || exit $? +shift +out="${CASE_WORK}/genome/sample1/clinvar" +case "$*" in + *"index -s "*) printf 'chr1\t248956422\t3\n' ;; + *"index -n "*) echo 3 ;; + *"view -H -f PASS "*) echo 1 ;; + *"view -h "*) printf '##fileformat=VCFv4.2\n##FORMAT=\n##FORMAT=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tsample1\n' ;; + *"--multi-overlaps"*) + if [ -e "${CASE_WORK}/mismatch" ]; then + printf 'REF_MISMATCH\tchr1\t40\tA\tT\nREF_MISMATCH\tchr1\t150\tGT\tCT\n' >&2 + fi + printf 'Lines total/split/joined/realigned/mismatch_removed/dup_removed/skipped:\t4/0/0/0/0/0/0\n' >&2 ;; + *"cdna check "*) : > "${out}/sample1_clinvar_cdna_check.tsv.part" ;; + *"cdna scan "*) : > "${out}/sample1_clinvar_cdna_scan.tsv.part" ;; +esac +exit 0 +HOOK +chmod +x "${CASE_WORK}/hook-step06" +export FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/hook-step06" + +COUNT_LINE='^Records whose REF differs from the reference \(REF_MISMATCH\): ' + +# --- scripts/06: two mismatched records --------------------------------------------- +touch "${CASE_WORK}/mismatch" +: > "$FAKE_DOCKER_LOG" +run_rc step06-mismatch "${SCRIPTS}/06-clinvar-screen.sh" sample1 +expect_rc step06-mismatch 0 +output_has step06-mismatch "^REF_MISMATCH chr1 40 A T$" +output_has step06-mismatch "${COUNT_LINE}2$" +output_has step06-mismatch '^NOTICE: 2 sample records have a REF that is not the reference'"'"'s' +DEEP_CALL=$(awk '/^run / && /og clinvar deep-intronic /' "$FAKE_DOCKER_LOG") +echo "deep-intronic call: ${DEEP_CALL:-none}" +grep -qE -- '--min-gq 30 --min-dp 10 ' <<<"$DEEP_CALL" \ + || fail "step 06 does not hand og clinvar deep-intronic its depth floor (--min-dp 10): ${DEEP_CALL:-no call}" +QUERY=$(awk '/^run / && /bcftools query -T /' "$FAKE_DOCKER_LOG") +grep -qF -- 'FILTER' <<<"$QUERY" || fail "step 06's always-report query reads no FILTER: ${QUERY:-no query}" + +# --- scripts/06: none, and another depth floor ----------------------------------- +rm -f "${CASE_WORK}/mismatch" +rm -rf "${GENOME_DIR}/sample1/clinvar" +: > "$FAKE_DOCKER_LOG" +CLINVAR_MIN_DP=8 run_rc step06-clean "${SCRIPTS}/06-clinvar-screen.sh" sample1 +expect_rc step06-clean 0 +output_has step06-clean "${COUNT_LINE}0$" +output_lacks step06-clean '^NOTICE: .* REF that is not the reference' +grep -qE -- '--min-dp 8 ' <<<"$(awk '/^run / && /og clinvar deep-intronic /' "$FAKE_DOCKER_LOG")" \ + || fail "step 06 does not pass CLINVAR_MIN_DP=8 to og clinvar deep-intronic" + +# --- both mirrors build the same query format, ending with FILTER ----------------- +WANT='%CHROM\t%POS\t%END\t%REF\t%ALT\t[%GT]\t[%GQ]\t[%DP]\t%FILTER\n' +HEADER="${CASE_WORK}/gq-dp-header.txt" +printf '##FORMAT=\n##FORMAT=\n' > "$HEADER" +# fn_text FILE: the query_format function of FILE, as text. +fn_text() { awk '/^ *query_format\(\) \{/ { on = 1 } on { print } on && /^ *\}$/ { exit }' "$1"; } +BASH_FN=$(fn_text "${SCRIPTS}/06-clinvar-screen.sh") +[ -n "$BASH_FN" ] || fail "scripts/06 has no query_format function" +# shellcheck disable=SC2016 # the function's text runs in the inner shell +GOT=$(HEADER=$HEADER bash -c 'set -euo pipefail; bcftools_run() { cat "$HEADER"; }; cpath() { printf %s "$1"; }; eval "$1"; query_format x.vcf.gz' _ "$BASH_FN") +[ "$GOT" = "$WANT" ] || fail "scripts/06 queries '${GOT}', want '${WANT}'" +MOD="${REPO_ROOT}/modules/local/clinvar_screen/main.nf" +MOD_FN=$(fn_text "$MOD" | sed -e 's/\\\$/$/g' -e 's/\\\\/\\/g') +[ -n "$MOD_FN" ] || fail "the ClinVar screen module has no query_format function" +# shellcheck disable=SC2016 # the function's text runs in the inner shell +GOT=$(cd "$CASE_WORK" && HEADER=$HEADER bash -c 'set -euo pipefail; bcftools() { cat "$HEADER"; }; eval "$1"; query_format x.vcf.gz' _ "$MOD_FN") +[ "$GOT" = "$WANT" ] || fail "the module queries '${GOT}', want '${WANT}'" + +# --- the module: depth floor and REF_MISMATCH count --------------------------------- +grep -qE -- 'og clinvar deep-intronic .*--min-gq \$\{min_gq\} --min-dp \$\{min_dp\} ' "$MOD" \ + || fail "the module does not hand og clinvar deep-intronic ext.min_dp" +grep -qF -- '--multi-overlaps . -c w -f ${reference} -Oz -o ${meta.id}_pass.vcf.gz - 2> norm.log' "$MOD" \ + || fail "the module's sample bcftools norm does not keep its REF_MISMATCH warnings" +grep -qF -- "REF_MISMATCH=\\\$(grep -c '^REF_MISMATCH' norm.log || true)" "$MOD" \ + || fail "the module does not count the REF_MISMATCH warnings" +grep -qF -- 'echo "Records whose REF differs from the reference (REF_MISMATCH): \${REF_MISMATCH}"' "$MOD" \ + || fail "the module's log has no REF_MISMATCH count" +echo "both mirrors read FILTER and the depth floor, and count REF_MISMATCH: PASS" From cda39714668ed30a5f2156c0bb990ad69fa16891 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:12:37 +0100 Subject: [PATCH 178/255] feat(dev): og dev licences notices writes the licences of every crate og links It walks cargo metadata per target from og along normal edges, checks each crate's licence against an allow list, and writes each licence text once, including the C library texts of crates that set links (htslib and htscodecs from hts-sys). Bead: lic-02 --- rust/crates/og-dev/src/licences.rs | 601 +++++++++++++++++++++- rust/crates/og-dev/tests/licences/main.rs | 118 ++++- 2 files changed, 692 insertions(+), 27 deletions(-) diff --git a/rust/crates/og-dev/src/licences.rs b/rust/crates/og-dev/src/licences.rs index 58319e4e5..74458dbae 100644 --- a/rust/crates/og-dev/src/licences.rs +++ b/rust/crates/og-dev/src/licences.rs @@ -1,13 +1,598 @@ -//! `og dev licences`: the licences of the data and code OwnGenome ships, -//! and the notices of the libraries in the binary. +//! `og dev licences`: the notices of the third-party code in the og binary. //! -//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so -//! on stderr and exits 2, before reading any argument. +//! ```text +//! og dev licences notices [--manifest-path FILE] [--package NAME] +//! (--target TRIPLE | --target-no-default TRIPLE)... [--out FILE] +//! ``` +//! +//! The og binaries link about seventy crates statically, and the Linux pair +//! also links htslib and htscodecs (C, built by `hts-sys`). Their licences +//! (MIT, BSD-3-Clause, Unicode-3.0 and others) ask that a binary carry their +//! notices, so every release ships the file this verb writes +//! (`rust/og.sh build`, `.github/workflows/release.yml`). +//! +//! For each target it asks `cargo metadata --filter-platform TRIPLE` for the +//! resolved graph (`--target-no-default` adds `--no-default-features`, as the +//! macOS build is made) and walks it from the package (default `og`) along +//! normal edges only: build and dev dependencies, and procedural macros with +//! everything below them, run at compile time and leave no code in the +//! binary. Each crate reached outside the workspace must have a licence +//! expression the allow list ([`ALLOWED`]) satisfies, and a licence text: +//! the LICENSE, LICENCE, COPYING, NOTICE, UNLICENSE or COPYRIGHT files at its +//! root and its `license-file`. A crate that builds C (it sets `links`) also +//! contributes every such file below its root, so the vendored htslib and +//! htscodecs texts come from the crate itself and cannot go stale on a bump. +//! A crate that ships no text, under a licence that MIT satisfies, gets the +//! MIT text with its authors; any other crate without a text is an error. +//! +//! The file lists every crate with its version, licence and, when it is not +//! in every binary, the targets it is in; then each distinct licence text +//! once, headed by the crates and files it covers. Errors name every crate +//! at fault and exit 1; usage errors exit 2. +use pycompat::json::{self, Value}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// The licences a linked crate may carry. Each is permissive and +/// compatible with GPL-3.0-or-later; an expression passes when these alone +/// satisfy it (an `OR` needs one side, an `AND` both). +pub const ALLOWED: &[&str] = &[ + "0BSD", + "Apache-2.0", + "Apache-2.0 WITH LLVM-exception", + "BSD-3-Clause", + "ISC", + "MIT", + "Unicode-3.0", + "Unlicense", + "Zlib", +]; + +const USAGE: &str = "usage: og dev licences notices [--manifest-path FILE] [--package NAME] (--target TRIPLE | --target-no-default TRIPLE)... [--out FILE]"; + +/// The MIT text, for a crate under MIT that ships no licence file. +const MIT_TEXT: &str = + "Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the \"Software\"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +"; + +/// `og dev licences ARGS`. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + match args.first().map(String::as_str) { + Some("notices") => notices_cli(&args[1..], out, err), + _ => { + let _ = writeln!(err, "{USAGE}"); + 2 + } + } +} + +/// One cargo build of the binary: a target triple and its feature set. +struct Build { + triple: String, + default_features: bool, +} + +struct Opts { + manifest: Option, + package: String, + builds: Vec, + out: Option, +} + +fn parse(args: &[String]) -> Result { + let mut o = Opts { + manifest: None, + package: "og".into(), + builds: Vec::new(), + out: None, + }; + let mut it = args.iter(); + while let Some(a) = it.next() { + let mut value = || { + it.next() + .cloned() + .ok_or_else(|| format!("{a} needs a value")) + }; + match a.as_str() { + "--manifest-path" => o.manifest = Some(value()?), + "--package" => o.package = value()?, + "--out" => o.out = Some(value()?), + "--target" | "--target-no-default" => o.builds.push(Build { + default_features: a == "--target", + triple: value()?, + }), + _ => return Err(format!("unknown argument {a:?}")), + } + } + if o.builds.is_empty() { + return Err("name at least one --target or --target-no-default".into()); + } + Ok(o) +} + +fn notices_cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let opts = match parse(args) { + Ok(o) => o, + Err(e) => { + let _ = writeln!(err, "og dev licences notices: {e}\n{USAGE}"); + return 2; + } + }; + match notices(&opts) { + Ok(text) => { + let written = match &opts.out { + Some(path) => std::fs::write(path, &text).map_err(|e| format!("{path}: {e}")), + None => out.write_all(text.as_bytes()).map_err(|e| e.to_string()), + }; + match written { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "og dev licences notices: {e}"); + 1 + } + } + } + Err(problems) => { + for p in problems { + let _ = writeln!(err, "og dev licences notices: {p}"); + } + 1 + } + } +} + +/// A third-party package the binary links, gathered over every build. +struct Crate { + name: String, + version: String, + license: Option, + license_file: Option, + authors: Vec, + dir: PathBuf, + builds_c: bool, + triples: BTreeSet, +} + +fn notices(opts: &Opts) -> Result> { + let mut crates: BTreeMap<(String, String), Crate> = BTreeMap::new(); + for b in &opts.builds { + let meta = cargo_metadata(opts, b).map_err(|e| vec![e])?; + for c in linked(&meta, &opts.package).map_err(|e| vec![e])? { + crates + .entry((c.name.clone(), c.version.clone())) + .or_insert(c) + .triples + .insert(b.triple.clone()); + } + } + let mut problems = Vec::new(); + // Each distinct text once, in the order its first crate comes. + let mut texts: Vec<(String, Vec)> = Vec::new(); + let mut index: HashMap = HashMap::new(); + for c in crates.values() { + let id = format!("{} {}", c.name, c.version); + let Some(expr) = &c.license else { + problems.push(format!("{id} declares no licence expression")); + continue; + }; + match satisfied(expr, &|l| ALLOWED.contains(&l)) { + Ok(true) => {} + Ok(false) => { + problems.push(format!( + "{id} is licensed {expr:?}, which the allow list ({}) does not satisfy", + ALLOWED.join(", ") + )); + continue; + } + Err(e) => { + problems.push(format!("{id}: cannot read its licence {expr:?}: {e}")); + continue; + } + } + let mut found = licence_files(c); + if found.is_empty() { + if satisfied(expr, &|l| l == "MIT") == Ok(true) { + let who = if c.authors.is_empty() { + format!("the {} authors", c.name) + } else { + c.authors.join(", ") + }; + found.push(( + "MIT, no licence file in the crate".into(), + format!("Copyright (c) {who}\n\n{MIT_TEXT}"), + )); + } else { + problems.push(format!( + "{id} ships no licence file, and {expr:?} cannot be met with the MIT text" + )); + continue; + } + } + for (label, text) in found { + let who = format!("{id}: {label}"); + match index.get(&text) { + Some(&i) => texts[i].1.push(who), + None => { + index.insert(text.clone(), texts.len()); + texts.push((text, vec![who])); + } + } + } + } + if !problems.is_empty() { + return Err(problems); + } + Ok(render(opts, &crates, &texts)) +} + +fn render( + opts: &Opts, + crates: &BTreeMap<(String, String), Crate>, + texts: &[(String, Vec)], +) -> String { + let all: BTreeSet = opts.builds.iter().map(|b| b.triple.clone()).collect(); + let mut s = String::new(); + s.push_str("Third-party licences of the og binaries\n"); + s.push_str("=======================================\n\n"); + s.push_str("OwnGenome is GPL-3.0-or-later (LICENSE in its source). The og binaries\n"); + s.push_str("also hold the code of the crates below, linked statically. A crate that\n"); + s.push_str("builds C carries that library's licence in its own folder: hts-sys gives\n"); + s.push_str("htslib/LICENSE and htslib/htscodecs/LICENSE.md, the licences of htslib and\n"); + s.push_str("htscodecs. Each licence text comes once below, after the crates and files\n"); + s.push_str("it covers. `og dev licences notices` writes this file from cargo metadata\n"); + s.push_str("and the files each crate ships.\n\n"); + s.push_str(&format!( + "Targets: {}\n\n", + all.iter().cloned().collect::>().join(", ") + )); + let title = format!("Crates ({})", crates.len()); + s.push_str(&format!("{title}\n{}\n", "-".repeat(title.len()))); + for c in crates.values() { + s.push_str(&format!( + "{} {} {}", + c.name, + c.version, + c.license.as_deref().unwrap_or("") + )); + if c.triples != all { + let only: Vec<&str> = c.triples.iter().map(String::as_str).collect(); + s.push_str(&format!(" (only {})", only.join(", "))); + } + s.push('\n'); + } + let title = format!("Licence texts ({})", texts.len()); + s.push_str(&format!("\n{title}\n{}\n", "-".repeat(title.len()))); + for (i, (text, who)) in texts.iter().enumerate() { + s.push_str(&format!("\n=== {} ===\n", i + 1)); + for w in who { + s.push_str(&format!("For {w}\n")); + } + s.push('\n'); + s.push_str(text.trim_end()); + s.push('\n'); + } + s +} + +fn cargo_metadata(opts: &Opts, b: &Build) -> Result { + let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); + let mut cmd = Command::new(cargo); + cmd.args([ + "metadata", + "--format-version", + "1", + "--locked", + "--filter-platform", + ]) + .arg(&b.triple); + if !b.default_features { + cmd.arg("--no-default-features"); + } + if let Some(m) = &opts.manifest { + cmd.arg("--manifest-path").arg(m); + } + let run = cmd + .output() + .map_err(|e| format!("cargo metadata did not start: {e}"))?; + if !run.status.success() { + return Err(format!( + "cargo metadata for {} failed: {}", + b.triple, + String::from_utf8_lossy(&run.stderr).trim() + )); + } + let text = + String::from_utf8(run.stdout).map_err(|_| "cargo metadata wrote non-UTF-8".to_string())?; + json::loads(&text) + .map_err(|e| format!("cargo metadata wrote unreadable JSON: {}", e.exception())) +} + +fn field<'a>(v: &'a Value, key: &str) -> Option<&'a Value> { + match v { + Value::Object(m) => m.get(key), + _ => None, + } +} + +fn text<'a>(v: &'a Value, key: &str) -> Option<&'a str> { + match field(v, key) { + Some(Value::Str(s)) => Some(s), + _ => None, + } +} + +fn list<'a>(v: &'a Value, key: &str) -> &'a [Value] { + match field(v, key) { + Some(Value::Array(a)) => a, + _ => &[], + } +} + +/// The third-party packages `package` links on this build's target. +fn linked(meta: &Value, package: &str) -> Result, String> { + let packages: HashMap<&str, &Value> = list(meta, "packages") + .iter() + .filter_map(|p| Some((text(p, "id")?, p))) + .collect(); + let members: HashSet<&str> = list(meta, "workspace_members") + .iter() + .filter_map(|v| match v { + Value::Str(s) => Some(s.as_str()), + _ => None, + }) + .collect(); + let resolve = field(meta, "resolve").ok_or("cargo metadata gave no resolve graph")?; + let nodes: HashMap<&str, &Value> = list(resolve, "nodes") + .iter() + .filter_map(|n| Some((text(n, "id")?, n))) + .collect(); + let root = members + .iter() + .copied() + .find(|id| packages.get(id).and_then(|p| text(p, "name")) == Some(package)) + .ok_or_else(|| format!("no workspace package is named {package}"))?; + let mut seen = HashSet::from([root]); + let mut stack = vec![root]; + while let Some(id) = stack.pop() { + let Some(node) = nodes.get(id) else { continue }; + for dep in list(node, "deps") { + let Some(pkg) = text(dep, "pkg") else { + continue; + }; + // kind null is a normal dependency; "build" and "dev" are not linked. + let normal = list(dep, "dep_kinds") + .iter() + .any(|k| matches!(field(k, "kind"), None | Some(Value::Null))); + let proc_macro = packages.get(pkg).is_some_and(|p| { + list(p, "targets").iter().any(|t| { + list(t, "kind") + .iter() + .any(|k| matches!(k, Value::Str(s) if s == "proc-macro")) + }) + }); + if normal && !proc_macro && seen.insert(pkg) { + stack.push(pkg); + } + } + } + let mut out = Vec::new(); + for id in seen { + if members.contains(id) { + continue; + } + let p = packages.get(id).ok_or_else(|| { + format!("cargo metadata lists {id} in its graph but not as a package") + })?; + let manifest = + text(p, "manifest_path").ok_or_else(|| format!("{id} has no manifest_path"))?; + out.push(Crate { + name: text(p, "name").unwrap_or(id).into(), + version: text(p, "version").unwrap_or("").into(), + license: text(p, "license").map(str::to_owned), + license_file: text(p, "license_file").map(str::to_owned), + authors: list(p, "authors") + .iter() + .filter_map(|a| match a { + Value::Str(s) => Some(s.clone()), + _ => None, + }) + .collect(), + dir: Path::new(manifest) + .parent() + .unwrap_or(Path::new(".")) + .to_path_buf(), + builds_c: text(p, "links").is_some(), + triples: BTreeSet::new(), + }); + } + Ok(out) +} + +fn is_licence_name(name: &str) -> bool { + let n = name.to_ascii_lowercase(); + [ + "license", + "licence", + "copying", + "notice", + "unlicense", + "copyright", + ] + .iter() + .any(|p| n.starts_with(p)) +} + +/// (label, text) of every licence file of a crate, label = path in the crate. +fn licence_files(c: &Crate) -> Vec<(String, String)> { + let mut paths: BTreeSet = BTreeSet::new(); + collect(&c.dir, c.builds_c, &mut paths); + if let Some(f) = &c.license_file { + let p = c.dir.join(f); + if p.is_file() { + paths.insert(p); + } + } + paths + .into_iter() + .filter_map(|p| { + let bytes = std::fs::read(&p).ok()?; + let label = p.strip_prefix(&c.dir).unwrap_or(&p).display().to_string(); + Some((label, String::from_utf8_lossy(&bytes).into_owned())) + }) + .collect() +} + +/// Licence files in `dir`, and below it when `deep` (a crate that builds C). +fn collect(dir: &Path, deep: bool, out: &mut BTreeSet) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for e in entries.flatten() { + let Ok(kind) = e.file_type() else { continue }; + let path = e.path(); + if kind.is_file() && is_licence_name(&e.file_name().to_string_lossy()) { + out.insert(path); + } else if kind.is_dir() && deep { + collect(&path, deep, out); + } + } +} + +/// Whether an SPDX expression is met by the licences `ok` accepts: `OR` +/// (or the old `/`) needs one side, `AND` both, `X WITH Y` is one licence. +pub fn satisfied(expr: &str, ok: &dyn Fn(&str) -> bool) -> Result { + let spaced = expr + .replace('(', " ( ") + .replace(')', " ) ") + .replace('/', " OR "); + let tokens: Vec<&str> = spaced.split_whitespace().collect(); + let mut p = Parser { + tokens: &tokens, + at: 0, + ok, + }; + let v = p.or()?; + if p.at != tokens.len() { + return Err(format!("unexpected {:?}", tokens[p.at])); + } + Ok(v) +} + +struct Parser<'a> { + tokens: &'a [&'a str], + at: usize, + ok: &'a dyn Fn(&str) -> bool, +} + +impl<'a> Parser<'a> { + fn peek(&self) -> Option<&'a str> { + self.tokens.get(self.at).copied() + } + fn or(&mut self) -> Result { + let mut v = self.and()?; + while self.peek().is_some_and(|t| t.eq_ignore_ascii_case("OR")) { + self.at += 1; + v |= self.and()?; + } + Ok(v) + } + fn and(&mut self) -> Result { + let mut v = self.one()?; + while self.peek().is_some_and(|t| t.eq_ignore_ascii_case("AND")) { + self.at += 1; + v &= self.one()?; + } + Ok(v) + } + fn one(&mut self) -> Result { + let t = self.peek().ok_or("the expression ends early")?; + self.at += 1; + if t == "(" { + let v = self.or()?; + if self.peek() != Some(")") { + return Err("a ( is not closed".into()); + } + self.at += 1; + return Ok(v); + } + if [")", "AND", "OR", "WITH"] + .iter() + .any(|k| t.eq_ignore_ascii_case(k)) + { + return Err(format!("unexpected {t:?}")); + } + if self.peek().is_some_and(|w| w.eq_ignore_ascii_case("WITH")) { + let exception = self + .tokens + .get(self.at + 1) + .ok_or("WITH needs an exception")?; + self.at += 2; + return Ok((self.ok)(&format!("{t} WITH {exception}"))); + } + Ok((self.ok)(t)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn allowed(expr: &str) -> Result { + satisfied(expr, &|l| ALLOWED.contains(&l)) + } + + #[test] + fn expressions_the_locked_crates_use_pass() { + for e in [ + "MIT", + "MIT OR Apache-2.0", + "MIT/Apache-2.0", + "(MIT OR Apache-2.0) AND Unicode-3.0", + "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT", + "MIT OR Apache-2.0 OR LGPL-2.1-or-later", + "0BSD OR MIT OR Apache-2.0", + "Unlicense OR MIT", + ] { + assert_eq!(allowed(e), Ok(true), "{e}"); + } + } + + #[test] + fn a_licence_outside_the_list_fails() { + for e in [ + "SSPL-1.0", + "MIT AND SSPL-1.0", + "LGPL-2.1-or-later", + "GPL-2.0 WITH Classpath-exception-2.0", + "(MIT OR Apache-2.0) AND MPL-2.0", + ] { + assert_eq!(allowed(e), Ok(false), "{e}"); + } + } -/// `og dev licences ARGS`: not ported yet, so one line on stderr and exit 2. -pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { - let _ = writeln!(err, "og dev licences: not ported yet"); - 2 + #[test] + fn an_unreadable_expression_is_an_error() { + for e in ["", "MIT OR", "(MIT", "MIT)", "AND MIT", "MIT WITH"] { + assert!(allowed(e).is_err(), "{e:?} should not parse"); + } + } } diff --git a/rust/crates/og-dev/tests/licences/main.rs b/rust/crates/og-dev/tests/licences/main.rs index 6aadf1f5f..ae7fdcad4 100644 --- a/rust/crates/og-dev/tests/licences/main.rs +++ b/rust/crates/og-dev/tests/licences/main.rs @@ -45,7 +45,9 @@ const GOOD_TEXT: &str = "Copyright (c) the good authors\nshared permissive text\ fn vendor(t: &Temp, name: &str, manifest: &str, files: &[(&str, &str)]) { t.write( &format!("vendor/{name}/Cargo.toml"), - &format!("[package]\nname = \"{name}\"\nversion = \"0.1.0\"\nedition = \"2021\"\n{manifest}\n"), + &format!( + "[package]\nname = \"{name}\"\nversion = \"0.1.0\"\nedition = \"2021\"\n{manifest}\n" + ), ); t.write(&format!("vendor/{name}/src/lib.rs"), ""); for (rel, text) in files { @@ -57,7 +59,10 @@ fn vendor(t: &Temp, name: &str, manifest: &str, files: &[(&str, &str)]) { /// vendor/, outside the workspace folder so they are not members. fn workspace() -> Temp { let t = Temp::new(); - t.write("ws/Cargo.toml", "[workspace]\nmembers = [\"app\"]\nresolver = \"2\"\n"); + t.write( + "ws/Cargo.toml", + "[workspace]\nmembers = [\"app\"]\nresolver = \"2\"\n", + ); t.write( "ws/app/Cargo.toml", r#"[package] @@ -91,7 +96,12 @@ notext = { path = "../../vendor/notext" } t.write("ws/app/src/lib.rs", ""); vendor(&t, "good", "license = \"MIT\"", &[("LICENSE", GOOD_TEXT)]); // The same text as good: it must come once, for both. - vendor(&t, "extra", "license = \"MIT OR Apache-2.0\"", &[("LICENSE-MIT", GOOD_TEXT)]); + vendor( + &t, + "extra", + "license = \"MIT OR Apache-2.0\"", + &[("LICENSE-MIT", GOOD_TEXT)], + ); vendor( &t, "dual", @@ -111,7 +121,12 @@ notext = { path = "../../vendor/notext" } ], ); let sspl = [("LICENSE", "server side text\n")]; - vendor(&t, "pm", "license = \"SSPL-1.0\"\n[lib]\nproc-macro = true\n[dependencies]\npmdep = { path = \"../pmdep\" }", &sspl); + vendor( + &t, + "pm", + "license = \"SSPL-1.0\"\n[lib]\nproc-macro = true\n[dependencies]\npmdep = { path = \"../pmdep\" }", + &sspl, + ); vendor(&t, "pmdep", "license = \"SSPL-1.0\"", &sspl); vendor(&t, "buildonly", "license = \"SSPL-1.0\"", &sspl); vendor(&t, "devonly", "license = \"SSPL-1.0\"", &sspl); @@ -123,16 +138,29 @@ notext = { path = "../../vendor/notext" } .arg(Path::new(&t.path("ws/Cargo.toml"))) .output() .unwrap(); - assert!(lock.status.success(), "{}", String::from_utf8_lossy(&lock.stderr)); + assert!( + lock.status.success(), + "{}", + String::from_utf8_lossy(&lock.stderr) + ); t } fn notices(t: &Temp, targets: &[&str]) -> (u8, String, String) { let out_file = t.path("THIRD-PARTY-LICENSES.txt"); let _ = std::fs::remove_file(&out_file); - let mut args: Vec = vec!["notices".into(), "--manifest-path".into(), t.path("ws/Cargo.toml")]; + let mut args: Vec = vec![ + "notices".into(), + "--manifest-path".into(), + t.path("ws/Cargo.toml"), + ]; args.extend(targets.iter().map(|s| s.to_string())); - args.extend(["--package".into(), "app".into(), "--out".into(), out_file.clone()]); + args.extend([ + "--package".into(), + "app".into(), + "--out".into(), + out_file.clone(), + ]); let mut out = Vec::new(); let mut err = Vec::new(); let code = licences::cli(&args, &mut out, &mut err); @@ -165,44 +193,96 @@ fn every_linked_crate_is_credited_and_nothing_else() { ); // The shared text once, for both crates that ship it. assert_eq!(text.matches("shared permissive text").count(), 1, "{text}"); - assert!(text.contains("For extra 0.1.0: LICENSE-MIT\nFor good 0.1.0: LICENSE\n"), "{text}"); + assert!( + text.contains("For extra 0.1.0: LICENSE-MIT\nFor good 0.1.0: LICENSE\n"), + "{text}" + ); // A crate that builds C carries its library's notice from below its root. - assert!(text.contains("For csys 0.1.0: vendored/clib/COPYING\n\nthe C library's own notice"), "{text}"); + assert!( + text.contains("For csys 0.1.0: vendored/clib/COPYING\n\nthe C library's own notice"), + "{text}" + ); // No file, MIT allowed: the MIT text with the crate's authors. assert!(text.contains("For dual 0.1.0: MIT, no licence file in the crate\n\nCopyright (c) Dual Author \n\nPermission is hereby granted"), "{text}"); - for absent in ["app 0.1.0", "pm 0.1.0", "pmdep", "buildonly", "devonly", "bad 0.1.0", "notext", "server side text"] { - assert!(!text.contains(absent), "{absent} must not be credited:\n{text}"); + for absent in [ + "app 0.1.0", + "pm 0.1.0", + "pmdep", + "buildonly", + "devonly", + "bad 0.1.0", + "notext", + "server side text", + ] { + assert!( + !text.contains(absent), + "{absent} must not be credited:\n{text}" + ); } } #[test] fn a_build_without_default_features_leaves_the_optional_crate_out() { let t = workspace(); - let (code, text, err) = notices(&t, &["--target", LINUX, "--target-no-default", "aarch64-apple-darwin"]); + let (code, text, err) = notices( + &t, + &[ + "--target", + LINUX, + "--target-no-default", + "aarch64-apple-darwin", + ], + ); assert_eq!(code, 0, "{err}"); - assert!(text.contains("Targets: aarch64-apple-darwin, x86_64-unknown-linux-gnu\n"), "{text}"); - assert!(text.contains("extra 0.1.0 MIT OR Apache-2.0 (only x86_64-unknown-linux-gnu)\n"), "{text}"); + assert!( + text.contains("Targets: aarch64-apple-darwin, x86_64-unknown-linux-gnu\n"), + "{text}" + ); + assert!( + text.contains("extra 0.1.0 MIT OR Apache-2.0 (only x86_64-unknown-linux-gnu)\n"), + "{text}" + ); assert!(text.contains("good 0.1.0 MIT\n"), "{text}"); } #[test] fn a_crate_outside_the_allow_list_or_without_a_text_fails() { let t = workspace(); - let (code, text, err) = notices(&t, &["--target", LINUX, "--target", "x86_64-pc-windows-gnu"]); + let (code, text, err) = notices( + &t, + &["--target", LINUX, "--target", "x86_64-pc-windows-gnu"], + ); assert_eq!(code, 1, "{err}"); assert!(text.is_empty(), "no file on an error, got:\n{text}"); - assert!(err.contains("bad 0.1.0 is licensed \"SSPL-1.0\", which the allow list"), "{err}"); - assert!(err.contains("notext 0.1.0 ships no licence file, and \"Apache-2.0\" cannot be met with the MIT text"), "{err}"); + assert!( + err.contains("bad 0.1.0 is licensed \"SSPL-1.0\", which the allow list"), + "{err}" + ); + assert!( + err.contains( + "notext 0.1.0 ships no licence file, and \"Apache-2.0\" cannot be met with the MIT text" + ), + "{err}" + ); assert_eq!(err.lines().count(), 2, "{err}"); } #[test] fn usage_errors_exit_2() { let mut out = Vec::new(); - for args in [&[][..], &["notices"][..], &["notices", "--target"][..], &["assets"][..]] { + for args in [ + &[][..], + &["notices"][..], + &["notices", "--target"][..], + &["assets"][..], + ] { let mut err = Vec::new(); let args: Vec = args.iter().map(|s| s.to_string()).collect(); assert_eq!(licences::cli(&args, &mut out, &mut err), 2, "{args:?}"); - assert!(String::from_utf8(err).unwrap().contains("usage: og dev licences notices")); + assert!( + String::from_utf8(err) + .unwrap() + .contains("usage: og dev licences notices") + ); } } From 3b3fbd3062512a9ccbd7796b127fed99c5d80bc4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:12:42 +0100 Subject: [PATCH 179/255] build(release): ship THIRD-PARTY-LICENSES.txt with the og binaries rust/og.sh build writes it in OG_BUILDER with the new og; release.yml copies it to dist, adds it to SHA256SUMS and the attestation, and checks the upload with check-release.sh dist. Bead: lic-02 --- .github/workflows/release.yml | 23 +++++++++++++++-------- CHANGELOG.md | 1 + docs/rust.md | 7 +++++-- rust/og.sh | 23 +++++++++++++++++++++-- scripts/ci/check-release.sh | 28 +++++++++++++++++++++++++++- 5 files changed, 69 insertions(+), 13 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 936471704..f7d3e728f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,8 @@ on: required: true type: string # A dry run: a pull request that changes this file or rust/ builds the og - # binaries and their SHA256SUMS as an artifact; nothing is released. + # binaries, THIRD-PARTY-LICENSES.txt and their SHA256SUMS as an artifact; + # nothing is released. pull_request: branches: [main] paths: ['.github/workflows/release.yml', 'rust/**'] @@ -21,7 +22,9 @@ permissions: jobs: # The og binaries every release carries (docs/rust.md): both musl builds # and the macOS pair (without BAM, for a live-site host), made by - # scripts/setup.sh --og as a checkout makes them, with SHA256SUMS. + # scripts/setup.sh --og as a checkout makes them, with + # THIRD-PARTY-LICENSES.txt (the notices of every crate and C library they + # link) and SHA256SUMS over all five. # A checkout of the tag downloads them instead of building (rust/og.sh). og-binaries: runs-on: ubuntu-latest @@ -44,16 +47,17 @@ jobs: ref: ${{ inputs.tag || github.ref }} persist-credentials: false - - name: Build og for Linux and macOS (scripts/setup.sh --og builds in OG_BUILDER) + - name: Build og for Linux and macOS, and its licence notices (scripts/setup.sh --og builds in OG_BUILDER) run: | # On a tag, setup.sh --og would download the release's own binaries, # which this job is about to make: build them. rust/og.sh build rust/og.sh check --required mkdir -p dist - cp bin/.og/og-linux-x86_64 bin/.og/og-linux-aarch64 bin/.og/og-darwin-arm64 bin/.og/og-darwin-x86_64 dist/ + cp bin/.og/og-linux-x86_64 bin/.og/og-linux-aarch64 bin/.og/og-darwin-arm64 bin/.og/og-darwin-x86_64 bin/.og/THIRD-PARTY-LICENSES.txt dist/ file dist/og-* - (cd dist && sha256sum og-* > SHA256SUMS && cat SHA256SUMS) + (cd dist && sha256sum og-* THIRD-PARTY-LICENSES.txt > SHA256SUMS) + scripts/ci/check-release.sh dist dist - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -108,7 +112,7 @@ jobs: echo "" echo "## og binaries" echo "" - echo "og-linux-x86_64 and og-linux-aarch64 are this release's Rust binaries, with their SHA256SUMS and a build attestation. og-darwin-arm64 and og-darwin-x86_64 run the live site's genome map on a Mac; they leave out the tools that read BAMs. \`./scripts/setup.sh --og\` on a checkout of this tag downloads and checks all four." + echo "og-linux-x86_64 and og-linux-aarch64 are this release's Rust binaries, with their SHA256SUMS and a build attestation. THIRD-PARTY-LICENSES.txt holds the licences of the crates and C libraries (htslib, htscodecs) the binaries link. og-darwin-arm64 and og-darwin-x86_64 run the live site's genome map on a Mac; they leave out the tools that read BAMs. \`./scripts/setup.sh --og\` on a checkout of this tag downloads and checks all four." } >> "${RUNNER_TEMP}/notes.md" # Only a commit that main's CI passed is released: it must be on main, @@ -135,10 +139,12 @@ jobs: # rust/og.sh install checks a downloaded binary with # `gh attestation verify` when gh is installed. - - name: Attest the og binaries + - name: Attest the og binaries and their licence notices uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: - subject-path: 'dist/og-*' + subject-path: | + dist/og-* + dist/THIRD-PARTY-LICENSES.txt - name: Create GitHub Release uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 @@ -152,4 +158,5 @@ jobs: dist/og-linux-aarch64 dist/og-darwin-arm64 dist/og-darwin-x86_64 + dist/THIRD-PARTY-LICENSES.txt dist/SHA256SUMS diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf6c9e6d..c1af0a35f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The variant calling page names the Genome Reference Consortium as the source of the PAR intervals in `assets/par_grch38.bed`. - Every verb the 1.0.0 release still has to port now exists in the `og` binary, each in its own module, as a stub that prints `og : not ported yet` and exits 2. The new crates are og-render, og-live, og-couple and og-admix. Each port then fills one module and never edits `rust/og/src/main.rs` or a crate's module list, so the ports can land in parallel. No step calls these verbs yet, and no output changes. - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. +- Each release now ships `THIRD-PARTY-LICENSES.txt` beside the og binaries. The binaries link about seventy crates and the C libraries htslib and htscodecs statically, and their licences (MIT, BSD-3-Clause, Unicode-3.0 and others) ask that a binary carry their notices; no release carried any. `rust/og.sh build` writes the file with the new `og dev licences notices`, from `cargo metadata` for each of the four targets and the licence files each crate ships, including htslib's and htscodecs' own from the `hts-sys` crate. The build stops when a linked crate's licence is outside the allow list in `docs/rust.md`. The release adds the file to `SHA256SUMS`, which `sha256sum og-*` alone left out, and to the build attestation, and `scripts/ci/check-release.sh dist` refuses a release whose files or sums do not match. No pipeline output changes. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. diff --git a/docs/rust.md b/docs/rust.md index bfe8ead4b..fec096372 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -42,7 +42,7 @@ tests/rust/ the differential test against CPython, the image test Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (were the commands of `chip_report.py`). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py`. Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. -Every verb the 1.0.0 release still has to port is already registered, each in its own module, as a stub that prints `og : not ported yet` on stderr and exits 2: `og report render|index|charts`, `og live serve|results|family|demo`, `og view serve`, `og couple screen|genes`, `og vcf index|shift-back`, `og setup chrm-shift|revel|roh-af`, `og sv survivor-prep|consensus-check`, `og archaic compare|genomes`, `og ancestry-fine freqs|project|paint|simulate`, `og longread clair3-model`, `og map svg`, `og dev demo-sample|demo-events|params-doc|python-gate|everything-check|licences|docs-check|stub-http|steps-check` and `og dev build roh-af|acmg-lof`. A port fills its module's body and does not edit `og/src/main.rs`, the workspace member list or a crate's module list. `rust/og/tests/port_scaffold.rs` lists these verbs and fails when one has no line in `og --help` or reaches a router's "unknown" error. +Every verb the 1.0.0 release still has to port is already registered, each in its own module, as a stub that prints `og : not ported yet` on stderr and exits 2: `og report render|index|charts`, `og live serve|results|family|demo`, `og view serve`, `og couple screen|genes`, `og vcf index|shift-back`, `og setup chrm-shift|revel|roh-af`, `og sv survivor-prep|consensus-check`, `og archaic compare|genomes`, `og ancestry-fine freqs|project|paint|simulate`, `og longread clair3-model`, `og map svg`, `og dev demo-sample|demo-events|params-doc|python-gate|everything-check|docs-check|stub-http|steps-check` and `og dev build roh-af|acmg-lof`. A port fills its module's body and does not edit `og/src/main.rs`, the workspace member list or a crate's module list. `rust/og/tests/port_scaffold.rs` lists these verbs and fails when one has no line in `og --help` or reaches a router's "unknown" error. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. @@ -59,6 +59,8 @@ The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, ` - `og-linux-x86_64` and `og-linux-aarch64`, two static musl builds for the pipeline's containers. Both are kept because an Apple Silicon host runs amd64 images under emulation, where `uname -m` says `x86_64`. - `og-darwin-arm64` and `og-darwin-x86_64`, for a Mac that hosts the live site (`scripts/owngenome-live.sh`), whose genome map runs `og map` on the host. They are built with `--no-default-features`, which leaves out the cargo feature `bam`: that host never reads a BAM, and BAM reading needs htslib, which is C and would have to cross-compile for Apple. The BAM tools (`og pgx cyp2d6-*`, `og str motif`, `og cdna check|scan`, `og comphet phase`, `og bam contigs`, `og bam-fetch-probe`) say so and exit 2 there. Every Linux build has the feature, since it is on by default. +Beside them the build writes `bin/.og/THIRD-PARTY-LICENSES.txt`, which every release ships. The binaries link about seventy crates statically (MIT, Apache-2.0, BSD-3-Clause, ISC, Zlib, Unicode-3.0 and others), and `hts-sys` builds the C libraries htslib and htscodecs into them; those licences ask that a binary carry their notices. The new `og` writes the file itself inside `OG_BUILDER`: `og dev licences notices` asks `cargo metadata --filter-platform` for each of the four targets (the macOS pair without default features, as it is built) and follows the normal dependencies from `og`, so build and dev dependencies and procedural macros, which leave no code in the binary, are not listed. The file lists every crate with its version and licence, and then each distinct licence text once. The texts come from each crate's own LICENSE, COPYING, NOTICE and similar files; a crate that builds C (it sets `links`) also gives the ones in its folders, which is how htslib's `htslib/LICENSE` and htscodecs' `htslib/htscodecs/LICENSE.md` get in without a copy that could go stale. A crate is credited for every target whose dependency graph reaches it, even where the linker then drops its unused code. The build stops when a crate's licence is outside the allow list (`ALLOWED` in `rust/crates/og-dev/src/licences.rs`: 0BSD, Apache-2.0, Apache-2.0 WITH LLVM-exception, BSD-3-Clause, ISC, MIT, Unicode-3.0, Unlicense, Zlib), so a new dependency under another licence needs a decision first. A crate that ships no licence file gets the MIT text with its authors when MIT is among its licences, and stops the build otherwise. + - **From a checkout of `main` or a branch,** it builds them from `rust/` inside `OG_BUILDER`, a pinned [cargo-zigbuild](https://github.com/rust-cross/cargo-zigbuild) image (`versions.env`). Docker is the only thing it needs; no Rust on the host. The cargo cache lives in `rust/target/og-build/`. - **From a checkout of a release tag** (`vX.Y.Z`), it downloads the release's four binaries, checks them against the release's `SHA256SUMS` and, when `gh` is installed and logged in, against the build attestation (`gh attestation verify`). A release without binaries falls back to a build. @@ -113,7 +115,8 @@ To add a primitive: write it in `pycompat` with unit tests, add an operation to - **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; the four builds through `scripts/setup.sh --og` (both musl ones and the macOS pair); and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. Then the fake-docker case `reports-stale-step` runs with `bin/og` and must check its `summary.json` with `og dev schema`, and every test that carries the line `# og-leg: OG_REQUIRED` (the tests that run `og`, such as the `og sv` and `og pgx` filter tests, the step 05 tests and `tests/test_ancestry_fine_report.py`) runs with `bin/og` and `OG_REQUIRED=1`; in `unit-tests`, which has no `og`, they skip with a note. A new test that needs `og` adds that line; the job has no list to edit. Before the real run the step plants a marked test with a failing assert, which must be picked and fail. The planned-verb test must fail once one verb's arm is dropped from `og/src/main.rs`. `cargo test` includes `og dev contract --static` and `og dev report-only` on the checkout (`rust/crates/og-dev/tests/repo.rs`). It also runs `og dev helpers --static` and `--list` on the checkout (`rust/crates/og-dev/tests/helpers_repo.rs`); the `helper-binaries` job runs the same verb through `scripts/ci/check-container-helpers.sh`, with the og the `og` job builds. - **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a fresh and a stale macOS pair, a tree hash that cannot be computed, a release download with a bad checksum or attestation, without the macOS pair, with `gh` not logged in, with a network error, without binaries, or whose copy into `bin/.og` fails), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. - **Rust** (`rust.yml`, on a pull request that touches the Rust code, the shim or `versions.env`): times a cold and a cached build, runs `og` by bare name in the four images on amd64 and arm64 (QEMU), runs that check once more with a wrong-arch binary planted, which must fail, runs the fetch smoke with both binaries (arm64 under QEMU) and with pysam in `PYPGX_IMAGE`, which must all match `fetch.pysam.tsv`, and keeps the binaries as the artifact `og-binaries`. Every night it runs the differential test at 10,000,000 cases per operation with a new seed. -- **Release** (`release.yml`): a tag builds the four binaries, writes `SHA256SUMS`, attests them and attaches all five files to the release. A pull request that changes `rust/` or the workflow builds them as a dry run. +- **Release** (`release.yml`): a tag builds the four binaries and `THIRD-PARTY-LICENSES.txt`, writes `SHA256SUMS` over all five, checks them with `scripts/ci/check-release.sh dist` (each file there, listed once and matching its sum, and the htslib and htscodecs licences in the notices), attests them and attaches all six files to the release. A pull request that changes `rust/` or the workflow builds them as a dry run, with the same check. +- **`og dev licences notices`**: `cargo test -p og-dev --test licences` runs it through real `cargo metadata` on a planted workspace. The crates the binary links must all be credited, each text once; build, dev, procedural-macro, other-target and feature-off crates must not be; and a crate under SSPL-1.0, or one with no licence text that MIT cannot cover, must stop it. [Testing](testing.md) has the rest of CI. diff --git a/rust/og.sh b/rust/og.sh index 70f5af058..8cfbd6031 100755 --- a/rust/og.sh +++ b/rust/og.sh @@ -12,7 +12,10 @@ # check them against SHA256SUMS (and the build # attestation when gh is logged in); otherwise, or # when the release has none, build them -# rust/og.sh build build all four from rust/ in OG_BUILDER (versions.env) +# rust/og.sh build build all four from rust/ in OG_BUILDER (versions.env), +# and bin/.og/THIRD-PARTY-LICENSES.txt: the licences of +# every crate and C library they link, which a +# release ships beside them (og dev licences notices) # rust/og.sh check [--required] # one line, OK, NOTE or ERROR; exit 1 on ERROR: a # binary built from another rust/, or (with @@ -186,8 +189,24 @@ build() { for out in "${outs[@]}"; do [ "$(embedded "$out")" = "$hash" ] || { echo "ERROR: ${out} does not carry tree ${hash}" >&2; return 1; } done + # The notices of everything the four link, written by the new og itself in + # the builder, where cargo and the crate sources are: the Linux binary of + # the builder's own arch, with the targets and feature sets built above. + local lic=() notices="${cache}/THIRD-PARTY-LICENSES.txt" + for a in $ARCHES; do lic+=(--target "$(triple "$a")"); done + for a in $DARWIN_ARCHES; do lic+=(--target-no-default "$(darwin_triple "$a")"); done + rm -f "$notices" + # shellcheck disable=SC2016 # $(uname -m) and $@ expand in the builder's sh + "$ENGINE" run --rm --user "$(id -u):$(id -g)" \ + -e HOME=/cache/home -e CARGO_HOME=/cache/cargo -e CARGO_TARGET_DIR=/cache/target \ + -v "${ROOT}/rust:/src:ro" -v "${cache}:/cache" -w /src "$OG_BUILDER" \ + sh -c 'exec "/cache/target/$(uname -m)-unknown-linux-musl/release/og" "$@"' og \ + dev licences notices "${lic[@]}" --out /cache/THIRD-PARTY-LICENSES.txt || return + [ -s "$notices" ] || { echo "ERROR: og dev licences notices wrote no ${notices}" >&2; return 1; } put "${outs[@]}" || return 1 - echo "Built bin/.og/og-linux-{${ARCHES// /,}} and bin/.og/og-darwin-{${DARWIN_ARCHES// /,}} in $(($(date +%s) - t0)) s" + { cp "$notices" "${BIN}/.THIRD-PARTY-LICENSES.txt.tmp" && mv -f "${BIN}/.THIRD-PARTY-LICENSES.txt.tmp" "${BIN}/THIRD-PARTY-LICENSES.txt"; } \ + || { echo "ERROR: could not install ${BIN}/THIRD-PARTY-LICENSES.txt" >&2; return 1; } + echo "Built bin/.og/og-linux-{${ARCHES// /,}}, bin/.og/og-darwin-{${DARWIN_ARCHES// /,}} and bin/.og/THIRD-PARTY-LICENSES.txt in $(($(date +%s) - t0)) s" } sha256() { diff --git a/scripts/ci/check-release.sh b/scripts/ci/check-release.sh index 481573ff9..c88ca0ecc 100755 --- a/scripts/ci/check-release.sh +++ b/scripts/ci/check-release.sh @@ -27,6 +27,14 @@ # tests/test_release.sh, so a bump to X.Y.0 cannot merge without its # notes. # +# check-release.sh dist DIR +# DIR is what a release uploads: the four og binaries, +# THIRD-PARTY-LICENSES.txt (rust/og.sh build) and SHA256SUMS. Each file +# must be there, not empty, listed once in SHA256SUMS and match its +# line. The licence file must list its crates and carry the htslib and +# htscodecs licences (hts-sys's htslib/LICENSE and +# htslib/htscodecs/LICENSE.md). Prints each file with its checksum. +# # --repo DIR runs against another git work tree (the tests plant one); the # default is the checkout this script is in. set -euo pipefail @@ -158,7 +166,25 @@ case "$MODE" in fi echo "OK: declared version ${declared}, ${count} notes file(s) well formed" ;; + dist) + DIST=${1:?Usage: $0 dist DIR} + sums="${DIST}/SHA256SUMS" + [ -s "$sums" ] || die "${DIST} has no SHA256SUMS" + for f in og-linux-x86_64 og-linux-aarch64 og-darwin-arm64 og-darwin-x86_64 THIRD-PARTY-LICENSES.txt; do + [ -s "${DIST}/${f}" ] || die "${DIST} has no ${f}, or it is empty" + want=$(awk -v f="$f" '$2 == f || $2 == "*" f {print $1}' "$sums") + [ "$(printf '%s\n' "$want" | grep -c .)" -eq 1 ] || die "SHA256SUMS must list ${f} once" + if command -v sha256sum > /dev/null 2>&1; then got=$(sha256sum "${DIST}/${f}"); else got=$(shasum -a 256 "${DIST}/${f}"); fi + [ "${got%% *}" = "$want" ] || die "${f} does not match its SHA256SUMS line" + echo "${want} ${f}" + done + lic="${DIST}/THIRD-PARTY-LICENSES.txt" + grep -qE '^Crates \([1-9][0-9]*\)$' "$lic" || die "THIRD-PARTY-LICENSES.txt lists no crates" + grep -qE '^For hts-sys [^:]+: htslib/LICENSE$' "$lic" || die "THIRD-PARTY-LICENSES.txt does not carry htslib's licence (hts-sys htslib/LICENSE)" + grep -qE '^For hts-sys [^:]+: htslib/htscodecs/LICENSE\.md$' "$lic" || die "THIRD-PARTY-LICENSES.txt does not carry htscodecs' licence (hts-sys htslib/htscodecs/LICENSE.md)" + echo "OK: the og binaries and THIRD-PARTY-LICENSES.txt match SHA256SUMS, and the licences of htslib and htscodecs are in it" + ;; *) - die "usage: $0 [--repo DIR] {tag TAG [--notes-out FILE] | checks RUNS.TSV [--ignore NAME]... | declared}" + die "usage: $0 [--repo DIR] {tag TAG [--notes-out FILE] | checks RUNS.TSV [--ignore NAME]... | declared | dist DIR}" ;; esac From 233517eb71d1a0aa5f71c9e4e2db04ca32623aa4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:13:49 +0100 Subject: [PATCH 180/255] docs(pgx): say a heteroplasmic MT-RNR1 allele gets the aminoglycoside warning Step 07, the PHARMCAT module comment, the step 07 and step 20 docs and the CHANGELOG now describe the AF 0.05 floor. The pharmcat-mito e2e case expects the outside call and the drug recommendation at AF 0.40, and adds a sample at AF 0.03 that stays No Result with a below-floor line. Bead: pgx-04 Bead: pgx-32 Bead: D10 --- CHANGELOG.md | 2 ++ docs/07-pharmacogenomics.md | 4 +-- docs/20-mtoolbox.md | 2 ++ modules/local/pharmcat/main.nf | 13 +++---- scripts/07-pharmacogenomics.sh | 13 +++---- .../pharmcat-mito-input-and-log-1-mtrnr1.sh | 35 +++++++++++++------ 6 files changed, 45 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85edb5eae..d669004aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- A heteroplasmic MT-RNR1 variant now gives the aminoglycoside warning. Step 07 and the Nextflow PHARMCAT process passed a named MT-RNR1 allele to PharmCAT only at an allele fraction (AF) of 0.95 or more, so m.1555A>G or m.1494C>T at AF 0.05 to 0.95 left MT-RNR1 No Result and the warning was lost. CPIC sets no heteroplasmy level at which aminoglycosides are safe, so `og pgx mito-merge` now passes a PASS named allele from AF 0.05, the pipeline's heteroplasmy floor, and its log says `heteroplasmic` with the AF. Below AF 0.05 the allele is not passed, the log says it is below the detection floor, and the gene is not called Reference either. The default of `--min-af` moves from 0.95 to 0.05; the option stays. A named allele whose AF reads `nan` is now held back as having no allele fraction; it was passed as homoplasmic. No output file or format changes: the MT-RNR1 line of the outside calls now appears in these cases. Part of #411 (pgx-04, pgx-32). + - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/07-pharmacogenomics.md b/docs/07-pharmacogenomics.md index 49641c30a..78ed16e08 100644 --- a/docs/07-pharmacogenomics.md +++ b/docs/07-pharmacogenomics.md @@ -97,7 +97,7 @@ docker run --rm \ - **gVCF.** PharmCAT refuses a gVCF, and decides by the file name too (`.g.vcf`, `.genomic.vcf`). Yet a gVCF is the better input: a variants-only VCF leaves about half of PharmCAT's genes Unknown, because PharmCAT cannot tell a reference call from a position that was not covered. So the script reads `vcf/${SAMPLE}.g.vcf.gz` when step 3 wrote one and expands its reference blocks into `${SAMPLE}.pgx_regions.vcf.gz` (step 0 above, deleted afterwards), which PharmCAT accepts; without a gVCF it reads `${SAMPLE}.vcf.gz`, and does not check whether that file is itself a gVCF (PharmCAT then stops on it). The Nextflow pipeline does the same with DeepVariant's gVCF, or with the gVCF a samplesheet row gives in its `gvcf` column (`run-all.sh` fills it from `vcf/${SAMPLE}.g.vcf.gz`); a gVCF given in the `vcf` column stops the run before any analysis when `pharmcat` is selected. For a vendor gVCF, remove the reference blocks and rename the file: [Starting from a Vendor VCF](vcf-first.md). - **Positions longer than one base.** The expansion writes one record per base, each with a one-base REF. 51 PharmCAT 3.4.0 positions have a REF or ALT longer than one base: the insertions and deletions of DPYD, UGT1A1 (\*28, the TA-repeat insertion at chr2:233760233), TPMT, CYP3A4, CYP3A5, CFTR, CYP2C9, NUDT15, RYR1, CYP2D6 and G6PD, and two two-base substitutions in RYR1. PharmCAT's preprocessor does not take a one-base record as a call at such a position: it sets the record aside (FILTER `PCATxINDEL`, or `PCATxREF` for the substitutions) and lists the position in `missing_pgx_var.vcf`. Where PharmCAT's REF is longer than one base (the deletions, UGT1A1 \*28, the substitutions), its matcher then discards the record ("REF in VCF (C) does not match expected reference (CAT)"), so the position has no call and the alleles defined there are uncallable, even where the gVCF shows you match the reference over the whole span. At an insertion with a one-base REF the matcher still reads the set-aside record as a reference call ("PharmCAT cannot validate this position"). So step 0 runs `og pgx refblock-fill` on the expanded file. For each of these positions in PharmCAT's positions file (`/pharmcat/pharmcat_positions.vcf.bgz` in the image), it writes a 0/0 record with PharmCAT's own ID, REF and ALT (INFO flag `PGX_REFBLOCK`) in place of the one-base record, when every base of the REF span lies in reference blocks with GQ 20 or more (GQ is Phred-scaled: 20 means a 1 in 100 chance the reference call is wrong; genotype 0/0, or 0 on a haploid chrX; FILTER `PASS` or `.`) and no other record overlaps the span: no variant call, no `./.` block, no block below GQ 20. Otherwise the expanded file stays as it was there: the position stays missing, and at an insertion the matcher reads the one-base record as before. The step log names the positions it filled (`pgx_refblock_fill: N of 51 PharmCAT positions longer than one base called 0/0 from reference blocks with GQ >= 20: ...`). The GQ threshold applies to these positions only: a one-base position takes any 0/0 block, as before. Where the expanded file has a record at such a position that is not a one-base reference call, that record stays and no 0/0 record is added beside it; the log says so. The Nextflow module does the same. -- **MT-RNR1 from the chrM calls.** PharmCAT 3.4.0 has no MT-RNR1 definition for its matcher: it reads the gene only as an outside call, one allele (chrM is not a pair), so from a VCF alone MT-RNR1 is No Result on every sample. When [step 20](20-mtoolbox.md) wrote its Mutect2 chrM calls (`mito/${SAMPLE}_chrM_filtered.vcf.gz`), the script runs `og pgx mito-merge` (step 3 above). It reads PharmCAT's own MT-RNR1 alleles from the jar (`Reference` and names such as `m.1555A>G`) and adds one MT-RNR1 line to a copy of step 36's outside calls (`${SAMPLE}.pharmcat_outside_calls.tsv`, deleted afterwards): a named single-base allele with FILTER `PASS` and an allele fraction (AF) of 0.95 or more, the level the pipeline calls homoplasmic; the one PharmCAT ranks highest (increased risk, then uncertain, then normal) when there are several, with the others named in the step log; or `Reference` when no named allele is called. Mutect2 writes variant records only, so a position without reads has no record either: `Reference` needs read evidence, 20 reads or more (`samtools depth`, MAPQ and base quality 20) at every position where PharmCAT names an MT-RNR1 allele, from the sample's BAM (`aligned/${SAMPLE}_sorted.bam`, the one step 20 read). Without the BAM, or with a position below 20 reads, no `Reference` is passed and the log names the positions; a homoplasmic named allele needs no depth, its record is the evidence. Mutect2 writes the genotype 0/1 even at AF 1.0, so the AF decides, not the genotype. No line is added, and the gene stays No Result with the reason in the step log, when a named allele is heteroplasmic (AF below 0.95), filtered, or has no AF; when a homoplasmic `PASS` record at a position where PharmCAT names an allele (m.960 and m.961 included, named there only as deletions) has an ALT PharmCAT does not name; when a `PASS` insertion or deletion overlaps a named position or lies within 10 bases of m.960 or m.961, where PharmCAT names deletions in a C run that the script does not map; and when the file has no `PASS` chrM record. Without step 20's file the step says so and MT-RNR1 stays No Result: run step 20 before this step. The PHARMCAT module does the same with MITO_VARIANTS' calls. +- **MT-RNR1 from the chrM calls.** PharmCAT 3.4.0 has no MT-RNR1 definition for its matcher: it reads the gene only as an outside call, one allele (chrM is not a pair), so from a VCF alone MT-RNR1 is No Result on every sample. When [step 20](20-mtoolbox.md) wrote its Mutect2 chrM calls (`mito/${SAMPLE}_chrM_filtered.vcf.gz`), the script runs `og pgx mito-merge` (step 3 above). It reads PharmCAT's own MT-RNR1 alleles from the jar (`Reference` and names such as `m.1555A>G`) and adds one MT-RNR1 line to a copy of step 36's outside calls (`${SAMPLE}.pharmcat_outside_calls.tsv`, deleted afterwards): a named single-base allele with FILTER `PASS` and an allele fraction (AF) of 0.05 or more, the pipeline's heteroplasmy floor; the one PharmCAT ranks highest (increased risk, then uncertain, then normal) when there are several, with the others named in the step log; or `Reference` when no named allele is called. A heteroplasmic allele (AF 0.05 to 0.95) is passed like a homoplasmic one: CPIC's MT-RNR1 guideline sets no heteroplasmy level at which aminoglycosides become safe, and asks for the homoplasmic guidance whenever a relevant variant is detected. So a heteroplasmic m.1555A>G or m.1494C>T gives PharmCAT's aminoglycoside recommendation, and the step log says `heteroplasmic` with the AF (the [step 20](20-mtoolbox.md) table and the report's chrM section give the level too). Mutect2 writes variant records only, so a position without reads has no record either: `Reference` needs read evidence, 20 reads or more (`samtools depth`, MAPQ and base quality 20) at every position where PharmCAT names an MT-RNR1 allele, from the sample's BAM (`aligned/${SAMPLE}_sorted.bam`, the one step 20 read). Without the BAM, or with a position below 20 reads, no `Reference` is passed and the log names the positions; a named allele needs no depth, its record is the evidence. Mutect2 writes the genotype 0/1 even at AF 1.0, so the AF decides, not the genotype. No line is added, and the gene stays No Result with the reason in the step log, when a named allele is below the detection floor (AF below 0.05; the log says so, and the gene is not called Reference either), filtered, or has no AF (or an AF that reads `nan`); when a `PASS` record at AF 0.05 or more at a position where PharmCAT names an allele (m.960 and m.961 included, named there only as deletions) has an ALT PharmCAT does not name; when a `PASS` insertion or deletion overlaps a named position or lies within 10 bases of m.960 or m.961, where PharmCAT names deletions in a C run that the script does not map; and when the file has no `PASS` chrM record. Without step 20's file the step says so and MT-RNR1 stays No Result: run step 20 before this step. The PHARMCAT module does the same with MITO_VARIANTS' calls. - **PharmCAT's own log file.** The jar's built-in `logback.xml` writes `pharmcat.log` into the working directory. In the image that is `/pharmcat`, which the calling user cannot write, so every run printed a logback status dump with a `pharmcat.log (Permission denied)` stack trace (the calls were not affected). The script runs the jar with `-w /tmp`, a writable directory that goes away with the container. A Nextflow task runs in its own writable work directory, so the module needs nothing: the log lands there. - **A backslash in a `##` header line.** PharmCAT up to 3.4.0 bundles vcf-parser 0.3.1, which stops with "Error parsing metadata: character to be escaped is missing" on one. The line is valid VCF; bcftools writes it for a soft filter with a quoted string (`bcftools filter -s LowDP -e 'FORMAT/DP<10 && GT!="0/0"'`). The script and the Nextflow module rewrite the header of PharmCAT's own copy (`${SAMPLE}.pharmcat_input.vcf`, deleted afterwards by the script): on `##` lines `\"` becomes `'` and any other `\` becomes `/`. PharmCAT's calls are the same with and without the rewrite. A newer PharmCAT is no fix yet: 3.4.0 still bundles vcf-parser 0.3.1 and fails the same way. @@ -120,7 +120,7 @@ docker run --rm \ ## Limitations - **HLA-A, HLA-B and CYP2D6** are not called from a VCF: PharmCAT 3.4.0 reports them with no result (`callSource` `NONE`). [Step 36](36-pgx-consensus.md) gives PharmCAT T1K's HLA types (step 8) and a CYP2D6 call only when pypgx (step 32) and Cyrius (step 21, opt-in) agree on depth that passed its check; PharmCAT reads them with `-po` (`callSource` `OUTSIDE`). Run step 36 before this step, or run this step again after it. A disagreement, or one caller alone, leaves CYP2D6 without a result, on purpose. -- **MT-RNR1** is an outside call from step 20's chrM calls (see above): PharmCAT 3.4.0 does not call it from a VCF. A heteroplasmic named allele, such as m.1555A>G at AF 0.4, is not passed, so the gene stays No Result and the step log names the allele and its AF: read it there. +- **MT-RNR1** is an outside call from step 20's chrM calls (see above): PharmCAT 3.4.0 does not call it from a VCF. A heteroplasmic named allele, such as m.1555A>G at AF 0.4, is passed and gets the aminoglycoside recommendation; the PharmCAT report does not show its AF, the step log does. A named allele below AF 0.05 is not passed: the gene stays No Result and the step log names the allele and its AF as below the detection floor. At that level the call cannot be told from noise or NuMT reads, so it is not called absent either. - **CYP4F2 can come out No Result** when unphased heterozygous calls elsewhere in the gene, a weak one among them, leave no diplotype that matches. CPIC's warfarin guidance needs only the \*3 position (rs2108622). Step 27 then reads the genotype PharmCAT itself lists at that position in `report.json` (the gene's `variants`) and prints it in a section headed "Single-Position Fallback (not a PharmCAT call)", with the CPIC warfarin rule when you carry the variant. It does the same for VKORC1 at rs9923231 (-1639G>A). PharmCAT's own result for the gene stays No Result in the phenotypes table. A position with no genotype is reported as not callable. The report carries no GQ or read depth for the position: check it in your VCF before acting on it. - PharmCAT may disagree with lab reports on complex haplotypes (e.g., NAT2). Discrepancies can arise from different genome builds (hg19 vs hg38), different star allele definitions, or different variant calling pipelines. When a discrepancy matters clinically, compare both sets of raw variant calls and consult the PharmVar database for the current allele definitions — do not blindly trust either source. - PharmCAT output structure changes across releases. If you upgrade PharmCAT, re-test step 27 (`27-cpic-lookup.sh`) because it parses the JSON output directly. diff --git a/docs/20-mtoolbox.md b/docs/20-mtoolbox.md index 90951b026..4105c002f 100644 --- a/docs/20-mtoolbox.md +++ b/docs/20-mtoolbox.md @@ -119,6 +119,8 @@ NuMTs are copies of mitochondrial DNA in the nuclear genome. Their reads can map | 0.03-0.10 | Low-level heteroplasmy — may be age-related somatic | | <0.03 | Near detection limit | +Step 07 reads these calls for the aminoglycoside risk gene MT-RNR1: a PASS allele PharmCAT names (m.1555A>G, m.1494C>T and others) at AF 0.05 or more is passed to PharmCAT, heteroplasmic or not, because CPIC sets no heteroplasmy level at which aminoglycosides are safe ([Pharmacogenomics](07-pharmacogenomics.md#input-the-preprocessor-or-pharmcat-refuses)). + Count only PASS calls: a call marked `possible_numt` or with another filter is not evidence of heteroplasmy. Calls near the two ends of the control region (about chrM:1-500 and 16,000-16,569) are less reliable: the reference is circular, so reads that span its end align poorly at both edges. Length variants in the D-loop poly-C tracts (m.303-315 and m.16184-16193), one C more or less, are found in most people and often read as a mix of lengths. The reports list them as "length heteroplasmy in the D-loop poly-C tracts" and leave them out of the heteroplasmic count. A call counts as in a tract when the bases it changes (an indel: the bases after its anchor base) overlap one. diff --git a/modules/local/pharmcat/main.nf b/modules/local/pharmcat/main.nf index d2c07c4f2..27e78dc8d 100644 --- a/modules/local/pharmcat/main.nf +++ b/modules/local/pharmcat/main.nf @@ -27,12 +27,13 @@ PharmCAT 3.4.0 reads MT-RNR1 only as an outside call: its matcher has no MT-RNR1 definition, so the gene is No Result from any VCF. When MITO_VARIANTS called the sample's chrM, og pgx mito-merge adds - an MT-RNR1 line to a copy of the outside calls: a homoplasmic (AF >= - 0.95) PASS allele PharmCAT names, or Reference when there is none and - the BAM has 20 reads or more (samtools depth, MAPQ and base quality 20) - at every position PharmCAT names an allele at; no line, with the reason - in the task log, for a heteroplasmic or filtered named allele or a - position without that depth. + an MT-RNR1 line to a copy of the outside calls: a PASS allele PharmCAT + names at AF 0.05 or more, heteroplasmic included (CPIC sets no safe + heteroplasmy level; the log gives the AF), or Reference when there is + none and the BAM has 20 reads or more (samtools depth, MAPQ and base + quality 20) at every position PharmCAT names an allele at; no line, with + the reason in the task log, for a named allele below AF 0.05 or + filtered, or a position without that depth. The jar's built-in logback.xml writes pharmcat.log into the working directory. A task runs in its own writable work directory, so the log diff --git a/scripts/07-pharmacogenomics.sh b/scripts/07-pharmacogenomics.sh index 6aa3956f4..65f2f37f7 100755 --- a/scripts/07-pharmacogenomics.sh +++ b/scripts/07-pharmacogenomics.sh @@ -105,12 +105,13 @@ run_in \ # PharmCAT 3.4.0 reads MT-RNR1 only as an outside call too: its matcher has no # MT-RNR1 definition, so the gene is No Result from any VCF. When step 20 # wrote its chrM calls, og pgx mito-merge adds an MT-RNR1 line to a copy -# of step 36's calls: a homoplasmic (AF >= 0.95) PASS allele PharmCAT names, -# or Reference when there is none and the BAM has 20 reads or more (samtools -# depth, MAPQ and base quality 20) at every position PharmCAT names an allele -# at; no line, with the reason in the log, for a heteroplasmic or filtered -# named allele or a position without that depth (the PHARMCAT module does the -# same). +# of step 36's calls: a PASS allele PharmCAT names at AF 0.05 or more, +# heteroplasmic included (CPIC sets no safe heteroplasmy level; the log gives +# the AF), or Reference when there is none and the BAM has 20 reads or more +# (samtools depth, MAPQ and base quality 20) at every position PharmCAT names +# an allele at; no line, with the reason in the log, for a named allele below +# AF 0.05 or filtered, or a position without that depth (the PHARMCAT module +# does the same). OUTSIDE="${GENOME_DIR}/${SAMPLE}/pgx_consensus/${SAMPLE}_outside_calls.tsv" MITO="${GENOME_DIR}/${SAMPLE}/mito/${SAMPLE}_chrM_filtered.vcf.gz" MERGED="${OUTPUT_DIR}/${SAMPLE}.pharmcat_outside_calls.tsv" diff --git a/tests/e2e/pharmcat-mito-input-and-log-1-mtrnr1.sh b/tests/e2e/pharmcat-mito-input-and-log-1-mtrnr1.sh index 2e1dc16b6..5be2f817b 100755 --- a/tests/e2e/pharmcat-mito-input-and-log-1-mtrnr1.sh +++ b/tests/e2e/pharmcat-mito-input-and-log-1-mtrnr1.sh @@ -2,8 +2,8 @@ # PharmCAT's MT-RNR1 call from step 20's chrM calls, and no logback noise. # # PharmCAT 3.4.0 reads MT-RNR1 only as an outside call. Step 07 runs -# bin/pgx_mito_merge.py on mito/_chrM_filtered.vcf.gz when it exists -# and adds the MT-RNR1 line to the outside calls. Four samples, each HG002's +# og pgx mito-merge on mito/_chrM_filtered.vcf.gz when it exists +# and adds the MT-RNR1 line to the outside calls. Five samples, each HG002's # variants-only VCF (case 21) with an invented Mutect2-style chrM call set, # in a GENOME_DIR of their own (the fixture reference hard-linked into it), # so no later case sees them: @@ -12,8 +12,12 @@ # MTREF PASS chrM records, none at an MT-RNR1 allele, and HG002's BAM # (case 20; about 500x on chrM) to show the positions are covered: # MT-RNR1 Reference; -# MTHET m.1555A>G at AF 0.40: nothing passed, the log says heteroplasmic, -# MT-RNR1 stays No Result; +# MTHET m.1555A>G at AF 0.40: passed all the same (CPIC sets no safe +# heteroplasmy level), the log says heteroplasmic with the AF, and +# PharmCAT gives the aminoglycoside recommendation (before issue +# #411's fix it was held back and MT-RNR1 stayed No Result); +# MTLOW m.1555A>G at AF 0.03, below the detection floor of 0.05: nothing +# passed, the log says below the floor, MT-RNR1 stays No Result; # MTNONE no chrM call set (the control, the state before the fix for every # sample): the log says so and MT-RNR1 stays No Result. # The jar ran in /pharmcat, which the calling user cannot write, and logback @@ -99,10 +103,12 @@ for e in "" .bai; do done sample MTHET "${BACKGROUND} $(row 1555 A G PASS 0.400)" +sample MTLOW "${BACKGROUND} +$(row 1555 A G PASS 0.030)" sample MTNONE declare -A LOGS=() -for p in MTHOM MTREF MTHET MTNONE; do +for p in MTHOM MTREF MTHET MTLOW MTNONE; do GENOME_DIR="$G" run_step 07-pharmacogenomics.sh "$p" check_step_exit "07-pharmacogenomics.sh (${p})" LOGS[$p]=$(cat "$STEP_LOG") @@ -132,11 +138,20 @@ check_eq "MTREF: PharmCAT reports MT-RNR1 Reference as an outside call" "$(head R=$(mt "${G}/MTHET/vcf/MTHET.report.json") echo "MTHET MT-RNR1: ${R}" -check "MTHET: the merge holds m.1555A>G back as heteroplasmic" \ - has '^pgx_mito_merge: held back: m\.1555A>G \(chrM:1555 A>G AF 0\.400\) is heteroplasmic' "${LOGS[MTHET]}" -check "MTHET: and says MT-RNR1 stays No Result" has '^pgx_mito_merge: no MT-RNR1 call passed to PharmCAT: it stays No Result' "${LOGS[MTHET]}" -check "MTHET: PharmCAT got no outside call" has '^No outside calls ' "${LOGS[MTHET]}" -check "MTHET: PharmCAT has no MT-RNR1 call" has '^NONE' "$(head -n 1 <<< "$R")" +check "MTHET: the merge passed m.1555A>G, heteroplasmic, with its AF" \ + has '^pgx_mito_merge: MT-RNR1 m\.1555A>G passed to PharmCAT as an outside call \(heteroplasmic, chrM:1555 A>G AF 0\.400; .*homoplasmic guidance applies$' "${LOGS[MTHET]}" +check "MTHET: PharmCAT got the outside calls" has '^Outside calls \(step 36, and MT-RNR1 from step 20\): ' "${LOGS[MTHET]}" +check_eq "MTHET: PharmCAT reports MT-RNR1 m.1555A>G as an outside call" "$(head -n 1 <<< "$R")" $'OUTSIDE\tm.1555A>G' +check_ge "MTHET: drugs with a recommendation for MT-RNR1 m.1555A>G" "$(grep -c '^drug' <<< "$R" || true)" 1 +check "MTHET: amikacin is one of them" has $'^drug\tamikacin\t' "$R" + +R=$(mt "${G}/MTLOW/vcf/MTLOW.report.json") +echo "MTLOW MT-RNR1: ${R}" +check "MTLOW: the merge holds m.1555A>G back as below the detection floor" \ + has '^pgx_mito_merge: held back: m\.1555A>G \(chrM:1555 A>G AF 0\.030\) is below the detection floor \(AF below 0\.05\)' "${LOGS[MTLOW]}" +check "MTLOW: and says MT-RNR1 stays No Result" has '^pgx_mito_merge: no MT-RNR1 call passed to PharmCAT: it stays No Result' "${LOGS[MTLOW]}" +check "MTLOW: PharmCAT got no outside call" has '^No outside calls ' "${LOGS[MTLOW]}" +check "MTLOW: PharmCAT has no MT-RNR1 call" has '^NONE' "$(head -n 1 <<< "$R")" R=$(mt "${G}/MTNONE/vcf/MTNONE.report.json") echo "MTNONE MT-RNR1 (control): ${R}" From 0ab9a59404ee58b505b0d89650ff97d66ed45571 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:16:57 +0100 Subject: [PATCH 181/255] test(live): the run_eta and run_headroom checks as cargo tests with planted faults Every check of tests/test_run_eta.py and tests/test_run_headroom.py now runs on og-runlog in og-live/tests/ops_eta.rs and ops_headroom.rs, and each of their source mutants is a named fault of the fault-inject feature whose planted_* test must fail. Bead: D1 --- rust/Cargo.lock | 1 + rust/crates/og-live/Cargo.toml | 3 + rust/crates/og-live/tests/ops_eta.rs | 692 ++++++++++++++++++++++ rust/crates/og-live/tests/ops_headroom.rs | 530 +++++++++++++++++ rust/crates/og-runlog/src/eta.rs | 6 + 5 files changed, 1232 insertions(+) create mode 100644 rust/crates/og-live/tests/ops_eta.rs create mode 100644 rust/crates/og-live/tests/ops_headroom.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 4b446571b..0bb469b44 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -753,6 +753,7 @@ dependencies = [ name = "og-live" version = "0.1.0" dependencies = [ + "indexmap", "jiff", "og-nf", "og-runlog", diff --git a/rust/crates/og-live/Cargo.toml b/rust/crates/og-live/Cargo.toml index 85508abff..e23bd6872 100644 --- a/rust/crates/og-live/Cargo.toml +++ b/rust/crates/og-live/Cargo.toml @@ -16,5 +16,8 @@ og-nf.workspace = true og-runlog.workspace = true jiff.workspace = true +[dev-dependencies] +indexmap.workspace = true + [lints] workspace = true diff --git a/rust/crates/og-live/tests/ops_eta.rs b/rust/crates/og-live/tests/ops_eta.rs new file mode 100644 index 000000000..14899fbac --- /dev/null +++ b/rust/crates/og-live/tests/ops_eta.rs @@ -0,0 +1,692 @@ +//! og-runlog's step history and hours left, as the live site's ops card +//! reads them: the synthetic traces of `tests/fixtures/run_eta`, synthetic +//! histories and synthetic run states; `nextflow.config`'s trace columns; +//! and the old Python command lines that must refuse to run. +//! +//! With `--features fault-inject` each `planted_*` test turns on one planted +//! mistake and runs the check that must catch it: every one of them must +//! fail (Guard's rust job requires it). A check that cannot fail is no check. + +use indexmap::{IndexMap, IndexSet}; +use jiff::tz::TimeZone; +use og_runlog::eta::{ + self, EtaArgs, HISTORY_COLUMNS, HistoryRow, Range, Source, TraceTask, box_name, history_path, + history_rows, ingest, read_trace, remaining, step_ranges, +}; +use og_runlog::state::{RunState, State, StepOut, StepState}; +use og_runlog::steps::{Step, process_steps, read_steps}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicUsize, Ordering}; + +const H: f64 = 3600.0; +/// 2026-01-02T10:00:00Z. +const NOW: f64 = 1_767_348_000.0; +const UTC: TimeZone = TimeZone::UTC; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn steps() -> &'static [Step] { + static S: OnceLock> = OnceLock::new(); + S.get_or_init(|| read_steps(&repo().join("assets/steps.tsv")).expect("assets/steps.tsv reads")) +} + +/// A new empty folder of this test run. +fn tmp() -> PathBuf { + static N: AtomicUsize = AtomicUsize::new(0); + let d = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!( + "ops-eta-{}-{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// A copy written now (a new modification time, as Python's shutil.copy). +fn copy(from: &Path, to: &Path) { + std::fs::write(to, std::fs::read(from).unwrap()).unwrap(); +} + +/// A fresh outdir whose `pipeline_info` holds the named fixture traces. +fn outdir(traces: &[(&str, &str)]) -> PathBuf { + let d = tmp(); + std::fs::create_dir_all(d.join("pipeline_info")).unwrap(); + for (name, fx) in traces { + copy( + &repo().join("tests/fixtures/run_eta").join(fx), + &d.join("pipeline_info").join(name), + ); + } + d +} + +fn ingest_as(d: &Path, box_: &str) -> Vec { + ingest(d, &box_name(Some(box_)), steps(), 16, &UTC).unwrap() +} + +/// `{step: sorted spans}` of history rows. +fn spans(rows: &[HistoryRow]) -> BTreeMap> { + let mut out: BTreeMap> = BTreeMap::new(); + for r in rows { + if let Ok(v) = r[10].parse::() { + let e = out.entry(r[3].clone()).or_default(); + if !e.contains(&v) { + e.push(v); + } + } + } + for v in out.values_mut() { + v.sort_by(f64::total_cmp); + } + out +} + +const OLD: &str = "trace_2026-01-02_09-59-00.txt"; + +// --- the history --------------------------------------------------------------------------- + +#[test] +fn an_old_and_a_new_trace_give_the_same_spans() { + let old = ingest_as(&outdir(&[(OLD, "trace_old.txt")]), "t"); + let new = ingest_as(&outdir(&[(OLD, "trace_new.txt")]), "t"); + let want: BTreeMap> = [ + ("01b", vec![1200.0]), + ("02", vec![120.0, 4.0 * H]), + ("03", vec![11.0 * H + 10.0]), + ("04", vec![1800.0]), + ] + .into_iter() + .map(|(k, v)| (k.to_owned(), v)) + .collect(); + assert_eq!(spans(&old), want); + assert_eq!(spans(&new), want); +} + +#[test] +fn human_and_raw_units_read_the_same() { + use og_nf::units::{duration, memory, time}; + let raw_18 = ((18.3 * 1024f64.powi(3)) as i128).to_string(); + assert_eq!(duration(Some("1h 2m 3s")), Some(3723.0)); + assert_eq!(duration(Some("3723000")), Some(3723.0)); + assert_eq!(duration(Some("850ms")), Some(0.85)); + assert_eq!(duration(Some("850")), Some(0.85)); + assert_eq!(memory(Some("18.3 GB")), memory(Some(&raw_18))); + assert_eq!(memory(Some("40 MB")), memory(Some(&(40 << 20).to_string()))); + assert_eq!(time(Some("2026-01-02 10:00:00.000"), &UTC), Some(NOW)); + assert_eq!(time(Some("1767348000000"), &UTC), Some(NOW)); + // And as the history reads a trace: a size and its bytes are one value. + let d = tmp(); + let head = "task_id\thash\tname\tstatus\texit\tmemory\n"; + std::fs::write(d.join("a.txt"), format!("{head}1\tab/1\tFASTP (s)\tCOMPLETED\t0\t18.3 GB\n")).unwrap(); + std::fs::write(d.join("b.txt"), format!("{head}1\tab/1\tFASTP (s)\tCOMPLETED\t0\t{raw_18}\n")).unwrap(); + let (a, b) = (read_trace(&d.join("a.txt"), &UTC), read_trace(&d.join("b.txt"), &UTC)); + assert!(a[0].memory_bytes.is_some()); + assert_eq!(a[0].memory_bytes, b[0].memory_bytes); +} + +#[test] +fn cached_failed_and_unknown_process_tasks_never_enter_the_history() { + let rows = ingest_as(&outdir(&[(OLD, "trace_new.txt")]), "t"); + let mut hashes: Vec<&str> = rows.iter().map(|r| r[1].as_str()).collect(); + hashes.sort_unstable(); + assert_eq!(hashes, ["aa000001", "aa000002", "aa000003", "aa000006", "aa000007"]); +} + +#[test] +fn ingesting_twice_or_a_copied_trace_adds_nothing() { + let d = outdir(&[(OLD, "trace_new.txt")]); + ingest_as(&d, "t"); + let path = history_path(&d, "t"); + let first = std::fs::read_to_string(&path).unwrap(); + ingest_as(&d, "t"); + copy( + &repo().join("tests/fixtures/run_eta/trace_new.txt"), + &d.join("pipeline_info/trace_2026-01-03_08-00-00.txt"), + ); + ingest_as(&d, "t"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), first); +} + +#[test] +fn two_boxes_write_two_files() { + let d = outdir(&[(OLD, "trace_new.txt")]); + ingest_as(&d, "t"); + ingest_as(&d, "box-b"); + let mut files: Vec = std::fs::read_dir(d.join("pipeline_info/step_history")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|f| f.ends_with(".tsv")) + .collect(); + files.sort(); + assert_eq!(files, ["box-b.tsv", "t.tsv"]); +} + +#[test] +fn the_history_holds_no_sample_id_and_no_slash() { + let d = outdir(&[(OLD, "trace_new.txt")]); + ingest_as(&d, "t"); + let body = std::fs::read_to_string(history_path(&d, "t")).unwrap(); + let head: Vec<&str> = body.lines().next().unwrap().split('\t').collect(); + assert!( + !body.contains("demo1") && !body.contains('/') && head == HISTORY_COLUMNS, + "{body}" + ); +} + +#[test] +fn the_box_name_keeps_only_letters_digits_underscore_and_dash() { + assert_eq!(box_name(Some("my box.local")), "my_box_local"); +} + +// --- the ranges --------------------------------------------------------------------------- + +/// History rows of one step: one run per span in hours. +fn hist(step: &str, runs: &[f64]) -> Vec { + runs.iter() + .enumerate() + .map(|(i, hours)| { + [ + format!("trace_{i}.txt"), + format!("h{i}"), + "1".into(), + step.into(), + "X".into(), + "no".into(), + "8".into(), + String::new(), + String::new(), + String::new(), + format!("{:?}", hours * H), + format!("2026-01-{:02}T00:00:00Z", i + 1), + "16".into(), + ] + }) + .collect() +} + +fn ranges(history: &[HistoryRow], cores: usize) -> IndexMap { + step_ranges(history, steps(), cores, &IndexSet::new()) +} + +fn step(id: &str) -> &'static Step { + steps().iter().find(|s| s.step == id).expect("a catalogue step") +} + +#[test] +fn three_runs_give_min_max_with_n_3() { + let r = &ranges(&hist("02", &[4.0, 3.0, 5.0]), 16)["02"]; + assert_eq!( + (r.low_s, r.high_s, r.n, r.source), + (Some(3.0 * H), Some(5.0 * H), 3, Source::Box) + ); +} + +#[test] +fn only_the_last_5_runs_count() { + let r = &ranges(&hist("02", &[9.0, 3.0, 4.0, 5.0, 6.0, 7.0]), 16)["02"]; + assert_eq!((r.low_s, r.high_s, r.n), (Some(3.0 * H), Some(7.0 * H), 5)); +} + +fn docs_hours(id: &str) -> (f64, f64) { + let s = step(id); + (s.hours.unwrap() * H, s.hours_low.parse::().unwrap() * H) +} + +#[test] +fn an_8_core_box_takes_the_measured_low_end_and_doubles_the_high_end() { + let (dv, low) = docs_hours("03"); + let r = &ranges(&[], 8)["03"]; + assert!( + r.source == Source::Docs + && r.low_s == Some(low) + && r.high_s == Some(2.0 * dv) + && r.label.contains("16 threads"), + "{r:?}" + ); +} + +#[test] +fn a_32_core_box_is_not_scaled_below_the_docs_hours() { + let (dv, low) = docs_hours("03"); + let r = &ranges(&[], 32)["03"]; + assert!(r.low_s == Some(low) && r.high_s == Some(dv), "{r:?}"); +} + +#[test] +fn on_16_and_32_cores_every_measured_step_has_low_below_high() { + let measured: Vec<&str> = steps() + .iter() + .filter(|s| s.hours_from == "measured") + .map(|s| s.step.as_str()) + .collect(); + assert!(measured.len() >= 25, "{measured:?}"); + for cores in [16, 32] { + let r = ranges(&[], cores); + let one: Vec<&&str> = measured + .iter() + .filter(|k| { + let x = &r[**k]; + !(x.source == Source::Docs && matches!((x.low_s, x.high_s), (Some(l), Some(h)) if l < h)) + }) + .collect(); + assert!(one.is_empty(), "{cores} cores: {one:?}"); + } +} + +/// A run state with these steps: (step, state, started_utc); upstream from +/// the catalogue. +fn state(rows: &[(&str, StepState, Option<&str>)], run: RunState) -> State<'static> { + State { + sample: "s1".into(), + generated: NOW, + run, + started: None, + finished_utc: None, + declared_sex: None, + status_file: true, + last_heard: None, + steps: rows + .iter() + .map(|(k, st, t)| StepOut { + step: step(k), + state: *st, + reason: None, + note: None, + earlier: None, + started_utc: t.map(str::to_owned), + ended_utc: None, + processes: Vec::new(), + }) + .collect(), + } +} + +use StepState::{Done, Running, Waiting}; + +#[test] +fn a_step_the_table_only_estimates_gets_no_number() { + let est = ranges(&[], 16); + let unk = remaining( + &state(&[("03", Done, None), ("14", Waiting, None)], RunState::Running), + &est, + NOW, + 16, + ); + assert!( + est["14"].source == Source::None + && est["04b"].source == Source::None + && est["14"].low_s.is_none() + && unk.unknown_steps == ["14"] + && unk.low_s == 0.0, + "{:?} {:?} {unk:?}", + est["14"], + est["04b"] + ); +} + +fn task(hash: &str, process: &str, status: &str, exit: &str, start: f64, end: f64, attempt: &str) -> TraceTask { + TraceTask { + trace_file: "trace_x.txt".into(), + hash: hash.into(), + attempt: attempt.into(), + process: Some(process.into()), + tag: Some("s1".into()), + status: status.into(), + exit: exit.into(), + cpus: Some(1), + memory_bytes: None, + peak_rss_bytes: None, + realtime_s: Some(end - start), + start: Some(start), + complete: Some(end), + } +} + +#[test] +fn a_step_with_a_task_that_failed_or_was_stopped_gives_no_span() { + // Step 07: the preprocessor ended, then PharmCAT was stopped. Step 03: a + // failed first attempt that a second one replaced still counts. + let rows = history_rows( + &[ + task("p1", "PHARMCAT_PREPROCESS", "COMPLETED", "0", 0.0, 40.0, "1"), + task("p2", "PHARMCAT", "ABORTED", "-", 41.0, 50.0, "1"), + task("d1", "DEEPVARIANT", "FAILED", "137", 0.0, 100.0, "1"), + task("d2", "DEEPVARIANT", "COMPLETED", "0", 110.0, 300.0, "2"), + ], + &process_steps(steps()), + 16, + ); + let got: BTreeMap<(String, String), Option> = rows + .iter() + .map(|r| ((r[3].clone(), r[4].clone()), r[10].parse::().ok())) + .collect(); + let want: BTreeMap<(String, String), Option> = [ + (("07".to_owned(), "PHARMCAT_PREPROCESS".to_owned()), None), + (("03".to_owned(), "DEEPVARIANT".to_owned()), Some(300.0)), + ] + .into_iter() + .collect(); + assert_eq!(got, want); +} + +// --- the hours left ----------------------------------------------------------------------- + +fn range(low: Option, high: Option, cpus: Option) -> Range { + Range { + low_s: low, + high_s: high, + n: 1, + source: Source::Box, + label: String::new(), + cpus, + } +} + +fn fixed_ranges() -> IndexMap { + [ + ("02", range(Some(4.0 * H), Some(5.0 * H), Some(8))), + ("03", range(Some(10.0 * H), Some(12.0 * H), Some(8))), + ("13", range(Some(4.0 * H), Some(5.0 * H), Some(8))), + ("23", range(Some(0.1 * H), Some(0.2 * H), Some(1))), + ("04", range(Some(0.5 * H), Some(H), Some(8))), + ("33", range(None, None, None)), + ] + .into_iter() + .map(|(k, v)| (k.to_owned(), v)) + .collect() +} + +fn close(a: Option, b: f64) -> bool { + a.is_some_and(|a| (a - b).abs() < 1e-6) +} + +fn chain_state() -> State<'static> { + state( + &[ + ("02", Done, None), + ("03", Waiting, None), + ("13", Waiting, None), + ("23", Waiting, None), + ("04", Waiting, None), + ], + RunState::Running, + ) +} + +#[test] +fn the_hours_left_are_the_longest_chain() { + // 03 -> 13 -> 23 is the longest chain: low 10 + 4 + 0.1, high 12 + 5 + 0.2. + let big = remaining(&chain_state(), &fixed_ranges(), NOW, 64); + assert!(close(Some(big.low_s), 14.1 * H) && close(big.high_s, 17.2 * H), "{big:?}"); +} + +#[test] +fn the_cpu_work_bound_when_it_is_larger() { + // CPU work: 12*8 + 5*8 + 0.2*1 + 1*8 = 144.2 CPU-hours; over 4 cores 36.05 h. + let small = remaining(&chain_state(), &fixed_ranges(), NOW, 4); + assert!(close(Some(small.low_s), 14.1 * H) && close(small.high_s, 36.05 * H), "{small:?}"); +} + +#[test] +fn a_running_step_past_its_high_end_makes_the_high_end_unknown_and_says_so() { + let late = remaining( + &state( + &[("02", Done, None), ("03", Running, Some("2026-01-01T21:00:00Z")), ("13", Waiting, None)], + RunState::Running, + ), + &fixed_ranges(), + NOW, + 64, + ); + assert!( + late.high_s.is_none() + && late.late_steps == ["03"] + && late.note.as_deref() == Some("step 03 is taking longer than usual") + && close(Some(late.low_s), 4.0 * H), + "{late:?}" + ); +} + +#[test] +fn a_running_step_counts_only_what_is_left_of_it() { + let going = remaining( + &state( + &[("02", Done, None), ("03", Running, Some("2026-01-02T06:00:00Z")), ("13", Waiting, None)], + RunState::Running, + ), + &fixed_ranges(), + NOW, + 64, + ); + assert!(close(Some(going.low_s), 10.0 * H) && close(going.high_s, 13.0 * H), "{going:?}"); +} + +#[test] +fn a_step_with_no_time_lands_in_unknown_steps_never_0() { + let r = ranges(&[], 16); + let unk = remaining( + &state(&[("02", Done, None), ("33", Waiting, None)], RunState::Running), + &r, + NOW, + 16, + ); + assert!( + unk.unknown_steps == ["33"] && r["33"].low_s.is_none() && r["33"].source == Source::None, + "{unk:?} {:?}", + r["33"] + ); +} + +#[test] +fn a_finished_run_has_nothing_left() { + let done = remaining( + &state(&[("03", Waiting, None)], RunState::Finished), + &fixed_ranges(), + NOW, + 16, + ); + assert!(done.low_s == 0.0 && done.high_s == Some(0.0), "{done:?}"); +} + +// --- the estimate of an outdir ------------------------------------------------------------ + +fn now() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs_f64() +} + +fn estimate(d: &Path, cores: usize) -> eta::Estimate<'static> { + eta::estimate( + d, + steps(), + &EtaArgs { + sample: None, + cores: Some(cores), + box_: Some("ci"), + now: now(), + tz: &UTC, + }, + ) + .unwrap() +} + +#[test] +fn the_estimate_gives_the_ranges_and_writes_the_history() { + let d = outdir(&[(OLD, "trace_new.txt")]); + let est = estimate(&d, 8); + assert!( + est.box_ == "ci" + && est.ranges["02"].source == Source::Box + && est.ranges["02"].high_s == Some(4.0 * H) + && est.ranges["13"].source == Source::Docs + && est.remaining.is_none() + && history_path(&d, "ci").is_file(), + "{} {:?}", + est.box_, + est.ranges["02"] + ); +} + +/// An outdir with a run that run-all.sh started a minute ago. +fn going(d: &Path) -> PathBuf { + let logs = d.join("demo1/logs"); + std::fs::create_dir_all(&logs).unwrap(); + let status = logs.join("run_status.tsv"); + std::fs::write( + &status, + format!("meta\tstarted_epoch\t{}\nstep\t02\tplanned\n", now() as i64 - 60), + ) + .unwrap(); + status +} + +#[test] +fn without_a_sample_the_trace_of_a_run_not_finished_is_left_out() { + let d = outdir(&[(OLD, "trace_new.txt")]); + let status = going(&d); + let before = estimate(&d, 16).ranges["02"].source; + let mut text = std::fs::read_to_string(&status).unwrap(); + text.push_str("meta\tfinished_utc\t2026-01-02 12:00:00 UTC\n"); + std::fs::write(&status, text).unwrap(); + let after = estimate(&d, 16).ranges["02"].source; + assert_eq!((before, after), (Source::Docs, Source::Box)); +} + +#[test] +fn an_unreadable_run_status_leaves_out_the_newest_trace() { + use std::os::unix::fs::PermissionsExt; + let d = outdir(&[(OLD, "trace_new.txt")]); + let status = going(&d); + let mut text = std::fs::read_to_string(&status).unwrap(); + text.push_str("meta\tfinished_utc\t2026-01-02 12:00:00 UTC\n"); + std::fs::write(&status, text).unwrap(); + std::fs::set_permissions(&status, std::fs::Permissions::from_mode(0o000)).unwrap(); + assert!( + std::fs::read(&status).is_err(), + "run this test as a user that file modes bind (not root)" + ); + let got = estimate(&d, 16).ranges["02"].source; + std::fs::set_permissions(&status, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert_eq!(got, Source::Docs); +} + +// --- what the Python modules and nextflow.config must still say --------------------------- + +/// `trace.fields` of nextflow.config. +fn trace_fields(config: &str) -> Option> { + let start = config.find("\ntrace {\n")?; + let block = &config[start..]; + let block = &block[..block.find("\n}")?]; + let line = block.lines().find(|l| l.trim_start().starts_with("fields"))?; + let value = line.split_once('\'')?.1.split_once('\'')?.0; + Some(value.split(',').map(str::to_owned).collect()) +} + +/// Nextflow's 14 default trace columns, then the 7 the history reads. +const TRACE_FIELDS: [&str; 21] = [ + "task_id", "hash", "native_id", "name", "status", "exit", "submit", "duration", "realtime", + "%cpu", "peak_rss", "peak_vmem", "rchar", "wchar", "process", "tag", "attempt", "cpus", + "memory", "start", "complete", +]; + +#[test] +fn nextflow_config_traces_the_14_default_columns_then_the_7_added() { + let cfg = std::fs::read_to_string(repo().join("nextflow.config")).unwrap(); + assert_eq!(trace_fields(&cfg), Some(TRACE_FIELDS.map(str::to_owned).to_vec())); + // Control: the same reading of a config without memory must differ. + let broken = cfg.replace(",memory,start", ",start"); + assert_ne!(broken, cfg, "the control found no trace.fields to break"); + assert_ne!(trace_fields(&broken), Some(TRACE_FIELDS.map(str::to_owned).to_vec())); +} + +/// An old `bin/run_*.py` exits 1 and names its og command, never a silent 0. +fn refuses(script: &Path, cmd: &str) -> Result<(), String> { + let exe = std::env::var("OG_PYTHON").unwrap_or_else(|_| "python3".into()); + let o = Command::new(&exe) + .arg(script) + .args(["--outdir", env!("CARGO_TARGET_TMPDIR")]) + .output() + .map_err(|e| format!("cannot run {exe}: {e}; set OG_PYTHON"))?; + let err = String::from_utf8_lossy(&o.stderr); + if o.status.code() == Some(1) && err.contains(&format!("bin/og run {cmd}")) && o.stdout.is_empty() { + Ok(()) + } else { + Err(format!("{:?} {err}", o.status)) + } +} + +/// While the live site still imports them, the modules' command lines stay +/// shut (they go with the Python server, and so does this check). +#[test] +fn the_old_python_command_lines_refuse_and_name_og_run() { + for (name, cmd) in [("run_eta", "eta"), ("run_state", "state"), ("run_headroom", "headroom")] { + let script = repo().join("bin").join(format!("{name}.py")); + if !script.is_file() { + continue; + } + refuses(&script, cmd).unwrap_or_else(|e| panic!("bin/{name}.py: {e}")); + // Control: the module without its refusing entry point must pass the run. + let src = std::fs::read_to_string(&script).unwrap(); + let cut = src + .rfind("\nif __name__ == \"__main__\":") + .expect("the module has an entry point"); + let planted = tmp().join(format!("{name}.py")); + std::fs::write(&planted, &src[..cut]).unwrap(); + assert!( + refuses(&planted, cmd).is_err(), + "bin/{name}.py: the check passed a module with no entry point" + ); + } +} + +// --- the planted mistakes: each test below must FAIL -------------------------------------- + +#[cfg(feature = "fault-inject")] +mod planted { + use og_runlog::eta::fault::plant; + + macro_rules! planted { + ($($name:ident: $fault:literal => $check:path;)*) => {$( + #[test] + fn $name() { + plant($fault); + $check(); + } + )*}; + } + + planted! { + planted_eta_old_start_is_submit: "eta-old-start-is-submit" => super::an_old_and_a_new_trace_give_the_same_spans; + planted_eta_memory_decimal_gb: "eta-memory-decimal-gb" => super::human_and_raw_units_read_the_same; + planted_eta_forced_ok: "eta-forced-ok" => super::cached_failed_and_unknown_process_tasks_never_enter_the_history; + planted_eta_ingest_adds_again: "eta-ingest-adds-again" => super::ingesting_twice_or_a_copied_trace_adds_nothing; + planted_eta_box_same: "eta-box-same" => super::two_boxes_write_two_files; + planted_eta_history_path_in_name: "eta-history-path-in-name" => super::the_history_holds_no_sample_id_and_no_slash; + planted_eta_history_sample_in_process: "eta-history-sample-in-process" => super::the_history_holds_no_sample_id_and_no_slash; + planted_eta_box_raw: "eta-box-raw" => super::the_box_name_keeps_only_letters_digits_underscore_and_dash; + planted_eta_last_runs_2: "eta-last-runs-2" => super::three_runs_give_min_max_with_n_3; + planted_eta_last_runs_6: "eta-last-runs-6" => super::only_the_last_5_runs_count; + planted_eta_docs_threads_8: "eta-docs-threads-8" => super::an_8_core_box_takes_the_measured_low_end_and_doubles_the_high_end; + planted_eta_docs_scale_down: "eta-docs-scale-down" => super::a_32_core_box_is_not_scaled_below_the_docs_hours; + planted_eta_no_hours_low: "eta-no-hours-low" => super::on_16_and_32_cores_every_measured_step_has_low_below_high; + planted_eta_estimates_count: "eta-estimates-count" => super::a_step_the_table_only_estimates_gets_no_number; + planted_eta_finished_always: "eta-finished-always" => super::a_step_with_a_task_that_failed_or_was_stopped_gives_no_span; + planted_eta_chain_ignores_upstream: "eta-chain-ignores-upstream" => super::the_hours_left_are_the_longest_chain; + planted_eta_no_cpu_work: "eta-no-cpu-work" => super::the_cpu_work_bound_when_it_is_larger; + planted_eta_late_not_flagged: "eta-late-not-flagged" => super::a_running_step_past_its_high_end_makes_the_high_end_unknown_and_says_so; + planted_eta_gone_ignored: "eta-gone-ignored" => super::a_running_step_counts_only_what_is_left_of_it; + planted_eta_none_is_zero: "eta-none-is-zero" => super::a_step_with_no_time_lands_in_unknown_steps_never_0; + planted_eta_finished_ignored: "eta-finished-ignored" => super::a_finished_run_has_nothing_left; + planted_eta_box_same_estimate: "eta-box-same" => super::the_estimate_gives_the_ranges_and_writes_the_history; + planted_eta_live_traces_none: "eta-live-traces-none" => super::without_a_sample_the_trace_of_a_run_not_finished_is_left_out; + planted_eta_unreadable_status_ignored: "eta-unreadable-status-ignored" => super::an_unreadable_run_status_leaves_out_the_newest_trace; + } +} diff --git a/rust/crates/og-live/tests/ops_headroom.rs b/rust/crates/og-live/tests/ops_headroom.rs new file mode 100644 index 000000000..43da95cd4 --- /dev/null +++ b/rust/crates/og-live/tests/ops_headroom.rs @@ -0,0 +1,530 @@ +//! og-runlog's headroom, as the live site's ops card reads it: synthetic +//! disks on a fake host, the tasks of `tests/fixtures/run_headroom`, the +//! weblog metadata, and the retried exits against `conf/base.config`. +//! +//! With `--features fault-inject` each `planted_*` test turns on one planted +//! mistake and runs the check that must catch it: every one of them must +//! fail (Guard's rust job requires it). A check that cannot fail is no check. + +use jiff::tz::TimeZone; +use og_runlog::eta::{TraceTask, read_trace}; +use og_runlog::headroom::{ + self, Disk, MEMORY_EXITS, Request, Snapshot, TaskMemory, TaskRow, event_rows, metadata, + task_memory, warn_percent, +}; +use og_runlog::host::{FsStat, Host, System}; +use og_runlog::steps::{Step, read_steps}; +use pycompat::json::{Value, loads}; +use std::cell::RefCell; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicUsize, Ordering}; + +const G: i128 = 1 << 30; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn fixture(name: &str) -> PathBuf { + repo().join("tests/fixtures/run_headroom").join(name) +} + +fn steps() -> Vec { + read_steps(&repo().join("assets/steps.tsv")).expect("assets/steps.tsv reads") +} + +/// A new empty folder of this test run. +fn tmp() -> PathBuf { + static N: AtomicUsize = AtomicUsize::new(0); + let d = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!( + "ops-headroom-{}-{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// The warn percents the checks expect come from no environment variable. +fn no_warn_env() { + for k in ["OWNGENOME_DISK_WARN_PERCENT", "OWNGENOME_MEM_WARN_PERCENT"] { + assert!( + std::env::var_os(k).is_none(), + "unset {k}: the checks test the default warn percents" + ); + } +} + +/// A host whose every file system is `used_pct`% used; its device ids are +/// `devs`, else 1; existence is the real one. +struct Fake { + used_pct: u64, + files: u64, + ffree: u64, + devs: Vec<(PathBuf, u64)>, + phys: Option, + seen: RefCell>, +} + +fn fake(used_pct: u64) -> Fake { + Fake { + used_pct, + files: 1000, + ffree: 900, + devs: Vec::new(), + phys: Some(64 << 30), + seen: RefCell::new(Vec::new()), + } +} + +impl Host for Fake { + fn statvfs(&self, path: &Path) -> io::Result { + self.seen.borrow_mut().push(path.to_path_buf()); + let free = 1000 - self.used_pct * 10; + Ok(FsStat { + frsize: 4096, + bsize: 4096, + blocks: 1000, + bfree: free, + bavail: free, + files: self.files, + ffree: self.ffree, + }) + } + fn dev(&self, path: &Path) -> io::Result { + Ok(self + .devs + .iter() + .find(|(p, _)| p == path) + .map_or(1, |(_, d)| *d)) + } + fn exists(&self, path: &Path) -> bool { + path.exists() + } + fn physical_memory(&self) -> Option { + self.phys + } +} + +fn s(p: &Path) -> &str { + p.to_str().unwrap() +} + +fn default_warn() -> f64 { + warn_percent(None, None).unwrap() +} + +fn snapshot_of(host: &dyn Host, r: &Request<'_>) -> Snapshot { + headroom::snapshot(host, &steps(), r).expect("the warn percents are good") +} + +fn request<'a>(work: Option<&'a str>, outdir: Option<&'a str>) -> Request<'a> { + Request { + events: &[], + trace_rows: &[], + work_dir: work, + outdir, + max_memory: None, + disk_warn_percent: None, + mem_warn_percent: None, + } +} + +fn by_process(ts: Vec) -> std::collections::HashMap { + ts.into_iter().map(|t| (t.process.clone(), t)).collect() +} + +fn close(a: Option, b: f64) -> bool { + a.is_some_and(|a| (a - b).abs() < 1e-6) +} + +fn trace(name: &str) -> Vec { + read_trace(&fixture(name), &TimeZone::UTC) +} + +fn events() -> Vec { + og_runlog::files::read_events(Some(&fixture("events.jsonl"))).unwrap() +} + +fn trace_memory(rows: &[TraceTask]) -> Vec { + let rows: Vec> = rows.iter().map(TaskRow::Trace).collect(); + task_memory(&rows, default_warn(), &steps()) +} + +fn weblog_memory(rows: &[&Value]) -> Vec { + let rows: Vec> = rows.iter().map(|v| TaskRow::Weblog(v)).collect(); + task_memory(&rows, default_warn(), &steps()) +} + +// --- disks -------------------------------------------------------------------------------- + +#[test] +fn disk_91_warns_at_the_default_89_does_not() { + let d = tmp(); + let hi = headroom::disk(&fake(91), &[("work", Some(s(&d)))], default_warn()); + let lo = headroom::disk(&fake(89), &[("work", Some(s(&d)))], default_warn()); + assert!( + hi.len() == 1 && hi[0].warn && close(hi[0].used_ratio, 0.91) && lo.len() == 1 && !lo[0].warn, + "{hi:?} {lo:?}" + ); +} + +/// The environment half runs in a child process (the test binary itself): +/// a test may not set a variable of its own process. +#[test] +fn disk_warn_percent_80_and_its_environment_variable_warn_at_85() { + no_warn_env(); + let d = tmp(); + let host = fake(85); + let plain = headroom::lines(&snapshot_of(&host, &request(Some(s(&d)), None))); + let mut with = request(Some(s(&d)), None); + with.disk_warn_percent = Some("80"); + let arg = headroom::lines(&snapshot_of(&host, &with)).join("\n"); + let mut child = Command::new(std::env::current_exe().unwrap()); + child + .args(["child_disk_warn_from_the_environment", "--exact", "--ignored", "--quiet"]) + .env("OWNGENOME_DISK_WARN_PERCENT", "80") + .env("OPS_CHILD_DIR", &d); + #[cfg(feature = "fault-inject")] + if let Some(f) = og_runlog::eta::fault::planted() { + child.env("OG_PLANTED", f); + } + let out = child.output().unwrap(); + assert!( + !plain[0].contains("WARN") && arg.contains("WARN: at or above 80%") && out.status.success(), + "{plain:?} {arg} {}", + String::from_utf8_lossy(&out.stdout) + ); +} + +#[test] +#[ignore = "run by disk_warn_percent_80_and_its_environment_variable_warn_at_85"] +fn child_disk_warn_from_the_environment() { + #[cfg(feature = "fault-inject")] + if let Ok(f) = std::env::var("OG_PLANTED") { + og_runlog::eta::fault::plant(Box::leak(f.into_boxed_str())); + } + let d = std::env::var("OPS_CHILD_DIR").expect("run by its parent test"); + let text = headroom::lines(&snapshot_of(&fake(85), &request(Some(&d), None))).join("\n"); + assert!(text.contains("WARN: at or above 80%"), "{text}"); +} + +#[test] +fn a_missing_work_folder_is_measured_on_its_parent() { + let d = tmp(); + let host = fake(50); + let miss = headroom::disk(&host, &[("work", Some(s(&d.join("not/yet"))))], default_warn()); + let seen = host.seen.borrow().clone(); + assert!( + miss.len() == 1 && miss[0].missing && seen == [d.clone()], + "{miss:?} {seen:?}" + ); +} + +#[test] +fn one_device_reports_once_two_report_twice() { + let d = tmp(); + let (a, b) = (d.join("a"), d.join("b")); + std::fs::create_dir_all(&a).unwrap(); + std::fs::create_dir_all(&b).unwrap(); + let paths = [("work", Some(s(&a))), ("results", Some(s(&b)))]; + let mut host = fake(50); + host.devs = vec![(a.clone(), 7), (b.clone(), 7)]; + let one = headroom::disk(&host, &paths, default_warn()); + host.devs = vec![(a.clone(), 7), (b.clone(), 8)]; + let two = headroom::disk(&host, &paths, default_warn()); + let fs = |v: &[Disk]| v.iter().map(|d| d.fs.clone()).collect::>(); + assert!( + fs(&one) == ["work and results"] && fs(&two) == ["work", "results"], + "{one:?} {two:?}" + ); +} + +#[test] +fn f_files_0_gives_no_inode_ratio() { + let d = tmp(); + let mut none = fake(50); + none.files = 0; + none.ffree = 0; + let noi = headroom::disk(&none, &[("work", Some(s(&d)))], default_warn()); + let withi = headroom::disk(&fake(50), &[("work", Some(s(&d)))], default_warn()); + assert!( + noi[0].inodes_used_ratio.is_none() && close(withi[0].inodes_used_ratio, 0.1), + "{noi:?} {withi:?}" + ); +} + +#[test] +fn a_real_statvfs_gives_a_ratio_in_0_to_1_and_a_disk_line() { + no_warn_env(); + let d = tmp(); + let real = headroom::disk(&System, &[("work", Some(s(&d)))], default_warn()); + let lines = headroom::lines(&snapshot_of(&System, &request(Some(s(&d)), Some(s(&d))))); + assert!( + real.len() == 1 + && real[0].used_ratio.is_some_and(|r| (0.0..=1.0).contains(&r)) + && real[0].size_bytes > 0 + && lines[0].starts_with("disk work and results: "), + "{real:?} {lines:?}" + ); +} + +// --- memory ------------------------------------------------------------------------------- + +#[test] +fn seven_point_six_of_8_gb_warns_6_of_8_does_not() { + let t = by_process(trace_memory(&trace("trace.txt"))); + assert!( + t["FASTP"].warn && close(t["FASTP"].ratio, 0.95) && !t["VEP"].warn && close(t["VEP"].ratio, 0.75), + "{t:?}" + ); +} + +#[test] +fn a_trace_size_and_weblog_bytes_give_the_same_ratio() { + let t = by_process(trace_memory(&trace("trace.txt"))); + let evs = events(); + let e = by_process(weblog_memory(&event_rows(&evs))); + assert!( + t["FASTP"].ratio.is_some() + && close(e["FASTP"].ratio, t["FASTP"].ratio.unwrap()) + && e["FASTP"].step.as_deref() == Some("01b"), + "{:?} {:?}", + t["FASTP"], + e["FASTP"] + ); +} + +#[test] +fn no_memory_column_gives_no_ratio() { + let t = trace_memory(&trace("trace_no_memory.txt")); + assert!(t.len() == 1 && t[0].ratio.is_none() && !t[0].warn, "{t:?}"); +} + +#[test] +fn exit_137_then_attempt_2_is_one_retry_and_a_running_task_shows_its_request() { + let evs = events(); + let e = by_process(weblog_memory(&event_rows(&evs))); + let (v, f, dv) = (&e["VEP"], &e["FASTP"], &e["DEEPVARIANT"]); + assert!( + v.oom_retry == 1 + && v.attempt == 2 + && close(v.ratio, 9.0 / 16.0) + && f.oom_retry == 0 + && dv.state == "running" + && dv.ratio.is_none() + && dv.request_bytes == Some(16 * G) + && dv.peak_rss_bytes.is_none(), + "{e:?}" + ); +} + +fn task(hash: &str, status: &str, exit: &str, peak: Option) -> TraceTask { + TraceTask { + trace_file: String::new(), + hash: hash.into(), + attempt: "1".into(), + process: Some("FASTP".into()), + tag: Some("demo".into()), + status: status.into(), + exit: exit.into(), + cpus: None, + memory_bytes: Some(8 * G), + peak_rss_bytes: peak, + realtime_s: None, + start: None, + complete: None, + } +} + +#[test] +fn a_running_row_after_a_finished_one_of_the_same_attempt_keeps_the_finished_ratio() { + let rows = [ + task("ab/1", "COMPLETED", "0", Some(6 * G)), + task("ab/2", "RUNNING", "", None), + ]; + let m = trace_memory(&rows); + assert!( + m.len() == 1 && m[0].state == "completed" && close(m[0].ratio, 0.75), + "{m:?}" + ); +} + +#[test] +fn a_running_message_after_a_finished_one_of_the_same_task_id_keeps_the_finished_ratio() { + let fin = loads(&format!( + r#"{{"task_id":7,"process":"VEP","name":"VEP (demo)","attempt":1,"status":"COMPLETED","exit":0,"memory":{},"peak_rss":{}}}"#, + 8 * G, + 6 * G + )) + .unwrap(); + let run = loads(&format!( + r#"{{"task_id":7,"process":"VEP","name":"VEP (demo)","attempt":1,"status":"RUNNING","memory":{}}}"#, + 8 * G + )) + .unwrap(); + let m = weblog_memory(&[&fin, &run]); + assert!( + m.len() == 1 && m[0].state == "completed" && close(m[0].ratio, 0.75), + "{m:?}" + ); +} + +#[test] +fn max_memory_64_gb_on_a_32_gb_host_warns_on_128_it_does_not() { + no_warn_env(); + let mut r = request(None, None); + r.max_memory = Some("64.GB"); + let mut host = fake(50); + host.phys = Some(32 << 30); + let small = snapshot_of(&host, &r).host; + host.phys = Some(128 << 30); + let big = snapshot_of(&host, &r).host; + assert!( + small.warn && small.max_memory_bytes == Some(64 * G) && !big.warn, + "{small:?} {big:?}" + ); +} + +// --- the snapshot and the weblog metadata --------------------------------------------------- + +fn strings(snap: &Snapshot) -> Vec { + let mut out: Vec = snap.disk.iter().map(|d| d.fs.clone()).collect(); + for t in &snap.tasks { + out.extend([t.process.clone(), t.state.clone()]); + out.extend(t.step.clone()); + } + out +} + +#[test] +fn the_snapshot_holds_no_path_and_takes_its_folders_from_the_metadata() { + no_warn_env(); + let d = tmp(); + let (work, res) = (d.join("work"), d.join("results")); + std::fs::create_dir_all(&work).unwrap(); + std::fs::create_dir_all(&res).unwrap(); + let meta = loads(&format!( + r#"{{"event":"started","metadata":{{"workflow":{{"workDir":"{}"}},"parameters":{{"outdir":"{}","max_memory":"64.GB"}}}}}}"#, + s(&work), + s(&res) + )) + .unwrap(); + let mut evs = vec![meta]; + evs.extend(events()); + let rows = trace("trace.txt"); + let mut host = fake(50); + host.devs = vec![(work.clone(), 1), (res.clone(), 2)]; + host.phys = Some(32 << 30); + let mut r = request(None, None); + r.events = &evs; + r.trace_rows = &rows; + let snap = snapshot_of(&host, &r); + let bad: Vec = strings(&snap).into_iter().filter(|x| x.contains('/')).collect(); + let fs: Vec<&str> = snap.disk.iter().map(|d| d.fs.as_str()).collect(); + let mut procs: Vec<&str> = snap.tasks.iter().map(|t| t.process.as_str()).collect(); + procs.sort_unstable(); + procs.dedup(); + assert!( + bad.is_empty() + && fs == ["work", "results"] + && snap.host.warn + && snap.warn_used_ratio == 0.9 + && snap.mem_warn_ratio == 0.9 + && procs == ["DEEPVARIANT", "FASTP", "VEP"], + "{bad:?} {snap:?}" + ); +} + +fn md(text: &str) -> (Option, Option, Option) { + metadata(&[loads(text).unwrap()]) +} + +fn st(v: &str) -> Option { + Some(Value::Str(v.into())) +} + +#[test] +fn a_relative_outdir_is_under_launch_dir_an_absolute_one_is_kept() { + let rel = md(r#"{"metadata":{"workflow":{"workDir":"/x/work","launchDir":"/x"},"parameters":{"outdir":"./results"}}}"#); + let ab = md(r#"{"metadata":{"workflow":{"workDir":"/x/work","launchDir":"/x"},"parameters":{"outdir":"/y/results"}}}"#); + assert_eq!(rel, (st("/x/work"), st("/x/./results"), None)); + assert_eq!(ab, (st("/x/work"), st("/y/results"), None)); +} + +#[test] +fn metadata_under_params_gives_the_outdir_and_max_memory() { + let alt = md(r#"{"metadata":{"workflow":{"workDir":"/x/work"},"params":{"outdir":"/y/r","max_memory":"8.GB"}}}"#); + assert_eq!(alt, (st("/x/work"), st("/y/r"), st("8.GB"))); +} + +/// The exits of `errorStrategy = { task.exitStatus in [...] ? 'retry' ...`. +fn retried_exits(config: &str) -> Option> { + let at = config.find("errorStrategy")?; + let rest = &config[at..]; + let open = rest.find("task.exitStatus in [")? + "task.exitStatus in [".len(); + let close = rest[open..].find(']')? + open; + Some( + rest[open..close] + .split(',') + .map(|x| x.trim().to_owned()) + .collect(), + ) +} + +#[test] +fn memory_exits_are_the_exits_base_config_retries() { + let base = std::fs::read_to_string(repo().join("conf/base.config")).unwrap(); + let mut want: Vec = MEMORY_EXITS.iter().map(|x| (*x).to_owned()).collect(); + want.sort(); + let sorted = |v: Option>| { + v.map(|mut v| { + v.sort(); + v + }) + }; + assert_eq!(sorted(retried_exits(&base)), Some(want.clone())); + // Control: the same reading of a config without 137 must differ. + let broken = base.replace("[143,137,", "[143,"); + assert_ne!(broken, base, "the control found no exit list to break"); + assert_ne!(sorted(retried_exits(&broken)), Some(want)); +} + +// --- the planted mistakes: each test below must FAIL -------------------------------------- + +#[cfg(feature = "fault-inject")] +mod planted { + use og_runlog::eta::fault::plant; + + macro_rules! planted { + ($($name:ident: $fault:literal => $check:path;)*) => {$( + #[test] + fn $name() { + plant($fault); + $check(); + } + )*}; + } + + planted! { + planted_headroom_default_95: "headroom-default-95" => super::disk_91_warns_at_the_default_89_does_not; + planted_headroom_percent_ignored: "headroom-percent-ignored" => super::disk_warn_percent_80_and_its_environment_variable_warn_at_85; + planted_headroom_missing_not_parent: "headroom-missing-not-parent" => super::a_missing_work_folder_is_measured_on_its_parent; + planted_headroom_no_merge: "headroom-no-merge" => super::one_device_reports_once_two_report_twice; + planted_headroom_inodes_always: "headroom-inodes-always" => super::f_files_0_gives_no_inode_ratio; + planted_headroom_used_above_size: "headroom-used-above-size" => super::a_real_statvfs_gives_a_ratio_in_0_to_1_and_a_disk_line; + planted_headroom_warn_strict: "headroom-warn-strict" => super::seven_point_six_of_8_gb_warns_6_of_8_does_not; + planted_headroom_weblog_decimal_peak: "headroom-weblog-decimal-peak" => super::a_trace_size_and_weblog_bytes_give_the_same_ratio; + planted_headroom_request_default_8g: "headroom-request-default-8g" => super::no_memory_column_gives_no_ratio; + planted_headroom_exits_143_only: "headroom-exits-143-only" => super::exit_137_then_attempt_2_is_one_retry_and_a_running_task_shows_its_request; + planted_headroom_running_wins: "headroom-running-wins" => super::a_running_row_after_a_finished_one_of_the_same_attempt_keeps_the_finished_ratio; + planted_headroom_replaces_always: "headroom-replaces-always" => super::a_running_message_after_a_finished_one_of_the_same_task_id_keeps_the_finished_ratio; + planted_headroom_host_never_warns: "headroom-host-never-warns" => super::max_memory_64_gb_on_a_32_gb_host_warns_on_128_it_does_not; + planted_headroom_label_is_path: "headroom-label-is-path" => super::the_snapshot_holds_no_path_and_takes_its_folders_from_the_metadata; + planted_headroom_no_launch_dir: "headroom-no-launch-dir" => super::a_relative_outdir_is_under_launch_dir_an_absolute_one_is_kept; + planted_headroom_parameters_only: "headroom-parameters-only" => super::metadata_under_params_gives_the_outdir_and_max_memory; + } +} diff --git a/rust/crates/og-runlog/src/eta.rs b/rust/crates/og-runlog/src/eta.rs index 57579bdd3..9773d1b04 100644 --- a/rust/crates/og-runlog/src/eta.rs +++ b/rust/crates/og-runlog/src/eta.rs @@ -77,6 +77,12 @@ pub mod fault { pub fn plant(name: &'static str) { ON.with(|c| c.set(Some(name))); } + + /// The planted mistake on in this thread, to hand to a child process. + #[cfg(feature = "fault-inject")] + pub fn planted() -> Option<&'static str> { + ON.with(std::cell::Cell::get) + } } /// One trace row, as the history and the headroom read it. From 6147aee42a6e0c7fa4e57e9a51ba12fbf2062a58 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:18:54 +0100 Subject: [PATCH 182/255] test(clinvar): parity goldens for the always-report words that change on purpose The three clinvar-deep-intronic cases differ from the awk oracle where the reader now says more: a ./. gVCF block has no genotype, a block without a GQ is a low-confidence reference call, a gVCF variant record is not a reference call (no PASS claim), and a VCF ./. without a gVCF names the VCF record. The summary test's sites-mode check takes the new no-genotype text. The fake-docker case gets shellcheck notes on the module text it greps for. Bead: clin-13 Bead: clin-14 --- tests/fake-docker/i411-site-states.sh | 3 +++ .../golden/clinvar-deep-intronic/gvcf/stdout | 14 ++++++++++++++ .../golden/clinvar-deep-intronic/min-gq-45/stdout | 14 ++++++++++++++ .../golden/clinvar-deep-intronic/no-gvcf/stdout | 14 ++++++++++++++ tests/test_collect_summary.py | 2 +- 5 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 tests/parity/golden/clinvar-deep-intronic/gvcf/stdout create mode 100644 tests/parity/golden/clinvar-deep-intronic/min-gq-45/stdout create mode 100644 tests/parity/golden/clinvar-deep-intronic/no-gvcf/stdout diff --git a/tests/fake-docker/i411-site-states.sh b/tests/fake-docker/i411-site-states.sh index 5980c5c6d..123ded1df 100755 --- a/tests/fake-docker/i411-site-states.sh +++ b/tests/fake-docker/i411-site-states.sh @@ -94,12 +94,15 @@ GOT=$(cd "$CASE_WORK" && HEADER=$HEADER bash -c 'set -euo pipefail; bcftools() { [ "$GOT" = "$WANT" ] || fail "the module queries '${GOT}', want '${WANT}'" # --- the module: depth floor and REF_MISMATCH count --------------------------------- +# shellcheck disable=SC2016 # the module's own text, dollar signs included grep -qE -- 'og clinvar deep-intronic .*--min-gq \$\{min_gq\} --min-dp \$\{min_dp\} ' "$MOD" \ || fail "the module does not hand og clinvar deep-intronic ext.min_dp" +# shellcheck disable=SC2016 # the module's own text grep -qF -- '--multi-overlaps . -c w -f ${reference} -Oz -o ${meta.id}_pass.vcf.gz - 2> norm.log' "$MOD" \ || fail "the module's sample bcftools norm does not keep its REF_MISMATCH warnings" grep -qF -- "REF_MISMATCH=\\\$(grep -c '^REF_MISMATCH' norm.log || true)" "$MOD" \ || fail "the module does not count the REF_MISMATCH warnings" +# shellcheck disable=SC2016 # the module's own text grep -qF -- 'echo "Records whose REF differs from the reference (REF_MISMATCH): \${REF_MISMATCH}"' "$MOD" \ || fail "the module's log has no REF_MISMATCH count" echo "both mirrors read FILTER and the depth floor, and count REF_MISMATCH: PASS" diff --git a/tests/parity/golden/clinvar-deep-intronic/gvcf/stdout b/tests/parity/golden/clinvar-deep-intronic/gvcf/stdout new file mode 100644 index 000000000..30fb08427 --- /dev/null +++ b/tests/parity/golden/clinvar-deep-intronic/gvcf/stdout @@ -0,0 +1,14 @@ +chrom pos ref alt gene hgvs clinvar_id call detail +chr1 100 A G GENE0 c.0A>T 0 het GQ 40, DP 30 +chr1 200 C T GENE1 c.1A>T 1 ref GQ 50, DP 30 +chr1 300 G A GENE2 c.2A>T 2 ref (low confidence) GQ 12, DP 30, below the call floor of GQ 30 +chr1 400 T C GENE3 c.3A>T 3 other allele T>G het +chr1 500 A T GENE4 c.4A>T 4 ref GQ 40 (gVCF block) +chr1 600 A T GENE5 c.5A>T 5 ref (low confidence) GQ 5 (gVCF block), below the call floor of GQ 30 +chr1 700 A T GENE6 c.6A>T 6 no call the gVCF record over the site has no genotype (GT ./.) +chr1 800 A T GENE7 c.7A>T 7 no call no record in the VCF or the gVCF covers the site +chr1 900 A T GENE8 c.8A>T 8 ref (low confidence) (gVCF block), no GQ to check against the call floor of GQ 30 +chr2 0100 A G GENE9 c.9A>T 9 het DP 20 +chr1 1000 A T GENE10 c.10A>T 10 one copy, other copy not called ref (./1) GQ 40, DP 30 +chr1 1100 A T GENE11 c.11A>T 11 no call the gVCF record over the site is not a reference call (GT 0/1) +chr1 1200 A T GENE12 c.12A>T 12 ref GQ 30 (gVCF block) diff --git a/tests/parity/golden/clinvar-deep-intronic/min-gq-45/stdout b/tests/parity/golden/clinvar-deep-intronic/min-gq-45/stdout new file mode 100644 index 000000000..1dd5086d7 --- /dev/null +++ b/tests/parity/golden/clinvar-deep-intronic/min-gq-45/stdout @@ -0,0 +1,14 @@ +chrom pos ref alt gene hgvs clinvar_id call detail +chr1 100 A G GENE0 c.0A>T 0 het GQ 40, DP 30 +chr1 200 C T GENE1 c.1A>T 1 ref GQ 50, DP 30 +chr1 300 G A GENE2 c.2A>T 2 ref (low confidence) GQ 12, DP 30, below the call floor of GQ 45 +chr1 400 T C GENE3 c.3A>T 3 other allele T>G het +chr1 500 A T GENE4 c.4A>T 4 ref (low confidence) GQ 40 (gVCF block), below the call floor of GQ 45 +chr1 600 A T GENE5 c.5A>T 5 ref (low confidence) GQ 5 (gVCF block), below the call floor of GQ 45 +chr1 700 A T GENE6 c.6A>T 6 no call the gVCF record over the site has no genotype (GT ./.) +chr1 800 A T GENE7 c.7A>T 7 no call no record in the VCF or the gVCF covers the site +chr1 900 A T GENE8 c.8A>T 8 ref (low confidence) (gVCF block), no GQ to check against the call floor of GQ 45 +chr2 0100 A G GENE9 c.9A>T 9 het DP 20 +chr1 1000 A T GENE10 c.10A>T 10 one copy, other copy not called ref (./1) GQ 40, DP 30 +chr1 1100 A T GENE11 c.11A>T 11 no call the gVCF record over the site is not a reference call (GT 0/1) +chr1 1200 A T GENE12 c.12A>T 12 ref (low confidence) GQ 30 (gVCF block), below the call floor of GQ 45 diff --git a/tests/parity/golden/clinvar-deep-intronic/no-gvcf/stdout b/tests/parity/golden/clinvar-deep-intronic/no-gvcf/stdout new file mode 100644 index 000000000..4be9be3cf --- /dev/null +++ b/tests/parity/golden/clinvar-deep-intronic/no-gvcf/stdout @@ -0,0 +1,14 @@ +chrom pos ref alt gene hgvs clinvar_id call detail +chr1 100 A G GENE0 c.0A>T 0 het GQ 40, DP 30 +chr1 200 C T GENE1 c.1A>T 1 ref GQ 50, DP 30 +chr1 300 G A GENE2 c.2A>T 2 ref (low confidence) GQ 12, DP 30, below the call floor of GQ 30 +chr1 400 T C GENE3 c.3A>T 3 other allele T>G het +chr1 500 A T GENE4 c.4A>T 4 no call no record in the VCF, and no gVCF to tell a reference call from no call +chr1 600 A T GENE5 c.5A>T 5 no call no record in the VCF, and no gVCF to tell a reference call from no call +chr1 700 A T GENE6 c.6A>T 6 no call no record in the VCF, and no gVCF to tell a reference call from no call +chr1 800 A T GENE7 c.7A>T 7 no call no record in the VCF, and no gVCF to tell a reference call from no call +chr1 900 A T GENE8 c.8A>T 8 no call the VCF record at the site has no genotype (GT ./.) +chr2 0100 A G GENE9 c.9A>T 9 het DP 20 +chr1 1000 A T GENE10 c.10A>T 10 one copy, other copy not called ref (./1) GQ 40, DP 30 +chr1 1100 A T GENE11 c.11A>T 11 no call no record in the VCF, and no gVCF to tell a reference call from no call +chr1 1200 A T GENE12 c.12A>T 12 no call no record in the VCF, and no gVCF to tell a reference call from no call diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index 20eb59617..b082641c5 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -638,7 +638,7 @@ def construct_scan_section(work, schema): check("sites mode: the calls", calls == { "G1": ("ref", "GQ 50 (gVCF block)"), "G2": ("hom", "GQ 60, DP 30"), "G3": ("ref", "GQ 35"), "G4": ("other allele", "G>C het"), - "G5": ("no call", "the gVCF record over the site is not a PASS reference call (GT ./.)")}, calls) + "G5": ("no call", "the gVCF record over the site has no genotype (GT ./.)")}, calls) # The call floor: a reference call below min_gq (step 6's CLINVAR_MIN_GQ) is no confident ref. out = og_clinvar("deep-intronic", "--min-gq", "40", sites, vcf_calls, gvcf_calls).splitlines() low = {r.split("\t")[4]: tuple(r.split("\t")[7:9]) for r in out[1:]} From f4ae05918bd8d75a509a4048b64d41309637d89a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:18:54 +0100 Subject: [PATCH 183/255] docs(clinvar): every always-report call state, the depth floor and the REF_MISMATCH count docs/06 lists each call word of the always-report list with when it applies, says a no call is not a negative result, and says the summary counts REF_MISMATCH records. The CHANGELOG names the output change. Bead: clin-09 Bead: clin-10 Bead: clin-11 Bead: clin-12 Bead: clin-13 Bead: clin-14 --- CHANGELOG.md | 2 ++ docs/06-clinvar-screen.md | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85edb5eae..eca20e722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. +- The always-report list of step 06 (`clinvar/_clinvar_deep_intronic.tsv`) now says what each call really is. A `0/0` with a filter such as `LowQual`, a `0/0` without a GQ, and a `0/0` read from one or two reads all read as plain `ref`, and a male's one-copy chrX call (`1` or `0`, as some callers write it) read as the raw `1` or as no call. One state table in `og clinvar deep-intronic` now decides each site. Only FILTER `PASS`, `.` and `RefCall` leave a reference call plain; any other FILTER makes it `ref (low confidence)` and names the FILTER. A reference call without a GQ is `ref (low confidence)` too. A depth below the step's depth floor (`CLINVAR_MIN_DP`, `ext.min_dp`, default 10, passed with the new `--min-dp`) reads `ref (shallow)`. On chrX and chrY outside the pseudoautosomal regions a one-copy `1` reads `hemizygous` and a one-copy `0` reads `ref`; a one-copy GT anywhere else is no call, with why. A no call says what the record holds (`has no genotype`, `is not a reference call`), and no longer claims a PASS check that never ran or that no record exists when a VCF record without a genotype does. Both mirrors of step 06 add FILTER as a ninth column of the query the reader takes. Output change: the file keeps its columns; the `call` column gains the words `ref (shallow)` and `hemizygous`, and some `detail` texts change. The parity goldens of `clinvar-deep-intronic` record the deliberate changes. +- Step 06 now counts the sample records whose REF is not the reference's base. `bcftools norm -c w` keeps such a record as written and only warns `REF_MISMATCH` in the log, and the record cannot match a ClinVar allele. The step's summary, and the Nextflow module's log, now print the count, and a notice to check the VCF's reference when there are any. - Each ClinVar hit now keeps the condition ClinVar names for it. Step 06 copied only ClinVar's ID, gene, significance and review status onto each hit, so a later step knew the gene but not the disease: a RET hit could be Hirschsprung disease or MEN2. The hits VCF (`clinvar/_clinvar_hits.vcf`) now also carries `CLNDN`, ClinVar's disease name, and `CLNDISDB`, its disease database ids, in both the bash step and the Nextflow module. Two INFO fields are added; nothing is renamed or removed, and `clinvar/_clinvar_hits.tsv` is unchanged. A hand-built ClinVar file that does not define the two fields still runs, without them and with a notice. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/06-clinvar-screen.md b/docs/06-clinvar-screen.md index 3a8e5873a..94f8f7832 100644 --- a/docs/06-clinvar-screen.md +++ b/docs/06-clinvar-screen.md @@ -51,14 +51,14 @@ In the Nextflow pipeline the same floor is `ext.min_gq` and `ext.min_dp` of `CLI 1. Lists the contigs each file holds records on (`bcftools index -s`) and keeps those the reference has too: ClinVar's contigs that are in the reference, and the sample's contigs among those. `bcftools norm` stops at the first record whose contig the reference lacks, such as a provider's unplaced scaffold or one of the `NT_` contigs the full ClinVar file has, and a record on a contig the other file lacks cannot match anyway. The step prints how many of the sample's records it leaves out and on which contigs. When no contig is shared it stops with an error instead of reporting zero hits, and says which file is named the other way (`1, 2, ...` against `chr1, chr2, ...`). 2. Builds `clinvar/clinvar_pathogenic_chr.norm.vcf.gz` beside the ClinVar file the first time, and again whenever the source file is newer: ClinVar's records on the reference's contigs, multiallelic records split (`bcftools norm -m -any`) and indels left-aligned against the reference. Every sample then reuses it. The Nextflow module reuses this file when it sits beside the `--clinvar` file and is newer; otherwise it normalises ClinVar inside the task. 3. Filters the sample VCF to PASS records. If the VCF has no PASS record at all (callers that leave FILTER as `.`), it uses `-f .,PASS` instead and prints a notice saying so. If no record is left, the step stops with an error. -4. Splits and left-aligns the sample the same way. `bcftools isec` matches only identical REF/ALT, so a pathogenic allele inside a multiallelic record (genotype `1/2`) is found only after this split. The split writes the other alt allele as missing, not as the reference (`bcftools norm --multi-overlaps .`): a `1/2` record gives `1/.` and `./1`, where the default would give `1/0` and `0/1`. Both copies of such a call carry an alt allele, so neither part may read as het. A poly-T tract is where this shows most: a call of 5 T on one copy and 9 T on the other, against a reference of 7, is one `1/2` record, and its 5 T allele is a ClinVar hit. The reports word such a hit "one copy, other copy not called ref (1/.)", never het; the ACMG and carrier lists count it as one copy of two. This step does not size the tract: a short-read call of a homopolymer length can be off by one or two bases, so check the reads. +4. Splits and left-aligns the sample the same way. `bcftools isec` matches only identical REF/ALT, so a pathogenic allele inside a multiallelic record (genotype `1/2`) is found only after this split. The split writes the other alt allele as missing, not as the reference (`bcftools norm --multi-overlaps .`): a `1/2` record gives `1/.` and `./1`, where the default would give `1/0` and `0/1`. Both copies of such a call carry an alt allele, so neither part may read as het. A poly-T tract is where this shows most: a call of 5 T on one copy and 9 T on the other, against a reference of 7, is one `1/2` record, and its 5 T allele is a ClinVar hit. The reports word such a hit "one copy, other copy not called ref (1/.)", never het; the ACMG and carrier lists count it as one copy of two. This step does not size the tract: a short-read call of a homopolymer length can be off by one or two bases, so check the reads. A sample record whose REF is not the reference's base at that position is kept as written (`bcftools norm -c w`), with a `REF_MISMATCH` warning in the log. Such a record cannot match a ClinVar allele, so the summary of item 11 counts these records, and prints a notice to check that the VCF was called on this reference when there are any. 5. Builds the combined alleles of neighbouring records. A caller can write one complex allele, a delins, as two records a few bases apart on the same haplotype, for example a deletion and an insertion. Matched one by one, the parts can hit two single-submitter ClinVar records and miss the better reviewed record ClinVar holds for the whole allele. `bcftools norm -m +any` joins only records at the same position, so it cannot build that allele. `og clinvar join-adjacent` pairs two records when the second starts at most 10 bases after the first ends (records that overlap are never paired) and their genotypes allow both ALT alleles on one haplotype: phased with the same genotype in the same phase set (`FORMAT/PS`), or unphased with the same genotype (`0/1` and `1/0` count as the same het call; an unphased call with a missing allele such as `./1` is never paired). A phased call with a missing allele, such as `1|.`, has its ALT on a known haplotype: it pairs with a record in the same phase set whose ALT sits on that haplotype (`1|0` or `1|.`), and the combined record keeps `.` on the haplotype where either part is missing. That `.` means the copy was not called there; when the part comes from a split multiallelic record, it holds the other alt allele. Two unphased het calls can also sit on different haplotypes, and the genotype cannot tell which, so the notes of a combined allele and its parts built from them end with "unphased, so the parts may sit on different haplotypes". The combined record runs from the first part's start to the second part's end; `og clinvar join-adjacent` takes the bases between the parts from the reference (an uncompressed FASTA with its `.fai`), and `bcftools norm` trims and left-aligns it the way ClinVar was. The gene is not checked here, because the sample VCF has no gene annotation yet: a combined allele only counts when ClinVar holds a record for it. The step prints how many combined alleles it built. Only pairs are built: an allele split into three or more records is matched by its pairs only. 6. Keeps the sample's records and combined alleles whose allele is in ClinVar (`bcftools isec -n=2 -w1`), copies ClinVar's `ID`, `GENEINFO`, `CLNSIG`, `CLNREVSTAT`, `CLNDN` and `CLNDISDB` onto them (`bcftools annotate --pair-logic exact`), and keeps only those whose genotype carries an ALT allele (`bcftools view -i 'GT~"[1-9]"'`, a non-zero allele index anywhere in the genotype). The match is by allele, so without this a `0/0` or `./.` record, the `0/0` half of a split multiallelic record and a reference-only row with ALT `.` would be listed as hits. A half call such as `./1` carries the ALT allele and is a hit; `GT="alt"` would drop it. The filter runs after the split, where it sees each allele's own genotype. No command line goes into the hits file's header (`--no-version`), so ClinVar's path on your machine stays out of it. `CLNDN` is the condition ClinVar names for the allele and `CLNDISDB` its ids in disease databases such as MONDO, OMIM and MedGen. A gene can cause more than one disease, so a later step reads these to know which one ClinVar asserted. Every ClinVar release defines both. `bcftools annotate` stops on a tag the ClinVar file does not define, so a hand-built ClinVar file without them gives hits without them, and the step prints a notice. 7. Marks the hits that belong together. A matched combined allele is a hit of its own and carries `INFO/PGP_JOINED_FROM`, its two parts as `CHROM:POS:REF:ALT`. Each of its parts that is a hit too keeps its row and gets `INFO/PGP_PART_OF`, the combined record it belongs to, so the reader sees one allele with the combined record's own review status, not two unrelated hits. Two hits in one gene that could be one allele by the rule of item 5 but matched no combined record each get `INFO/PGP_ONE_ALLELE`, the other hit: possibly one allele, check ClinVar for a combined record. 8. Writes a call floor other than the default into the hits file's header, for example `##PGP_CALL_FLOOR=GQ=20,DP=8` (below). At the default the file has no such line and reads as GQ 30 and DP 10. 9. Writes the always-report list (below): the sample's call at every site of `assets/deep_intronic_sites.tsv`. 10. Runs the cDNA read check on each hit and the cDNA scan of every exon (below). -11. Prints the hits grouped by ClinVar's review stars (`og clinvar hits`): 4 practice guideline, 3 expert panel, 2 multiple submitters with no conflict, 1 a single submitter or conflicting classifications, 0 no assertion criteria, each with its combined-allele note when it has one and "LOW-CONFIDENCE CALL (GQ .., DP ..): check the reads" when the call is below the floor, and the count of low-confidence calls. The screening file keeps every Pathogenic/Likely_pathogenic submission whatever its review status, so a zero-star hit counts as a hit; the stars say how much weight it deserves. The Nextflow module runs the same steps and prints the same counts; its `CLINVAR_CDNA_CHECK` process runs the cDNA read check and the cDNA scan. +11. Prints the hits grouped by ClinVar's review stars (`og clinvar hits`): 4 practice guideline, 3 expert panel, 2 multiple submitters with no conflict, 1 a single submitter or conflicting classifications, 0 no assertion criteria, each with its combined-allele note when it has one and "LOW-CONFIDENCE CALL (GQ .., DP ..): check the reads" when the call is below the floor, and the count of low-confidence calls, then the count of `REF_MISMATCH` records of item 4. The screening file keeps every Pathogenic/Likely_pathogenic submission whatever its review status, so a zero-star hit counts as a hit; the stars say how much weight it deserves. The Nextflow module runs the same steps and prints the same counts; its `CLINVAR_CDNA_CHECK` process runs the cDNA read check and the cDNA scan. A delins that the caller writes as two records, while ClinVar holds it as one, is matched exactly once, as the combined record of item 5: `tests/test_clinvar_join_adjacent.py` check 8 runs the real step on such a fixture and counts the hits. @@ -66,7 +66,18 @@ A delins that the caller writes as two records, while ClinVar holds it as one, i Some pathogenic alleles sit deep inside an intron, far from any exon. Filters built around exons and splice sites skip them, and a report that lists only hits cannot say whether such a site was read as the reference or not read at all ([Vaz-Drago 2017](https://pubmed.ncbi.nlm.nih.gov/28497172/)). `assets/deep_intronic_sites.tsv` lists the best known ones: CFTR c.3718-2477C>T and c.1680-886A>G, CEP290 c.2991+1655A>G, USH2A c.7595-2144A>G, ATM c.5763-1050A>G and GLA c.640-801G>A, each a ClinVar Pathogenic record with an expert-panel or multi-submitter review, at the GRCh38 position and alleles of ClinVar's canonical SPDI (checked 2026-11-04). -Every run writes `clinvar/${SAMPLE}_clinvar_deep_intronic.tsv`, one row per site, carrier or not. Its `call` is the genotype in the words the hits use (het, hom, ...) when the sample carries the site's allele; `ref` for a reference call, with the GQ of the `0/0` record or of the gVCF reference block over the site (`ref (low confidence)` when that GQ is below the call floor, `CLINVAR_MIN_GQ` or `ext.min_gq`, default 30); `other allele` when the sample carries another allele there; or `no call`, with the reason (no record in the VCF or the gVCF, a gVCF record that is not a reference call, or no gVCF at all). `detail` holds the GQ and DP, the other allele or the reason. The records read are the split PASS records of item 4 at the site's position (`bcftools query -T`) and the gVCF records over it (`bcftools query -R`). Both reports print every row under the ClinVar hits. +Every run writes `clinvar/${SAMPLE}_clinvar_deep_intronic.tsv`, one row per site, carrier or not. The records read are the split PASS records of item 4 at the site's position (`bcftools query -T`) and the gVCF records over it (`bcftools query -R`), each with its GT, GQ, DP and FILTER. The `call` column is one of these: + +| `call` | When | +|---|---| +| a genotype (`het`, `hom`, ...) | The sample carries the site's allele, in the words the hits use. One copy on chrX or chrY outside the pseudoautosomal regions (GT `1`, as some callers write a male's X) reads `hemizygous`. | +| `ref` | A reference call: a record whose alleles are all `0` (`0/0`, or `0` on chrX or chrY outside the pseudoautosomal regions), or a gVCF reference block over the site. Its FILTER is `PASS`, `.` or `RefCall` (DeepVariant's reference call), its GQ is at or above the call floor (`CLINVAR_MIN_GQ` or `ext.min_gq`, default 30), and its depth is at or above the depth floor (`CLINVAR_MIN_DP` or `ext.min_dp`, default 10). | +| `ref (low confidence)` | A reference call with any other FILTER (`LowQual`, for example), a GQ below the call floor, or no GQ at all, so nothing checked it against the floor. | +| `ref (shallow)` | A reference call whose only fault is a depth (`FORMAT/DP`) below the depth floor. A DeepVariant reference block has no DP (it writes `MIN_DP`), and its GQ already falls with the depth, so only its GQ is checked. | +| `other allele` | The sample carries another allele at the site. | +| `no call` | Nothing says reference or carrier: no record in the VCF or the gVCF, no gVCF at all, a record with no genotype (`./.`), a record that is not a reference call (`0/.`, or a filtered call of the allele that only the gVCF holds), or a one-copy GT where the genome has two copies (an autosome, or chrX inside a pseudoautosomal region). | + +`detail` holds the call's GQ and DP (`GQ n (gVCF block)` for a block, with its DP when that is below the depth floor), each reason a reference call is not plain `ref` (`FILTER LowQual`, `below the call floor of GQ 30`, `no GQ to check against the call floor of GQ 30`, `below the depth floor of DP 10`), the other allele, or what the record at the site holds when there is no call. A no call is not a negative result: it says only that the site was not read. These floors apply to reference calls only. A carrier's GQ and DP are shown, not judged: the same allele is a ClinVar hit, and the hits table marks it low confidence when it is below the floor. Both reports print every row under the ClinVar hits. ### Lab-construct sites and the cDNA scan From 92339d64bdab0ae2510f3d5e4e251f30627a7033 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:21:34 +0100 Subject: [PATCH 184/255] test(og-dev): og dev fixture kinds for the genotype and repeat e2e inputs append-n, rfc1, str-extra, tas2r38, mito-sam, hba-tiles, cut-haplotype and contains write the bytes the inline Python of the e2e cases wrote, pinned by FNV-1a hashes of the Python output. Bead: D1 --- rust/crates/og-dev/src/fixture.rs | 11 +- rust/crates/og-dev/src/fixture/e2e.rs | 280 ++++++++++++++ .../crates/og-dev/src/fixture/e2e/append_n.rs | 61 +++ .../crates/og-dev/src/fixture/e2e/contains.rs | 83 ++++ .../og-dev/src/fixture/e2e/cut_haplotype.rs | 274 +++++++++++++ .../og-dev/src/fixture/e2e/hba_tiles.rs | 145 +++++++ .../crates/og-dev/src/fixture/e2e/mito_sam.rs | 252 ++++++++++++ rust/crates/og-dev/src/fixture/e2e/rfc1.rs | 365 ++++++++++++++++++ .../og-dev/src/fixture/e2e/str_extra.rs | 294 ++++++++++++++ rust/crates/og-dev/src/fixture/e2e/tas2r38.rs | 290 ++++++++++++++ 10 files changed, 2054 insertions(+), 1 deletion(-) create mode 100644 rust/crates/og-dev/src/fixture/e2e.rs create mode 100644 rust/crates/og-dev/src/fixture/e2e/append_n.rs create mode 100644 rust/crates/og-dev/src/fixture/e2e/contains.rs create mode 100644 rust/crates/og-dev/src/fixture/e2e/cut_haplotype.rs create mode 100644 rust/crates/og-dev/src/fixture/e2e/hba_tiles.rs create mode 100644 rust/crates/og-dev/src/fixture/e2e/mito_sam.rs create mode 100644 rust/crates/og-dev/src/fixture/e2e/rfc1.rs create mode 100644 rust/crates/og-dev/src/fixture/e2e/str_extra.rs create mode 100644 rust/crates/og-dev/src/fixture/e2e/tas2r38.rs diff --git a/rust/crates/og-dev/src/fixture.rs b/rust/crates/og-dev/src/fixture.rs index ec74f8dd3..9db7a3a1c 100644 --- a/rust/crates/og-dev/src/fixture.rs +++ b/rust/crates/og-dev/src/fixture.rs @@ -9,6 +9,9 @@ //! `report.json` cut down to the fields `og pgx` reads. It replaces //! `tests/fixtures/pharmcat/trim_report.py` and writes the same bytes. //! +//! - `og dev fixture ` for the kinds of [`e2e`]: the inputs the +//! genotype, repeat and copy-number e2e cases make for themselves. +//! //! Exit 0 when written, 1 when an input cannot be read or has the wrong //! shape, 2 on a usage error. @@ -20,6 +23,8 @@ use std::fs; use std::io::Write; use std::path::Path; +pub mod e2e; + const USAGE: &str = "usage: og dev fixture comphet OUT_DIR [--swap]\n og dev fixture trim-pharmcat REPORT.json OUT.json [MAX_ANNOTATIONS]\n"; /// Runs `og dev fixture ARGS`; returns the exit code. @@ -58,9 +63,13 @@ pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { } Some("-h" | "--help") => { let _ = out.write_all(USAGE.as_bytes()); + let _ = out.write_all(e2e::USAGE.as_bytes()); return 0; } - Some(other) => return usage(err, &format!("unknown fixture '{other}'")), + Some(other) => { + return e2e::cli(other, &args[1..], out, err) + .unwrap_or_else(|| usage(err, &format!("unknown fixture '{other}'"))); + } None => return usage(err, "the following arguments are required: fixture"), }; match result { diff --git a/rust/crates/og-dev/src/fixture/e2e.rs b/rust/crates/og-dev/src/fixture/e2e.rs new file mode 100644 index 000000000..1af9e630b --- /dev/null +++ b/rust/crates/og-dev/src/fixture/e2e.rs @@ -0,0 +1,280 @@ +//! The inputs the genotype, repeat and copy-number e2e cases make for +//! themselves: made-up references, reads and records with fixed seeds. +//! Each kind writes the bytes the inline Python of its case wrote, so a +//! case keeps what it asserts. +//! +//! - `append-n SRC OUT CONTIG LENGTH`: SRC, then CONTIG as LENGTH `N`s, 60 +//! a line (a stand-in contig a reference lacks). +//! - `rfc1 KIND OUT.sam [--samd12 UNIT]`: made-up reads at the GRCh38 RFC1 +//! repeat ([`rfc1`]); `--samd12` adds reads of UNIT and reference reads at +//! SAMD12 on chr8. +//! - `str-extra REF.fa HET_NAME HET.sam HOM_NAME HOM.sam`: a reference that +//! is N except a window around the CFTR TG/poly-T and UGT1A1 TA tracts, +//! and read pairs of two samples (`e2e/str_extra.rs`). +//! - `tas2r38 REF_DIR GENOME_DIR`: a chr7 that is N except a window around +//! TAS2R38, with gVCFs, VCFs and reads of four samples (`e2e/tas2r38.rs`). +//! - `mito-sam --fasta FA --header SAM --sample S --out OUT [--depth X] +//! [--deletion FIRST-LAST] [--level F]`: single-end chrM reads, some across +//! a deletion's junction (`e2e/mito_sam.rs`). +//! - `hba-tiles FASTA HEADER SAMPLE OUT`: reads tiled every 5 bases over a +//! chr16 window, MAPQ 0 where a read fits twice; a sample whose name ends +//! in `3` loses every other read over HBA2 (`e2e/hba_tiles.rs`). +//! - `cut-haplotype START END FQ NAMES`: SAM on stdin; the haplotype-1 reads +//! lose bases START..END-1 and go to FQ as FASTQ (`e2e/cut_haplotype.rs`). +//! - `contains FILE PART`: exit 0 when FILE holds PART's bytes, 1 when not. + +mod append_n; +mod contains; +mod cut_haplotype; +mod hba_tiles; +mod mito_sam; +pub mod rfc1; +mod str_extra; +mod tas2r38; + +use super::Mt19937; +use std::fs::File; +use std::io::{self, BufWriter, Write}; +use std::path::Path; + +/// The usage lines of these kinds, one a line. +pub const USAGE: &str = " og dev fixture append-n SRC OUT CONTIG LENGTH\n og dev fixture rfc1 KIND OUT.sam [--samd12 UNIT]\n og dev fixture str-extra REF.fa HET_NAME HET.sam HOM_NAME HOM.sam\n og dev fixture tas2r38 REF_DIR GENOME_DIR\n og dev fixture mito-sam --fasta FA --header SAM --sample S --out OUT [--depth X] [--deletion FIRST-LAST] [--level F]\n og dev fixture hba-tiles FASTA HEADER SAMPLE OUT\n og dev fixture cut-haplotype START END FQ NAMES < SAM\n og dev fixture contains FILE PART\n"; + +/// What a kind ended in: written, a usage error, or a failure. +pub enum Fail { + Usage(String), + Run(String), +} + +impl From for Fail { + fn from(e: io::Error) -> Self { + Fail::Run(e.to_string()) + } +} + +/// Runs kind `kind` with `args`, or `None` when no kind here has that name. +pub fn cli(kind: &str, args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> Option { + let result = match kind { + "append-n" => append_n::run(args), + "rfc1" => rfc1::run(args), + "str-extra" => str_extra::run(args), + "tas2r38" => tas2r38::run(args), + "mito-sam" => mito_sam::run(args, out), + "hba-tiles" => hba_tiles::run(args), + "cut-haplotype" => cut_haplotype::run(args, &mut io::stdin().lock(), out, err), + "contains" => return Some(contains::run(args, err)), + _ => return None, + }; + Some( + match result.and_then(|()| out.flush().map_err(Fail::from)) { + Ok(()) => 0, + Err(Fail::Usage(msg)) => { + let _ = writeln!(err, "usage:\n{USAGE}og dev fixture: error: {msg}"); + 2 + } + Err(Fail::Run(msg)) => { + let _ = writeln!(err, "og dev fixture {kind}: {msg}"); + 1 + } + }, + ) +} + +/// `n` arguments exactly, or a usage error naming them. +fn exactly<'a>(args: &'a [String], n: usize, names: &str) -> Result<&'a [String], Fail> { + if args.len() == n { + Ok(args) + } else { + Err(Fail::Usage(format!("takes {names}"))) + } +} + +/// A whole number argument. +fn number(s: &str, what: &str) -> Result { + s.parse() + .map_err(|_| Fail::Usage(format!("{what} '{s}' is not a whole number"))) +} + +/// A file to write, buffered; the error names the path. +fn create(path: &Path) -> Result, Fail> { + File::create(path) + .map(BufWriter::new) + .map_err(|e| Fail::Run(format!("{}: {e}", path.display()))) +} + +/// A file to read, whole; the error names the path. +fn read(path: &Path) -> Result, Fail> { + std::fs::read(path).map_err(|e| Fail::Run(format!("{}: {e}", path.display()))) +} + +/// CPython's `random.Random(seed)`, as far as the cases use it. +pub struct PyRandom(Mt19937); + +impl PyRandom { + /// `random.Random(seed)` (and `random.seed(seed)`). + pub fn new(seed: u32) -> Self { + PyRandom(Mt19937::new(seed)) + } + + /// `randrange(0, n)`, for `0 < n < 2^32`. + pub fn below(&mut self, n: usize) -> usize { + let n = u32::try_from(n).expect("a range below 2^32"); + self.0.below(n) as usize + } + + /// `choice(seq)`. + pub fn choice(&mut self, seq: &[T]) -> T { + seq[self.below(seq.len())] + } + + /// `random()`: 53 random bits in [0, 1). + pub fn random(&mut self) -> f64 { + let a = f64::from(self.0.next_u32() >> 5); + let b = f64::from(self.0.next_u32() >> 6); + (a * 67_108_864.0 + b) * (1.0 / 9_007_199_254_740_992.0) + } + + /// `"".join(choice("ACGT") for _ in range(n))`. + pub fn bases(&mut self, n: usize) -> Vec { + (0..n).map(|_| self.choice(b"ACGT")).collect() + } +} + +/// The reverse complement over `ACGTN`; other bytes stay as they are. +pub fn revcomp(s: &[u8]) -> Vec { + s.iter() + .rev() + .map(|&b| match b { + b'A' => b'T', + b'C' => b'G', + b'G' => b'C', + b'T' => b'A', + x => x, + }) + .collect() +} + +/// Writes `n` `N`s as FASTA lines of 60, continuing a line that already +/// holds `col` bases. +fn n_lines(w: &mut dyn Write, mut n: usize, col: usize) -> io::Result { + const NS: [u8; 60] = [b'N'; 60]; + let mut col = col; + while n > 0 { + let k = n.min(60 - col); + w.write_all(&NS[..k])?; + col += k; + n -= k; + if col == 60 { + w.write_all(b"\n")?; + col = 0; + } + } + Ok(col) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// FNV-1a, 64 bits: a pin of the bytes a kind writes. + pub fn fnv(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |h, &b| { + (h ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3) + }) + } + + /// A writer that keeps only an FNV-1a hash, the line count and the + /// longest line of what it is given. + pub struct Hashed { + pub hash: u64, + pub lines: usize, + pub longest: usize, + col: usize, + } + + impl Default for Hashed { + fn default() -> Self { + Hashed { + hash: fnv(b""), + lines: 0, + longest: 0, + col: 0, + } + } + } + + impl Write for Hashed { + fn write(&mut self, buf: &[u8]) -> io::Result { + for &b in buf { + self.hash = (self.hash ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3); + if b == b'\n' { + self.lines += 1; + self.longest = self.longest.max(self.col); + self.col = 0; + } else { + self.col += 1; + } + } + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + pub fn scratch(name: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("og-e2e-fixture-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + pub fn run(kind: &str, args: &[&str]) -> (Option, String, String) { + let args: Vec = args.iter().map(|s| (*s).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(kind, &args, &mut out, &mut err); + ( + code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) + } + + #[test] + fn random_is_cpythons() { + // python3 -c 'import random; r = random.Random(1); + // print(r.random(), r.randrange(0, 16569), r.choice((0, 16)))' + let mut r = PyRandom::new(1); + assert_eq!(r.random(), 0.134_364_244_112_401_22); + assert_eq!(r.below(16569), 2067); + assert_eq!(r.choice(&[0, 16]), 16); + // r = random.Random(7); "".join(r.choice("ACGT") for _ in range(12)) + assert_eq!(PyRandom::new(7).bases(12), b"GCTAAAGACAAT"); + } + + #[test] + fn n_lines_fill_60_a_line_and_carry_a_partial_line() { + let mut v = Vec::new(); + assert_eq!(n_lines(&mut v, 130, 0).unwrap(), 10); + assert_eq!(v.len(), 132); + assert_eq!(n_lines(&mut v, 50, 10).unwrap(), 0); + assert_eq!(v.iter().filter(|&&b| b == b'\n').count(), 3); + } + + #[test] + fn unknown_kinds_are_left_to_the_caller_and_bad_arguments_exit_2() { + assert_eq!(run("no-such-kind", &[]).0, None); + let (code, _, err) = run("append-n", &["a", "b"]); + assert_eq!(code, Some(2)); + assert!( + err.contains("og dev fixture: error: takes SRC OUT CONTIG LENGTH"), + "{err}" + ); + let (code, _, err) = run( + "append-n", + &["/no/such/file", "/no/such/out", "chr11", "60"], + ); + assert_eq!(code, Some(1), "{err}"); + } +} diff --git a/rust/crates/og-dev/src/fixture/e2e/append_n.rs b/rust/crates/og-dev/src/fixture/e2e/append_n.rs new file mode 100644 index 000000000..17ab6f123 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/e2e/append_n.rs @@ -0,0 +1,61 @@ +//! `append-n SRC OUT CONTIG LENGTH`: a copy of the reference SRC with one +//! more contig of LENGTH `N`s, 60 a line. Step 35's Parascopy cases need a +//! chr11 the fixture reference lacks, because a model names it. + +use super::{Fail, create, exactly, n_lines, number}; +use std::fs::File; +use std::io::{self, Write}; +use std::path::Path; + +pub fn run(args: &[String]) -> Result<(), Fail> { + let a = exactly(args, 4, "SRC OUT CONTIG LENGTH")?; + let length = number(&a[3], "LENGTH")?; + let src = Path::new(&a[0]); + let mut input = File::open(src).map_err(|e| Fail::Run(format!("{}: {e}", src.display())))?; + let mut out = create(Path::new(&a[1]))?; + io::copy(&mut input, &mut out)?; + writeln!(out, ">{}", a[2])?; + if n_lines(&mut out, length, 0)? != 0 { + out.write_all(b"\n")?; + } + out.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::fixture::e2e::tests::{run, scratch}; + use std::fs; + + #[test] + fn copies_the_reference_then_the_n_contig() { + let d = scratch("append-n"); + fs::write(d.join("ref.fa"), ">chr1\nACGT\n").unwrap(); + let out = d.join("out.fa"); + let (code, _, err) = run( + "append-n", + &[ + d.join("ref.fa").to_str().unwrap(), + out.to_str().unwrap(), + "chr11", + "125", + ], + ); + assert_eq!(code, Some(0), "{err}"); + let want = format!(">chr1\nACGT\n>chr11\n{n}\n{n}\nNNNNN\n", n = "N".repeat(60)); + assert_eq!(fs::read_to_string(&out).unwrap(), want); + // 120 is two whole lines: no empty line after them. + run( + "append-n", + &[ + d.join("ref.fa").to_str().unwrap(), + out.to_str().unwrap(), + "chr11", + "120", + ], + ); + assert!(fs::read_to_string(&out).unwrap().ends_with("NNN\n")); + assert!(!fs::read_to_string(&out).unwrap().ends_with("\n\n")); + fs::remove_dir_all(d).unwrap(); + } +} diff --git a/rust/crates/og-dev/src/fixture/e2e/contains.rs b/rust/crates/og-dev/src/fixture/e2e/contains.rs new file mode 100644 index 000000000..951bd740f --- /dev/null +++ b/rust/crates/og-dev/src/fixture/e2e/contains.rs @@ -0,0 +1,83 @@ +//! `contains FILE PART`: exit 0 when FILE holds the bytes of PART anywhere, +//! 1 when it does not, 2 when either cannot be read. The paralog case asks +//! it whether the pinned Paraphase binary embeds its config byte for byte. + +use super::{Fail, read}; +use std::io::Write; +use std::path::Path; + +/// Whether `part` occurs in `hay`. An empty part occurs everywhere. +pub fn holds(hay: &[u8], part: &[u8]) -> bool { + let Some((&first, rest)) = part.split_first() else { + return true; + }; + if part.len() > hay.len() { + return false; + } + let last = hay.len() - part.len(); + let mut i = 0; + while i <= last { + match hay[i..=last].iter().position(|&b| b == first) { + None => return false, + Some(k) => { + let at = i + k; + if &hay[at + 1..at + part.len()] == rest { + return true; + } + i = at + 1; + } + } + } + false +} + +pub fn run(args: &[String], err: &mut dyn Write) -> u8 { + if args.len() != 2 { + let _ = writeln!(err, "og dev fixture contains: takes FILE PART"); + return 2; + } + let load = |p: &String| { + read(Path::new(p)).map_err(|f| match f { + Fail::Run(e) | Fail::Usage(e) => e, + }) + }; + match load(&args[0]).and_then(|hay| load(&args[1]).map(|part| (hay, part))) { + Ok((hay, part)) => u8::from(!holds(&hay, &part)), + Err(e) => { + let _ = writeln!(err, "og dev fixture contains: {e}"); + 2 + } + } +} + +#[cfg(test)] +mod tests { + use super::holds; + use crate::fixture::e2e::tests::{run, scratch}; + use std::fs; + + #[test] + fn finds_a_part_anywhere_and_only_whole() { + assert!(holds(b"xxsmn1:\n genes: Xyy", b"smn1:\n genes: X")); + assert!(holds(b"abc", b"abc")); + assert!(holds(b"abc", b"")); + assert!(holds(b"aab", b"ab")); + assert!(!holds(b"abc", b"abd")); + assert!(!holds(b"ab", b"abc")); + assert!(!holds(b"a\nb", b"a b")); + } + + #[test] + fn exit_codes() { + let d = scratch("contains"); + fs::write(d.join("bin"), b"\x00\x01data/38/config.yaml\n\xff").unwrap(); + fs::write(d.join("in"), b"config.yaml\n").unwrap(); + fs::write(d.join("out"), b"other.yaml\n").unwrap(); + let p = |n: &str| d.join(n).to_str().unwrap().to_owned(); + assert_eq!(run("contains", &[&p("bin"), &p("in")]).0, Some(0)); + assert_eq!(run("contains", &[&p("bin"), &p("out")]).0, Some(1)); + assert_eq!(run("contains", &[&p("bin"), &p("none")]).0, Some(2)); + assert_eq!(run("contains", &[&p("bin")]).0, Some(2)); + fs::remove_dir_all(d).unwrap(); + } +} diff --git a/rust/crates/og-dev/src/fixture/e2e/cut_haplotype.rs b/rust/crates/og-dev/src/fixture/e2e/cut_haplotype.rs new file mode 100644 index 000000000..3306a4038 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/e2e/cut_haplotype.rs @@ -0,0 +1,274 @@ +//! `cut-haplotype START END FQ NAMES < SAM`: plants a deletion of the +//! reference bases START..END-1 (1-based) in the haplotype-1 reads of a +//! long-read slice (`tests/e2e/paralogs-paraphase-1.sh`). +//! +//! Each primary alignment tagged `HP:i:1` that overlaps the bases loses +//! them: a read across the gap is joined over it, one that starts or ends +//! inside keeps the part outside if that is 1 kb or more, else it is +//! dropped. Kept reads go to FQ as FASTQ, their RG, rq, HP and PS tags in the +//! comment (for `minimap2 -y`); every read that overlaps goes to NAMES, so +//! the caller can leave the originals out. Prints the count of joined reads, +//! and on stderr the three counts. + +use super::{Fail, create, exactly, number}; +use std::io::{BufRead, Write}; +use std::path::Path; + +/// What happened to one read. +#[derive(Debug, PartialEq, Eq)] +pub enum Cut { + /// Not haplotype 1, not primary, or not over the bases. + Skip, + /// Joined across the gap, or trimmed: the FASTQ record. + Spanning(String), + Trimmed(String), + Dropped, +} + +/// The query offsets of the first base at or after `a` and `b`, and the +/// alignment's end (exclusive, 1-based). +fn offsets( + pos: usize, + cigar: &str, + a: usize, + b: usize, +) -> Result<(Option, Option, usize), String> { + let (mut rf, mut qry) = (pos, 0usize); + let (mut qa, mut qb) = (None, None); + let mut n = 0usize; + for c in cigar.bytes() { + if c.is_ascii_digit() { + n = n * 10 + usize::from(c - b'0'); + continue; + } + match c { + b'M' | b'=' | b'X' | b'D' | b'N' => { + let consumes = matches!(c, b'M' | b'=' | b'X'); + for (x, q) in [(a, &mut qa), (b, &mut qb)] { + if q.is_none() && rf + n > x { + *q = Some(if consumes { + qry + x.saturating_sub(rf) + } else { + qry + }); + } + } + rf += n; + if consumes { + qry += n; + } + } + b'I' | b'S' => qry += n, + b'H' | b'P' => {} + _ => return Err(format!("CIGAR '{cigar}' has an unknown operation")), + } + n = 0; + } + Ok((qa, qb, rf)) +} + +/// Cuts one SAM line. +pub fn cut(line: &str, a: usize, b: usize) -> Result { + let f: Vec<&str> = line.split('\t').collect(); + if f.len() < 11 { + return Err(format!("a SAM line with {} fields", f.len())); + } + let tags = &f[11..]; + let flag: u32 = f[1].parse().map_err(|_| format!("flag '{}'", f[1]))?; + if !tags.contains(&"HP:i:1") || flag & 0x900 != 0 { + return Ok(Cut::Skip); + } + let pos: usize = f[3].parse().map_err(|_| format!("POS '{}'", f[3]))?; + let (qa, qb, end) = offsets(pos, f[5], a, b)?; + if end <= a || pos >= b { + return Ok(Cut::Skip); + } + let head = |s: &str| -> String { + s.get(..qa.unwrap_or(0).min(s.len())) + .unwrap_or("") + .to_owned() + }; + let tail = |s: &str| -> String { + s.get(qb.unwrap_or(s.len()).min(s.len())..) + .unwrap_or("") + .to_owned() + }; + let (left, lq) = if pos < a { + (head(f[9]), head(f[10])) + } else { + (String::new(), String::new()) + }; + let (right, rq) = if end > b { + (tail(f[9]), tail(f[10])) + } else { + (String::new(), String::new()) + }; + let keep: Vec<&str> = tags + .iter() + .copied() + .filter(|t| { + ["RG", "rq", "HP", "PS"] + .iter() + .any(|k| t.get(..2) == Some(k)) + }) + .collect(); + let record = format!( + "@{}\t{}\n{left}{right}\n+\n{lq}{rq}\n", + f[0], + keep.join("\t") + ); + Ok(if !left.is_empty() && !right.is_empty() { + Cut::Spanning(record) + } else if left.len() + right.len() >= 1000 { + Cut::Trimmed(record) + } else { + Cut::Dropped + }) +} + +pub fn run( + args: &[String], + input: &mut dyn BufRead, + out: &mut dyn Write, + err: &mut dyn Write, +) -> Result<(), Fail> { + let x = exactly(args, 4, "START END FQ NAMES")?; + let (a, b) = (number(&x[0], "START")?, number(&x[1], "END")?); + let (mut fq, mut names) = (create(Path::new(&x[2]))?, create(Path::new(&x[3]))?); + let (mut spanning, mut trimmed, mut dropped) = (0u64, 0u64, 0u64); + let mut line = String::new(); + while { + line.clear(); + input.read_line(&mut line)? > 0 + } { + let l = line.strip_suffix('\n').unwrap_or(&line); + let c = cut(l, a, b).map_err(Fail::Run)?; + if c != Cut::Skip { + writeln!(names, "{}", l.split('\t').next().unwrap_or(""))?; + } + match c { + Cut::Skip => {} + Cut::Dropped => dropped += 1, + Cut::Spanning(r) => { + spanning += 1; + fq.write_all(r.as_bytes())?; + } + Cut::Trimmed(r) => { + trimmed += 1; + fq.write_all(r.as_bytes())?; + } + } + } + fq.flush()?; + names.flush()?; + writeln!( + err, + "haplotype-1 reads joined across the deletion: {spanning}, trimmed: {trimmed}, dropped: {dropped}" + )?; + writeln!(out, "{spanning}")?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sam(name: &str, flag: u32, pos: usize, cigar: &str, seq: &str, tags: &str) -> String { + format!( + "{name}\t{flag}\tchr16\t{pos}\t60\t{cigar}\t*\t0\t0\t{seq}\t{}\t{tags}", + "#".repeat(seq.len()) + ) + } + + #[test] + fn a_read_across_the_gap_is_joined_over_it() { + // Bases 100..199 (1-based) of a read at 1 are reference 1..200; cut 101..150. + let seq: String = (0..200).map(|i| char::from(b"ACGT"[i % 4])).collect(); + let line = sam("r", 0, 1, "200M", &seq, "RG:Z:x\tHP:i:1\tNM:i:0\tPS:i:5"); + let Cut::Spanning(r) = cut(&line, 101, 151).unwrap() else { + panic!() + }; + let want = format!( + "@r\tRG:Z:x\tHP:i:1\tPS:i:5\n{}{}\n+\n{}\n", + &seq[..100], + &seq[150..], + "#".repeat(150) + ); + assert_eq!(r, want); + } + + #[test] + fn insertions_and_deletions_move_the_offsets() { + // 10M 5I 10M 5D 10M from 1: ref 1-10, ins, 11-20, del 21-25, 26-35. + assert_eq!( + offsets(1, "10M5I10M5D10M", 13, 30).unwrap(), + (Some(17), Some(29), 36) + ); + // A cut point inside the deletion maps to the query base after it. + assert_eq!(offsets(1, "10M5I10M5D10M", 22, 30).unwrap().0, Some(25)); + assert_eq!(offsets(1, "5S10M", 3, 8).unwrap(), (Some(7), Some(12), 11)); + assert!(offsets(1, "10Q", 3, 8).is_err()); + } + + #[test] + fn others_are_skipped_short_ends_dropped_long_ends_trimmed() { + let seq = "A".repeat(1200); + let hp1 = "HP:i:1"; + assert_eq!( + cut(&sam("r", 0, 1, "1200M", &seq, "HP:i:2"), 500, 600).unwrap(), + Cut::Skip + ); + assert_eq!( + cut(&sam("r", 256, 1, "1200M", &seq, hp1), 500, 600).unwrap(), + Cut::Skip + ); + assert_eq!( + cut(&sam("r", 2048, 1, "1200M", &seq, hp1), 500, 600).unwrap(), + Cut::Skip + ); + assert_eq!( + cut(&sam("r", 0, 2000, "1200M", &seq, hp1), 500, 600).unwrap(), + Cut::Skip + ); + // Starts inside, keeps 1200 - 99 bases after 600: trimmed. + assert!(matches!( + cut(&sam("r", 0, 501, "1200M", &seq, hp1), 500, 600).unwrap(), + Cut::Trimmed(_) + )); + // Starts inside, keeps 1200 - 399 bases after 900: dropped. + assert_eq!( + cut(&sam("r", 0, 501, "1200M", &seq, hp1), 500, 900).unwrap(), + Cut::Dropped + ); + } + + #[test] + fn run_writes_the_fastq_the_names_and_the_count() { + let d = crate::fixture::e2e::tests::scratch("cut"); + let seq = "C".repeat(3000); + let input = [ + sam("span", 0, 1, "3000M", &seq, "HP:i:1"), + sam("other", 0, 1, "3000M", &seq, "HP:i:2"), + sam("drop", 0, 1500, "700M", &seq[..700], "HP:i:1"), + ] + .join("\n") + + "\n"; + let (mut out, mut err) = (Vec::new(), Vec::new()); + let p = |n: &str| d.join(n).to_str().unwrap().to_owned(); + let args = ["1100".to_owned(), "2100".to_owned(), p("fq"), p("names")]; + assert!(run(&args, &mut input.as_bytes(), &mut out, &mut err).is_ok()); + assert_eq!(String::from_utf8(out).unwrap(), "1\n"); + assert_eq!( + std::fs::read_to_string(d.join("names")).unwrap(), + "span\ndrop\n" + ); + let fq = std::fs::read_to_string(d.join("fq")).unwrap(); + assert_eq!(fq.lines().nth(1).unwrap().len(), 2000); + assert!( + String::from_utf8(err) + .unwrap() + .contains("joined across the deletion: 1, trimmed: 0, dropped: 1") + ); + std::fs::remove_dir_all(d).unwrap(); + } +} diff --git a/rust/crates/og-dev/src/fixture/e2e/hba_tiles.rs b/rust/crates/og-dev/src/fixture/e2e/hba_tiles.rs new file mode 100644 index 000000000..fecf78d52 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/e2e/hba_tiles.rs @@ -0,0 +1,145 @@ +//! `hba-tiles FASTA HEADER SAMPLE OUT`: forward 150-base reads every 5 +//! bases over the alpha-globin window chr16:95001-235000 (FASTA, as +//! `samtools faidx REF chr16:95001-235000` writes it), under HEADER's lines, +//! for step 35's HBA case (`tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh`). +//! +//! A read whose bases, or their reverse complement, occur more than once in +//! the window gets MAPQ 0, as an aligner gives a read that fits two places; +//! every other read MAPQ 60. A SAMPLE whose name ends in `3` loses every +//! other read that overlaps HBA2 (0-based 172875-173710), so HBA2 sits at +//! about half depth. Each read carries the first `@RG` of HEADER, if any. + +use super::{Fail, create, exactly, read, revcomp}; +use std::collections::HashMap; +use std::io::Write; +use std::path::Path; + +const START0: usize = 95_000; +const HBA2: (usize, usize) = (172_875, 173_710); +const L: usize = 150; + +/// The SAM records (no header) of the window `seq`. +pub fn records(seq: &[u8], sample: &str, rg: Option<&str>) -> String { + let mut seen: HashMap<&[u8], u32> = HashMap::new(); + for w in seq.windows(L) { + *seen.entry(w).or_default() += 1; + } + let count = |r: &[u8]| seen.get(r).copied().unwrap_or(0); + let half = sample.ends_with('3'); + let tag = rg + .filter(|g| !g.is_empty()) + .map(|g| format!("\tRG:Z:{g}")) + .unwrap_or_default(); + let q = "I".repeat(L); + let mut out = String::new(); + let mut n = 0u64; + for (i, r) in seq.windows(L).enumerate().step_by(5) { + let s = START0 + i; + if half && s < HBA2.1 && s + L > HBA2.0 { + n += 1; + if n % 2 == 1 { + continue; + } + } + let mapq = if count(r) + count(&revcomp(r)) > 1 { + 0 + } else { + 60 + }; + out += &format!( + "syn{s}\t0\tchr16\t{}\t{mapq}\t{L}M\t*\t0\t0\t{}\t{q}{tag}\n", + s + 1, + String::from_utf8_lossy(r) + ); + } + out +} + +pub fn run(args: &[String]) -> Result<(), Fail> { + let a = exactly(args, 4, "FASTA HEADER SAMPLE OUT")?; + let fasta = String::from_utf8_lossy(&read(Path::new(&a[0]))?).into_owned(); + let seq: Vec = fasta + .lines() + .filter(|l| !l.starts_with('>')) + .flat_map(|l| l.trim().bytes()) + .map(|b| b.to_ascii_uppercase()) + .collect(); + let header = read(Path::new(&a[1]))?; + let text = String::from_utf8_lossy(&header); + let rg = text + .lines() + .find(|l| l.starts_with("@RG")) + .and_then(|l| l.split('\t').nth(1)) + .map(|f| f.get(3..).unwrap_or("")); + let mut out = create(Path::new(&a[3]))?; + out.write_all(&header)?; + out.write_all(records(&seq, &a[2], rg).as_bytes())?; + out.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixture::e2e::PyRandom; + use crate::fixture::e2e::tests::fnv; + + /// 2000 random bases with a 300-base repeat, then its reverse complement. + fn window() -> Vec { + let mut s = PyRandom::new(9).bases(2000); + let rep = s[100..400].to_vec(); + s[1000..1300].copy_from_slice(&rep); + s[1500..1800].copy_from_slice(&revcomp(&rep)); + s + } + + #[test] + fn a_read_that_fits_twice_gets_mapq_0() { + let recs = records(&window(), "S4", Some("rg1")); + let mapq = |pos: usize| { + recs.lines() + .find(|l| l.split('\t').nth(3) == Some(&(START0 + pos + 1).to_string())) + .map(|l| l.split('\t').nth(4).unwrap().to_owned()) + }; + assert_eq!(mapq(0).as_deref(), Some("60")); + assert_eq!(mapq(150).as_deref(), Some("0")); // [150, 300) is inside the repeat + assert_eq!(mapq(1000).as_deref(), Some("0")); + assert_eq!(mapq(1550).as_deref(), Some("0")); // the reverse complement copy + assert_eq!(mapq(600).as_deref(), Some("60")); + assert!(recs.lines().all(|l| l.ends_with("\tRG:Z:rg1"))); + assert_eq!(recs.lines().count(), (2000 - L) / 5 + 1); + } + + #[test] + fn a_sample_ending_in_3_loses_every_other_read_over_hba2() { + let seq = PyRandom::new(9).bases(80_000); + let over = |r: &str| { + r.lines() + .filter(|l| { + let s: usize = l.split('\t').nth(3).unwrap().parse::().unwrap() - 1; + s < HBA2.1 && s + L > HBA2.0 + }) + .count() + }; + let (four, three) = ( + records(&seq, "HG002hba4", None), + records(&seq, "HG002hba3", None), + ); + assert_eq!(over(&four), 196); + assert_eq!(over(&three), 98); + assert_eq!(four.lines().count() - three.lines().count(), 98); + assert!(!four.contains("RG:Z")); + } + + // A pin of what the inline Python of + // tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh writes on window(). + #[test] + fn writes_the_pythons_records() { + assert_eq!( + fnv(records(&window(), "HG002hba3", Some("HG002")).as_bytes()), + PIN + ); + } + + const PIN: u64 = 12_081_885_455_233_452_406; +} diff --git a/rust/crates/og-dev/src/fixture/e2e/mito_sam.rs b/rust/crates/og-dev/src/fixture/e2e/mito_sam.rs new file mode 100644 index 000000000..70f11aa54 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/e2e/mito_sam.rs @@ -0,0 +1,252 @@ +//! `mito-sam --fasta FA --header SAM --sample S --out OUT [--depth X] +//! [--deletion FIRST-LAST] [--level F]`: single-end 150-base chrM reads at +//! depth X (default 400) for step 20's case on large deletions +//! (`tests/e2e/mito-extended-1-deletions.sh`), the synth-sam of +//! `tests/test_mito_extended.py` with its seed (`random.Random(1)`). +//! +//! With `--deletion`, a share F (default 0.1) of the molecules lack the +//! bases FIRST..LAST (1-based). A read across the junction is written as an +//! aligner writes it: aligned on its longer side, the rest soft-clipped (a +//! forward read clipped on the right, a reverse read on the left); one with +//! a side under 30 bases is left out. The reads go under FA's `@HD` and +//! `@SQ` lines from SAM and one `@RG` of S. Prints how many reads it wrote +//! and how many are soft-clipped. + +use super::{Fail, PyRandom, create, read}; +use std::io::Write; +use std::path::Path; + +const READ_LEN: usize = 150; + +/// One read: 1-based position, CIGAR, flag and bases. +type Synth = (usize, String, u16, Vec); + +/// The reads, sorted as Python sorts the tuples. +pub fn reads(seq: &[u8], depth: f64, deletion: Option<(usize, usize)>, level: f64) -> Vec { + let mut rnd = PyRandom::new(1); + // int(len(seq) * depth / read_len): truncation of a float, as Python. + let n = (seq.len() as f64 * depth / READ_LEN as f64) as usize; + let mut out = Vec::with_capacity(n); + let molecule = deletion.map(|(a, b)| ([&seq[..a - 1], &seq[b..]].concat(), a - 1)); + for _ in 0..n { + if let (Some((a, b)), Some((mol, junction))) = (deletion, &molecule) + && rnd.random() < level + { + let s = rnd.below(mol.len() - READ_LEN); + let r = mol[s..s + READ_LEN].to_vec(); + if *junction <= s { + out.push(( + s + 1 + (b - a + 1), + format!("{READ_LEN}M"), + rnd.choice(&[0, 16]), + r, + )); + continue; + } + let left = junction - s; + if left >= READ_LEN { + out.push((s + 1, format!("{READ_LEN}M"), rnd.choice(&[0, 16]), r)); + } else if left.min(READ_LEN - left) < 30 { + continue; + } else if left >= READ_LEN - left { + out.push((s + 1, format!("{left}M{}S", READ_LEN - left), 0, r)); + } else { + out.push((b + 1, format!("{left}S{}M", READ_LEN - left), 16, r)); + } + continue; + } + let s = rnd.below(seq.len() - READ_LEN); + out.push(( + s + 1, + format!("{READ_LEN}M"), + rnd.choice(&[0, 16]), + seq[s..s + READ_LEN].to_vec(), + )); + } + out.sort(); + out +} + +struct Opts { + fasta: String, + header: String, + sample: String, + out: String, + depth: f64, + deletion: Option<(usize, usize)>, + level: f64, +} + +fn parse(args: &[String]) -> Result { + let (mut fasta, mut header, mut sample, mut out) = (None, None, None, None); + let (mut depth, mut deletion, mut level) = (400.0, None, 0.1); + let float = |v: &str, k: &str| { + v.parse::() + .ok() + .filter(|x| x.is_finite()) + .ok_or_else(|| Fail::Usage(format!("{k} '{v}' is not a number"))) + }; + let mut it = args.iter(); + while let Some(k) = it.next() { + let v = it + .next() + .ok_or_else(|| Fail::Usage(format!("{k} takes a value")))? + .clone(); + match k.as_str() { + "--fasta" => fasta = Some(v), + "--header" => header = Some(v), + "--sample" => sample = Some(v), + "--out" => out = Some(v), + "--depth" => depth = float(&v, k)?, + "--level" => level = float(&v, k)?, + "--deletion" => { + let span = v + .split_once('-') + .and_then(|(a, b)| Some((a.parse::().ok()?, b.parse::().ok()?))) + .filter(|&(a, b)| 1 < a && a <= b) + .ok_or_else(|| Fail::Usage(format!("--deletion '{v}' is not FIRST-LAST")))?; + deletion = Some(span); + } + _ => return Err(Fail::Usage(format!("unknown option '{k}'"))), + } + } + let need = + |o: Option, k: &str| o.ok_or_else(|| Fail::Usage(format!("{k} is required"))); + Ok(Opts { + fasta: need(fasta, "--fasta")?, + header: need(header, "--header")?, + sample: need(sample, "--sample")?, + out: need(out, "--out")?, + depth, + deletion, + level, + }) +} + +pub fn run(args: &[String], stdout: &mut dyn Write) -> Result<(), Fail> { + let o = parse(args)?; + let fasta = String::from_utf8_lossy(&read(Path::new(&o.fasta))?).into_owned(); + let seq: Vec = fasta + .lines() + .filter(|l| !l.starts_with('>')) + .flat_map(|l| l.trim().bytes()) + .map(|b| b.to_ascii_uppercase()) + .collect(); + if let Some((_, b)) = o.deletion + && b + READ_LEN >= seq.len() + { + return Err(Fail::Run(format!( + "--deletion ends past {} bases of chrM", + seq.len() + ))); + } + if seq.len() <= READ_LEN { + return Err(Fail::Run(format!( + "{}: fewer than {READ_LEN} bases", + o.fasta + ))); + } + let header = String::from_utf8_lossy(&read(Path::new(&o.header))?).into_owned(); + let reads = reads(&seq, o.depth, o.deletion, o.level); + let mut w = create(Path::new(&o.out))?; + for line in header.split_inclusive('\n') { + if line.starts_with("@HD") || line.starts_with("@SQ") { + w.write_all(line.as_bytes())?; + } + } + let s = &o.sample; + writeln!(w, "@RG\tID:{s}\tSM:{s}\tPL:ILLUMINA\tLB:{s}")?; + for (i, (pos, cigar, flag, r)) in reads.iter().enumerate() { + writeln!( + w, + "r{i}\t{flag}\tchrM\t{pos}\t60\t{cigar}\t*\t0\t0\t{}\t{}\tRG:Z:{s}", + String::from_utf8_lossy(r), + "I".repeat(r.len()) + )?; + } + w.flush()?; + let clipped = reads.iter().filter(|r| r.1.contains('S')).count(); + writeln!( + stdout, + "{} reads, {clipped} soft-clipped at the junction", + reads.len() + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixture::e2e::tests::fnv; + + /// A made-up chrM: 16569 bases of `random.Random(5)`. + fn chrm() -> Vec { + PyRandom::new(5).bases(16569) + } + + fn text(r: &[Synth]) -> String { + r.iter() + .map(|(p, c, f, s)| format!("{p} {c} {f} {}\n", String::from_utf8_lossy(s))) + .collect() + } + + // Pins of what tests/test_mito_extended.py's synth_reads gives on the + // same chrM, with and without the common deletion. + #[test] + fn gives_the_pythons_reads() { + let seq = chrm(); + let del = reads(&seq, 50.0, Some((8470, 13446)), 0.1); + let wt = reads(&seq, 50.0, None, 0.1); + assert_eq!((del.len(), wt.len()), PIN_COUNTS); + assert_eq!( + (fnv(text(&del).as_bytes()), fnv(text(&wt).as_bytes())), + PIN_HASHES + ); + } + + #[test] + fn junction_reads_are_clipped_on_their_shorter_side() { + let seq = chrm(); + let del = reads(&seq, 200.0, Some((8470, 13446)), 0.5); + let clipped: Vec<&Synth> = del.iter().filter(|r| r.1.contains('S')).collect(); + assert!(clipped.len() > 10, "{}", clipped.len()); + for (pos, cigar, flag, _) in clipped { + if cigar.ends_with('S') { + assert_eq!(*flag, 0); + assert!(*pos < 8470); + } else { + assert_eq!((*flag, *pos), (16, 13447)); + } + } + assert!(reads(&seq, 200.0, None, 0.5).iter().all(|r| r.1 == "150M")); + } + + #[test] + fn bad_options() { + use crate::fixture::e2e::tests::run; + assert_eq!(run("mito-sam", &["--fasta", "a"]).0, Some(2)); + assert_eq!( + run( + "mito-sam", + &[ + "--deletion", + "9-3", + "--fasta", + "a", + "--header", + "h", + "--sample", + "s", + "--out", + "o" + ] + ) + .0, + Some(2) + ); + assert_eq!(run("mito-sam", &["--bogus", "1"]).0, Some(2)); + } + + const PIN_COUNTS: (usize, usize) = (5520, 5523); + const PIN_HASHES: (u64, u64) = (1_728_179_969_328_416_176, 8_620_160_239_877_651_625); +} diff --git a/rust/crates/og-dev/src/fixture/e2e/rfc1.rs b/rust/crates/og-dev/src/fixture/e2e/rfc1.rs new file mode 100644 index 000000000..e604241f5 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/e2e/rfc1.rs @@ -0,0 +1,365 @@ +//! `rfc1 KIND OUT.sam [--samd12 UNIT]`: made-up reads at the GRCh38 RFC1 +//! repeat (chr4:39348425-39348479, the reference (AAAAG)11) with random +//! flanks, the read sets of `tests/test_rfc1_motif.py`, as unsorted SAM: +//! +//! - `benign`: reads across an expanded AAAAG allele, and reads whose AAAAG +//! part is soft-clipped; +//! - `pathogenic`: AAGGG reads soft-clipped next to the repeat, unmapped and +//! placed at their mate, mates mapped to chr1, and reverse-strand reads +//! across the repeat; three reads of an expanded AAAAG allele; eight reads +//! across the unexpanded allele; a duplicate, a secondary, a QC-failed +//! alignment, a homopolymer and a run of three units; +//! - `mixed` (AAAAG and AAAGG), `acagg`, `unknown` (AAGAG) and `few` (two +//! AAGGG reads). +//! +//! `--samd12 UNIT` adds chr8 and, at SAMD12, five reads of UNIT x30 and six +//! reads of the reference allele (TAAAA)20 TGAAA with a random flank. + +use super::{Fail, PyRandom, create, revcomp}; +use std::io::Write; +use std::path::Path; + +pub const CHROM: &str = "chr4"; +pub const OTHER: &str = "chr1"; +/// 0-based start and end (exclusive) of the repeat: 55 bases. +pub const START: usize = 39_348_424; +pub const END: usize = 39_348_479; +const KINDS: [&str; 6] = ["benign", "pathogenic", "mixed", "acagg", "unknown", "few"]; + +/// One alignment, as the SAM line needs it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Read { + pub name: String, + pub contig: &'static str, + /// 0-based. + pub start: usize, + /// Empty for an unmapped read. + pub cigar: String, + pub seq: Vec, + pub read1: bool, + pub unmapped: bool, + pub reverse: bool, + pub dup: bool, + pub secondary: bool, + pub qcfail: bool, + pub mate: (&'static str, usize), +} + +impl Read { + fn new(name: String, contig: &'static str, start: usize, cigar: &str, seq: Vec) -> Self { + Read { + name, + contig, + start, + cigar: cigar.to_owned(), + seq, + read1: true, + unmapped: false, + reverse: false, + dup: false, + secondary: false, + qcfail: false, + mate: (contig, start), + } + } + + pub fn flag(&self) -> u16 { + 1 | if self.read1 { 64 } else { 128 } + | if self.unmapped { 4 } else { 0 } + | if self.reverse { 16 } else { 0 } + | if self.secondary { 256 } else { 0 } + | if self.qcfail { 512 } else { 0 } + | if self.dup { 1024 } else { 0 } + } + + /// The SAM line, with no tags and a newline. + pub fn sam(&self) -> String { + let mate = if self.mate.0 == self.contig { + "=" + } else { + self.mate.0 + }; + format!( + "{}\t{}\t{}\t{}\t{}\t{}\t{mate}\t{}\t0\t{}\t{}\n", + self.name, + self.flag(), + self.contig, + self.start + 1, + if self.unmapped { 0 } else { 60 }, + if self.cigar.is_empty() { + "*" + } else { + &self.cigar + }, + self.mate.1 + 1, + String::from_utf8_lossy(&self.seq), + "I".repeat(self.seq.len()), + ) + } +} + +/// The 300 random bases either side of the repeat: `random.Random(7)`. +pub struct Flanks { + pub left: Vec, + pub right: Vec, +} + +impl Flanks { + pub fn new() -> Self { + let mut rng = PyRandom::new(7); + let left = rng.bases(300); + let right = rng.bases(300); + Flanks { left, right } + } + + fn left_tail(&self, n: usize) -> &[u8] { + &self.left[self.left.len() - n..] + } + + /// A 150-base read aligned across the repeat: `offset` flank bases, the + /// repeat (55 reference bases plus an insertion of the rest), flank. + fn spanning(&self, name: String, unit: &str, units: usize) -> Read { + let offset = 40; + let rep = unit.repeat(units); + let right = 150 - offset - rep.len(); + let ins = rep.len() - (END - START); + let mut cigar = format!("{offset}M{}M", END - START); + if ins > 0 { + cigar += &format!("{ins}I"); + } + cigar += &format!("{right}M"); + let seq = [self.left_tail(offset), rep.as_bytes(), &self.right[..right]].concat(); + Read::new(name, CHROM, START - offset, &cigar, seq) + } + + /// 60 aligned bases that end at the repeat start, 90 soft-clipped bases. + fn clipped(&self, name: String, unit: &str) -> Read { + self.clip(name, unit.repeat(18).as_bytes()) + } + + fn clip(&self, name: String, tail: &[u8]) -> Read { + let seq = [self.left_tail(60), tail].concat(); + Read::new(name, CHROM, START - 60, "60M90S", seq) + } + + /// (reads at the locus, mates elsewhere) of one kind. + pub fn case(&self, kind: &str) -> (Vec, Vec) { + let (mut at, mut elsewhere) = (Vec::new(), Vec::new()); + let n = |p: &str, i: usize| format!("{p}{i}"); + match kind { + "benign" => { + at.extend((0..6).map(|i| self.spanning(n("b", i), "AAAAG", 20))); + at.extend((0..4).map(|i| self.clipped(n("bc", i), "AAAAG"))); + } + "pathogenic" => { + at.extend((0..6).map(|i| self.clipped(n("pc", i), "AAGGG"))); + for i in 0..2 { + let mut r = Read::new( + n("pu", i), + CHROM, + START - 100, + "", + "AAGGG".repeat(30).into_bytes(), + ); + r.read1 = false; + r.unmapped = true; + at.push(r); + } + for i in 0..3 { + let mut a = Read::new( + n("pm", i), + CHROM, + START - 120, + "100M", + self.left_tail(120)[..100].to_vec(), + ); + a.mate = (OTHER, 5000); + let mut b = Read::new( + n("pm", i), + OTHER, + 5000, + "150M", + "AAGGG".repeat(30).into_bytes(), + ); + b.read1 = false; + b.mate = (CHROM, START - 120); + at.push(a); + elsewhere.push(b); + } + for i in 0..2 { + let fwd = [self.left_tail(40), "AAGGG".repeat(22).as_bytes()].concat(); + let mut r = Read::new(n("pr", i), CHROM, START - 40, "150M", revcomp(&fwd)); + r.reverse = true; + at.push(r); + } + at.extend((0..3).map(|i| self.spanning(n("aaaag", i), "AAAAG", 20))); + at.extend((0..8).map(|i| self.spanning(n("ref", i), "AAAAG", 11))); + at.push(self.clipped("pc0".into(), "AAGGG")); + let tail = "AAGGG".repeat(18); + let mut dup = self.clip("dup".into(), tail.as_bytes()); + dup.dup = true; + let mut sec = self.clip("sec".into(), tail.as_bytes()); + sec.secondary = true; + let mut qc = self.clip("qc".into(), tail.as_bytes()); + qc.qcfail = true; + at.extend([dup, sec, qc]); + at.push(self.clip("poly".into(), &[b'A'; 90])); + let short = ["AAGGG".repeat(3).as_bytes(), &self.right[..75]].concat(); + at.push(self.clip("short".into(), &short)); + } + "mixed" => { + at.extend((0..6).map(|i| self.spanning(n("m", i), "AAAAG", 20))); + at.extend((0..4).map(|i| self.spanning(n("n", i), "AAAGG", 20))); + } + "acagg" => at.extend((0..5).map(|i| self.clipped(n("a", i), "ACAGG"))), + "unknown" => at.extend((0..5).map(|i| self.clipped(n("u", i), "AAGAG"))), + "few" => at.extend((0..2).map(|i| self.clipped(n("f", i), "AAGGG"))), + _ => {} + } + (at, elsewhere) + } +} + +impl Default for Flanks { + fn default() -> Self { + Self::new() + } +} + +/// The SAMD12 reads: five of `unit` x30 and six of the reference allele, +/// with a 60-base flank of `random.Random(3)`. +fn samd12(unit: &str) -> String { + let start = 118_366_812; + let mut s = String::new(); + let seq = unit.repeat(30); + for i in 0..5 { + s += &format!( + "f{i}\t0\tchr8\t{}\t60\t150M\t*\t0\t0\t{seq}\t{}\n", + start + 1, + "I".repeat(seq.len()) + ); + } + let left = String::from_utf8(PyRandom::new(3).bases(60)).expect("ACGT"); + let allele = format!("{left}{}TGAAA{}", "TAAAA".repeat(20), &left[..30]); + for i in 0..6 { + s += &format!( + "fr{i}\t0\tchr8\t{}\t60\t{}M\t*\t0\t0\t{allele}\t{}\n", + start - 60 + 1, + allele.len(), + "I".repeat(allele.len()) + ); + } + s +} + +/// The whole SAM of one kind. +pub fn sam(kind: &str, samd12_unit: Option<&str>) -> String { + let mut s = format!( + "@HD\tVN:1.6\tSO:unsorted\n@SQ\tSN:{CHROM}\tLN:190214555\n@SQ\tSN:{OTHER}\tLN:248956422\n" + ); + if samd12_unit.is_some() { + s += "@SQ\tSN:chr8\tLN:145138636\n"; + } + let (at, elsewhere) = Flanks::new().case(kind); + for r in at.iter().chain(&elsewhere) { + s += &r.sam(); + } + if let Some(unit) = samd12_unit { + s += &samd12(unit); + } + s +} + +pub fn run(args: &[String]) -> Result<(), Fail> { + let (unit, rest) = match args { + [k, o, flag, u] if flag == "--samd12" => (Some(u.as_str()), [k, o]), + [k, o] => (None, [k, o]), + _ => { + return Err(Fail::Usage( + "rfc1 takes KIND OUT.sam [--samd12 UNIT]".into(), + )); + } + }; + if !KINDS.contains(&rest[0].as_str()) { + return Err(Fail::Usage(format!( + "rfc1 KIND '{}' is not one of {}", + rest[0], + KINDS.join(", ") + ))); + } + if unit.is_some_and(|u| u.is_empty() || !u.bytes().all(|b| b"ACGT".contains(&b))) { + return Err(Fail::Usage("--samd12 UNIT is bases of ACGT".into())); + } + let mut out = create(Path::new(rest[1]))?; + out.write_all(sam(rest[0], unit).as_bytes())?; + out.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixture::e2e::tests::fnv; + + // The pins are of the bytes the inline Python of + // tests/e2e/expansions-and-rfc1-1-motif.sh and str-catalog-1-strchive.sh + // wrote, from tests/test_rfc1_motif.py's case(). + #[test] + fn writes_the_pythons_bytes() { + assert_eq!(fnv(sam("pathogenic", None).as_bytes()), PIN_PATHOGENIC); + assert_eq!(fnv(sam("benign", None).as_bytes()), PIN_BENIGN); + assert_eq!(fnv(sam("pathogenic", Some("TTTCA")).as_bytes()), PIN_SAMD12); + } + + #[test] + fn the_pathogenic_case_has_what_the_e2e_case_counts() { + let (at, elsewhere) = Flanks::new().case("pathogenic"); + let clipped = at + .iter() + .filter(|r| r.cigar == "60M90S" && r.seq.ends_with(b"AAGGGAAGGG")) + .count(); + // 6 soft-clipped plus the same read again, the duplicate, the secondary and the QC-failed one. + assert_eq!(clipped, 10); + assert_eq!(at.iter().filter(|r| r.unmapped).count(), 2); + assert_eq!(elsewhere.len(), 3); + assert!( + elsewhere + .iter() + .all(|r| r.contig == OTHER && r.mate == (CHROM, START - 120)) + ); + assert_eq!(at.iter().filter(|r| r.reverse).count(), 2); + assert!(at.iter().all(|r| r.seq.len() == 150 || r.cigar == "100M")); + let line = at.iter().find(|r| r.name == "pu0").unwrap().sam(); + assert!( + line.starts_with("pu0\t133\tchr4\t39348325\t0\t*\t=\t39348325\t0\t"), + "{line}" + ); + let line = at.iter().find(|r| r.name == "pm0").unwrap().sam(); + assert!( + line.starts_with("pm0\t65\tchr4\t39348305\t60\t100M\tchr1\t5001\t0\t"), + "{line}" + ); + } + + #[test] + fn spanning_reads_insert_what_the_reference_lacks() { + let f = Flanks::new(); + assert_eq!(f.spanning("x".into(), "AAAAG", 20).cigar, "40M55M45I10M"); + assert_eq!(f.spanning("x".into(), "AAAAG", 11).cigar, "40M55M55M"); + } + + #[test] + fn bad_arguments() { + use crate::fixture::e2e::tests::run; + assert_eq!(run("rfc1", &["nope", "/dev/null"]).0, Some(2)); + assert_eq!( + run("rfc1", &["benign", "/dev/null", "--samd12", "XY"]).0, + Some(2) + ); + assert_eq!(run("rfc1", &["benign"]).0, Some(2)); + } + + const PIN_PATHOGENIC: u64 = 7_022_458_020_176_947_217; + const PIN_BENIGN: u64 = 5_018_501_953_293_861_024; + const PIN_SAMD12: u64 = 5_496_064_447_239_138_924; +} diff --git a/rust/crates/og-dev/src/fixture/e2e/str_extra.rs b/rust/crates/og-dev/src/fixture/e2e/str_extra.rs new file mode 100644 index 000000000..8246dbbec --- /dev/null +++ b/rust/crates/og-dev/src/fixture/e2e/str_extra.rs @@ -0,0 +1,294 @@ +//! `str-extra REF.fa HET_NAME HET.sam HOM_NAME HOM.sam`: the made-up +//! reference and reads of the two cases on the repeats STRchive leaves out +//! (`assets/str_extra_loci.json`). +//! +//! REF.fa holds chr7 and chr2 of the given lengths, N everywhere except 1500 +//! random bases (`random.Random(11)`) either side of the GRCh38 tract at its +//! GRCh38 place: CFTR TG11 T7 and UGT1A1 A(TA)7. Each sample's SAM holds +//! read pairs every 4 bases from each of its two haplotypes, the CIGAR +//! putting the tract's length change at the tract's end: +//! +//! - HET: CFTR TG12-T5 and TG11-T7, UGT1A1 TA7 and TA8; +//! - HOM: CFTR TG11-T7 and UGT1A1 TA7 on both copies. + +use super::{Fail, PyRandom, create, exactly}; +use std::fmt::Write as _; +use std::io::{self, Write}; +use std::path::Path; + +const W: usize = 1500; + +/// A locus: contig, contig length, 0-based tract start, the bases before +/// and after the tract, and the reference tract. +struct Locus { + contig: &'static str, + length: usize, + start: usize, + left: &'static str, + right: &'static str, + tract: &'static [(&'static str, usize)], +} + +const LOCI: [Locus; 2] = [ + Locus { + contig: "chr7", + length: 117_551_000, + start: 117_548_606, + left: "TTTTGA", + right: "AACAGG", + tract: &[("TG", 11), ("T", 7)], + }, + Locus { + contig: "chr2", + length: 233_763_000, + start: 233_760_234, + left: "TTGCCA", + right: "AGTAGG", + tract: &[("TA", 7)], + }, +]; + +type Hap = &'static [(&'static str, usize)]; +const HET: [[Hap; 2]; 2] = [ + [&[("TG", 12), ("T", 5)], &[("TG", 11), ("T", 7)]], + [&[("TA", 7)], &[("TA", 8)]], +]; +const HOM: [[Hap; 2]; 2] = [ + [&[("TG", 11), ("T", 7)], &[("TG", 11), ("T", 7)]], + [&[("TA", 7)], &[("TA", 7)]], +]; + +fn tract(units: Hap) -> String { + units.iter().map(|(m, n)| m.repeat(*n)).collect() +} + +/// The random flanks of each locus, in order. +fn flanks() -> Vec<(Vec, Vec)> { + let mut rng = PyRandom::new(11); + LOCI.iter() + .map(|l| { + let mut lf = rng.bases(W - l.left.len()); + lf.extend_from_slice(l.left.as_bytes()); + let mut rf = l.right.as_bytes().to_vec(); + rf.extend(rng.bases(W - l.right.len())); + (lf, rf) + }) + .collect() +} + +/// Writes FASTA lines of 60, a run of `N`s or bases at a time. +pub(crate) struct Lines<'a> { + w: &'a mut dyn Write, + col: usize, +} + +impl<'a> Lines<'a> { + pub(crate) fn new(w: &'a mut dyn Write) -> Self { + Lines { w, col: 0 } + } + + pub(crate) fn ns(&mut self, n: usize) -> io::Result<()> { + self.col = super::n_lines(self.w, n, self.col)?; + Ok(()) + } + + pub(crate) fn bases(&mut self, mut s: &[u8]) -> io::Result<()> { + while !s.is_empty() { + let k = s.len().min(60 - self.col); + self.w.write_all(&s[..k])?; + self.col += k; + s = &s[k..]; + if self.col == 60 { + self.w.write_all(b"\n")?; + self.col = 0; + } + } + Ok(()) + } + + /// Ends a partial last line. + pub(crate) fn end(&mut self) -> io::Result<()> { + if self.col > 0 { + self.w.write_all(b"\n")?; + self.col = 0; + } + Ok(()) + } +} + +fn reference(w: &mut dyn Write, flanks: &[(Vec, Vec)]) -> io::Result<()> { + for (l, (lf, rf)) in LOCI.iter().zip(flanks) { + writeln!(w, ">{}", l.contig)?; + let t = tract(l.tract); + let mut lines = Lines::new(w); + lines.ns(l.start - W)?; + lines.bases(lf)?; + lines.bases(t.as_bytes())?; + lines.bases(rf)?; + lines.ns(l.length - (l.start - W + lf.len() + t.len() + rf.len()))?; + lines.end()?; + } + Ok(()) +} + +/// The 1-based POS and CIGAR of haplotype bases [a, a+150), or `None` when +/// the read starts or ends inside the tract. +fn place( + a: usize, + w0: usize, + t0: usize, + t1: usize, + hap_len: usize, + ref_len: usize, +) -> Option<(usize, String)> { + const N: usize = 150; + if a + N <= t0 { + return Some((w0 + a + 1, format!("{N}M"))); + } + if a >= t1 { + return Some(((w0 + a + 1 + ref_len) - hap_len, format!("{N}M"))); + } + if a >= t0 || a + N <= t1 { + return None; + } + let pre = t0 - a + hap_len.min(ref_len); + let post = N - (t0 - a) - hap_len; + let cigar = match hap_len.cmp(&ref_len) { + std::cmp::Ordering::Equal => format!("{N}M"), + std::cmp::Ordering::Greater => format!("{pre}M{}I{post}M", hap_len - ref_len), + std::cmp::Ordering::Less => format!("{pre}M{}D{post}M", ref_len - hap_len), + }; + Some((w0 + a + 1, cigar)) +} + +fn reads(out: &mut String, l: &Locus, (lf, rf): &(Vec, Vec), haps: &[Hap; 2], tag: &str) { + let w0 = l.start - W; + let ref_len = tract(l.tract).len(); + let q = "I".repeat(150); + for (h, units) in haps.iter().enumerate() { + let t = tract(units); + let seq = [lf.as_slice(), t.as_bytes(), rf.as_slice()].concat(); + let (t0, t1) = (lf.len(), lf.len() + t.len()); + let mut k = 0; + for a in (0..seq.len() - 400).step_by(4) { + let b = a + 300 + (a % 7) * 10 - 150; + let (Some((p1, c1)), Some((p2, c2))) = ( + place(a, w0, t0, t1, t.len(), ref_len), + place(b, w0, t0, t1, t.len(), ref_len), + ) else { + continue; + }; + k += 1; + let name = format!("{tag}{}h{h}r{k}", l.contig); + let (r1, r2) = ( + String::from_utf8_lossy(&seq[a..a + 150]), + String::from_utf8_lossy(&seq[b..b + 150]), + ); + let tlen = p2 + 150 - p1; + let c = l.contig; + let _ = writeln!( + out, + "{name}\t99\t{c}\t{p1}\t60\t{c1}\t=\t{p2}\t{tlen}\t{r1}\t{q}\tRG:Z:{tag}" + ); + let _ = writeln!( + out, + "{name}\t147\t{c}\t{p2}\t60\t{c2}\t=\t{p1}\t-{tlen}\t{r2}\t{q}\tRG:Z:{tag}" + ); + } + } +} + +/// The SAM of one sample. +fn sam(name: &str, haps: &[[Hap; 2]; 2], flanks: &[(Vec, Vec)]) -> String { + let mut s = String::from("@HD\tVN:1.6\tSO:unsorted\n"); + for l in &LOCI { + let _ = writeln!(s, "@SQ\tSN:{}\tLN:{}", l.contig, l.length); + } + let _ = writeln!(s, "@RG\tID:{name}\tSM:{name}"); + for ((l, f), h) in LOCI.iter().zip(flanks).zip(haps) { + reads(&mut s, l, f, h, name); + } + s +} + +pub fn run(args: &[String]) -> Result<(), Fail> { + let a = exactly(args, 5, "REF.fa HET_NAME HET.sam HOM_NAME HOM.sam")?; + let f = flanks(); + let mut out = create(Path::new(&a[0]))?; + reference(&mut out, &f)?; + out.flush()?; + for (name, path, haps) in [(&a[1], &a[2], &HET), (&a[3], &a[4], &HOM)] { + let mut out = create(Path::new(path))?; + out.write_all(sam(name, haps, &f).as_bytes())?; + out.flush()?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixture::e2e::tests::{Hashed, fnv}; + + // Pins of the bytes the inline Python of + // tests/e2e/genotype-rule-rows-1-str-extra.sh wrote (samples EXA, EXB). + #[test] + fn writes_the_pythons_reads() { + let f = flanks(); + assert_eq!(fnv(sam("EXA", &HET, &f).as_bytes()), PIN_EXA); + assert_eq!(fnv(sam("EXB", &HOM, &f).as_bytes()), PIN_EXB); + } + + #[test] + fn the_window_holds_the_grch38_tract_at_its_place() { + let f = flanks(); + let (lf, rf) = &f[0]; + let win = [ + lf.as_slice(), + tract(LOCI[0].tract).as_bytes(), + rf.as_slice(), + ] + .concat(); + // chr7:117548601-117548641, 1-based, starts 5 bases before the tract. + let at = 117_548_600 - (LOCI[0].start - W); + assert_eq!( + &win[at..at + 41], + b"TTTTGATGTGTGTGTGTGTGTGTGTGTGTTTTTTTAACAGG" + ); + } + + #[test] + fn the_reference_is_n_outside_the_windows_and_60_a_line() { + // Both contigs are 351 MB: hash them on the way, line by line. + let mut h = Hashed::default(); + reference(&mut h, &flanks()).unwrap(); + assert_eq!( + h.lines, + 2 + 117_551_000_usize.div_ceil(60) + 233_763_000_usize.div_ceil(60) + ); + assert_eq!(h.longest, 60); + assert_eq!(h.hash, PIN_REF); + } + + #[test] + fn a_read_across_a_longer_tract_carries_the_insertion() { + // TG12-T5 is 29 bases against the reference's 29: no change; TA8 is 2 longer. + assert_eq!( + place(1400, 0, 1500, 1516, 16, 14), + Some((1401, "114M2I34M".into())) + ); + assert_eq!( + place(1400, 0, 1500, 1512, 12, 14), + Some((1401, "112M2D38M".into())) + ); + assert_eq!( + place(1520, 0, 1500, 1516, 16, 14), + Some((1519, "150M".into())) + ); + assert_eq!(place(1505, 0, 1500, 1516, 16, 14), None); + } + + const PIN_EXA: u64 = 12_879_245_652_337_382_645; + const PIN_EXB: u64 = 10_035_011_117_999_463_469; + const PIN_REF: u64 = 8_076_056_084_048_732_156; +} diff --git a/rust/crates/og-dev/src/fixture/e2e/tas2r38.rs b/rust/crates/og-dev/src/fixture/e2e/tas2r38.rs new file mode 100644 index 000000000..c5ecf1936 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/e2e/tas2r38.rs @@ -0,0 +1,290 @@ +//! `tas2r38 REF_DIR GENOME_DIR`: the made-up inputs of the step 42 rule +//! case on TAS2R38 (`tests/e2e/genotype-rules-1-phase.sh`). +//! +//! - `REF_DIR/chr7.fa`: chr7 of GRCh38's length, N everywhere except a 2.6 +//! kb window of random bases (`random.seed(42)`) whose three TAS2R38 sites +//! carry the GRCh38 bases: rs10246939 T, rs1726866 G, rs713598 C. +//! - GRSPAN, GRSWAP and GRNOSPAN: a gVCF heterozygous at the three sites +//! between reference blocks of GQ 42, the variant-only VCF, and reads: +//! pairs spanning the sites from the PAV and AVI copies (GRSPAN), from the +//! G,A,C and C,G,T copies (GRSWAP), or single 60-base reads that each cover +//! one site (GRNOSPAN). +//! - GRREF: a gVCF that is one reference block, and no reads. +//! +//! Each goes to `GENOME_DIR//vcf/.g.vcf`, `.vcf` and +//! `GENOME_DIR//aligned/.sam`. + +use super::str_extra::Lines; +use super::{Fail, PyRandom, create, exactly}; +use std::fmt::Write as _; +use std::io::Write; +use std::path::Path; + +/// GRCh38 chr7's length, and the window [W0, W1), 1-based. +const LEN: usize = 159_345_973; +const W0: usize = 141_972_000; +const W1: usize = 141_974_600; +/// rs10246939, rs1726866 and rs713598, in the order of their positions. +const S3: usize = 141_972_804; +const S2: usize = 141_972_905; +const S1: usize = 141_973_545; +const SITES: [(usize, u8, u8); 3] = [(S3, b'T', b'C'), (S2, b'G', b'A'), (S1, b'C', b'G')]; + +/// The window's bases, with the GRCh38 bases at the three sites. +fn window() -> Vec { + let mut win = PyRandom::new(42).bases(W1 - W0); + for (p, b, _) in SITES { + win[p - W0] = b; + } + win +} + +fn base(win: &[u8], p: usize) -> char { + char::from(win[p - W0]) +} + +/// The window with `alleles` at S1, S2 and S3 (in that order). +fn hap(win: &[u8], alleles: &[u8; 3]) -> Vec { + let mut h = win.to_vec(); + for (p, &b) in [S1, S2, S3].iter().zip(alleles) { + h[p - W0] = b; + } + h +} + +fn gvcf(win: &[u8], het: bool) -> String { + let mut s = format!( + "##fileformat=VCFv4.2\n##contig=\n\ + ##ALT=\n\ + ##INFO=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + ##FORMAT=\n\ + #CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + ); + let block = |s: &mut String, start: usize, end: usize| { + let _ = writeln!( + s, + "chr7\t{start}\t.\t{}\t<*>\t0\t.\tEND={end}\tGT:GQ:MIN_DP:PL\t0/0:42:30:0,90,900", + base(win, start) + ); + }; + let mut start = W0; + if het { + for (p, _, alt) in SITES { + block(&mut s, start, p - 1); + let _ = writeln!( + s, + "chr7\t{p}\t.\t{}\t{},<*>\t40\tPASS\t.\tGT:GQ:DP:PL\t0/1:35:40:400,0,400,990,990,990", + base(win, p), + char::from(alt) + ); + start = p + 1; + } + } + block(&mut s, start, W1 - 1); + s +} + +fn vcf(win: &[u8]) -> String { + let mut s = format!( + "##fileformat=VCFv4.2\n##contig=\n\ + ##FORMAT=\n\ + #CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + ); + for (p, _, alt) in SITES { + let _ = writeln!( + s, + "chr7\t{p}\t.\t{}\t{}\t40\tPASS\t.\tGT\t0/1", + base(win, p), + char::from(alt) + ); + } + s +} + +fn bases(h: &[u8], at: usize, n: usize) -> std::borrow::Cow<'_, str> { + String::from_utf8_lossy(&h[at - W0..at - W0 + n]) +} + +/// 20 read pairs: read 1 covers S3 and S2, its mate S1. +fn pairs(s: &mut String, h: &[u8], tag: &str) { + let q = "I".repeat(150); + for i in 0..20 { + let (a, b) = (S3 - 40 + i, S1 - 60 + i); + let tlen = b + 150 - a; + let _ = writeln!( + s, + "{tag}{i}\t99\tchr7\t{a}\t60\t150M\t=\t{b}\t{tlen}\t{}\t{q}", + bases(h, a, 150) + ); + let _ = writeln!( + s, + "{tag}{i}\t147\tchr7\t{b}\t60\t150M\t=\t{a}\t-{tlen}\t{}\t{q}", + bases(h, b, 150) + ); + } +} + +/// 20 single 60-base reads around each site: none covers two. +fn singles(s: &mut String, h: &[u8], tag: &str) { + let q = "I".repeat(60); + for (p, _, _) in SITES { + for i in 0..20 { + let a = p - 35 + (i % 10); + let flag = if i % 2 == 1 { 16 } else { 0 }; + let _ = writeln!( + s, + "{tag}{p}_{i}\t{flag}\tchr7\t{a}\t60\t60M\t*\t0\t0\t{}\t{q}", + bases(h, a, 60) + ); + } + } +} + +fn sam(win: &[u8], sample: &str) -> String { + let mut s = format!("@HD\tVN:1.6\tSO:unsorted\n@SQ\tSN:chr7\tLN:{LEN}\n"); + let (pav, avi) = (hap(win, b"GGC"), hap(win, b"CAT")); + match sample { + "GRSPAN" => { + pairs(&mut s, &pav, "pav"); + pairs(&mut s, &avi, "avi"); + } + "GRSWAP" => { + pairs(&mut s, &hap(win, b"GAC"), "gac"); + pairs(&mut s, &hap(win, b"CGT"), "cgt"); + } + _ => { + singles(&mut s, &pav, "pav"); + singles(&mut s, &avi, "avi"); + } + } + s +} + +fn write(path: &Path, text: &str) -> Result<(), Fail> { + if let Some(d) = path.parent() { + std::fs::create_dir_all(d).map_err(|e| Fail::Run(format!("{}: {e}", d.display())))?; + } + let mut out = create(path)?; + out.write_all(text.as_bytes())?; + out.flush()?; + Ok(()) +} + +fn reference(w: &mut dyn Write, win: &[u8]) -> std::io::Result<()> { + w.write_all(b">chr7\n")?; + let mut lines = Lines::new(w); + lines.ns(W0 - 1)?; + lines.bases(win)?; + lines.ns(LEN - (W1 - 1))?; + lines.end() +} + +pub fn run(args: &[String]) -> Result<(), Fail> { + let a = exactly(args, 2, "REF_DIR GENOME_DIR")?; + let (rd, g) = (Path::new(&a[0]), Path::new(&a[1])); + let win = window(); + std::fs::create_dir_all(rd).map_err(|e| Fail::Run(format!("{}: {e}", rd.display())))?; + let mut out = create(&rd.join("chr7.fa"))?; + reference(&mut out, &win)?; + out.flush()?; + for s in ["GRSPAN", "GRSWAP", "GRNOSPAN"] { + write( + &g.join(s).join("vcf").join(format!("{s}.g.vcf")), + &gvcf(&win, true), + )?; + write(&g.join(s).join("vcf").join(format!("{s}.vcf")), &vcf(&win))?; + write( + &g.join(s).join("aligned").join(format!("{s}.sam")), + &sam(&win, s), + )?; + } + write( + &g.join("GRREF").join("vcf").join("GRREF.g.vcf"), + &gvcf(&win, false), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fixture::e2e::tests::{Hashed, fnv}; + + // Pins of the bytes the inline Python of + // tests/e2e/genotype-rules-1-phase.sh wrote. + #[test] + fn writes_the_pythons_bytes() { + let win = window(); + let got: Vec = [ + gvcf(&win, true), + gvcf(&win, false), + vcf(&win), + sam(&win, "GRSPAN"), + sam(&win, "GRSWAP"), + sam(&win, "GRNOSPAN"), + ] + .iter() + .map(|t| fnv(t.as_bytes())) + .collect(); + assert_eq!(got, PINS); + let mut h = Hashed::default(); + reference(&mut h, &win).unwrap(); + assert_eq!(h.lines, 1 + LEN.div_ceil(60)); + assert_eq!(h.hash, PIN_REF); + } + + #[test] + fn the_copies_carry_their_alleles_at_the_three_sites() { + let win = window(); + let pav = hap(&win, b"GGC"); + assert_eq!([pav[S3 - W0], pav[S2 - W0], pav[S1 - W0]], *b"CGG"); + assert_eq!([win[S3 - W0], win[S2 - W0], win[S1 - W0]], *b"TGC"); + let spans = sam(&win, "GRSPAN"); + assert_eq!(spans.lines().filter(|l| !l.starts_with('@')).count(), 80); + // Read 1 of the first pair starts 40 bases before S3 and covers S2. + let r1: Vec<&str> = spans.lines().nth(2).unwrap().split('\t').collect(); + assert_eq!(r1[3], (S3 - 40).to_string()); + assert_eq!(r1[9].as_bytes()[40], b'C'); + assert_eq!(r1[9].as_bytes()[40 + S2 - S3], b'G'); + } + + #[test] + fn writes_every_file() { + let d = crate::fixture::e2e::tests::scratch("tas2r38"); + let (code, _, err) = crate::fixture::e2e::tests::run( + "tas2r38", + &[ + d.join("ref").to_str().unwrap(), + d.join("g").to_str().unwrap(), + ], + ); + assert_eq!(code, Some(0), "{err}"); + for f in [ + "GRSPAN/vcf/GRSPAN.g.vcf", + "GRSWAP/aligned/GRSWAP.sam", + "GRNOSPAN/vcf/GRNOSPAN.vcf", + "GRREF/vcf/GRREF.g.vcf", + ] { + assert!(d.join("g").join(f).is_file(), "{f}"); + } + assert!(!d.join("g/GRREF/aligned/GRREF.sam").exists()); + assert_eq!( + std::fs::metadata(d.join("ref/chr7.fa")).unwrap().len() as usize, + 6 + LEN + LEN.div_ceil(60) + ); + std::fs::remove_dir_all(d).unwrap(); + } + + const PINS: [u64; 6] = [ + 15_484_360_573_342_373_983, + 9_756_800_734_539_015_102, + 16_850_025_374_021_543_854, + 12_356_292_411_453_828_175, + 143_055_192_633_579_987, + 6_433_961_028_917_585_143, + ]; + const PIN_REF: u64 = 12_401_145_397_378_992_347; +} From c87ee3ded2aa8928d2845c024d2328c04f346b89 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:22:25 +0100 Subject: [PATCH 185/255] ci(freshness): let a manual run do the pull-request check, so a train car gets it before its PR can run A stacked train car conflicts with main until the car before it lands, so its pull request gets no pull_request runs until then. The landing train now runs the fast checks on the car's own tree on the fork, while its E2E runs. Every required check can be run that way except pull-request, which ran only on a pull_request event. A manual run with the new pr_base input (full shas, each an ancestor of the ref) now runs the pull-request job too: the changed files and the added URLs are taken against each base, and the job checks the union. Pull request runs do the same as before. keepalive skips a pr_base run. --- .github/workflows/freshness.yml | 34 +++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/freshness.yml b/.github/workflows/freshness.yml index 2df8e31e8..144b3f886 100644 --- a/.github/workflows/freshness.yml +++ b/.github/workflows/freshness.yml @@ -23,6 +23,13 @@ name: Freshness # whole-tree link check runs monthly, so a link that dies upstream shows in the # issue instead of failing an unrelated pull request. # +# A manual run with pr_base runs the pull-request job too, as a pull request +# from those base commits would: the files changed and the URLs added are taken +# against each base, and the job checks the union. The landing train uses it to +# get this check on a branch's own tree before the pull request can run (a +# stacked branch conflicts with main until the branch before it lands). Each +# base must be a full sha and an ancestor of the ref. +# # 60-day rule: GitHub disables scheduled workflows in a public repository after # 60 days without repository activity. Monthly Dependabot and weekly Renovate merges # normally count as activity. As a fallback the keepalive job re-enables this @@ -40,6 +47,11 @@ on: # The 2nd at 05:17 UTC: after the monthly image test on the 1st. - cron: '17 5 2 * *' workflow_dispatch: + inputs: + pr_base: + description: 'Also run the pull-request job against these base commits (full shas separated by spaces, each an ancestor of the ref). Empty: a plain manual run.' + type: string + default: '' pull_request: branches: [main] @@ -142,7 +154,7 @@ jobs: run: scripts/ci/check-pgs-labels.sh --self-test pull-request: - if: ${{ !cancelled() && github.event_name == 'pull_request' && needs.self-tests.result == 'success' }} + if: ${{ !cancelled() && (github.event_name == 'pull_request' || inputs.pr_base != '') && needs.self-tests.result == 'success' }} needs: self-tests runs-on: ubuntu-latest timeout-minutes: 20 @@ -154,9 +166,16 @@ jobs: - name: Files this pull request changes id: changed env: - BASE: ${{ github.event.pull_request.base.sha }} + EVENT: ${{ github.event_name }} + BASES: ${{ github.event.pull_request.base.sha || inputs.pr_base }} run: | - git diff --name-only "$BASE" HEAD > "${RUNNER_TEMP}/changed.txt" + if [ "$EVENT" != pull_request ]; then + for b in $BASES; do + [[ "$b" =~ ^[0-9a-f]{40}$ ]] || { echo "ERROR: pr_base takes full commit shas (the value is not echoed)"; exit 1; } + git merge-base --is-ancestor "$b" HEAD || { echo "ERROR: ${b} is not an ancestor of ${GITHUB_SHA}"; exit 1; } + done + fi + for b in $BASES; do git diff --name-only "$b" HEAD; done | awk '!seen[$0]++' > "${RUNNER_TEMP}/changed.txt" cat "${RUNNER_TEMP}/changed.txt" has() { grep -qxE "$1" "${RUNNER_TEMP}/changed.txt" && echo true || echo false; } { @@ -186,9 +205,12 @@ jobs: - name: Every download URL this pull request adds answers env: - BASE: ${{ github.event.pull_request.base.sha }} + BASES: ${{ github.event.pull_request.base.sha || inputs.pr_base }} OG: ${{ github.workspace }}/rust/target/release/og - run: scripts/ci/check-links.sh --since "$BASE" | tee -a "$GITHUB_STEP_SUMMARY" + # The exit is inside the braces: a variable set left of a pipe is lost, so + # the failure has to leave as the left side's status (pipefail). + run: | + { rc=0; for b in $BASES; do scripts/ci/check-links.sh --since "$b" || rc=1; done; exit "$rc"; } | tee -a "$GITHUB_STEP_SUMMARY" - name: Every PGS ID's label matches the PGS Catalog if: steps.changed.outputs.prs == 'true' @@ -294,7 +316,7 @@ jobs: # Fallback for the 60-day rule described at the top of this file. No `needs`: # it runs even when an upstream outage fails the checks. keepalive: - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && inputs.pr_base == '' runs-on: ubuntu-latest timeout-minutes: 5 permissions: From 08db2f7975a40a9241b40ee97f1f722971d911bf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:22:33 +0100 Subject: [PATCH 186/255] test(live): the ops card and /metrics checks as cargo tests, with parity against the Python plugin og-live/tests/ops.rs runs every check of tests/test_owngenome_live_ops.py on the Rust card, and compares the card (by meaning) and the metrics text (byte for byte) with bin/owngenome_live_ops.py on the ops fixture and on the run_eta and run_headroom fixtures. The plugin's eight source mutants are fault-inject faults; three more plant drift the parity cases must see. Bead: D1 --- rust/crates/og-live/src/ops.rs | 134 ++- rust/crates/og-live/tests/ops.rs | 1047 +++++++++++++++++++++ rust/crates/og-live/tests/ops_eta.rs | 146 ++- rust/crates/og-live/tests/ops_headroom.rs | 41 +- rust/crates/og-runlog/src/eta.rs | 3 +- 5 files changed, 1311 insertions(+), 60 deletions(-) create mode 100644 rust/crates/og-live/tests/ops.rs diff --git a/rust/crates/og-live/src/ops.rs b/rust/crates/og-live/src/ops.rs index 58474de8c..d9595a074 100644 --- a/rust/crates/og-live/src/ops.rs +++ b/rust/crates/og-live/src/ops.rs @@ -87,7 +87,11 @@ pub const METRICS: [(&str, &str, &str); 21] = [ "gauge", "Steps left with no usual time.", ), - ("pgp_run_seconds_left_low", "gauge", "Low end of the time left."), + ( + "pgp_run_seconds_left_low", + "gauge", + "Low end of the time left.", + ), ( "pgp_run_seconds_left_high", "gauge", @@ -119,7 +123,11 @@ pub const METRICS: [(&str, &str, &str); 21] = [ "gauge", "Used share of the filesystem's inodes.", ), - ("pgp_disk_free_bytes", "gauge", "Bytes free on the filesystem."), + ( + "pgp_disk_free_bytes", + "gauge", + "Bytes free on the filesystem.", + ), ("pgp_disk_size_bytes", "gauge", "Size of the filesystem."), ]; @@ -264,7 +272,11 @@ pub enum OpsState { /// is not the newest message, and from the last weblog `started`. pub fn current_run(events: &[Value]) -> &[Value] { let mut start = 0; - for (i, ev) in events.iter().enumerate().take(events.len().saturating_sub(1)) { + for (i, ev) in events + .iter() + .enumerate() + .take(events.len().saturating_sub(1)) + { if is_event(ev, "pgp_run_finished") { start = i + 1; } @@ -311,7 +323,11 @@ fn abs_dir(v: Option) -> Option { /// The newest sign of life: a message's `utcTime`, the run's /// `logs/run_status.tsv`, or a trace file of the run. -pub fn last_activity(run_dir: Option<&Path>, events: &[Value], started: Option) -> Option { +pub fn last_activity( + run_dir: Option<&Path>, + events: &[Value], + started: Option, +) -> Option { let mut times: Vec = events .iter() .filter_map(|ev| epoch_of(field(ev, "utcTime"))) @@ -359,7 +375,12 @@ fn started_of(state: &State<'_>) -> Option { } /// This run's own tasks from its trace files. -fn trace_rows(run_dir: Option<&Path>, sample: &str, started: Option, tz: &TimeZone) -> Vec { +fn trace_rows( + run_dir: Option<&Path>, + sample: &str, + started: Option, + tz: &TimeZone, +) -> Vec { let Some(dir) = is_dir(run_dir) else { return Vec::new(); }; @@ -391,7 +412,11 @@ fn exit_ok(v: &Value) -> bool { } fn plural(n: usize, one: &str, many: &str) -> String { - if n > 1 { many.to_owned() } else { one.to_owned() } + if n > 1 { + many.to_owned() + } else { + one.to_owned() + } } /// The ops card of the run in `run_dir` (none or missing for a run known @@ -420,7 +445,9 @@ pub fn run_ops( }; // run-all.sh posts pgp_run_finished as its last message; the site marks // the run finished on it too. - let ended = cur.last().is_some_and(|ev| is_event(ev, "pgp_run_finished")); + let ended = cur + .last() + .is_some_and(|ev| is_event(ev, "pgp_run_finished")); let run_st = if ended && state.run == RunState::Running { RunState::Finished } else { @@ -433,15 +460,22 @@ pub fn run_ops( let cores = cfg.eta_cores.filter(|c| *c != 0).unwrap_or_else(cpu_count); let history = root .as_deref() - .map(|r| eta::read_history(&eta::history_path(r, &eta::box_name(cfg.eta_box.as_deref())))) + .map(|r| { + eta::read_history(&eta::history_path( + r, + &eta::box_name(cfg.eta_box.as_deref()), + )) + }) .unwrap_or_default(); let mut exclude = root.as_deref().map(eta::live_traces).unwrap_or_default(); if let Some(dir) = is_dir(run_dir) && state.run == RunState::Running { - exclude.extend(trace_files(dir, started).iter().filter_map(|p| { - p.file_name().map(|n| n.to_string_lossy().into_owned()) - })); + exclude.extend( + trace_files(dir, started) + .iter() + .filter_map(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())), + ); } let ranges = eta::step_ranges(&history, ctx.steps, cores, &exclude); let rem = running.then(|| eta::remaining(state, &ranges, now, cores)); @@ -489,18 +523,27 @@ pub fn run_ops( let exit = cur .iter() .rfind(|ev| is_event(ev, "pgp_run_finished")) - .map_or(Value::Null, |ev| field(ev, "exit").cloned().unwrap_or(Value::Null)); + .map_or(Value::Null, |ev| { + field(ev, "exit").cloned().unwrap_or(Value::Null) + }); let error = cur.iter().any(|ev| is_event(ev, "error")); let finished = run_st == RunState::Finished; let bad_exit = !exit_ok(&exit) && !fault::on("ops-exit-ignored"); let failed = !failed_steps.is_empty() || (finished && (bad_exit || error)); let quiet = last_activity(run_dir, cur, started).map(|a| (now - a).max(0.0)); - let stall_s = cfg.stall_minutes.filter(|m| *m != 0.0).unwrap_or(DEFAULT_STALL_MINUTES) * 60.0; + let stall_s = cfg + .stall_minutes + .filter(|m| *m != 0.0) + .unwrap_or(DEFAULT_STALL_MINUTES) + * 60.0; // Running steps still inside their usual high end: a long task posts // nothing meanwhile. let covered = state.steps.iter().any(|s| { let high = ranges.get(&s.step.step).and_then(|r| r.high_s); - let st = s.started_utc.as_deref().and_then(|v| parse_utc(v, &[ISO_Z])); + let st = s + .started_utc + .as_deref() + .and_then(|v| parse_utc(v, &[ISO_Z])); s.state == StepState::Running && matches!((high, st), (Some(h), Some(st)) if now - st < h) && !fault::on("ops-stall-ignores-steps") @@ -538,7 +581,10 @@ pub fn run_ops( failed_steps.join(", ") ) } else if !exit_ok(&exit) { - format!("the run ended with exit {}", py_str(&exit).unwrap_or_default()) + format!( + "the run ended with exit {}", + py_str(&exit).unwrap_or_default() + ) } else { "Nextflow reported an error".to_owned() }; @@ -650,7 +696,13 @@ pub fn run_ops( /// The state hook: the card, or the line it shows instead. Only a warn /// percent outside 1 to 100 fails it (the readers take a missing or /// half-written file as no value), and the rest of the run's state stays. -pub fn hook(run_dir: Option<&Path>, state: &State<'_>, events: &[Value], cfg: &OpsConfig, ctx: &Ctx<'_>) -> OpsState { +pub fn hook( + run_dir: Option<&Path>, + state: &State<'_>, + events: &[Value], + cfg: &OpsConfig, + ctx: &Ctx<'_>, +) -> OpsState { match run_ops(run_dir, state, events, cfg, ctx) { Ok(o) => OpsState::Ok(Box::new(o)), Err(BadPercent(m)) => OpsState::Error(m.chars().take(200).collect()), @@ -691,7 +743,10 @@ impl RunOps { ("failed_steps", strs(&r.failed_steps)), ("exit", r.exit.clone()), ("stalled", Value::Bool(r.stalled)), - ("seconds_since_activity", opt_float(r.seconds_since_activity)), + ( + "seconds_since_activity", + opt_float(r.seconds_since_activity), + ), ("stall_seconds", Value::Float(r.stall_seconds)), ]), ), @@ -822,9 +877,12 @@ fn label(v: &str) -> String { /// `OWNGENOME_DISK_WARN_PERCENT`, else 90) over 100; none when it is not a /// number from 1 to 100. pub fn disk_warn_ratio(cfg: &OpsConfig) -> Option { - headroom::warn_percent(cfg.disk_warn_percent.as_deref(), Some("OWNGENOME_DISK_WARN_PERCENT")) - .ok() - .map(|p| p / 100.0) + headroom::warn_percent( + cfg.disk_warn_percent.as_deref(), + Some("OWNGENOME_DISK_WARN_PERCENT"), + ) + .ok() + .map(|p| p / 100.0) } /// The Prometheus text of `runs` (run name, card). A value that is not @@ -860,27 +918,51 @@ pub fn metrics_text( for (name, o) in runs { let run = [("run", *name)]; let (r, m) = (&o.run, &o.memory); - put("pgp_run_running", Some(Num::Flag(r.state == RunState::Running)), &run); + put( + "pgp_run_running", + Some(Num::Flag(r.state == RunState::Running)), + &run, + ); put("pgp_run_failed", Some(Num::Flag(r.failed)), &run); put("pgp_run_failed_steps", whole(r.failed_steps.len()), &run); put("pgp_run_stalled", Some(Num::Flag(r.stalled)), &run); - put("pgp_run_seconds_since_activity", r.seconds_since_activity.map(Num::Real), &run); + put( + "pgp_run_seconds_since_activity", + r.seconds_since_activity.map(Num::Real), + &run, + ); if let Some(h) = &o.hours_left { put("pgp_run_late_steps", whole(h.late_steps.len()), &run); put("pgp_run_unknown_steps", whole(h.unknown_steps.len()), &run); put("pgp_run_seconds_left_low", Some(Num::Real(h.low_s)), &run); put("pgp_run_seconds_left_high", h.high_s.map(Num::Real), &run); } - put("pgp_run_tasks_near_memory_request", whole(m.near_request), &run); + put( + "pgp_run_tasks_near_memory_request", + whole(m.near_request), + &run, + ); put("pgp_run_memory_retries", whole(m.retries), &run); - put("pgp_run_max_memory_bytes", m.max_memory_bytes.and_then(whole), &run); + put( + "pgp_run_max_memory_bytes", + m.max_memory_bytes.and_then(whole), + &run, + ); if m.max_memory_bytes.is_some() { - put("pgp_run_max_memory_above_ram", Some(Num::Flag(m.max_memory_above_ram)), &run); + put( + "pgp_run_max_memory_above_ram", + Some(Num::Flag(m.max_memory_above_ram)), + &run, + ); } for d in &o.disk { let at = [("run", *name), ("fs", d.fs.as_str())]; put("pgp_disk_used_ratio", d.used_ratio.map(Num::Real), &at); - put("pgp_disk_inodes_used_ratio", d.inodes_used_ratio.map(Num::Real), &at); + put( + "pgp_disk_inodes_used_ratio", + d.inodes_used_ratio.map(Num::Real), + &at, + ); put("pgp_disk_free_bytes", whole(d.free_bytes), &at); put("pgp_disk_size_bytes", whole(d.size_bytes), &at); } diff --git a/rust/crates/og-live/tests/ops.rs b/rust/crates/og-live/tests/ops.rs new file mode 100644 index 000000000..8a4ddf80c --- /dev/null +++ b/rust/crates/og-live/tests/ops.rs @@ -0,0 +1,1047 @@ +//! og-live's ops card and `/metrics` text on the made-up run of +//! `tests/fixtures/live/ops`, with a fake disk and a fake host; then the same +//! inputs through the Python plugin it replaces (`bin/owngenome_live_ops.py`, +//! while the Python site still runs it), which must give the same card by +//! meaning and the same metrics text byte for byte. A second parity case +//! runs on the traces and messages of `tests/fixtures/run_eta` and +//! `tests/fixtures/run_headroom`. +//! +//! With `--features fault-inject` each `planted_*` test turns on one planted +//! mistake and runs the behaviour checks: every one of them must fail +//! (Guard's rust job requires it). A check that cannot fail is no check. + +use jiff::tz::TimeZone; +use og_live::ops::{ + self, Asker, Ctx, METRICS, OpsConfig, OpsState, RunOps, current_run, metrics_allowed, + metrics_text, run_ops, +}; +use og_runlog::host::{FsStat, Host}; +use og_runlog::state::{Inputs, RunState, State, build}; +use og_runlog::steps::{Step, read_steps}; +use pycompat::json::{Value, loads}; +use std::collections::{BTreeMap, BTreeSet}; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, SystemTime}; + +const SAMPLE: &str = "OPS-001"; +/// 2026-01-02T10:00:00Z, the fixture's start. +const T0: f64 = 1_767_348_000.0; +const G: u128 = 1 << 30; +const UTC: TimeZone = TimeZone::UTC; +/// The last commit whose `bin/` still held the Python plugin, for the parity +/// cases once the Python site is gone. +const ORACLE: &str = "1ec630690e63796b4410d23ec65768200adb8dc0"; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn fix(name: &str) -> PathBuf { + repo().join("tests/fixtures/live/ops").join(name) +} + +fn steps() -> &'static [Step] { + static S: OnceLock> = OnceLock::new(); + S.get_or_init(|| read_steps(&repo().join("assets/steps.tsv")).expect("assets/steps.tsv reads")) +} + +/// A new empty folder of this test run. +fn tmp() -> PathBuf { + static N: AtomicUsize = AtomicUsize::new(0); + let d = Path::new(env!("CARGO_TARGET_TMPDIR")).join(format!( + "ops-{}-{}", + std::process::id(), + N.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn set_mtime(p: &Path, t: f64) { + std::fs::File::options() + .write(true) + .open(p) + .unwrap() + .set_modified(SystemTime::UNIX_EPOCH + Duration::from_secs_f64(t)) + .unwrap(); +} + +fn s(p: &Path) -> String { + p.to_string_lossy().into_owned() +} + +/// A host whose every file system is `ratio` used; device ids and +/// existence are the real ones. +struct Fake { + ratio: f64, + phys: u128, +} + +impl Host for Fake { + fn statvfs(&self, _: &Path) -> io::Result { + let blocks = 1_000_000u64; + let free = (blocks as f64 * (1.0 - self.ratio)) as u64; + Ok(FsStat { + frsize: 4096, + bsize: 4096, + blocks, + bfree: free, + bavail: free, + files: 1000, + ffree: 900, + }) + } + fn dev(&self, p: &Path) -> io::Result { + use std::os::unix::fs::MetadataExt; + Ok(std::fs::metadata(p)?.dev()) + } + fn exists(&self, p: &Path) -> bool { + p.exists() + } + fn physical_memory(&self) -> Option { + Some(self.phys) + } +} + +/// The genome folder of one case. +struct Tree { + root: PathBuf, + run_dir: PathBuf, + events: Vec, + events_file: PathBuf, + work: PathBuf, + out: PathBuf, + sample: String, +} + +/// GENOME_DIR in a new folder: the history of box `opsbox` and, for a +/// run-all.sh run, the run folder; `extra` messages after the fixture's. +fn make_tree(run_all: bool, extra: &[Value]) -> Tree { + let tmp = tmp(); + let root = tmp.join("genome"); + let (work, out) = (tmp.join("nfwork"), tmp.join("nfout")); + let hist = root.join("pipeline_info/step_history"); + for d in [&root, &work, &out, &hist] { + std::fs::create_dir_all(d).unwrap(); + } + std::fs::copy(fix("step_history.tsv"), hist.join("opsbox.tsv")).unwrap(); + let text = std::fs::read_to_string(fix("events.jsonl")) + .unwrap() + .replace("@WORK@", &s(&work)) + .replace("@OUT@", &s(&out)); + let mut events: Vec = text + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| loads(l).unwrap()) + .collect(); + events.extend(extra.iter().cloned()); + let run_dir = root.join(SAMPLE); + if run_all { + std::fs::create_dir_all(run_dir.join("logs")).unwrap(); + std::fs::create_dir_all(run_dir.join("nextflow/work")).unwrap(); + let status = run_dir.join("logs/run_status.tsv"); + std::fs::copy(fix("run_status.tsv"), &status).unwrap(); + set_mtime(&status, T0); + } + let events_file = tmp.join("events.jsonl"); + write_events(&events_file, &events); + Tree { + root, + run_dir, + events, + events_file, + work, + out, + sample: SAMPLE.into(), + } +} + +fn write_events(path: &Path, events: &[Value]) { + let opts = pycompat::json::Dump::default(); + let text: String = events + .iter() + .map(|e| pycompat::json::dumps(e, &opts).unwrap() + "\n") + .collect(); + std::fs::write(path, text).unwrap(); +} + +/// How one case runs. +#[derive(Clone)] +struct Case { + now: f64, + physical: u128, + disk: f64, + stopped: bool, + stall_minutes: f64, + disk_warn_percent: Option<&'static str>, +} + +fn case(now: f64) -> Case { + Case { + now, + physical: 64 * G, + disk: 0.50, + stopped: false, + stall_minutes: 60.0, + disk_warn_percent: None, + } +} + +fn cfg_for(t: &Tree, c: &Case) -> OpsConfig { + OpsConfig { + root: Some(t.root.clone()), + eta_box: Some("opsbox".into()), + eta_cores: Some(8), + disk_warn_percent: c.disk_warn_percent.map(str::to_owned), + mem_warn_percent: None, + stall_minutes: Some(c.stall_minutes), + } +} + +/// The state the site builds for the tree at `c.now`. +fn state_of(t: &Tree, c: &Case) -> State<'static> { + let status = og_runlog::files::read_run_status(&t.run_dir.join("logs/run_status.tsv")).unwrap(); + let cur = current_run(&t.events); + let mut st = build( + &t.sample, + steps(), + &Inputs { + status: status.as_ref(), + event_tasks: og_runlog::tasks::event_tasks(cur, &t.sample), + events: cur, + ..Default::default() + }, + c.now, + &UTC, + ); + if c.stopped { + st.run = RunState::Stopped; + } + st +} + +fn ops_of(t: &Tree, c: &Case) -> RunOps { + let host = Fake { + ratio: c.disk, + phys: c.physical, + }; + run_ops( + Some(&t.run_dir), + &state_of(t, c), + &t.events, + &cfg_for(t, c), + &Ctx { + now: c.now, + steps: steps(), + host: &host, + tz: &UTC, + }, + ) + .expect("the warn percents are good") +} + +fn ops_at(c: &Case, run_all: bool, extra: &[Value]) -> RunOps { + ops_of(&make_tree(run_all, extra), c) +} + +/// `{kind: level}` of the alerts. +fn kinds(o: &RunOps) -> BTreeMap<&'static str, &'static str> { + o.alerts + .iter() + .map(|a| (a.kind.as_str(), a.level.as_str())) + .collect() +} + +/// The series (name and labels, no value) of a Prometheus text. +fn series(text: &str) -> BTreeSet { + text.lines() + .filter(|l| !l.is_empty() && !l.starts_with('#')) + .map(|l| l.rsplit_once(' ').unwrap().0.to_owned()) + .collect() +} + +fn ev(text: &str) -> Value { + loads(text).unwrap() +} + +fn finished(code: i64) -> Value { + ev(&format!( + r#"{{"event":"pgp_run_finished","exit":{code},"utcTime":"2026-01-02T10:06:00Z"}}"# + )) +} + +fn failed_task() -> Value { + ev(&format!( + r#"{{"runName":"ops_demo","event":"process_completed","utcTime":"2026-01-02T10:06:00Z","trace":{{"task_id":4,"hash":"ab/000004","name":"CLINICAL:ROH ({SAMPLE})","process":"CLINICAL:ROH","tag":"{SAMPLE}","status":"FAILED","exit":1,"attempt":1,"complete":{}}}}}"#, + (T0 as i64 + 360) * 1000 + )) +} + +// --- the behaviour checks ------------------------------------------------------------------ + +#[test] +fn half_a_minute_into_step_11_not_late_not_stalled_and_both_ends() { + let o = ops_at(&case(T0 + 330.0), true, &[]); + let k = kinds(&o); + let h = o.hours_left.as_ref().expect("the run is going"); + assert!( + !k.contains_key("late") + && !k.contains_key("stalled") + && !o.run.stalled + && matches!(h.high_s, Some(hi) if h.low_s <= hi), + "{o:?}" + ); +} + +#[test] +fn two_hours_later_and_quiet_step_11_is_late_and_the_run_stalled() { + let o = ops_at(&case(T0 + 302.0 + 7200.0), true, &[]); + let h = o.hours_left.as_ref().expect("the run is going"); + let k = kinds(&o); + assert!( + h.late_steps == ["11"] + && h.high_s.is_none() + && k.get("late") == Some(&"warning") + && k.get("stalled") == Some(&"warning"), + "{:?}", + o.alerts + ); +} + +#[test] +fn the_same_quiet_run_read_stopped_is_still_stalled() { + let mut c = case(T0 + 302.0 + 7200.0); + c.stopped = true; + let o = ops_at(&c, true, &[]); + assert!( + o.run.stalled && kinds(&o).get("stalled") == Some(&"warning"), + "{:?} {:?}", + o.run, + o.alerts + ); +} + +#[test] +fn quiet_past_stall_minutes_but_step_11_inside_its_usual_time_is_not_stalled() { + let mut c = case(T0 + 302.0 + 90.0); + c.stall_minutes = 1.0; + let o = ops_at(&c, true, &[]); + let k = kinds(&o); + assert!( + o.run.seconds_since_activity.is_some_and(|q| q >= 60.0) + && !k.contains_key("stalled") + && !k.contains_key("late"), + "{:?} {:?}", + o.run, + o.alerts + ); +} + +#[test] +fn stall_minutes_300_keeps_a_2_hour_quiet_run_from_being_stalled() { + let mut c = case(T0 + 302.0 + 7200.0); + c.stall_minutes = 300.0; + let o = ops_at(&c, true, &[]); + let k = kinds(&o); + assert!( + !k.contains_key("stalled") && k.contains_key("late"), + "{:?}", + o.alerts + ); +} + +#[test] +fn a_failed_task_fails_the_run_exit_1_fails_it_exit_0_does_not() { + let c = case(T0 + 400.0); + let f1 = ops_at(&c, true, &[failed_task()]); + let f2 = ops_at(&c, true, &[finished(1)]); + let f0 = ops_at(&c, true, &[finished(0)]); + assert!( + f1.run.failed + && kinds(&f1).get("failed") == Some(&"critical") + && f2.run.failed + && f2.run.exit == Value::Int(1) + && !f0.run.failed + && !kinds(&f0).contains_key("failed"), + "{:?} {:?} {:?}", + f1.run, + f2.run, + f0.run + ); +} + +#[test] +fn one_retry_after_exit_137_and_7_6_of_8_gb_asked_is_a_hint() { + let o = ops_at(&case(T0 + 330.0), true, &[]); + let m = &o.memory; + assert!( + m.retries == 1 && m.near_request == 1 && kinds(&o).get("memory") == Some(&"warning"), + "{m:?} {:?}", + o.alerts + ); +} + +#[test] +fn max_memory_30_gb_on_a_16_gb_host_warns_on_64_it_does_not() { + let big = ops_at(&case(T0 + 330.0), true, &[]); + let mut c = case(T0 + 330.0); + c.physical = 16 * G; + let small = ops_at(&c, true, &[]); + let text = small + .alerts + .iter() + .find(|a| a.kind.as_str() == "memory") + .map(|a| a.text.clone()) + .unwrap_or_default(); + assert!( + small.memory.max_memory_above_ram + && !big.memory.max_memory_above_ram + && text.contains("max_memory"), + "{:?}", + small.memory + ); +} + +#[test] +fn a_disk_95_used_warns_at_90_not_at_disk_warn_percent_96() { + let early = ops_at(&case(T0 + 330.0), true, &[]); + let mut c = case(T0 + 330.0); + c.disk = 0.95; + let full = ops_at(&c, true, &[]); + c.disk_warn_percent = Some("96"); + let full96 = ops_at(&c, true, &[]); + assert!( + kinds(&full).get("disk") == Some(&"warning") + && !kinds(&full96).contains_key("disk") + && !kinds(&early).contains_key("disk") + && full + .disk + .first() + .and_then(|d| d.used_ratio) + .is_some_and(|r| (r - 0.95).abs() < 0.001), + "{:?} {:?}", + full.disk, + full96.alerts + ); +} + +#[test] +fn a_bare_nextflow_run_gives_the_same_series_as_the_run_all_run() { + let early = ops_at(&case(T0 + 330.0), true, &[]); + let bare = ops_at(&case(T0 + 330.0), false, &[]); + let a = series(&metrics_text(&[(SAMPLE, &early)], Some(0.9), None, 0)); + let b = series(&metrics_text(&[(SAMPLE, &bare)], Some(0.9), None, 0)); + assert!( + a == b && a.len() > 15, + "{:?}", + a.symmetric_difference(&b).collect::>() + ); +} + +/// Checks 1 to 9: what each planted mistake must break. +#[cfg_attr(not(feature = "fault-inject"), allow(dead_code))] +fn behaviour() { + half_a_minute_into_step_11_not_late_not_stalled_and_both_ends(); + two_hours_later_and_quiet_step_11_is_late_and_the_run_stalled(); + the_same_quiet_run_read_stopped_is_still_stalled(); + quiet_past_stall_minutes_but_step_11_inside_its_usual_time_is_not_stalled(); + stall_minutes_300_keeps_a_2_hour_quiet_run_from_being_stalled(); + a_failed_task_fails_the_run_exit_1_fails_it_exit_0_does_not(); + one_retry_after_exit_137_and_7_6_of_8_gb_asked_is_a_hint(); + max_memory_30_gb_on_a_16_gb_host_warns_on_64_it_does_not(); + a_disk_95_used_warns_at_90_not_at_disk_warn_percent_96(); + a_bare_nextflow_run_gives_the_same_series_as_the_run_all_run(); +} + +// --- no path, two labels, a valid text ------------------------------------------------------ + +fn strings(v: &Value, out: &mut Vec) { + match v { + Value::Str(x) => out.push(x.clone()), + Value::Array(a) => a.iter().for_each(|x| strings(x, out)), + Value::Object(o) => o.values().for_each(|x| strings(x, out)), + _ => {} + } +} + +/// A sample line: `pgp_name{labels} value` or `pgp_name value`. +fn valid_sample(line: &str) -> bool { + let Some((head, value)) = line.rsplit_once(' ') else { + return false; + }; + let name = head.split_once('{').map_or(head, |(n, _)| n); + let labels_ok = head + .split_once('{') + .is_none_or(|(_, rest)| rest.ends_with('}') && !rest[..rest.len() - 1].contains('}')); + name.starts_with("pgp_") + && name.chars().all(|c| c.is_ascii_lowercase() || c == '_') + && labels_ok + && !value.is_empty() + && value + .chars() + .all(|c| c.is_ascii_digit() || ".e+-".contains(c)) +} + +fn label_names(text: &str) -> BTreeSet { + let mut out = BTreeSet::new(); + for line in text.lines().filter(|l| !l.starts_with('#')) { + if let Some((_, rest)) = line.split_once('{') + && let Some((labels, _)) = rest.split_once('}') + { + out.extend( + labels + .split(',') + .filter_map(|kv| kv.split_once('=')) + .map(|(k, _)| k.to_owned()), + ); + } + } + out +} + +#[test] +fn no_path_in_the_card_and_the_metrics_carry_only_run_and_fs_in_a_valid_text() { + let t = make_tree(true, &[]); + let o = ops_of(&t, &case(T0 + 330.0)); + let mut all = Vec::new(); + strings(&o.to_json(), &mut all); + let leaked: Vec<&String> = all + .iter() + .filter(|x| { + x.starts_with('/') || [&t.root, &t.work, &t.out].iter().any(|p| x.contains(&s(p))) + }) + .collect(); + let text = metrics_text(&[(SAMPLE, &o)], Some(0.9), Some(64 * G), 0); + let bad: Vec<&str> = text + .lines() + .filter(|l| !l.is_empty() && !l.starts_with('#') && !valid_sample(l)) + .collect(); + let labels = label_names(&text); + assert!( + leaked.is_empty() + && labels == BTreeSet::from(["fs".to_owned(), "run".to_owned()]) + && bad.is_empty(), + "{leaked:?} {labels:?} {bad:?}" + ); + // Control: a path label and a broken line must be seen. + let planted = format!("{text}pgp_runs{{path=\"/x\"}} 1\npgp_runs one\n"); + assert!(label_names(&planted).contains("path")); + assert!( + planted + .lines() + .any(|l| !l.starts_with('#') && !valid_sample(l)) + ); +} + +// --- who may read /metrics ------------------------------------------------------------------ + +fn who(password: bool, logged_in: bool, auth: Option<&str>) -> Asker<'_> { + Asker { + password_set: password, + logged_in, + authorization: auth, + } +} + +fn tok(t: Option<&str>) -> impl FnOnce() -> io::Result> + '_ { + move || Ok(t.map(str::to_owned)) +} + +fn unreadable() -> io::Result> { + Err(io::Error::new(io::ErrorKind::NotFound, "no token file")) +} + +#[test] +fn metrics_open_without_a_password_and_with_one_a_login_or_the_bearer_token_only() { + let yes = |w: &Asker<'_>, t: Option<&str>| metrics_allowed(w, tok(t)).unwrap(); + assert!(yes(&who(false, false, None), None)); + assert!(!yes(&who(true, false, None), None)); + assert!(!yes(&who(true, false, Some("Bearer s3cret")), None)); + assert!(yes( + &who(true, false, Some("Bearer s3cret")), + Some("s3cret") + )); + assert!(yes( + &who(true, false, Some("bearer s3cret ")), + Some("s3cret") + )); + assert!(!yes( + &who(true, false, Some("Bearer wrong")), + Some("s3cret") + )); + assert!(!yes( + &who(true, false, Some("Bearer s3cre")), + Some("s3cret") + )); + assert!(!yes( + &who(true, false, Some("Basic s3cret")), + Some("s3cret") + )); + assert!(yes(&who(true, true, None), None)); + // The token file is read only when a token is needed; unreadable is an error. + assert!(metrics_allowed(&who(true, false, Some("Bearer s3cret")), unreadable).is_err()); + assert!(metrics_allowed(&who(true, true, None), unreadable).unwrap()); + assert!(metrics_allowed(&who(false, false, None), unreadable).unwrap()); +} + +#[test] +fn the_token_file_is_read_stripped_and_an_empty_one_is_no_token() { + let d = tmp(); + std::fs::write(d.join("t"), " s3cret\n").unwrap(); + std::fs::write(d.join("e"), "\n").unwrap(); + assert_eq!( + ops::read_token(Some(&d.join("t"))).unwrap().as_deref(), + Some("s3cret") + ); + assert_eq!(ops::read_token(Some(&d.join("e"))).unwrap(), None); + assert_eq!(ops::read_token(None).unwrap(), None); + assert!(ops::read_token(Some(&d.join("missing"))).is_err()); +} + +// --- the docs and the example rules --------------------------------------------------------- + +/// The `pgp_*` names the rules' expressions use. +fn rule_metrics(rules: &str) -> BTreeSet { + let mut out = BTreeSet::new(); + for line in rules.lines() { + if !(line.trim_start().starts_with("expr:") || line.starts_with(&" ".repeat(10))) { + continue; + } + let code = line.split('#').next().unwrap_or(""); + out.extend( + code.split(|c: char| !(c.is_ascii_alphanumeric() || c == '_')) + .filter(|w| w.starts_with("pgp_")) + .map(str::to_owned), + ); + } + out +} + +fn missing_from_doc(doc: &str) -> Vec<&'static str> { + METRICS + .iter() + .map(|(n, _, _)| *n) + .filter(|n| !doc.contains(&format!("`{n}`"))) + .collect() +} + +fn unknown(used: &BTreeSet) -> Vec<&String> { + used.iter() + .filter(|n| !METRICS.iter().any(|(m, _, _)| m == n)) + .collect() +} + +#[test] +fn every_metric_is_in_the_docs_and_every_metric_the_rules_use_exists() { + let doc = std::fs::read_to_string(repo().join("docs/alerts.md")).unwrap(); + let rules = std::fs::read_to_string(repo().join("assets/monitoring/pgp-alerts.yml")).unwrap(); + let used = rule_metrics(&rules); + assert!( + missing_from_doc(&doc).is_empty() && unknown(&used).is_empty() && !used.is_empty(), + "{:?} {:?}", + missing_from_doc(&doc), + unknown(&used) + ); + // Control: a doc without one name, and a rule with a name /metrics never + // writes, must each be seen. + assert_eq!( + missing_from_doc(&doc.replace("`pgp_run_stalled`", "")), + ["pgp_run_stalled"] + ); + let planted = format!("{rules}\n expr: pgp_run_nonexistent > 0\n"); + assert_eq!( + unknown(&rule_metrics(&planted)), + [&"pgp_run_nonexistent".to_owned()] + ); +} + +// --- read only, and the state hook ---------------------------------------------------------- + +fn mtimes(root: &Path) -> BTreeMap { + let mut out = BTreeMap::new(); + let mut todo = vec![root.to_path_buf()]; + while let Some(d) = todo.pop() { + for e in std::fs::read_dir(&d).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + todo.push(p); + } else { + out.insert(p.clone(), pycompat::mtime::mtime(&p).unwrap()); + } + } + } + out +} + +#[test] +fn the_step_history_is_read_never_written() { + let t = make_tree(true, &[]); + let hist = t.root.join("pipeline_info/step_history/opsbox.tsv"); + std::fs::write( + t.root.join("pipeline_info/trace_2026-01-02_10-00-00.txt"), + "task_id\thash\tname\tstatus\texit\n1\tzz/1\tROH (OPS-001)\tCOMPLETED\t0\n", + ) + .unwrap(); + let before = mtimes(&t.root); + let text = std::fs::read_to_string(&hist).unwrap(); + ops_of(&t, &case(T0 + 330.0)); + assert_eq!(mtimes(&t.root), before); + assert_eq!(std::fs::read_to_string(&hist).unwrap(), text); +} + +fn hook_of(t: &Tree, cfg: &OpsConfig) -> OpsState { + let c = case(T0 + 330.0); + let host = Fake { + ratio: 0.5, + phys: 64 * G, + }; + ops::hook( + Some(&t.run_dir), + &state_of(t, &c), + &t.events, + cfg, + &Ctx { + now: c.now, + steps: steps(), + host: &host, + tz: &UTC, + }, + ) +} + +/// The Python plugin turned any exception into an error card; the one left +/// here is a warn percent outside 1 to 100. A history with no `trace_file` +/// column (the Python card's KeyError) now gives the card from what the +/// history holds. +#[test] +fn the_state_hook_gives_the_card_or_one_error_line() { + let t = make_tree(true, &[]); + std::fs::write( + t.root.join("pipeline_info/step_history/opsbox.tsv"), + "step\tseconds\nROH\t60\n", + ) + .unwrap(); + let c = case(T0 + 330.0); + let ok = hook_of(&t, &cfg_for(&t, &c)); + assert!( + matches!(&ok, OpsState::Ok(o) if o.hours_left.is_some()), + "{ok:?}" + ); + let mut bad = cfg_for(&t, &c); + bad.disk_warn_percent = Some("0".into()); + let err = hook_of(&t, &bad); + assert_eq!( + err.to_json(), + ev(r#"{"error":"warn percent must be a number from 1 to 100, got '0'"}"#) + ); +} + +// --- parity with the Python plugin ---------------------------------------------------------- + +fn git(args: &[&str]) -> std::process::Output { + Command::new("git") + .arg("-C") + .arg(repo()) + .args(args) + .output() + .expect("git runs") +} + +/// The folder holding `owngenome_live_ops.py` and the modules it imports: +/// `bin/` while the Python site lives, else those files at [`ORACLE`]. +fn oracle_bin() -> &'static Path { + static DIR: OnceLock = OnceLock::new(); + DIR.get_or_init(|| { + let bin = repo().join("bin"); + if bin.join("owngenome_live_ops.py").is_file() { + return bin; + } + if !git(&["cat-file", "-e", &format!("{ORACLE}^{{commit}}")]) + .status + .success() + { + let f = git(&["fetch", "-q", "--depth=1", "origin", ORACLE]); + assert!(f.status.success(), "commit {ORACLE} could not be fetched"); + } + let root = tmp().join("oracle"); + std::fs::create_dir_all(root.join("bin")).unwrap(); + std::fs::create_dir_all(root.join("assets")).unwrap(); + for f in ["owngenome_live_ops", "run_eta", "run_headroom", "run_state"] { + let o = git(&["show", &format!("{ORACLE}:bin/{f}.py")]); + assert!(o.status.success(), "git show {ORACLE}:bin/{f}.py"); + std::fs::write(root.join("bin").join(format!("{f}.py")), o.stdout).unwrap(); + } + std::fs::copy( + repo().join("assets/steps.tsv"), + root.join("assets/steps.tsv"), + ) + .unwrap(); + root.join("bin") + }) +} + +const PY: &str = r#" +import json, os, sys, types +a = json.loads(sys.argv[1]) +sys.path.insert(0, a["bin"]) +import run_headroom, run_state +import owngenome_live_ops as ops + +class FakeFs: + def __init__(self, used_ratio): + self.f_frsize = self.f_bsize = 4096 + self.f_blocks = 1000000 + self.f_bfree = self.f_bavail = int(self.f_blocks * (1 - used_ratio)) + self.f_files, self.f_ffree = 1000, 900 + +run_headroom._statvfs = lambda p: FakeFs(a["disk"]) +events = [json.loads(l) for l in open(a["events"]) if l.strip()] +steps = run_state.read_steps() +status_path = os.path.join(a["run_dir"], "logs", "run_status.tsv") +status = run_state.read_run_status(status_path) if os.path.isfile(status_path) else None +cur = ops.current_run(events) +state = run_state.build(a["sample"], steps, status, None, [], run_state.event_tasks(cur, a["sample"]), cur, None, a["now"]) +if a["stopped"]: + state["run"]["state"] = "stopped" +cfg = types.SimpleNamespace(root=a["root"], steps=run_state.STEPS_TSV, eta_box="opsbox", eta_cores=8, + disk_warn_percent=a["dwp"], mem_warn_percent=None, stall_minutes=a["stall"]) +o = ops.run_ops(a["run_dir"], state, events, cfg, now=a["now"], steps=steps, physical=a["physical"]) +sys.stdout.write(json.dumps(o) + "\n---\n" + ops.metrics_text([(a["sample"], o)], 0.9, a["physical"], 0)) +"#; + +/// The Python card and metrics text of the tree at `c`. +fn py_ops(t: &Tree, c: &Case) -> (Value, String) { + let mut args = indexmap::IndexMap::new(); + let mut put = |k: &str, v: Value| { + args.insert(k.to_owned(), v); + }; + put("bin", Value::Str(s(oracle_bin()))); + put("events", Value::Str(s(&t.events_file))); + put("run_dir", Value::Str(s(&t.run_dir))); + put("root", Value::Str(s(&t.root))); + put("sample", Value::Str(t.sample.clone())); + put("now", Value::Float(c.now)); + put("disk", Value::Float(c.disk)); + put("physical", Value::Int(c.physical as i128)); + put("stopped", Value::Bool(c.stopped)); + put("stall", Value::Float(c.stall_minutes)); + put( + "dwp", + c.disk_warn_percent + .map_or(Value::Null, |p| Value::Str(p.into())), + ); + let arg = + pycompat::json::dumps(&Value::Object(args), &pycompat::json::Dump::default()).unwrap(); + let exe = std::env::var("OG_PYTHON").unwrap_or_else(|_| "python3".into()); + let o = Command::new(&exe) + .args(["-c", PY, &arg]) + .env("TZ", "UTC") + .env_remove("PGP_BOX") + .env_remove("OWNGENOME_DISK_WARN_PERCENT") + .env_remove("OWNGENOME_MEM_WARN_PERCENT") + .output() + .unwrap_or_else(|e| panic!("cannot run the Python oracle {exe}: {e}; set OG_PYTHON")); + assert!( + o.status.success(), + "the Python plugin failed: {}", + String::from_utf8_lossy(&o.stderr) + ); + let out = String::from_utf8(o.stdout).unwrap(); + let (card, text) = out.split_once("\n---\n").expect("card and text"); + (loads(card).unwrap(), text.to_owned()) +} + +/// The same by meaning: numbers by value, the rest equal, keys in order. +fn same(a: &Value, b: &Value, at: &str) -> Result<(), String> { + let num = |v: &Value| match v { + Value::Int(i) => Some(*i as f64), + Value::Float(x) => Some(*x), + _ => None, + }; + match (a, b) { + (Value::Object(x), Value::Object(y)) => { + let (kx, ky): (Vec<_>, Vec<_>) = (x.keys().collect(), y.keys().collect()); + if kx != ky { + return Err(format!("{at}: keys {kx:?} != {ky:?}")); + } + x.iter() + .try_for_each(|(k, v)| same(v, &y[k], &format!("{at}.{k}"))) + } + (Value::Array(x), Value::Array(y)) if x.len() == y.len() => x + .iter() + .zip(y) + .enumerate() + .try_for_each(|(i, (v, w))| same(v, w, &format!("{at}[{i}]"))), + _ => match (num(a), num(b)) { + (Some(x), Some(y)) if (x - y).abs() <= 1e-9 * x.abs().max(1.0) => Ok(()), + (Some(_), Some(_)) => Err(format!("{at}: {a:?} != {b:?}")), + _ if a == b => Ok(()), + _ => Err(format!("{at}: {a:?} != {b:?}")), + }, + } +} + +fn parity(t: &Tree, c: &Case, what: &str) { + let o = ops_of(t, c); + let (card, text) = py_ops(t, c); + if let Err(e) = same(&o.to_json(), &card, what) { + panic!("og and the Python plugin differ: {e}"); + } + let mine = metrics_text(&[(&t.sample, &o)], Some(0.9), Some(c.physical), 0); + assert_eq!(mine, text, "{what}: the metrics text differs"); +} + +#[test] +fn the_card_and_metrics_match_the_python_plugin_on_the_ops_fixture() { + let quiet = T0 + 302.0 + 7200.0; + let mut cases: Vec<(&str, bool, Vec, Case)> = vec![ + ("early", true, vec![], case(T0 + 330.0)), + ("late and stalled", true, vec![], case(quiet)), + ("failed task", true, vec![failed_task()], case(T0 + 400.0)), + ("exit 1", true, vec![finished(1)], case(T0 + 400.0)), + ("exit 0", true, vec![finished(0)], case(T0 + 400.0)), + ( + "error then exit 0", + true, + vec![ + ev(r#"{"event":"error","utcTime":"2026-01-02T10:05:50Z"}"#), + finished(0), + ], + case(T0 + 400.0), + ), + ("bare nextflow run", false, vec![], case(T0 + 330.0)), + ]; + let mut c = case(quiet); + c.stopped = true; + cases.push(("stopped", true, vec![], c)); + let mut c = case(T0 + 302.0 + 90.0); + c.stall_minutes = 1.0; + cases.push(("inside its usual time", true, vec![], c)); + let mut c = case(quiet); + c.stall_minutes = 300.0; + cases.push(("stall 300", true, vec![], c)); + let mut c = case(T0 + 330.0); + c.physical = 16 * G; + cases.push(("16 GB host", true, vec![], c.clone())); + c.physical = 64 * G; + c.disk = 0.95; + cases.push(("disk 95%", true, vec![], c.clone())); + c.disk = 0.985; + cases.push(("disk 98.5%", true, vec![], c.clone())); + c.disk = 0.95; + c.disk_warn_percent = Some("96"); + cases.push(("disk warn 96", true, vec![], c)); + for (what, run_all, extra, c) in &cases { + parity(&make_tree(*run_all, extra), c, what); + } +} + +#[test] +fn the_card_and_metrics_match_the_python_plugin_on_the_eta_and_headroom_fixtures() { + let d = tmp(); + let root = d.join("genome"); + let (work, out) = (d.join("nfwork"), d.join("nfout")); + let info = root.join("pipeline_info"); + let run_dir = root.join("demo1"); + for p in [ + &info, + &work, + &out, + &run_dir.join("logs"), + &run_dir.join("nextflow/work"), + ] { + std::fs::create_dir_all(p).unwrap(); + } + // An earlier run's trace makes this box's history. + let old = info.join("trace_2026-01-01_09-59-00.txt"); + std::fs::copy(repo().join("tests/fixtures/run_eta/trace_new.txt"), &old).unwrap(); + set_mtime(&old, T0 - 86400.0); + og_runlog::eta::ingest(&root, "opsbox", steps(), 16, &UTC).unwrap(); + // This run: started at T0, its trace and its messages. + let new = info.join("trace_2026-01-02_10-00-00.txt"); + std::fs::copy(repo().join("tests/fixtures/run_headroom/trace.txt"), &new).unwrap(); + set_mtime(&new, T0 + 60.0); + let status = run_dir.join("logs/run_status.tsv"); + std::fs::write( + &status, + format!( + "meta\tstarted_epoch\t{}\nmeta\tstarted_utc\t2026-01-02 10:00:00 UTC\nstep\t01b\tplanned\nstep\t02\tplanned\nstep\t03\tplanned\nstep\t13\tplanned\n", + T0 as i64 + ), + ) + .unwrap(); + set_mtime(&status, T0); + let mut events = vec![ev(&format!( + r#"{{"event":"started","utcTime":"2026-01-02T10:00:00Z","metadata":{{"parameters":{{"outdir":"{}","max_memory":"64.GB"}},"workflow":{{"workDir":"{}"}}}}}}"#, + s(&out), + s(&work) + ))]; + events.extend( + og_runlog::files::read_events(Some( + &repo().join("tests/fixtures/run_headroom/events.jsonl"), + )) + .unwrap(), + ); + let events_file = d.join("events.jsonl"); + write_events(&events_file, &events); + let t = Tree { + root, + run_dir, + events, + events_file, + work, + out, + sample: "demo1".into(), + }; + let mut c = case(T0 + 3600.0); + c.physical = 32 * G; + parity(&t, &c, "eta and headroom fixtures"); + let o = ops_of(&t, &c); + // The case is worth comparing: hours left from the box, a retry, a host warning. + assert!( + o.hours_left.is_some() && o.memory.retries == 1 && o.memory.max_memory_above_ram, + "{o:?}" + ); + // Control: a card that differs in one number must be seen. + let (card, _) = py_ops(&t, &c); + let mut changed = o.clone(); + changed.memory.retries = 2; + assert!(same(&changed.to_json(), &card, "control").is_err()); +} + +// --- the planted mistakes: each test below must FAIL -------------------------------------- + +#[cfg(feature = "fault-inject")] +mod planted { + use og_runlog::eta::fault::plant; + + macro_rules! planted { + ($($name:ident: $fault:literal => $check:path;)*) => {$( + #[test] + fn $name() { + plant($fault); + $check(); + } + )*}; + } + + planted! { + planted_ops_stall_ignores_quiet: "ops-stall-ignores-quiet" => super::behaviour; + planted_ops_stopped_not_stalled: "ops-stopped-not-stalled" => super::behaviour; + planted_ops_stall_ignores_steps: "ops-stall-ignores-steps" => super::behaviour; + planted_ops_late_dropped: "ops-late-dropped" => super::behaviour; + planted_ops_exit_ignored: "ops-exit-ignored" => super::behaviour; + planted_ops_disk_warn_ignored: "ops-disk-warn-ignored" => super::behaviour; + planted_ops_retries_uncounted: "ops-retries-uncounted" => super::behaviour; + planted_ops_bare_run_no_work: "ops-bare-run-no-work" => super::behaviour; + // The parity cases must see a port that drifts from the Python plugin. + planted_ops_parity_retries: "ops-retries-uncounted" => super::the_card_and_metrics_match_the_python_plugin_on_the_ops_fixture; + planted_ops_parity_late: "ops-late-dropped" => super::the_card_and_metrics_match_the_python_plugin_on_the_ops_fixture; + planted_ops_parity_headroom: "headroom-exits-143-only" => super::the_card_and_metrics_match_the_python_plugin_on_the_eta_and_headroom_fixtures; + } +} diff --git a/rust/crates/og-live/tests/ops_eta.rs b/rust/crates/og-live/tests/ops_eta.rs index 14899fbac..c6beb0c0b 100644 --- a/rust/crates/og-live/tests/ops_eta.rs +++ b/rust/crates/og-live/tests/ops_eta.rs @@ -123,9 +123,20 @@ fn human_and_raw_units_read_the_same() { // And as the history reads a trace: a size and its bytes are one value. let d = tmp(); let head = "task_id\thash\tname\tstatus\texit\tmemory\n"; - std::fs::write(d.join("a.txt"), format!("{head}1\tab/1\tFASTP (s)\tCOMPLETED\t0\t18.3 GB\n")).unwrap(); - std::fs::write(d.join("b.txt"), format!("{head}1\tab/1\tFASTP (s)\tCOMPLETED\t0\t{raw_18}\n")).unwrap(); - let (a, b) = (read_trace(&d.join("a.txt"), &UTC), read_trace(&d.join("b.txt"), &UTC)); + std::fs::write( + d.join("a.txt"), + format!("{head}1\tab/1\tFASTP (s)\tCOMPLETED\t0\t18.3 GB\n"), + ) + .unwrap(); + std::fs::write( + d.join("b.txt"), + format!("{head}1\tab/1\tFASTP (s)\tCOMPLETED\t0\t{raw_18}\n"), + ) + .unwrap(); + let (a, b) = ( + read_trace(&d.join("a.txt"), &UTC), + read_trace(&d.join("b.txt"), &UTC), + ); assert!(a[0].memory_bytes.is_some()); assert_eq!(a[0].memory_bytes, b[0].memory_bytes); } @@ -135,7 +146,10 @@ fn cached_failed_and_unknown_process_tasks_never_enter_the_history() { let rows = ingest_as(&outdir(&[(OLD, "trace_new.txt")]), "t"); let mut hashes: Vec<&str> = rows.iter().map(|r| r[1].as_str()).collect(); hashes.sort_unstable(); - assert_eq!(hashes, ["aa000001", "aa000002", "aa000003", "aa000006", "aa000007"]); + assert_eq!( + hashes, + ["aa000001", "aa000002", "aa000003", "aa000006", "aa000007"] + ); } #[test] @@ -215,7 +229,10 @@ fn ranges(history: &[HistoryRow], cores: usize) -> IndexMap { } fn step(id: &str) -> &'static Step { - steps().iter().find(|s| s.step == id).expect("a catalogue step") + steps() + .iter() + .find(|s| s.step == id) + .expect("a catalogue step") } #[test] @@ -235,7 +252,10 @@ fn only_the_last_5_runs_count() { fn docs_hours(id: &str) -> (f64, f64) { let s = step(id); - (s.hours.unwrap() * H, s.hours_low.parse::().unwrap() * H) + ( + s.hours.unwrap() * H, + s.hours_low.parse::().unwrap() * H, + ) } #[test] @@ -272,7 +292,8 @@ fn on_16_and_32_cores_every_measured_step_has_low_below_high() { .iter() .filter(|k| { let x = &r[**k]; - !(x.source == Source::Docs && matches!((x.low_s, x.high_s), (Some(l), Some(h)) if l < h)) + !(x.source == Source::Docs + && matches!((x.low_s, x.high_s), (Some(l), Some(h)) if l < h)) }) .collect(); assert!(one.is_empty(), "{cores} cores: {one:?}"); @@ -313,7 +334,10 @@ use StepState::{Done, Running, Waiting}; fn a_step_the_table_only_estimates_gets_no_number() { let est = ranges(&[], 16); let unk = remaining( - &state(&[("03", Done, None), ("14", Waiting, None)], RunState::Running), + &state( + &[("03", Done, None), ("14", Waiting, None)], + RunState::Running, + ), &est, NOW, 16, @@ -330,7 +354,15 @@ fn a_step_the_table_only_estimates_gets_no_number() { ); } -fn task(hash: &str, process: &str, status: &str, exit: &str, start: f64, end: f64, attempt: &str) -> TraceTask { +fn task( + hash: &str, + process: &str, + status: &str, + exit: &str, + start: f64, + end: f64, + attempt: &str, +) -> TraceTask { TraceTask { trace_file: "trace_x.txt".into(), hash: hash.into(), @@ -354,7 +386,15 @@ fn a_step_with_a_task_that_failed_or_was_stopped_gives_no_span() { // failed first attempt that a second one replaced still counts. let rows = history_rows( &[ - task("p1", "PHARMCAT_PREPROCESS", "COMPLETED", "0", 0.0, 40.0, "1"), + task( + "p1", + "PHARMCAT_PREPROCESS", + "COMPLETED", + "0", + 0.0, + 40.0, + "1", + ), task("p2", "PHARMCAT", "ABORTED", "-", 41.0, 50.0, "1"), task("d1", "DEEPVARIANT", "FAILED", "137", 0.0, 100.0, "1"), task("d2", "DEEPVARIANT", "COMPLETED", "0", 110.0, 300.0, "2"), @@ -423,21 +463,31 @@ fn chain_state() -> State<'static> { fn the_hours_left_are_the_longest_chain() { // 03 -> 13 -> 23 is the longest chain: low 10 + 4 + 0.1, high 12 + 5 + 0.2. let big = remaining(&chain_state(), &fixed_ranges(), NOW, 64); - assert!(close(Some(big.low_s), 14.1 * H) && close(big.high_s, 17.2 * H), "{big:?}"); + assert!( + close(Some(big.low_s), 14.1 * H) && close(big.high_s, 17.2 * H), + "{big:?}" + ); } #[test] fn the_cpu_work_bound_when_it_is_larger() { // CPU work: 12*8 + 5*8 + 0.2*1 + 1*8 = 144.2 CPU-hours; over 4 cores 36.05 h. let small = remaining(&chain_state(), &fixed_ranges(), NOW, 4); - assert!(close(Some(small.low_s), 14.1 * H) && close(small.high_s, 36.05 * H), "{small:?}"); + assert!( + close(Some(small.low_s), 14.1 * H) && close(small.high_s, 36.05 * H), + "{small:?}" + ); } #[test] fn a_running_step_past_its_high_end_makes_the_high_end_unknown_and_says_so() { let late = remaining( &state( - &[("02", Done, None), ("03", Running, Some("2026-01-01T21:00:00Z")), ("13", Waiting, None)], + &[ + ("02", Done, None), + ("03", Running, Some("2026-01-01T21:00:00Z")), + ("13", Waiting, None), + ], RunState::Running, ), &fixed_ranges(), @@ -457,21 +507,31 @@ fn a_running_step_past_its_high_end_makes_the_high_end_unknown_and_says_so() { fn a_running_step_counts_only_what_is_left_of_it() { let going = remaining( &state( - &[("02", Done, None), ("03", Running, Some("2026-01-02T06:00:00Z")), ("13", Waiting, None)], + &[ + ("02", Done, None), + ("03", Running, Some("2026-01-02T06:00:00Z")), + ("13", Waiting, None), + ], RunState::Running, ), &fixed_ranges(), NOW, 64, ); - assert!(close(Some(going.low_s), 10.0 * H) && close(going.high_s, 13.0 * H), "{going:?}"); + assert!( + close(Some(going.low_s), 10.0 * H) && close(going.high_s, 13.0 * H), + "{going:?}" + ); } #[test] fn a_step_with_no_time_lands_in_unknown_steps_never_0() { let r = ranges(&[], 16); let unk = remaining( - &state(&[("02", Done, None), ("33", Waiting, None)], RunState::Running), + &state( + &[("02", Done, None), ("33", Waiting, None)], + RunState::Running, + ), &r, NOW, 16, @@ -542,7 +602,10 @@ fn going(d: &Path) -> PathBuf { let status = logs.join("run_status.tsv"); std::fs::write( &status, - format!("meta\tstarted_epoch\t{}\nstep\t02\tplanned\n", now() as i64 - 60), + format!( + "meta\tstarted_epoch\t{}\nstep\t02\tplanned\n", + now() as i64 - 60 + ), ) .unwrap(); status @@ -585,26 +648,52 @@ fn trace_fields(config: &str) -> Option> { let start = config.find("\ntrace {\n")?; let block = &config[start..]; let block = &block[..block.find("\n}")?]; - let line = block.lines().find(|l| l.trim_start().starts_with("fields"))?; + let line = block + .lines() + .find(|l| l.trim_start().starts_with("fields"))?; let value = line.split_once('\'')?.1.split_once('\'')?.0; Some(value.split(',').map(str::to_owned).collect()) } /// Nextflow's 14 default trace columns, then the 7 the history reads. const TRACE_FIELDS: [&str; 21] = [ - "task_id", "hash", "native_id", "name", "status", "exit", "submit", "duration", "realtime", - "%cpu", "peak_rss", "peak_vmem", "rchar", "wchar", "process", "tag", "attempt", "cpus", - "memory", "start", "complete", + "task_id", + "hash", + "native_id", + "name", + "status", + "exit", + "submit", + "duration", + "realtime", + "%cpu", + "peak_rss", + "peak_vmem", + "rchar", + "wchar", + "process", + "tag", + "attempt", + "cpus", + "memory", + "start", + "complete", ]; #[test] fn nextflow_config_traces_the_14_default_columns_then_the_7_added() { let cfg = std::fs::read_to_string(repo().join("nextflow.config")).unwrap(); - assert_eq!(trace_fields(&cfg), Some(TRACE_FIELDS.map(str::to_owned).to_vec())); + assert_eq!( + trace_fields(&cfg), + Some(TRACE_FIELDS.map(str::to_owned).to_vec()) + ); // Control: the same reading of a config without memory must differ. let broken = cfg.replace(",memory,start", ",start"); assert_ne!(broken, cfg, "the control found no trace.fields to break"); - assert_ne!(trace_fields(&broken), Some(TRACE_FIELDS.map(str::to_owned).to_vec())); + assert_ne!( + trace_fields(&broken), + Some(TRACE_FIELDS.map(str::to_owned).to_vec()) + ); } /// An old `bin/run_*.py` exits 1 and names its og command, never a silent 0. @@ -616,7 +705,10 @@ fn refuses(script: &Path, cmd: &str) -> Result<(), String> { .output() .map_err(|e| format!("cannot run {exe}: {e}; set OG_PYTHON"))?; let err = String::from_utf8_lossy(&o.stderr); - if o.status.code() == Some(1) && err.contains(&format!("bin/og run {cmd}")) && o.stdout.is_empty() { + if o.status.code() == Some(1) + && err.contains(&format!("bin/og run {cmd}")) + && o.stdout.is_empty() + { Ok(()) } else { Err(format!("{:?} {err}", o.status)) @@ -627,7 +719,11 @@ fn refuses(script: &Path, cmd: &str) -> Result<(), String> { /// shut (they go with the Python server, and so does this check). #[test] fn the_old_python_command_lines_refuse_and_name_og_run() { - for (name, cmd) in [("run_eta", "eta"), ("run_state", "state"), ("run_headroom", "headroom")] { + for (name, cmd) in [ + ("run_eta", "eta"), + ("run_state", "state"), + ("run_headroom", "headroom"), + ] { let script = repo().join("bin").join(format!("{name}.py")); if !script.is_file() { continue; diff --git a/rust/crates/og-live/tests/ops_headroom.rs b/rust/crates/og-live/tests/ops_headroom.rs index 43da95cd4..a6b61c97c 100644 --- a/rust/crates/og-live/tests/ops_headroom.rs +++ b/rust/crates/og-live/tests/ops_headroom.rs @@ -167,7 +167,11 @@ fn disk_91_warns_at_the_default_89_does_not() { let hi = headroom::disk(&fake(91), &[("work", Some(s(&d)))], default_warn()); let lo = headroom::disk(&fake(89), &[("work", Some(s(&d)))], default_warn()); assert!( - hi.len() == 1 && hi[0].warn && close(hi[0].used_ratio, 0.91) && lo.len() == 1 && !lo[0].warn, + hi.len() == 1 + && hi[0].warn + && close(hi[0].used_ratio, 0.91) + && lo.len() == 1 + && !lo[0].warn, "{hi:?} {lo:?}" ); } @@ -185,7 +189,12 @@ fn disk_warn_percent_80_and_its_environment_variable_warn_at_85() { let arg = headroom::lines(&snapshot_of(&host, &with)).join("\n"); let mut child = Command::new(std::env::current_exe().unwrap()); child - .args(["child_disk_warn_from_the_environment", "--exact", "--ignored", "--quiet"]) + .args([ + "child_disk_warn_from_the_environment", + "--exact", + "--ignored", + "--quiet", + ]) .env("OWNGENOME_DISK_WARN_PERCENT", "80") .env("OPS_CHILD_DIR", &d); #[cfg(feature = "fault-inject")] @@ -216,7 +225,11 @@ fn child_disk_warn_from_the_environment() { fn a_missing_work_folder_is_measured_on_its_parent() { let d = tmp(); let host = fake(50); - let miss = headroom::disk(&host, &[("work", Some(s(&d.join("not/yet"))))], default_warn()); + let miss = headroom::disk( + &host, + &[("work", Some(s(&d.join("not/yet"))))], + default_warn(), + ); let seen = host.seen.borrow().clone(); assert!( miss.len() == 1 && miss[0].missing && seen == [d.clone()], @@ -278,7 +291,10 @@ fn a_real_statvfs_gives_a_ratio_in_0_to_1_and_a_disk_line() { fn seven_point_six_of_8_gb_warns_6_of_8_does_not() { let t = by_process(trace_memory(&trace("trace.txt"))); assert!( - t["FASTP"].warn && close(t["FASTP"].ratio, 0.95) && !t["VEP"].warn && close(t["VEP"].ratio, 0.75), + t["FASTP"].warn + && close(t["FASTP"].ratio, 0.95) + && !t["VEP"].warn + && close(t["VEP"].ratio, 0.75), "{t:?}" ); } @@ -423,7 +439,10 @@ fn the_snapshot_holds_no_path_and_takes_its_folders_from_the_metadata() { r.events = &evs; r.trace_rows = &rows; let snap = snapshot_of(&host, &r); - let bad: Vec = strings(&snap).into_iter().filter(|x| x.contains('/')).collect(); + let bad: Vec = strings(&snap) + .into_iter() + .filter(|x| x.contains('/')) + .collect(); let fs: Vec<&str> = snap.disk.iter().map(|d| d.fs.as_str()).collect(); let mut procs: Vec<&str> = snap.tasks.iter().map(|t| t.process.as_str()).collect(); procs.sort_unstable(); @@ -449,15 +468,21 @@ fn st(v: &str) -> Option { #[test] fn a_relative_outdir_is_under_launch_dir_an_absolute_one_is_kept() { - let rel = md(r#"{"metadata":{"workflow":{"workDir":"/x/work","launchDir":"/x"},"parameters":{"outdir":"./results"}}}"#); - let ab = md(r#"{"metadata":{"workflow":{"workDir":"/x/work","launchDir":"/x"},"parameters":{"outdir":"/y/results"}}}"#); + let rel = md( + r#"{"metadata":{"workflow":{"workDir":"/x/work","launchDir":"/x"},"parameters":{"outdir":"./results"}}}"#, + ); + let ab = md( + r#"{"metadata":{"workflow":{"workDir":"/x/work","launchDir":"/x"},"parameters":{"outdir":"/y/results"}}}"#, + ); assert_eq!(rel, (st("/x/work"), st("/x/./results"), None)); assert_eq!(ab, (st("/x/work"), st("/y/results"), None)); } #[test] fn metadata_under_params_gives_the_outdir_and_max_memory() { - let alt = md(r#"{"metadata":{"workflow":{"workDir":"/x/work"},"params":{"outdir":"/y/r","max_memory":"8.GB"}}}"#); + let alt = md( + r#"{"metadata":{"workflow":{"workDir":"/x/work"},"params":{"outdir":"/y/r","max_memory":"8.GB"}}}"#, + ); assert_eq!(alt, (st("/x/work"), st("/y/r"), st("8.GB"))); } diff --git a/rust/crates/og-runlog/src/eta.rs b/rust/crates/og-runlog/src/eta.rs index 9773d1b04..5fe3c2c74 100644 --- a/rust/crates/og-runlog/src/eta.rs +++ b/rust/crates/og-runlog/src/eta.rs @@ -152,7 +152,8 @@ pub fn read_trace(path: &Path, tz: &TimeZone) -> Vec { }; let forced = fault::on("eta-forced-ok"); let mut memory_bytes = memory(r.get("memory")); - if fault::on("eta-memory-decimal-gb") && r.get("memory").is_some_and(|m| m.ends_with("GB")) { + if fault::on("eta-memory-decimal-gb") && r.get("memory").is_some_and(|m| m.ends_with("GB")) + { memory_bytes = memory_bytes.map(|m| m / 1024 / 1024 / 1024 * 1_000_000_000); } out.push(TraceTask { From 8ccacbde11fe7e0fffb3afd9a2f70868d8afdd3a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:25:40 +0100 Subject: [PATCH 187/255] feat(ancestry): add og-admix, the parts our ancestry engines share The engines that replace ADMIXTURE and RFMix in step 26b need one fixed interface to build against in parallel. og-admix now holds: - panel: a phased panel VCF read into bit-packed haplotypes for chosen people at chosen sites, biallelic SNVs only; an unphased, missing or haploid genotype of a chosen person stops the read. - map: the PLINK genetic map reader that chroms.rs had, plus the inverse (position by cM). A non-finite cM or a cM that goes down is refused. - geno: model/ref.geno, 2-bit genotypes with a magic, version, J and N header, its writer and reader. - rng: xoshiro256** seeded by SplitMix64, checked against the authors' reference code. - simulate: the admixed-genome simulator with a truth BED. og ancestry-fine chroms reads its maps with og-admix's reader. The feature planted-red feeds every red control its broken input. Bead: D5 Bead: D14 Bead: D1 --- rust/Cargo.lock | 7 + rust/crates/og-admix/Cargo.toml | 13 +- rust/crates/og-admix/src/geno.rs | 279 +++++++ rust/crates/og-admix/src/lib.rs | 130 ++- rust/crates/og-admix/src/map.rs | 232 ++++++ rust/crates/og-admix/src/panel.rs | 503 ++++++++++++ rust/crates/og-admix/src/rng.rs | 213 +++++ rust/crates/og-admix/src/simulate.rs | 768 ++++++++++++++++++ rust/crates/og-admix/tests/data/README.md | 20 + .../data/family-phased.mum-dad.bcftools.tsv | 3 + .../tests/data/panel.P4-P1-P3.bcftools.tsv | 6 + rust/crates/og-admix/tests/data/panel.vcf.gz | Bin 0 -> 590 bytes rust/crates/og-admix/tests/panel_bcftools.rs | 81 ++ rust/crates/og-ancestry/Cargo.toml | 1 + rust/crates/og-ancestry/src/chroms.rs | 66 +- 15 files changed, 2259 insertions(+), 63 deletions(-) create mode 100644 rust/crates/og-admix/src/geno.rs create mode 100644 rust/crates/og-admix/src/map.rs create mode 100644 rust/crates/og-admix/src/panel.rs create mode 100644 rust/crates/og-admix/src/rng.rs create mode 100644 rust/crates/og-admix/src/simulate.rs create mode 100644 rust/crates/og-admix/tests/data/README.md create mode 100644 rust/crates/og-admix/tests/data/family-phased.mum-dad.bcftools.tsv create mode 100644 rust/crates/og-admix/tests/data/panel.P4-P1-P3.bcftools.tsv create mode 100644 rust/crates/og-admix/tests/data/panel.vcf.gz create mode 100644 rust/crates/og-admix/tests/panel_bcftools.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 6bf5d012d..029aa66bd 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -628,12 +628,19 @@ dependencies = [ [[package]] name = "og-admix" version = "0.1.0" +dependencies = [ + "memchr", + "og-io", + "og-vcf", + "pycompat", +] [[package]] name = "og-ancestry" version = "0.1.0" dependencies = [ "flate2", + "og-admix", "og-intervals", "og-io", "og-tsv", diff --git a/rust/crates/og-admix/Cargo.toml b/rust/crates/og-admix/Cargo.toml index 784875681..cf646e56e 100644 --- a/rust/crates/og-admix/Cargo.toml +++ b/rust/crates/og-admix/Cargo.toml @@ -1,12 +1,23 @@ [package] name = "og-admix" -description = "What the fine-ancestry engines share: our own local-ancestry painting and the projection of ancestry fractions" +description = "What the fine-ancestry engines share: the panel's haplotypes, the genetic map, packed reference genotypes, a seeded RNG, the admixed-genome simulator, and the painting and projection built on them" version.workspace = true edition.workspace = true license.workspace = true publish.workspace = true +[features] +# Feeds each red control its broken input in place of the good one +# (a shuffled map, T doubled, a flipped byte, a swapped sample order, a +# second seed), so `cargo test -p og-admix --features planted-red +# --no-fail-fast` shows every control fail. Never on in a build. +planted-red = [] + [dependencies] +memchr.workspace = true +og-io.workspace = true +og-vcf.workspace = true +pycompat.workspace = true [lints] workspace = true diff --git a/rust/crates/og-admix/src/geno.rs b/rust/crates/og-admix/src/geno.rs new file mode 100644 index 000000000..6c2eb7131 --- /dev/null +++ b/rust/crates/og-admix/src/geno.rs @@ -0,0 +1,279 @@ +//! `model/ref.geno`: the reference people's genotypes at the model's SNVs, +//! 2 bits each, so a bootstrap can resample people and count their alleles +//! again without the panel. +//! +//! Layout, version 1, all numbers little-endian: +//! +//! | bytes | what | +//! |---|---| +//! | 8 | magic `OGREFGEN` | +//! | 4 | version, `1` | +//! | 8 | J, the SNVs | +//! | 8 | N, the people | +//! | J x ceil(N / 4) | one row per SNV, in `ref.bim` order | +//! +//! In a row, person i is bits `2 (i mod 4)` and up of byte `i / 4`: the +//! copies of the `.bim`'s A1 allele (0, 1 or 2), or 3 for a missing +//! genotype. The unused bits of a row's last byte are 0. About 1,000 +//! people at 100,000 SNVs is 25 MB. + +use crate::{Error, Result, cannot_read, cannot_write}; +use std::io::{self, Read, Write}; +use std::path::Path; + +pub const MAGIC: [u8; 8] = *b"OGREFGEN"; +pub const VERSION: u32 = 1; +/// The code of a missing genotype. +pub const MISSING: u8 = 3; +const HEADER: usize = 8 + 4 + 8 + 8; + +/// Genotypes of N people at J SNVs, packed 4 to a byte. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RefGeno { + people: usize, + sites: usize, + row: usize, + data: Vec, +} + +impl RefGeno { + /// No SNV yet, for `people` people. + pub fn new(people: usize) -> RefGeno { + RefGeno { + people, + sites: 0, + row: people.div_ceil(4), + data: Vec::new(), + } + } + + pub fn people(&self) -> usize { + self.people + } + + pub fn sites(&self) -> usize { + self.sites + } + + /// Adds the next SNV: one genotype per person, the copies of A1, `None` + /// when missing. + pub fn push_site(&mut self, genotypes: &[Option]) -> Result<()> { + if genotypes.len() != self.people { + return Err(Error(format!( + "a ref.geno row holds {} genotypes, not {}", + self.people, + genotypes.len() + ))); + } + let start = self.data.len(); + self.data.resize(start + self.row, 0); + for (i, g) in genotypes.iter().enumerate() { + let code = match *g { + None => MISSING, + Some(c @ 0..=2) => c, + Some(c) => { + self.data.truncate(start); + return Err(Error(format!( + "a genotype counts 0, 1 or 2 copies of A1, not {c}" + ))); + } + }; + self.data[start + i / 4] |= code << (2 * (i % 4)); + } + self.sites += 1; + Ok(()) + } + + /// The copies of A1 person `person` carries at SNV `site`, `None` when + /// missing. + pub fn get(&self, site: usize, person: usize) -> Option { + assert!( + site < self.sites && person < self.people, + "outside ref.geno" + ); + let code = (self.data[site * self.row + person / 4] >> (2 * (person % 4))) & 3; + (code != MISSING).then_some(code) + } + + /// At SNV `site`, the copies of A1 among `people` (a person listed twice + /// counts twice, as a bootstrap draw does) and the people called. + pub fn a1_count(&self, site: usize, people: &[usize]) -> (u32, u32) { + people + .iter() + .filter_map(|&p| self.get(site, p)) + .fold((0, 0), |(copies, called), g| { + (copies + u32::from(g), called + 1) + }) + } + + /// Writes the file's bytes. + pub fn write(&self, out: &mut impl Write) -> io::Result<()> { + out.write_all(&MAGIC)?; + out.write_all(&VERSION.to_le_bytes())?; + out.write_all(&(self.sites as u64).to_le_bytes())?; + out.write_all(&(self.people as u64).to_le_bytes())?; + out.write_all(&self.data) + } + + /// Reads the file's bytes: the magic, the version, a body of exactly + /// J rows and nothing after it, and 0 in every unused bit. + pub fn read(input: &mut impl Read) -> Result { + let bad = |what: &str| Error(format!("not a ref.geno file: {what}")); + let mut head = [0u8; HEADER]; + input + .read_exact(&mut head) + .map_err(|_| bad("shorter than its header"))?; + if head[..8] != MAGIC { + return Err(bad("no OGREFGEN magic")); + } + let word = |at: usize| u64::from_le_bytes(head[at..at + 8].try_into().expect("8 bytes")); + let version = u32::from_le_bytes(head[8..12].try_into().expect("4 bytes")); + if version != VERSION { + return Err(Error(format!( + "ref.geno version {version}: this og reads version {VERSION}; run setup.sh --ancestry-fine-panel again" + ))); + } + let (sites, people) = (word(12), word(20)); + let size = usize::try_from(sites) + .ok() + .zip(usize::try_from(people).ok()) + .and_then(|(j, n)| j.checked_mul(n.div_ceil(4)).map(|s| (j, n, s))); + let Some((sites, people, size)) = size else { + return Err(bad("its size does not fit in memory")); + }; + let mut data = Vec::new(); + input + .take((size as u64).saturating_add(1)) + .read_to_end(&mut data) + .map_err(|e| bad(&e.to_string()))?; + if data.len() != size { + return Err(bad(&format!( + "{} SNVs of {} people take {size} bytes after the header, not {}{}", + sites, + people, + data.len().min(size), + if data.len() > size { " and more" } else { "" } + ))); + } + let geno = RefGeno { + people, + sites, + row: people.div_ceil(4), + data, + }; + let used = 2 * (people % 4); + if used > 0 && geno.row > 0 { + let mask = !0u8 << used; + if let Some(j) = (0..sites).find(|j| geno.data[j * geno.row + geno.row - 1] & mask != 0) + { + return Err(bad(&format!("unused bits set in the row of SNV {}", j + 1))); + } + } + Ok(geno) + } + + /// Writes `path` in full under a temporary name, then renames it into + /// place. + pub fn save(&self, path: &Path) -> Result<()> { + let mut f = og_io::AtomicFile::create(path).map_err(|e| cannot_write(path, &e))?; + self.write(&mut f).map_err(|e| cannot_write(path, &e))?; + f.commit().map_err(|e| cannot_write(path, &e)) + } + + pub fn load(path: &Path) -> Result { + let f = std::fs::File::open(path).map_err(|e| cannot_read(path, &e))?; + RefGeno::read(&mut io::BufReader::new(f)) + .map_err(|e| Error(format!("{}: {e}", path.display()))) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rng::Rng; + use crate::testutil::Dir; + + fn random(rng: &mut Rng, people: usize, sites: usize) -> (RefGeno, Vec>>) { + let mut g = RefGeno::new(people); + let mut rows = Vec::new(); + for _ in 0..sites { + let row: Vec> = (0..people) + .map(|_| match rng.below(4) { + 3 => None, + c => Some(c as u8), + }) + .collect(); + g.push_site(&row).unwrap(); + rows.push(row); + } + (g, rows) + } + + #[test] + fn round_trips_random_matrices() { + let mut rng = Rng::new(11); + let d = Dir::new("geno"); + // Every remainder of N mod 4, an empty matrix and a single person. + for (people, sites) in [(0, 3), (1, 1), (4, 0), (5, 17), (6, 9), (7, 33), (1001, 40)] { + let (g, rows) = random(&mut rng, people, sites); + assert_eq!((g.people(), g.sites()), (people, sites)); + let mut bytes = Vec::new(); + g.write(&mut bytes).unwrap(); + assert_eq!(bytes.len(), HEADER + sites * people.div_ceil(4)); + if crate::PLANTED && bytes.len() > HEADER { + bytes[HEADER] ^= 0b01; + } + let back = RefGeno::read(&mut bytes.as_slice()).unwrap(); + assert_eq!(back, g, "{people} people, {sites} SNVs"); + for (j, row) in rows.iter().enumerate() { + for (i, want) in row.iter().enumerate() { + assert_eq!(back.get(j, i), *want); + } + } + let path = d.0.join(format!("ref-{people}-{sites}.geno")); + g.save(&path).unwrap(); + assert_eq!(RefGeno::load(&path).unwrap(), g); + } + } + + #[test] + fn counts_a1_with_repeats_and_gaps() { + let mut g = RefGeno::new(3); + g.push_site(&[Some(2), None, Some(1)]).unwrap(); + assert_eq!(g.a1_count(0, &[0, 1, 2]), (3, 2)); + assert_eq!(g.a1_count(0, &[0, 0, 2]), (5, 3)); + assert!(g.push_site(&[Some(3), None, None]).is_err()); + assert!(g.push_site(&[Some(0)]).is_err()); + assert_eq!(g.sites(), 1, "a refused row adds nothing"); + } + + #[test] + fn refuses_damaged_files() { + let mut g = RefGeno::new(5); + g.push_site(&[Some(0), Some(1), Some(2), None, Some(1)]) + .unwrap(); + g.push_site(&[None; 5]).unwrap(); + let mut good = Vec::new(); + g.write(&mut good).unwrap(); + let read = |b: &[u8]| RefGeno::read(&mut &b[..]).unwrap_err().0; + assert!(read(&good[..10]).contains("shorter than its header")); + let mut b = good.clone(); + b[0] = b'X'; + assert!(read(&b).contains("no OGREFGEN magic")); + let mut b = good.clone(); + b[8] = 2; + assert!(read(&b).contains("ref.geno version 2")); + assert!(read(&good[..good.len() - 1]).contains("take 4 bytes after the header, not 3")); + let mut b = good.clone(); + b.push(0); + assert!(read(&b).contains("not 4 and more")); + let mut b = good.clone(); + // Person 5 is in bits 0-1 of the row's second byte; set bit 2. + b[HEADER + 1] |= 0b100; + assert!(read(&b).contains("unused bits set in the row of SNV 1")); + let mut b = good.clone(); + b[12..20].copy_from_slice(&u64::MAX.to_le_bytes()); + assert!(read(&b).contains("does not fit in memory")); + assert_eq!(RefGeno::read(&mut &good[..]).unwrap(), g); + } +} diff --git a/rust/crates/og-admix/src/lib.rs b/rust/crates/og-admix/src/lib.rs index bf0a44088..39a0478b2 100644 --- a/rust/crates/og-admix/src/lib.rs +++ b/rust/crates/og-admix/src/lib.rs @@ -1,9 +1,129 @@ -//! What the fine-ancestry engines of step 26b share: our own -//! local-ancestry painting ([`paint`]) and the projection of ancestry -//! fractions on the panel's frequencies ([`project`]). +//! What the fine-ancestry engines of step 26b share, so each engine is +//! built against one fixed interface: //! -//! Empty until their units of the 1.0.0 plan land. The crate exists now so -//! those units add code without editing the same files. +//! - [`panel`]: a phased panel VCF read into bit-packed haplotypes, for +//! chosen people at chosen sites. +//! - [`map`]: a PLINK genetic map, cM by position and position by cM. +//! - [`geno`]: `model/ref.geno`, the reference people's genotypes, 2 bits +//! each, with its writer and reader. +//! - [`rng`]: a seeded xoshiro256** generator, so a seed gives the same +//! bytes on every machine. +//! - [`simulate`]: an admixed diploid genome with known truth, built from +//! held-out people's haplotypes; every engine's truth tests start from it. +//! - [`paint`] and [`project`]: our own local-ancestry painting and the +//! projection of ancestry fractions (empty until their units land). +use std::fmt; +use std::io; +use std::path::Path; + +pub mod geno; +pub mod map; pub mod paint; +pub mod panel; pub mod project; +pub mod rng; +pub mod simulate; + +/// Why a reader or a writer stopped; the text is a whole sentence for an +/// `ERROR:` line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error(pub String); + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for Error {} + +pub type Result = std::result::Result; + +/// An I/O error's text without Rust's ` (os error N)`. +fn reason(e: &io::Error) -> String { + let s = e.to_string(); + match s.rfind(" (os error ") { + Some(i) if s.ends_with(')') => s[..i].to_owned(), + _ => s, + } +} + +fn cannot_read(path: &Path, e: &io::Error) -> Error { + Error(format!("cannot read {}: {}", path.display(), reason(e))) +} + +fn cannot_write(path: &Path, e: &io::Error) -> Error { + Error(format!("cannot write {}: {}", path.display(), reason(e))) +} + +/// Python's `repr()` of a str, as the messages of og ancestry-fine quote a +/// value. +fn repr(s: &str) -> String { + let quote = if s.contains('\'') && !s.contains('"') { + '"' + } else { + '\'' + }; + let mut out = String::with_capacity(s.len() + 2); + out.push(quote); + for c in s.chars() { + match c { + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if c == quote => { + out.push('\\'); + out.push(c); + } + c if (c as u32) < 0x20 || c as u32 == 0x7f => { + out.push_str(&format!("\\x{:02x}", c as u32)); + } + c => out.push(c), + } + } + out.push(quote); + out +} + +/// True when the red controls are planted (feature `planted-red`). +#[cfg(test)] +pub(crate) const PLANTED: bool = cfg!(feature = "planted-red"); + +#[cfg(test)] +pub(crate) mod testutil { + use std::path::PathBuf; + + /// A fresh directory under the temp dir, removed on drop. + pub struct Dir(pub PathBuf); + + impl Dir { + pub fn new(name: &str) -> Dir { + let p = std::env::temp_dir().join(format!( + "og-admix-{name}-{}-{:?}", + std::process::id(), + std::thread::current().id() + )); + let _ = std::fs::remove_dir_all(&p); + std::fs::create_dir_all(&p).unwrap(); + Dir(p) + } + + /// Writes `text` at `name` (folders made as needed); its path. + pub fn put(&self, name: &str, text: &str) -> PathBuf { + let p = self.0.join(name); + if let Some(d) = p.parent() { + std::fs::create_dir_all(d).unwrap(); + } + std::fs::write(&p, text).unwrap(); + p + } + } + + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } +} diff --git a/rust/crates/og-admix/src/map.rs b/rust/crates/og-admix/src/map.rs new file mode 100644 index 000000000..71894b8a8 --- /dev/null +++ b/rust/crates/og-admix/src/map.rs @@ -0,0 +1,232 @@ +//! A PLINK genetic map (`chromosome id cM position`, the layout of Beagle's +//! maps in `maps/chrN.map`): the genetic position of a base position, and +//! back. +//! +//! Between two markers the map is linear; before the first marker and after +//! the last it is flat, at the end marker's cM. + +use crate::{Error, Result, cannot_read, repr}; +use std::path::Path; + +/// Markers sorted by position, with cM never going down. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct GeneticMap { + pos: Vec, + cm: Vec, +} + +impl GeneticMap { + /// Reads a PLINK `.map`: the third column is cM, the fourth the + /// position, split on whitespace; a line with fewer than 4 columns is + /// skipped. gzip is read through. + /// + /// A position that is not a whole number, a cM that is not a finite + /// number, or a cM that goes down from one position to the next is an + /// error: a map like that cannot place a base or a crossover. + pub fn read(path: &Path) -> Result { + let input = og_io::open(path).map_err(|e| cannot_read(path, &e))?; + let mut lines = og_io::Lines::new(input, og_io::Utf8::Strict); + let mut line = String::new(); + let mut markers = Vec::new(); + let mut n = 0u64; + while lines + .read_line(&mut line) + .map_err(|e| cannot_read(path, &e))? + { + n += 1; + let text = line.strip_suffix('\n').unwrap_or(&line); + let c: Vec<&str> = text + .split(pycompat::num::py_isspace) + .filter(|w| !w.is_empty()) + .collect(); + if c.len() < 4 { + continue; + } + let bad = |what: &str, v: &str| { + Error(format!("{}: line {n}: {what}: {}", path.display(), repr(v))) + }; + let pos = pycompat::num::parse_int(c[3]) + .ok() + .and_then(|p| i64::try_from(p).ok()) + .ok_or_else(|| bad("position is not a whole number", c[3]))?; + let cm = + pycompat::num::parse_float(c[2]).map_err(|_| bad("cM is not a number", c[2]))?; + if !cm.is_finite() { + return Err(bad("cM is not a finite number", c[2])); + } + markers.push((pos, cm)); + } + GeneticMap::from_markers(markers).map_err(|e| Error(format!("{}: {e}", path.display()))) + } + + /// A map from (position, cM) markers in any order. The cM must be + /// finite and must not go down as the position goes up. + pub fn from_markers(mut markers: Vec<(i64, f64)>) -> Result { + if let Some(&(p, c)) = markers.iter().find(|(_, c)| !c.is_finite()) { + return Err(Error(format!("cM {c} at position {p} is not finite"))); + } + markers.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.total_cmp(&b.1))); + if let Some(w) = markers.windows(2).find(|w| w[1].1 < w[0].1) { + return Err(Error(format!( + "cM goes down from {} at position {} to {} at position {}", + w[0].1, w[0].0, w[1].1, w[1].0 + ))); + } + let (pos, cm) = markers.into_iter().unzip(); + Ok(GeneticMap { pos, cm }) + } + + /// The number of markers. + pub fn len(&self) -> usize { + self.pos.len() + } + + pub fn is_empty(&self) -> bool { + self.pos.is_empty() + } + + /// The markers, (position, cM), in position order. + pub fn markers(&self) -> impl Iterator + '_ { + self.pos.iter().copied().zip(self.cm.iter().copied()) + } + + /// The genetic position of `pos` in cM: linear between the markers + /// around it, flat beyond the ends, 0 on an empty map. At a position + /// with two markers, the last one counts. + pub fn cm_at(&self, pos: i64) -> f64 { + let (Some(&p_first), Some(&p_last)) = (self.pos.first(), self.pos.last()) else { + return 0.0; + }; + if pos <= p_first { + return self.cm[0]; + } + if pos >= p_last { + return self.cm[self.cm.len() - 1]; + } + // The last marker at or before pos, and the one after it. + let hi = self.pos.partition_point(|&p| p <= pos); + let (p0, c0, p1, c1) = (self.pos[hi - 1], self.cm[hi - 1], self.pos[hi], self.cm[hi]); + if p1 > p0 { + c0 + (c1 - c0) * (pos - p0) as f64 / (p1 - p0) as f64 + } else { + c0 + } + } + + /// The base position (not rounded) at genetic position `cm`: the + /// inverse of [`cm_at`](GeneticMap::cm_at) where the map rises, the + /// first position of a flat stretch, the end positions beyond the ends, + /// 0 on an empty map. + pub fn pos_at(&self, cm: f64) -> f64 { + let (Some(&c_first), Some(&c_last)) = (self.cm.first(), self.cm.last()) else { + return 0.0; + }; + if cm <= c_first { + return self.pos[0] as f64; + } + if cm >= c_last { + return self.pos[self.pos.len() - 1] as f64; + } + // The first marker above cm, and the last one at or below it. + let hi = self.cm.partition_point(|&c| c <= cm); + let (p0, c0, p1, c1) = (self.pos[hi - 1], self.cm[hi - 1], self.pos[hi], self.cm[hi]); + p0 as f64 + (p1 - p0) as f64 * (cm - c0) / (c1 - c0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::Dir; + + /// Three markers: 1 cM per Mb, then 3 cM per Mb, written out of order. + const MAP: &str = "chr1 . 3.0 2000000\nchr1 . 0.0 1000000\n\nchr1 . 6.0 3000000\n"; + + /// The same markers with the cM column shuffled across them: what a + /// map read from the wrong column or the wrong file looks like. + const SHUFFLED: &str = "chr1 . 6.0 2000000\nchr1 . 3.0 1000000\n\nchr1 . 0.0 3000000\n"; + + #[test] + fn interpolates_hand_values_at_both_ends_and_between() { + let d = Dir::new("map"); + let text = if crate::PLANTED { SHUFFLED } else { MAP }; + let map = GeneticMap::read(&d.put("chr1.map", text)).unwrap(); + assert_eq!(map.len(), 3); + // Before the first marker, at it, between, at the last, after it. + assert_eq!(map.cm_at(1), 0.0); + assert_eq!(map.cm_at(1_000_000), 0.0); + assert_eq!(map.cm_at(1_500_000), 1.5); + assert_eq!(map.cm_at(2_000_000), 3.0); + assert_eq!(map.cm_at(2_250_000), 3.75); + assert_eq!(map.cm_at(3_000_000), 6.0); + assert_eq!(map.cm_at(250_000_000), 6.0); + // And back. + assert_eq!(map.pos_at(-1.0), 1_000_000.0); + assert_eq!(map.pos_at(0.0), 1_000_000.0); + assert_eq!(map.pos_at(1.5), 1_500_000.0); + assert_eq!(map.pos_at(3.75), 2_250_000.0); + assert_eq!(map.pos_at(6.0), 3_000_000.0); + assert_eq!(map.pos_at(60.0), 3_000_000.0); + for pos in [1_000_000, 1_234_567, 2_000_001, 2_999_999] { + assert!((map.pos_at(map.cm_at(pos)) - pos as f64).abs() < 1e-6); + } + } + + #[test] + fn a_shuffled_cm_column_is_refused() { + let d = Dir::new("map-shuffled"); + let err = GeneticMap::read(&d.put("chr1.map", SHUFFLED)).unwrap_err(); + assert!( + err.0.ends_with( + "chr1.map: cM goes down from 6 at position 2000000 to 0 at position 3000000" + ), + "{err}" + ); + // Lines out of order are only sorted: the map is the same. + let a = GeneticMap::read(&d.put("a.map", MAP)).unwrap(); + let sorted = "chr1 . 0.0 1000000\nchr1 . 3.0 2000000\nchr1 . 6.0 3000000\n"; + let b = GeneticMap::read(&d.put("b.map", sorted)).unwrap(); + assert_eq!(a, b); + } + + #[test] + fn flat_stretches_and_empty_maps() { + let map = + GeneticMap::from_markers(vec![(100, 1.0), (200, 1.0), (300, 2.0), (300, 2.5)]).unwrap(); + assert_eq!(map.cm_at(150), 1.0); + // Two markers at one position: the higher cM counts from there on. + assert_eq!(map.cm_at(300), 2.5); + // A flat stretch maps back to its first position. + assert_eq!(map.pos_at(1.0), 100.0); + assert_eq!(map.pos_at(1.5), 250.0); + let empty = GeneticMap::default(); + assert!(empty.is_empty()); + assert_eq!(empty.cm_at(5), 0.0); + assert_eq!(empty.pos_at(5.0), 0.0); + } + + #[test] + fn bad_values_name_the_line() { + let d = Dir::new("map-bad"); + let p = d.put("x.map", "chr1 . 0.0 10\nchr1 . 1.0 2x\n"); + let err = GeneticMap::read(&p).unwrap_err().0; + assert!( + err.ends_with("x.map: line 2: position is not a whole number: '2x'"), + "{err}" + ); + let p = d.put("y.map", "chr1 . one 10\n"); + let err = GeneticMap::read(&p).unwrap_err().0; + assert!( + err.ends_with("y.map: line 1: cM is not a number: 'one'"), + "{err}" + ); + let p = d.put("z.map", "chr1 . nan 10\n"); + let err = GeneticMap::read(&p).unwrap_err().0; + assert!( + err.ends_with("z.map: line 1: cM is not a finite number: 'nan'"), + "{err}" + ); + let err = GeneticMap::read(&d.0.join("missing.map")).unwrap_err().0; + assert!(err.starts_with("cannot read "), "{err}"); + } +} diff --git a/rust/crates/og-admix/src/panel.rs b/rust/crates/og-admix/src/panel.rs new file mode 100644 index 000000000..7a95fda91 --- /dev/null +++ b/rust/crates/og-admix/src/panel.rs @@ -0,0 +1,503 @@ +//! The phased reference panel (`panel/chrN.vcf.gz`, one chromosome per +//! file) read into bit-packed haplotypes, for chosen people at chosen +//! sites. +//! +//! Only biallelic SNVs are read: one REF base and one ALT base, each A, C, +//! G or T. Every other record (an indel, a multi-allelic site, no ALT, a +//! `*` allele) is skipped and counted. A chosen person's genotype must be +//! phased, diploid and called (`0|0`, `0|1`, `1|0` or `1|1`), as the +//! panel Beagle phases against is: anything else stops the read, naming +//! the person and the site, so a painting never runs on a guessed allele. +//! People nobody asked for are not read. +//! +//! Haplotype `2 i` is the first allele of person `i` (in the order asked +//! for), haplotype `2 i + 1` the second. A site's alleles are a row of +//! `ceil(2 N / 64)` words, haplotype `h` in bit `h mod 64` of word +//! `h / 64`, 1 for ALT: chromosome 1 of a 470-haplotype panel at 100,000 +//! sites is 6 MB. + +use crate::{Error, Result, cannot_read, repr}; +use memchr::memchr_iter; +use std::io::BufRead; +use std::path::{Path, PathBuf}; + +/// A biallelic SNV. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Site { + pub pos: u32, + pub ref_: u8, + pub alt: u8, +} + +impl Site { + /// The site of a record's POS, REF and ALT when they make a biallelic + /// SNV. + pub fn snv(pos: u32, ref_: &[u8], alt: &[u8]) -> Option { + let base = |b: &[u8]| match b { + [c @ (b'A' | b'C' | b'G' | b'T')] => Some(*c), + _ => None, + }; + let (r, a) = (base(ref_)?, base(alt)?); + (r != a).then_some(Site { + pos, + ref_: r, + alt: a, + }) + } +} + +/// The words a row of `haplotypes` bits takes. +pub fn words(haplotypes: usize) -> usize { + haplotypes.div_ceil(64) +} + +/// Bit `h` of a row. +pub fn bit(row: &[u64], h: usize) -> u8 { + ((row[h / 64] >> (h % 64)) & 1) as u8 +} + +/// A panel read one site at a time, so a file of any size streams. +pub struct PanelReader { + path: PathBuf, + lines: og_vcf::Lines>, + people: Vec, + /// (sample column, slot in `people`), by column. + columns: Vec<(usize, usize)>, + chrom: Option, + /// The position read last and the alleles read there. + at: Option<(u32, Vec<(u8, u8)>)>, + skipped: u64, + line: Vec, +} + +impl PanelReader { + /// Opens `path` (plain, gzip or BGZF) and finds `people` among its + /// samples. A person missing from the panel, or asked for twice, or + /// named twice in the panel, is an error. + pub fn open(path: &Path, people: &[impl AsRef]) -> Result { + let input = + og_io::bgzf::open(path, og_io::bgzf::workers()).map_err(|e| cannot_read(path, &e))?; + let mut lines = og_vcf::Lines::new(input); + let header = og_vcf::Header::read(&mut lines) + .map_err(|e| Error(format!("{}: {e}", path.display())))?; + let samples: Vec<&[u8]> = header.samples().collect(); + let mut columns = Vec::with_capacity(people.len()); + let mut names = Vec::with_capacity(people.len()); + for (slot, person) in people.iter().enumerate() { + let person = person.as_ref(); + if names.iter().any(|n| n == person) { + return Err(Error(format!( + "{}: {} is asked for twice", + path.display(), + repr(person) + ))); + } + let mut found = samples + .iter() + .enumerate() + .filter(|(_, s)| **s == person.as_bytes()); + let Some((column, _)) = found.next() else { + return Err(Error(format!( + "{}: no sample {} in the panel", + path.display(), + repr(person) + ))); + }; + if found.next().is_some() { + return Err(Error(format!( + "{}: the panel names sample {} twice", + path.display(), + repr(person) + ))); + } + columns.push((column, slot)); + names.push(person.to_owned()); + } + columns.sort_unstable(); + Ok(PanelReader { + path: path.to_path_buf(), + lines, + people: names, + columns, + chrom: None, + at: None, + skipped: 0, + line: Vec::new(), + }) + } + + /// The people read, in the order asked for. + pub fn people(&self) -> &[String] { + &self.people + } + + /// Two per person. + pub fn haplotypes(&self) -> usize { + 2 * self.people.len() + } + + /// The chromosome of the sites read so far. + pub fn chrom(&self) -> Option<&str> { + self.chrom.as_deref() + } + + /// Records skipped so far: not a biallelic SNV, or the same SNV again. + pub fn skipped(&self) -> u64 { + self.skipped + } + + fn fail(&self, what: String) -> Error { + Error(format!( + "{}: line {}: {what}", + self.path.display(), + self.lines.number() + )) + } + + /// The next biallelic SNV, its alleles written into `row` (which holds + /// [`words`]`(haplotypes)` words); `None` at the end of the file. + pub fn next_site(&mut self, row: &mut [u64]) -> Result> { + assert_eq!(row.len(), words(self.haplotypes()), "row size"); + loop { + // The line is copied out of the reader, so the reader can say + // its number while the record is read. + match self.lines.next_line() { + Ok(Some(line)) => { + self.line.clear(); + self.line.extend_from_slice(line); + } + Ok(None) => return Ok(None), + Err(e) => return Err(cannot_read(&self.path, &e)), + } + if self.line.is_empty() || self.line.starts_with(b"#") { + continue; + } + let rec = og_vcf::Record::parse(&self.line).map_err(|e| { + Error(format!( + "{}: line {}: {e}", + self.path.display(), + self.lines.number() + )) + })?; + let pos = rec.pos().map_err(|e| { + Error(format!( + "{}: line {}: {e}", + self.path.display(), + self.lines.number() + )) + })?; + let chrom = String::from_utf8_lossy(rec.chrom()); + match &self.chrom { + None => self.chrom = Some(chrom.into_owned()), + Some(c) if *c == chrom => {} + Some(c) => { + let msg = format!( + "a record on {chrom} after records on {c}; the panel has one file per chromosome" + ); + return Err(self.fail(msg)); + } + } + if let Some((last, _)) = &self.at + && pos < *last + { + let msg = format!("position {pos} after {last}: the panel is not sorted"); + return Err(self.fail(msg)); + } + let Some(site) = Site::snv(pos, rec.ref_(), rec.alt()) else { + self.skipped += 1; + continue; + }; + match &mut self.at { + Some((p, seen)) if *p == pos => { + if seen.contains(&(site.ref_, site.alt)) { + self.skipped += 1; + continue; + } + seen.push((site.ref_, site.alt)); + } + at => *at = Some((pos, vec![(site.ref_, site.alt)])), + } + self.genotypes(&rec, site, row)?; + return Ok(Some(site)); + } + } + + /// The chosen people's alleles at `site` into `row`. + fn genotypes(&self, rec: &og_vcf::Record<'_>, site: Site, row: &mut [u64]) -> Result<()> { + row.fill(0); + if self.columns.is_empty() { + return Ok(()); + } + if rec.format_keys().next() != Some(b"GT") { + return Err(self.fail("GT is not the first FORMAT field".into())); + } + let Some(samples) = rec.samples() else { + return Err(self.fail("no sample columns".into())); + }; + let mut starts = std::iter::once(0).chain(memchr_iter(b'\t', samples).map(|t| t + 1)); + let mut column = 0usize; + let mut start = starts.next().expect("the first column"); + for &(want, slot) in &self.columns { + while column < want { + let Some(next) = starts.next() else { + break; + }; + start = next; + column += 1; + } + if column < want { + let msg = format!("{} sample columns, fewer than the header names", column + 1); + return Err(self.fail(msg)); + } + let field = &samples[start..]; + let end = field + .iter() + .position(|&c| c == b':' || c == b'\t') + .unwrap_or(field.len()); + let gt = &field[..end]; + let (a, b) = match gt { + [a @ (b'0' | b'1'), b'|', b @ (b'0' | b'1')] => (a - b'0', b - b'0'), + _ => { + let msg = format!( + "{} has genotype {} at {}:{}; a panel genotype is phased, diploid and called (0|0, 0|1, 1|0 or 1|1)", + self.people[slot], + repr(&String::from_utf8_lossy(gt)), + self.chrom.as_deref().unwrap_or(""), + site.pos + ); + return Err(self.fail(msg)); + } + }; + for (h, allele) in [(2 * slot, a), (2 * slot + 1, b)] { + row[h / 64] |= u64::from(allele) << (h % 64); + } + } + Ok(()) + } +} + +/// One chromosome of the panel, held whole: the sites and, per site, a row +/// of haplotype bits. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Haplotypes { + chrom: String, + people: Vec, + sites: Vec, + words: usize, + bits: Vec, +} + +impl Haplotypes { + /// Reads the biallelic SNVs of `people` from `path`, every one, or only + /// those in `keep` (in any order). A site of `keep` the panel lacks is + /// simply not there: compare [`sites`](Haplotypes::sites) with it. + pub fn read( + path: &Path, + people: &[impl AsRef], + keep: Option<&[Site]>, + ) -> Result { + let keep = keep.map(|k| { + let mut k = k.to_vec(); + k.sort_unstable(); + k + }); + let mut reader = PanelReader::open(path, people)?; + let words = words(reader.haplotypes()); + let mut row = vec![0u64; words]; + let (mut sites, mut bits) = (Vec::new(), Vec::new()); + while let Some(site) = reader.next_site(&mut row)? { + if keep.as_ref().is_none_or(|k| k.binary_search(&site).is_ok()) { + sites.push(site); + bits.extend_from_slice(&row); + } + } + Ok(Haplotypes { + chrom: reader.chrom().unwrap_or("").to_owned(), + people: reader.people().to_vec(), + sites, + words, + bits, + }) + } + + /// The chromosome; empty when no record was read. + pub fn chrom(&self) -> &str { + &self.chrom + } + + pub fn people(&self) -> &[String] { + &self.people + } + + pub fn sites(&self) -> &[Site] { + &self.sites + } + + /// Two per person. + pub fn haplotypes(&self) -> usize { + 2 * self.people.len() + } + + /// The alleles of every haplotype at site `m`. + pub fn row(&self, m: usize) -> &[u64] { + &self.bits[m * self.words..(m + 1) * self.words] + } + + /// The allele (0 REF, 1 ALT) of haplotype `h` at site `m`. + pub fn allele(&self, m: usize, h: usize) -> u8 { + assert!( + h < self.haplotypes(), + "haplotype {h} of {}", + self.haplotypes() + ); + bit(self.row(m), h) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::Dir; + + const HEAD: &str = + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\tC\n"; + + fn panel(d: &Dir, body: &str) -> PathBuf { + d.put("chr1.vcf.gz", &format!("{HEAD}{body}")) + } + + #[test] + fn packs_many_haplotypes_across_words() { + // 40 people, 80 haplotypes: two words per row. + let d = Dir::new("panel-wide"); + let names: Vec = (0..40).map(|i| format!("S{i}")).collect(); + let mut text = format!( + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{}\n", + names.join("\t") + ); + for pos in 1..=3u32 { + let gts: Vec = (0..40u32) + .map(|i| format!("{}|{}", (i + pos) % 2, (i * pos) % 3 % 2)) + .collect(); + text.push_str(&format!( + "chr1\t{pos}\t.\tA\tC\t.\t.\t.\tGT\t{}\n", + gts.join("\t") + )); + } + let p = d.put("chr1.vcf", &text); + let mut asked = names.clone(); + asked.reverse(); + let h = Haplotypes::read(&p, &asked, None).unwrap(); + assert_eq!(h.haplotypes(), 80); + assert_eq!(h.row(0).len(), 2); + for m in 0..3 { + let pos = m as u32 + 1; + for (slot, name) in asked.iter().enumerate() { + let i: u32 = name[1..].parse().unwrap(); + assert_eq!(u32::from(h.allele(m, 2 * slot)), (i + pos) % 2); + assert_eq!(u32::from(h.allele(m, 2 * slot + 1)), (i * pos) % 3 % 2); + } + } + } + + #[test] + fn keeps_only_the_chosen_sites() { + let d = Dir::new("panel-keep"); + let p = panel( + &d, + "chr1\t10\t.\tA\tG\t.\t.\t.\tGT\t0|1\t0|0\t1|1\n\ + chr1\t20\t.\tA\tG\t.\t.\t.\tGT\t1|1\t0|0\t1|1\n\ + chr1\t20\t.\tA\tT\t.\t.\t.\tGT\t0|0\t1|0\t0|0\n\ + chr1\t20\t.\tA\tG\t.\t.\t.\tGT\t0|0\t0|0\t0|0\n\ + chr1\t30\t.\tC\tT\t.\t.\t.\tGT\t1|0\t0|1\t0|1\n", + ); + let all = Haplotypes::read(&p, &["C", "A"], None).unwrap(); + assert_eq!(all.chrom(), "chr1"); + assert_eq!(all.sites().len(), 4, "the second A>G at 20 is skipped"); + let keep = [ + Site::snv(30, b"C", b"T").unwrap(), + Site::snv(20, b"A", b"T").unwrap(), + Site::snv(25, b"A", b"T").unwrap(), + ]; + let h = Haplotypes::read(&p, &["C", "A"], Some(&keep)).unwrap(); + let positions: Vec<(u32, u8)> = h.sites().iter().map(|s| (s.pos, s.alt)).collect(); + assert_eq!(positions, [(20, b'T'), (30, b'T')]); + // C then A: C is 0|0 at 20 A>T, A is 0|0; at 30 C is 0|1, A 1|0. + assert_eq!(h.row(0), [0b0000]); + assert_eq!(h.row(1), [0b0110]); + } + + #[test] + fn refuses_what_it_cannot_read_truly() { + let d = Dir::new("panel-bad"); + let err = |body: &str, people: &[&str]| { + let p = panel(&d, body); + Haplotypes::read(&p, people, None).unwrap_err().0 + }; + let ok = "chr1\t10\t.\tA\tG\t.\t.\t.\tGT\t0|1\t0|0\t1|1\n"; + let e = err(ok, &["D"]); + assert!(e.ends_with("no sample 'D' in the panel"), "{e}"); + let e = err(ok, &["A", "A"]); + assert!(e.ends_with("'A' is asked for twice"), "{e}"); + let e = err("chr1\t10\t.\tA\tG\t.\t.\t.\tGT\t0/1\t0|0\t1|1\n", &["A"]); + assert!( + e.ends_with("line 3: A has genotype '0/1' at chr1:10; a panel genotype is phased, diploid and called (0|0, 0|1, 1|0 or 1|1)"), + "{e}" + ); + let e = err("chr1\t10\t.\tA\tG\t.\t.\t.\tGT\t0|1\t.|.\t1|1\n", &["B"]); + assert!(e.contains("B has genotype '.|.'"), "{e}"); + let e = err("chr1\t10\t.\tA\tG\t.\t.\t.\tGT\t0|1\t1\t1|1\n", &["B"]); + assert!(e.contains("B has genotype '1'"), "{e}"); + let e = err("chr1\t10\t.\tA\tG\t.\t.\t.\tGT\t0|1\t0|0\n", &["C"]); + assert!( + e.contains("2 sample columns, fewer than the header names"), + "{e}" + ); + let e = err( + "chr1\t10\t.\tA\tG\t.\t.\t.\tDP:GT\t3:0|1\t3:0|0\t3:1|1\n", + &["C"], + ); + assert!(e.contains("GT is not the first FORMAT field"), "{e}"); + let e = err( + &format!("{ok}chr2\t5\t.\tA\tG\t.\t.\t.\tGT\t0|1\t0|0\t1|1\n"), + &["A"], + ); + assert!( + e.contains("line 4: a record on chr2 after records on chr1"), + "{e}" + ); + let e = err( + &format!("{ok}chr1\t9\t.\tA\tG\t.\t.\t.\tGT\t0|1\t0|0\t1|1\n"), + &["A"], + ); + assert!( + e.contains("position 9 after 10: the panel is not sorted"), + "{e}" + ); + // A person not asked for may be anything. + let p = panel(&d, "chr1\t10\t.\tA\tG\t.\t.\t.\tGT\t0|1\t./.\t1|1\n"); + assert_eq!( + Haplotypes::read(&p, &["A", "C"], None).unwrap().row(0), + [0b1110] + ); + // No one asked for: the sites alone. + let h = Haplotypes::read(&p, &[] as &[&str], None).unwrap(); + assert_eq!((h.sites().len(), h.haplotypes()), (1, 0)); + } + + #[test] + fn snv_rule() { + assert!(Site::snv(1, b"A", b"G").is_some()); + for (r, a) in [ + (&b"A"[..], &b"A"[..]), + (b"AT", b"A"), + (b"A", b"G,T"), + (b"A", b"."), + (b"A", b"*"), + (b"a", b"g"), + (b"N", b"A"), + ] { + assert!(Site::snv(1, r, a).is_none(), "{r:?} {a:?}"); + } + } +} diff --git a/rust/crates/og-admix/src/rng.rs b/rust/crates/og-admix/src/rng.rs new file mode 100644 index 000000000..a0687c735 --- /dev/null +++ b/rust/crates/og-admix/src/rng.rs @@ -0,0 +1,213 @@ +//! A seeded random number generator of our own: xoshiro256** (Blackman and +//! Vigna), its state filled by SplitMix64 from a 64-bit seed, as the +//! authors' reference code seeds it. No dependency, and the same seed gives +//! the same numbers on every machine and every build, which is what a +//! simulation with known truth and a bootstrap with fixed seeds need. + +/// xoshiro256**: 256 bits of state, period 2^256 - 1. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Rng { + s: [u64; 4], +} + +/// SplitMix64's step: the next output, advancing `state`. +fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +/// FNV-1a, 64 bits: a stable hash of a label, the same on every build. +fn fnv1a(label: &str) -> u64 { + label.bytes().fold(0xcbf2_9ce4_8422_2325, |h, b| { + (h ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3) + }) +} + +impl Rng { + /// The generator for `seed`: four SplitMix64 outputs from it. + pub fn new(seed: u64) -> Rng { + let mut sm = seed; + Rng { + s: [ + splitmix64(&mut sm), + splitmix64(&mut sm), + splitmix64(&mut sm), + splitmix64(&mut sm), + ], + } + } + + /// An independent stream for one part of a job (a chromosome and + /// haplotype, a bootstrap replicate), so a part's numbers do not change + /// when other parts are added, removed or run in another order. + pub fn stream(seed: u64, label: &str) -> Rng { + Rng::new(seed ^ fnv1a(label)) + } + + /// The next 64 random bits. + pub fn next_u64(&mut self) -> u64 { + let s = &mut self.s; + let result = s[1].wrapping_mul(5).rotate_left(7).wrapping_mul(9); + let t = s[1] << 17; + s[2] ^= s[0]; + s[3] ^= s[1]; + s[1] ^= s[2]; + s[0] ^= s[3]; + s[2] ^= t; + s[3] = s[3].rotate_left(45); + result + } + + /// A uniform number in [0, 1), from the top 53 bits. + pub fn uniform(&mut self) -> f64 { + (self.next_u64() >> 11) as f64 * (1.0 / (1u64 << 53) as f64) + } + + /// A uniform whole number in [0, n), without bias (Lemire's method). + /// `n` must be at least 1. + pub fn below(&mut self, n: u64) -> u64 { + assert!(n > 0, "below(0) has no value to give"); + let threshold = n.wrapping_neg() % n; + loop { + let m = u128::from(self.next_u64()) * u128::from(n); + if (m as u64) >= threshold { + return (m >> 64) as u64; + } + } + } + + /// An exponential waiting time with this `rate` (mean `1 / rate`); + /// infinite for a rate of 0 or less, so an event that never happens + /// never comes. + pub fn exponential(&mut self, rate: f64) -> f64 { + if rate <= 0.0 { + return f64::INFINITY; + } + // 1 - u is in (0, 1], so the log is finite. + -(1.0 - self.uniform()).ln() / rate + } + + /// An index drawn with probability proportional to its weight, from the + /// running totals of the weights (`cumulative[i]` is the sum of weights + /// 0..=i; the last one is the total, above 0). + pub fn pick(&mut self, cumulative: &[f64]) -> usize { + let total = *cumulative.last().expect("at least one weight"); + let u = self.uniform() * total; + cumulative + .partition_point(|&c| c <= u) + .min(cumulative.len() - 1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The first five outputs of the authors' reference C code + /// (xoshiro256starstar.c, state from splitmix64.c), compiled and run + /// for these seeds. + #[test] + fn matches_the_reference_code() { + let seed0 = if crate::PLANTED { 1 } else { 0 }; + let mut r = Rng::new(seed0); + let got: Vec = (0..5).map(|_| r.next_u64()).collect(); + assert_eq!( + got, + [ + 0x99ec_5f36_cb75_f2b4, + 0xbf6e_1f78_4956_452a, + 0x1a5f_849d_4933_e6e0, + 0x6aa5_94f1_262d_2d2c, + 0xbba5_ad4a_1f84_2e59 + ] + ); + let mut r = Rng::new(42); + let got: Vec = (0..5).map(|_| r.next_u64()).collect(); + assert_eq!( + got, + [ + 0x1578_0b2e_0c2e_c716, + 0x6104_d986_6d11_3a7e, + 0xae17_5332_39e4_99a1, + 0xecb8_ad47_03b3_60a1, + 0xfde6_dc7f_e2ec_5e64 + ] + ); + // The bare step, from the state 1, 2, 3, 4. + let mut r = Rng { s: [1, 2, 3, 4] }; + let got: Vec = (0..5).map(|_| r.next_u64()).collect(); + assert_eq!( + got, + [ + 0x2d00, + 0, + 0x5a00_7080, + 0x10e0_0000_0000_9d80, + 0x10e0_b61c_e100_9d80 + ] + ); + } + + #[test] + fn streams_are_stable_and_distinct() { + let a: Vec = { + let mut r = Rng::stream(7, "chr22/0"); + (0..4).map(|_| r.next_u64()).collect() + }; + let b: Vec = { + let mut r = Rng::stream(7, "chr22/0"); + (0..4).map(|_| r.next_u64()).collect() + }; + let c: Vec = { + let mut r = Rng::stream(7, "chr22/1"); + (0..4).map(|_| r.next_u64()).collect() + }; + assert_eq!(a, b); + assert_ne!(a, c); + assert_eq!(fnv1a(""), 0xcbf2_9ce4_8422_2325); + assert_eq!(fnv1a("a"), 0xaf63_dc4c_8601_ec8c); + } + + #[test] + fn draws_have_their_laws() { + let mut r = Rng::new(3); + let n = 200_000; + // uniform: in [0, 1), mean 1/2. + let mut sum = 0.0; + for _ in 0..n { + let u = r.uniform(); + assert!((0.0..1.0).contains(&u)); + sum += u; + } + assert!((sum / f64::from(n) - 0.5).abs() < 0.005); + // below: every value, none outside, about equally often. + let mut counts = [0u32; 7]; + for _ in 0..n { + counts[r.below(7) as usize] += 1; + } + let want = f64::from(n) / 7.0; + for c in counts { + assert!((f64::from(c) - want).abs() < 0.03 * want, "{counts:?}"); + } + assert_eq!(r.below(1), 0); + // exponential: mean 1 / rate, and P(X > mean) = 1/e. + let rate = 4.0; + let draws: Vec = (0..n).map(|_| r.exponential(rate)).collect(); + let mean = draws.iter().sum::() / f64::from(n); + assert!((mean * rate - 1.0).abs() < 0.01, "{mean}"); + let above = draws.iter().filter(|&&x| x > 1.0 / rate).count() as f64 / f64::from(n); + assert!((above - (-1.0f64).exp()).abs() < 0.005, "{above}"); + assert_eq!(r.exponential(0.0), f64::INFINITY); + // pick: in proportion to the weights. + let cum = [0.2, 0.2, 1.0]; + let mut got = [0u32; 3]; + for _ in 0..n { + got[r.pick(&cum)] += 1; + } + assert_eq!(got[1], 0, "a zero weight is never picked"); + assert!((f64::from(got[0]) / f64::from(n) - 0.2).abs() < 0.005); + } +} diff --git a/rust/crates/og-admix/src/simulate.rs b/rust/crates/og-admix/src/simulate.rs new file mode 100644 index 000000000..be354dcba --- /dev/null +++ b/rust/crates/og-admix/src/simulate.rs @@ -0,0 +1,768 @@ +//! An admixed diploid genome with known truth, the input every engine's +//! truth tests start from (`og ancestry-fine simulate`). +//! +//! Each of the two haplotypes is built on its own along the genetic map: +//! +//! - **Ancestry tracts.** Ancestry events come as a Poisson process of +//! rate T per Morgan (T is about the generations since admixture). At +//! each event the new ancestry is drawn from the proportions mu, so it +//! may be the ancestry the haplotype already has, as in the model the +//! painting fits. The raw tract lengths are exponential with mean 1/T +//! Morgan. +//! - **Copying inside a tract.** The haplotype copies one haplotype of a +//! held-out person of that ancestry, drawn at the start of the tract and +//! drawn again at copying switches, which come at `switch_rate` per +//! Morgan (1 by default: one crossover per Morgan per meiosis). A draw +//! never takes the haplotype the other copy is copying at that moment +//! when its group has another one, so a group of one held-out person +//! gives back that person's own genotype instead of a run of +//! homozygosity. +//! +//! The genome is written as an unphased VCF with sorted alleles (`0/1`, +//! never `1/0`), so nothing in it says which copy an allele came from and +//! Beagle phases it as it would a real person. Every biallelic SNV of the +//! panel is written, the first one at a position where there are two. The +//! truth is a BED of each haplotype's ancestry: per chromosome and +//! haplotype the rows run from the first written site to the last with no +//! gap and no overlap, and two touching rows never share an ancestry. A +//! breakpoint between two sites sits where the map puts the event, kept +//! between the two sites, so every site's truth is the row that holds it. +//! +//! Each chromosome and haplotype draws from its own [`Rng::stream`] of the +//! seed, so a chromosome's genome does not change when other chromosomes +//! are added or left out, and the same seed gives the same bytes. + +use crate::map::GeneticMap; +use crate::panel::{PanelReader, bit, words}; +use crate::rng::Rng; +use crate::{Error, Result, repr}; +use std::io::Write; +use std::path::Path; + +/// How a genome is drawn. +#[derive(Debug, Clone, PartialEq)] +pub struct Config { + /// T, ancestry events per Morgan. + pub generations: f64, + /// Copying switches per Morgan inside a tract. + pub switch_rate: f64, + /// The ancestries and their proportions mu, in the order the truth + /// numbers them. + pub mu: Vec<(String, f64)>, + pub seed: u64, +} + +impl Config { + /// Checks the numbers: T above 0, a switch rate of 0 or more, and + /// proportions above 0 that sum to 1 (within 1e-6), one per ancestry. + pub fn check(&self) -> Result<()> { + if !(self.generations.is_finite() && self.generations > 0.0) { + return Err(Error(format!( + "T is {}: ancestry events per Morgan must be above 0", + self.generations + ))); + } + if !(self.switch_rate.is_finite() && self.switch_rate >= 0.0) { + return Err(Error(format!( + "the switch rate is {}: copying switches per Morgan must be 0 or more", + self.switch_rate + ))); + } + if self.mu.is_empty() { + return Err(Error("no ancestry proportions".into())); + } + for (i, (g, w)) in self.mu.iter().enumerate() { + if self.mu[..i].iter().any(|(h, _)| h == g) { + return Err(Error(format!("ancestry {} is given twice", repr(g)))); + } + if !(w.is_finite() && *w > 0.0) { + return Err(Error(format!( + "the proportion of {} is {w}: each must be above 0", + repr(g) + ))); + } + } + let total: f64 = self.mu.iter().map(|(_, w)| w).sum(); + if (total - 1.0).abs() > 1e-6 { + return Err(Error(format!( + "the ancestry proportions sum to {total}, not 1" + ))); + } + Ok(()) + } + + /// Running totals of mu, for [`Rng::pick`]. + fn cumulative(&self) -> Vec { + self.mu + .iter() + .scan(0.0, |s, (_, w)| { + *s += w; + Some(*s) + }) + .collect() + } +} + +/// The ancestry events of one haplotype along the map: a Poisson process +/// of rate T per Morgan, each event drawing the ancestry from mu. +#[derive(Debug, Clone)] +pub struct AncestryProcess { + per_cm: f64, + cumulative: Vec, + next_cm: f64, + ancestry: usize, +} + +impl AncestryProcess { + /// Starts at `start_cm`: draws the first ancestry and the first event. + /// `cumulative` holds the running totals of mu. + pub fn new(rng: &mut Rng, generations: f64, cumulative: &[f64], start_cm: f64) -> Self { + let per_cm = generations / 100.0; + let ancestry = rng.pick(cumulative); + AncestryProcess { + per_cm, + cumulative: cumulative.to_vec(), + next_cm: start_cm + rng.exponential(per_cm), + ancestry, + } + } + + pub fn ancestry(&self) -> usize { + self.ancestry + } + + /// Where the next event is, in cM. + pub fn next_cm(&self) -> f64 { + self.next_cm + } + + /// Moves to the next event: its place in cM and the ancestry drawn + /// there. + pub fn step(&mut self, rng: &mut Rng) -> (f64, usize) { + let at = self.next_cm; + self.ancestry = rng.pick(&self.cumulative); + self.next_cm = at + rng.exponential(self.per_cm); + (at, self.ancestry) + } +} + +/// One row of the truth: `[start, end)` in 0-based BED coordinates. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TruthRow { + pub start: i64, + pub end: i64, + /// 0 or 1. + pub haplotype: u8, + /// An index into [`Config::mu`]. + pub ancestry: usize, +} + +/// What one chromosome gave. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Chrom { + pub name: String, + /// Sites written. + pub sites: u64, + /// Records of the panel left out: not a biallelic SNV, or a second SNV + /// at a position. + pub skipped: u64, + /// The truth, by start, haplotype 0 before 1. + pub rows: Vec, +} + +/// One haplotype being built. +struct Copy { + rng: Rng, + tracts: AncestryProcess, + switch_per_cm: f64, + next_switch: f64, + /// An index into the haplotypes of the current ancestry's group. + source: usize, + start: i64, + first: usize, + /// (breakpoint, ancestry from there on). + changes: Vec<(i64, usize)>, +} + +/// A source among `n`, never `exclude` when there is another. +fn draw_source(rng: &mut Rng, n: usize, exclude: Option) -> usize { + match exclude { + Some(x) if n > 1 => { + let i = rng.below(n as u64 - 1) as usize; + if i >= x { i + 1 } else { i } + } + _ => rng.below(n as u64) as usize, + } +} + +impl Copy { + fn new( + mut rng: Rng, + cfg: &Config, + cumulative: &[f64], + groups: &[Vec], + start_cm: f64, + start: i64, + other: Option<(usize, usize)>, + ) -> Copy { + let tracts = AncestryProcess::new(&mut rng, cfg.generations, cumulative, start_cm); + let a = tracts.ancestry(); + let exclude = other.and_then(|(oa, os)| (oa == a).then_some(os)); + let source = draw_source(&mut rng, groups[a].len(), exclude); + let switch_per_cm = cfg.switch_rate / 100.0; + let next_switch = start_cm + rng.exponential(switch_per_cm); + Copy { + rng, + tracts, + switch_per_cm, + next_switch, + source, + start, + first: a, + changes: Vec::new(), + } + } + + fn state(&self) -> (usize, usize) { + (self.tracts.ancestry(), self.source) + } + + /// Takes every event up to `cm`, the place of the site at `pos`; the + /// site before it is at `prev`. + fn advance( + &mut self, + cm: f64, + prev: i64, + pos: i64, + map: &GeneticMap, + groups: &[Vec], + other: (usize, usize), + ) { + loop { + let tract_next = self.tracts.next_cm(); + if tract_next.min(self.next_switch) > cm { + return; + } + let at = if tract_next <= self.next_switch { + let (at, a) = self.tracts.step(&mut self.rng); + // The site before keeps the old ancestry, this one the new. + let b = (map.pos_at(at).round() as i64).clamp(prev, pos - 1); + self.changes.push((b, a)); + at + } else { + self.next_switch + }; + let a = self.tracts.ancestry(); + let exclude = (other.0 == a).then_some(other.1); + self.source = draw_source(&mut self.rng, groups[a].len(), exclude); + self.next_switch = at + self.rng.exponential(self.switch_per_cm); + } + } + + /// The truth rows up to `end`, empty rows dropped and touching rows of + /// one ancestry joined. + fn rows(&self, haplotype: u8, end: i64) -> Vec { + let mut out: Vec = Vec::new(); + let (mut start, mut ancestry) = (self.start, self.first); + let mut push = |start: i64, end: i64, ancestry: usize| { + if end <= start { + return; + } + match out.last_mut() { + Some(last) if last.ancestry == ancestry => last.end = end, + _ => out.push(TruthRow { + start, + end, + haplotype, + ancestry, + }), + } + }; + for &(b, a) in &self.changes { + if b > start { + push(start, b, ancestry); + start = b; + } + ancestry = a; + } + push(start, end, ancestry); + out + } +} + +/// Writes the VCF header: one `##contig` per chromosome, the settings, and +/// one sample. +pub fn write_header( + out: &mut dyn Write, + sample: &str, + chroms: &[&str], + cfg: &Config, +) -> Result<()> { + let mu: Vec = cfg.mu.iter().map(|(g, w)| format!("{g}:{w}")).collect(); + let mut text = String::from("##fileformat=VCFv4.2\n##source=og ancestry-fine simulate\n"); + text.push_str(&format!( + "##og_simulate=seed={};generations={};switch_rate={};mu={}\n", + cfg.seed, + cfg.generations, + cfg.switch_rate, + mu.join(",") + )); + for c in chroms { + text.push_str(&format!("##contig=\n")); + } + text.push_str("##FORMAT=\n"); + text.push_str(&format!( + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample}\n" + )); + out.write_all(text.as_bytes()) + .map_err(|e| Error(format!("cannot write the VCF: {e}"))) +} + +/// Simulates one chromosome from the panel file `panel` and its `map`: +/// writes its records to `vcf` and gives back its truth. `people` holds the +/// held-out people as (sample, ancestry); only those of an ancestry in mu +/// are read, and each ancestry in mu needs at least one. +pub fn simulate_chrom( + panel: &Path, + map: &GeneticMap, + people: &[(String, String)], + cfg: &Config, + vcf: &mut dyn Write, +) -> Result { + cfg.check()?; + if map.is_empty() { + return Err(Error(format!( + "no genetic map markers for {}", + panel.display() + ))); + } + let cumulative = cfg.cumulative(); + let mut ids: Vec<&str> = Vec::new(); + let mut groups: Vec> = vec![Vec::new(); cfg.mu.len()]; + for (id, group) in people { + if let Some(g) = cfg.mu.iter().position(|(m, _)| m == group) { + groups[g].extend([2 * ids.len(), 2 * ids.len() + 1]); + ids.push(id); + } + } + if let Some(((g, _), _)) = cfg.mu.iter().zip(&groups).find(|(_, h)| h.is_empty()) { + return Err(Error(format!( + "no held-out person of ancestry {} to copy", + repr(g) + ))); + } + let mut reader = PanelReader::open(panel, &ids)?; + let mut row = vec![0u64; words(reader.haplotypes())]; + let mut copies: Vec = Vec::with_capacity(2); + let (mut prev, mut sites, mut doubles) = (0i64, 0u64, 0u64); + let mut line = Vec::with_capacity(64); + let mut name = String::new(); + while let Some(site) = reader.next_site(&mut row)? { + let pos = i64::from(site.pos); + if sites > 0 && pos == prev { + doubles += 1; + continue; + } + let cm = map.cm_at(pos); + if copies.is_empty() { + name = reader.chrom().unwrap_or("").to_owned(); + for h in 0..2u8 { + let rng = Rng::stream(cfg.seed, &format!("{name}/{h}")); + let other = copies.first().map(Copy::state); + copies.push(Copy::new( + rng, + cfg, + &cumulative, + &groups, + cm, + pos - 1, + other, + )); + } + } else { + for h in 0..2 { + let other = copies[1 - h].state(); + copies[h].advance(cm, prev, pos, map, &groups, other); + } + } + let mut alleles = [0u8; 2]; + for (h, copy) in copies.iter().enumerate() { + let (a, s) = copy.state(); + alleles[h] = bit(&row, groups[a][s]); + } + let gt: &[u8] = match alleles[0] + alleles[1] { + 0 => b"0/0", + 1 => b"0/1", + _ => b"1/1", + }; + line.clear(); + line.extend_from_slice(name.as_bytes()); + line.extend_from_slice(format!("\t{pos}\t.\t").as_bytes()); + line.extend_from_slice(&[site.ref_, b'\t', site.alt]); + line.extend_from_slice(b"\t.\tPASS\t.\tGT\t"); + line.extend_from_slice(gt); + line.push(b'\n'); + vcf.write_all(&line) + .map_err(|e| Error(format!("cannot write the VCF: {e}")))?; + prev = pos; + sites += 1; + } + if sites == 0 { + return Err(Error(format!( + "{}: no biallelic SNV to copy", + panel.display() + ))); + } + let mut rows: Vec = copies + .iter() + .enumerate() + .flat_map(|(h, c)| c.rows(h as u8, prev)) + .collect(); + rows.sort_by_key(|r| (r.start, r.haplotype)); + Ok(Chrom { + name, + sites, + skipped: reader.skipped() + doubles, + rows, + }) +} + +/// Writes the truth BED of every chromosome: a header line, then +/// `chrom start end haplotype ancestry` rows. +pub fn write_truth(out: &mut dyn Write, chroms: &[Chrom], cfg: &Config) -> Result<()> { + let mut text = String::from("#chrom\tstart\tend\thaplotype\tancestry\n"); + for c in chroms { + for r in &c.rows { + text.push_str(&format!( + "{}\t{}\t{}\t{}\t{}\n", + c.name, r.start, r.end, r.haplotype, cfg.mu[r.ancestry].0 + )); + } + } + out.write_all(text.as_bytes()) + .map_err(|e| Error(format!("cannot write the truth BED: {e}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testutil::Dir; + + /// Whether `lengths` (cM) follow the exponential law of mean 100/T cM: + /// the mean within 5%, and the share longer than the mean within 0.02 + /// of 1/e. + fn exponential_law(lengths: &[f64], generations: f64) -> std::result::Result<(), String> { + let mean_want = 100.0 / generations; + let n = lengths.len() as f64; + let mean = lengths.iter().sum::() / n; + let above = lengths.iter().filter(|&&l| l > mean_want).count() as f64 / n; + if (mean / mean_want - 1.0).abs() > 0.05 { + return Err(format!("mean tract {mean} cM, the law says {mean_want}")); + } + if (above - (-1.0f64).exp()).abs() > 0.02 { + return Err(format!( + "{above} of tracts above the mean, the law says 1/e" + )); + } + Ok(()) + } + + fn tract_lengths(generations: f64, seed: u64, n: usize) -> Vec { + let mut rng = Rng::new(seed); + let mut p = AncestryProcess::new(&mut rng, generations, &[0.3, 1.0], 0.0); + let mut last = 0.0; + (0..n) + .map(|_| { + let (at, _) = p.step(&mut rng); + let l = at - last; + last = at; + l + }) + .collect() + } + + #[test] + fn tract_lengths_follow_the_exponential_law() { + let t = 10.0; + for seed in 1..=5 { + let drawn_t = if crate::PLANTED { 2.0 * t } else { t }; + let lengths = tract_lengths(drawn_t, seed, 10_000); + exponential_law(&lengths, t).unwrap(); + } + } + + #[test] + fn the_law_check_fails_when_t_is_doubled() { + for seed in 1..=5 { + let lengths = tract_lengths(20.0, seed, 10_000); + let err = exponential_law(&lengths, 10.0).unwrap_err(); + assert!(err.starts_with("mean tract"), "{err}"); + } + } + + /// A panel where group A carries REF and group B carries ALT at every + /// site, so a site's ALT count is the number of copies of B there; plus + /// a person of no group. `n` sites, 1 kb apart, from position 1001. + fn two_group_panel(d: &Dir, n: u32) -> std::path::PathBuf { + let mut text = String::from( + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA1\tB1\tX\tA2\tB2\n", + ); + for i in 1..=n { + text.push_str(&format!( + "chr7\t{}\t.\tC\tT\t.\t.\t.\tGT\t0|0\t1|1\t0|1\t0|0\t1|1\n", + 1000 + i * 1000 + )); + if i % 97 == 0 { + // A second SNV at the position, left out. + text.push_str(&format!( + "chr7\t{}\t.\tC\tG\t.\t.\t.\tGT\t0|0\t1|1\t0|1\t0|0\t1|1\n", + 1000 + i * 1000 + )); + } + } + d.put("chr7.vcf.gz", &text) + } + + /// 100 cM per Mb over the panel: 3 Morgans for 3,000 sites. + fn steep_map() -> GeneticMap { + GeneticMap::from_markers(vec![(1, 0.0), (3_001_001, 300.0)]).unwrap() + } + + fn people() -> Vec<(String, String)> { + [ + ("A1", "A"), + ("B1", "B"), + ("X", "Z"), + ("A2", "A"), + ("B2", "B"), + ] + .iter() + .map(|(p, g)| ((*p).to_owned(), (*g).to_owned())) + .collect() + } + + fn cfg(seed: u64) -> Config { + Config { + generations: 5.0, + switch_rate: 1.0, + mu: vec![("A".into(), 0.6), ("B".into(), 0.4)], + seed, + } + } + + /// Every haplotype's rows run from the first site to the last with no + /// gap or overlap, and touching rows differ in ancestry. + fn check_cover(rows: &[TruthRow], start: i64, end: i64) -> std::result::Result<(), String> { + for h in 0..2u8 { + let mine: Vec<&TruthRow> = rows.iter().filter(|r| r.haplotype == h).collect(); + let mut at = start; + let mut last = None; + for r in &mine { + if r.start != at { + return Err(format!( + "haplotype {h}: a row starts at {}, not {at}", + r.start + )); + } + if r.end <= r.start { + return Err(format!("haplotype {h}: an empty row at {}", r.start)); + } + if last == Some(r.ancestry) { + return Err(format!( + "haplotype {h}: two touching rows of one ancestry at {at}" + )); + } + at = r.end; + last = Some(r.ancestry); + } + if at != end { + return Err(format!("haplotype {h}: the rows end at {at}, not {end}")); + } + } + Ok(()) + } + + fn ancestry_at(rows: &[TruthRow], h: u8, pos: i64) -> usize { + rows.iter() + .find(|r| r.haplotype == h && r.start < pos && pos <= r.end) + .map(|r| r.ancestry) + .expect("a row holds every site") + } + + #[test] + fn truth_covers_the_chromosome_and_explains_every_genotype() { + let d = Dir::new("sim-truth"); + let panel = two_group_panel(&d, 3000); + let map = steep_map(); + let (mut bp_a, mut bp_all) = (0i64, 0i64); + let mut breaks = 0usize; + for seed in 1..=20 { + let mut vcf = Vec::new(); + let chrom = simulate_chrom(&panel, &map, &people(), &cfg(seed), &mut vcf).unwrap(); + assert_eq!( + (chrom.name.as_str(), chrom.sites, chrom.skipped), + ("chr7", 3000, 30) + ); + let mut rows = chrom.rows.clone(); + if crate::PLANTED && rows.len() > 2 { + rows.remove(1); + } + check_cover(&rows, 2000 - 1, 3_001_000).unwrap(); + for line in String::from_utf8(vcf).unwrap().lines() { + let c: Vec<&str> = line.split('\t').collect(); + let pos: i64 = c[1].parse().unwrap(); + let alt = c[9].bytes().filter(|&b| b == b'1').count(); + let want = (0..2).filter(|&h| ancestry_at(&rows, h, pos) == 1).count(); + assert_eq!(alt, want, "seed {seed}, {line}"); + assert!(["0/0", "0/1", "1/1"].contains(&c[9]), "{line}"); + assert_eq!(c[3..5], ["C", "T"]); + } + for r in &rows { + bp_all += r.end - r.start; + if r.ancestry == 0 { + bp_a += r.end - r.start; + } + } + breaks += rows.len() - 2; + } + // 40 haplotypes of 3 Morgans: share of A near mu, and the joined + // breakpoints near T (1 - sum mu^2) = 2.4 per Morgan. + let share = bp_a as f64 / bp_all as f64; + assert!((share - 0.6).abs() < 0.06, "share of A {share}"); + let per_morgan = breaks as f64 / 120.0; + assert!( + (per_morgan - 2.4).abs() < 0.5, + "{per_morgan} breakpoints per Morgan" + ); + } + + #[test] + fn the_cover_check_finds_a_gap_an_overlap_and_a_split() { + let r = |start, end, haplotype, ancestry| TruthRow { + start, + end, + haplotype, + ancestry, + }; + let good = [r(0, 5, 0, 0), r(0, 10, 1, 1), r(5, 10, 0, 1)]; + check_cover(&good, 0, 10).unwrap(); + let gap = [r(0, 4, 0, 0), r(0, 10, 1, 1), r(5, 10, 0, 1)]; + assert!( + check_cover(&gap, 0, 10) + .unwrap_err() + .contains("starts at 5, not 4") + ); + let overlap = [r(0, 6, 0, 0), r(0, 10, 1, 1), r(5, 10, 0, 1)]; + assert!(check_cover(&overlap, 0, 10).is_err()); + let split = [r(0, 5, 0, 1), r(0, 10, 1, 1), r(5, 10, 0, 1)]; + assert!( + check_cover(&split, 0, 10) + .unwrap_err() + .contains("one ancestry") + ); + assert!(check_cover(&good, 0, 11).unwrap_err().contains("not 11")); + } + + #[test] + fn one_held_out_person_comes_back_as_themselves() { + let d = Dir::new("sim-self"); + let mut text = String::from( + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tW\n", + ); + let gts = ["0|1", "1|0", "1|1", "0|0", "0|1"]; + for i in 0..2000u32 { + text.push_str(&format!( + "chr3\t{}\t.\tA\tG\t.\t.\t.\tGT\t{}\n", + 100 + i * 1000, + gts[(i * 7 % 5) as usize] + )); + } + let panel = d.put("chr3.vcf.gz", &text); + let map = GeneticMap::from_markers(vec![(1, 0.0), (2_000_100, 200.0)]).unwrap(); + let cfg = Config { + generations: 6.0, + switch_rate: 5.0, + mu: vec![("WEA".into(), 1.0)], + seed: 9, + }; + let mut vcf = Vec::new(); + let chrom = + simulate_chrom(&panel, &map, &[("W".into(), "WEA".into())], &cfg, &mut vcf).unwrap(); + assert_eq!(chrom.rows.len(), 2, "one row per haplotype"); + for (i, line) in String::from_utf8(vcf).unwrap().lines().enumerate() { + let want = match gts[i * 7 % 5] { + "0|0" => "0/0", + "1|1" => "1/1", + _ => "0/1", + }; + assert!(line.ends_with(want), "{line}"); + } + } + + fn whole_run(seed: u64) -> (Vec, Vec) { + let d = Dir::new(&format!("sim-run-{seed}")); + let panel = two_group_panel(&d, 500); + let map = steep_map(); + let cfg = cfg(seed); + let mut vcf = Vec::new(); + write_header(&mut vcf, "SIM", &["chr7"], &cfg).unwrap(); + let chrom = simulate_chrom(&panel, &map, &people(), &cfg, &mut vcf).unwrap(); + let mut bed = Vec::new(); + write_truth(&mut bed, &[chrom], &cfg).unwrap(); + (vcf, bed) + } + + #[test] + fn the_same_seed_gives_the_same_bytes() { + let a = whole_run(4); + let b = whole_run(if crate::PLANTED { 5 } else { 4 }); + assert_eq!(a, b); + let c = whole_run(5); + assert_ne!(a.1, c.1, "another seed, another truth"); + let vcf = String::from_utf8(a.0).unwrap(); + assert!(vcf.starts_with("##fileformat=VCFv4.2\n##source=og ancestry-fine simulate\n##og_simulate=seed=4;generations=5;switch_rate=1;mu=A:0.6,B:0.4\n##contig=\n")); + assert!(vcf.contains("\nchr7\t2000\t.\tC\tT\t.\tPASS\t.\tGT\t")); + let bed = String::from_utf8(a.1).unwrap(); + assert!( + bed.starts_with("#chrom\tstart\tend\thaplotype\tancestry\nchr7\t1999\t"), + "{bed}" + ); + } + + #[test] + fn settings_are_checked() { + let d = Dir::new("sim-check"); + let panel = two_group_panel(&d, 10); + let map = steep_map(); + let run = |cfg: Config, people: &[(String, String)]| { + simulate_chrom(&panel, &map, people, &cfg, &mut Vec::new()) + .unwrap_err() + .0 + }; + let mut c = cfg(1); + c.generations = 0.0; + assert!(run(c, &people()).contains("must be above 0")); + let mut c = cfg(1); + c.switch_rate = -1.0; + assert!(run(c, &people()).contains("must be 0 or more")); + let mut c = cfg(1); + c.mu[1].1 = 0.5; + assert!(run(c, &people()).contains("sum to 1.1, not 1")); + let mut c = cfg(1); + c.mu[1].0 = "A".into(); + assert!(run(c, &people()).contains("ancestry 'A' is given twice")); + let mut c = cfg(1); + c.mu = vec![("A".into(), 0.5), ("Q".into(), 0.5)]; + assert!(run(c, &people()).contains("no held-out person of ancestry 'Q'")); + let e = simulate_chrom( + &panel, + &GeneticMap::default(), + &people(), + &cfg(1), + &mut Vec::new(), + ) + .unwrap_err() + .0; + assert!(e.starts_with("no genetic map markers"), "{e}"); + } +} diff --git a/rust/crates/og-admix/tests/data/README.md b/rust/crates/og-admix/tests/data/README.md new file mode 100644 index 000000000..9bd5d4423 --- /dev/null +++ b/rust/crates/og-admix/tests/data/README.md @@ -0,0 +1,20 @@ +# og-admix test data + +- `panel.vcf.gz`: a bgzipped phased panel of five people on chr22 with the + records the panel reader must skip (a multi-allelic site, an indel, a + site with no ALT, a `*` allele) and the ones it must keep (two SNVs at + one position, extra FORMAT fields, a missing or unphased genotype in a + person nobody asked for, a failed FILTER). +- `*.bcftools.tsv`: what bcftools 1.25 + (`quay.io/biocontainers/bcftools:1.25--h118bc1c_1`) prints for the + biallelic SNVs of the named people, in the order named: + + ``` + bcftools query -s P4,P1,P3 -i 'TYPE="snp" && N_ALT=1' \ + -f '%CHROM\t%POS\t%REF\t%ALT[\t%GT]\n' panel.vcf.gz + bcftools query -s mum,dad -i 'TYPE="snp" && N_ALT=1' \ + -f '%CHROM\t%POS\t%REF\t%ALT[\t%GT]\n' tests/parity/inputs/family/phased.vcf + ``` + + `tests/panel_bcftools.rs` reads the same files with the panel reader and + compares its haplotypes with these lines. diff --git a/rust/crates/og-admix/tests/data/family-phased.mum-dad.bcftools.tsv b/rust/crates/og-admix/tests/data/family-phased.mum-dad.bcftools.tsv new file mode 100644 index 000000000..0c7e873ad --- /dev/null +++ b/rust/crates/og-admix/tests/data/family-phased.mum-dad.bcftools.tsv @@ -0,0 +1,3 @@ +chr2 100 A G 0|0 0|1 +chr2 200 C T 0|1 0|0 +chr2 300 C T 0|1 0|0 diff --git a/rust/crates/og-admix/tests/data/panel.P4-P1-P3.bcftools.tsv b/rust/crates/og-admix/tests/data/panel.P4-P1-P3.bcftools.tsv new file mode 100644 index 000000000..a294988c9 --- /dev/null +++ b/rust/crates/og-admix/tests/data/panel.P4-P1-P3.bcftools.tsv @@ -0,0 +1,6 @@ +chr22 16050075 A G 1|0 0|1 0|0 +chr22 16050319 C T 0|0 1|1 1|0 +chr22 16050527 C A 1|0 0|0 0|1 +chr22 16050527 C G 0|1 1|0 0|0 +chr22 16050607 G A 0|0 0|1 1|1 +chr22 16050739 T C 1|1 1|1 1|0 diff --git a/rust/crates/og-admix/tests/data/panel.vcf.gz b/rust/crates/og-admix/tests/data/panel.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..15efe35ea6341881521224f77df498a41e3d416f GIT binary patch literal 590 zcmb2|=3rp}f&Xj_PR>jWR*b0={qvg)L=K$)8f;QDN6PW$!L{B}8~a?AGCwPItjU;s zX;V<_o#pZgWdZs6@Aux)eo;NDl}M` zFWTg!eQ9~nqqa>6Suv@lyubfNAAS-QS@>ecpA4JzXIrJkN+nOR%?{ql?EFGuZ-~Q; zD>G|9G_k};{IF|0r9XG7>&IEk<|fblXMJb2@S>P` zUHa2d^4LirPSE^kh2r@OF4^LZIp^+J7uQ7hFqXdxQEz|h;-bRw?PGyMO;x~7=TjG) zHV8zC=y2)ovYe;;Qo*C+woOD{=#4#M!GEm|f4O(T$sqjOy zk#S(r!RFS3*CHRvRh>>~kKF2d?5w@_=S!C+e-%9|U{hbSw`SkI^{h44pMP!nYkpsv SE1Wxm3Deus49wsJ1R?<6x)8Jg literal 0 HcmV?d00001 diff --git a/rust/crates/og-admix/tests/panel_bcftools.rs b/rust/crates/og-admix/tests/panel_bcftools.rs new file mode 100644 index 000000000..274246783 --- /dev/null +++ b/rust/crates/og-admix/tests/panel_bcftools.rs @@ -0,0 +1,81 @@ +//! The panel reader against what bcftools prints for the same files +//! (`tests/data/README.md` holds the commands): every biallelic SNV, in +//! file order, with the chosen people's phased genotypes in the order +//! asked for. + +use og_admix::panel::Haplotypes; +use std::path::{Path, PathBuf}; + +fn here(rel: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join(rel) +} + +/// The panel read back as bcftools query's lines. +fn query(path: &Path, people: &[&str]) -> String { + let h = Haplotypes::read(path, people, None).unwrap(); + let mut out = String::new(); + for (m, s) in h.sites().iter().enumerate() { + out.push_str(&format!( + "{}\t{}\t{}\t{}", + h.chrom(), + s.pos, + s.ref_ as char, + s.alt as char + )); + for i in 0..people.len() { + out.push_str(&format!( + "\t{}|{}", + h.allele(m, 2 * i), + h.allele(m, 2 * i + 1) + )); + } + out.push('\n'); + } + out +} + +/// The red control reads the people in another order than bcftools was +/// asked for, so the comparison must fail. +fn asked<'a>(people: &[&'a str]) -> Vec<&'a str> { + let mut p = people.to_vec(); + if cfg!(feature = "planted-red") { + p.reverse(); + } + p +} + +#[test] +fn bgzipped_panel_matches_bcftools() { + let want = std::fs::read_to_string(here("tests/data/panel.P4-P1-P3.bcftools.tsv")).unwrap(); + assert_eq!(want.lines().count(), 6, "the golden itself"); + let got = query( + &here("tests/data/panel.vcf.gz"), + &asked(&["P4", "P1", "P3"]), + ); + assert_eq!(got, want); +} + +#[test] +fn parity_family_fixture_matches_bcftools() { + let want = + std::fs::read_to_string(here("tests/data/family-phased.mum-dad.bcftools.tsv")).unwrap(); + assert_eq!(want.lines().count(), 3, "the golden itself"); + let got = query( + &here("../../../tests/parity/inputs/family/phased.vcf"), + &asked(&["mum", "dad"]), + ); + assert_eq!(got, want); +} + +#[test] +fn skipped_records_are_counted() { + let path = here("tests/data/panel.vcf.gz"); + let mut r = og_admix::panel::PanelReader::open(&path, &["P2"]).unwrap(); + let mut row = vec![0u64; 1]; + let mut n = 0; + while r.next_site(&mut row).unwrap().is_some() { + n += 1; + } + // A multi-allelic site, an indel, no ALT and a `*` allele. + assert_eq!((n, r.skipped()), (6, 4)); +} diff --git a/rust/crates/og-ancestry/Cargo.toml b/rust/crates/og-ancestry/Cargo.toml index 4406d01ba..4ea905878 100644 --- a/rust/crates/og-ancestry/Cargo.toml +++ b/rust/crates/og-ancestry/Cargo.toml @@ -7,6 +7,7 @@ license.workspace = true publish.workspace = true [dependencies] +og-admix.workspace = true pycompat.workspace = true og-io.workspace = true og-intervals.workspace = true diff --git a/rust/crates/og-ancestry/src/chroms.rs b/rust/crates/og-ancestry/src/chroms.rs index 1305b9eb5..e3788be29 100644 --- a/rust/crates/og-ancestry/src/chroms.rs +++ b/rust/crates/og-ancestry/src/chroms.rs @@ -1,9 +1,11 @@ //! `chroms`: the chromosomes to phase and paint, those with N genotyped //! SNVs or more (default 100) spanning X cM or more (default 5). RFMix //! stops with a segmentation fault on a span of a few cM; on a whole -//! genome every autosome spans more than 40. +//! genome every autosome spans more than 40. The map is read by +//! [`og_admix::map`], which the engines share. use crate::{Args, Fail, Result, Spec}; +use og_admix::map::GeneticMap; use std::io::Write; use std::path::Path; @@ -14,55 +16,6 @@ pub(crate) const SPEC: Spec = Spec { required: &["--counts", "--maps"], }; -/// A PLINK `.map` (chromosome, id, cM, position) as (position, cM), -/// sorted. -pub fn read_map(path: &Path) -> Result> { - let mut out = Vec::new(); - let mut n = 0u64; - crate::for_each_line(path, |line| { - n += 1; - let c: Vec<&str> = crate::split_ws(line).collect(); - if c.len() >= 4 { - let bad = |what: &str, v: &str| { - Fail(format!( - "{}: line {n}: {what}: {}", - path.display(), - crate::repr(v) - )) - }; - let pos = - crate::int(c[3]).ok_or_else(|| bad("position is not a whole number", c[3]))?; - let cm = - pycompat::num::parse_float(c[2]).map_err(|_| bad("cM is not a number", c[2]))?; - out.push((pos, cm)); - } - Ok(()) - })?; - out.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.total_cmp(&b.1))); - Ok(out) -} - -/// The genetic position of `pos`, linear between the map's markers. -pub fn cm_at(map: &[(i64, f64)], pos: i64) -> f64 { - let (Some(&(p_first, c_first)), Some(&(p_last, c_last))) = (map.first(), map.last()) else { - return 0.0; - }; - if pos <= p_first { - return c_first; - } - if pos >= p_last { - return c_last; - } - // The last marker at or before pos, and the one after it. - let hi = map.partition_point(|&(p, _)| p <= pos); - let ((p0, c0), (p1, c1)) = (map[hi - 1], map[hi]); - if p1 > p0 { - c0 + (c1 - c0) * (pos - p0) as f64 / (p1 - p0) as f64 - } else { - c0 - } -} - /// One row of `counts.tsv`: chromosome, genotyped SNVs, first and last /// position. pub type Count = (String, i64, i64, i64); @@ -103,8 +56,8 @@ pub fn paint_chroms( if *n < min_sites || !path.is_file() { continue; } - let map = read_map(&path)?; - if cm_at(&map, *last) - cm_at(&map, *first) >= min_cm { + let map = GeneticMap::read(&path).map_err(|e| Fail(e.0))?; + if map.cm_at(*last) - map.cm_at(*first) >= min_cm { out.push(chrom.clone()); } } @@ -142,11 +95,10 @@ mod tests { ]; let maps = d.0.join("maps"); assert_eq!(paint_chroms(&counts, &maps, 100, 5.0).unwrap(), ["chr20"]); - let map = read_map(&maps.join("chr20.map")).unwrap(); - assert_eq!(cm_at(&map, 6_000_000), 5.0); - assert_eq!(cm_at(&map, 1), 0.0); - assert_eq!(cm_at(&map, 99_000_000), 10.0); - assert_eq!(cm_at(&[], 5), 0.0); + let map = GeneticMap::read(&maps.join("chr20.map")).unwrap(); + assert_eq!(map.cm_at(6_000_000), 5.0); + assert_eq!(map.cm_at(1), 0.0); + assert_eq!(map.cm_at(99_000_000), 10.0); let c = d.put( "counts.tsv", "chr22\t500\t2000000\t8000000\nchr20\t500\t2000000\t8000000\nhomref\t3\n", From 05352478bb01e181870bc0e0dcd9abe8f18316c0 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:25:40 +0100 Subject: [PATCH 188/255] feat(ancestry): og ancestry-fine simulate writes an admixed genome and its truth The verb copies held-out panel people's haplotypes into a diploid mosaic: ancestry tracts at T events per Morgan drawn from mu, copying switches inside a tract. It writes an unphased VCF (BGZF when the name ends in .gz) and a truth BED, the same bytes for the same seed. The crate docs and the changelog name it. Bead: D5 Bead: D14 Bead: D1 --- CHANGELOG.md | 2 + docs/rust.md | 6 +- rust/README.md | 2 +- rust/crates/og-ancestry/src/simulate.rs | 382 +++++++++++++++++++++++- 4 files changed, 380 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf6c9e6d..e9ece66b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The variant calling page names the Genome Reference Consortium as the source of the PAR intervals in `assets/par_grch38.bed`. - Every verb the 1.0.0 release still has to port now exists in the `og` binary, each in its own module, as a stub that prints `og : not ported yet` and exits 2. The new crates are og-render, og-live, og-couple and og-admix. Each port then fills one module and never edits `rust/og/src/main.rs` or a crate's module list, so the ports can land in parallel. No step calls these verbs yet, and no output changes. - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. +- `og ancestry-fine simulate` builds an admixed genome with known truth, for testing the fine-ancestry engines that replace ADMIXTURE and RFMix in step 26b. It copies haplotypes of held-out panel people: ancestry tracts come at T events per Morgan with each ancestry drawn from the given proportions, and inside a tract the copied haplotype switches at 1 per Morgan by default. It writes an unphased VCF (sorted alleles, so nothing says which copy an allele came from) and a truth BED of each copy's ancestry that covers every chromosome from its first site to its last with no gap or overlap. The same seed gives the same bytes. On a synthetic panel of 1,000 people and 100,000 sites it takes 1.2 to 2.3 s and 12 to 15 MB. No step calls it, and no output of a run changes. The new crate `og-admix` holds it and what the engines will share: the panel reader (phased biallelic SNVs, bit-packed), the genetic map, the `model/ref.geno` file of packed reference genotypes and a seeded xoshiro256** generator. `og ancestry-fine chroms` now reads its maps with this reader. What changes on purpose: + - A map with a cM value that is not finite, or with cM going down as the position goes up, now stops `og ancestry-fine chroms` with an error naming the file. Before, a `nan` cM dropped the chromosome without a word, and a decreasing map gave a wrong span. Beagle's GRCh38 maps never go down. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. diff --git a/docs/rust.md b/docs/rust.md index bfe8ead4b..18d0c4a8a 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -34,7 +34,7 @@ rust/ crates/og-render/ og report render|index|charts: the reports from summary.json (not ported yet) crates/og-live/ og live serve|results|family|demo, og view serve: the live site (not ported yet) crates/og-couple/ og couple screen|genes: step 39's couple screen (not ported yet) - crates/og-admix/ what the fine-ancestry engines share: painting and projection (not ported yet) + crates/og-admix/ what the fine-ancestry engines share: panel, map, ref.geno, RNG, simulator; painting and projection (not ported yet) og/ the binary: og [args] bin/og the shim tasks call; runs bin/.og/og-linux- (og-darwin- on a Mac) tests/rust/ the differential test against CPython, the image test @@ -42,7 +42,9 @@ tests/rust/ the differential test against CPython, the image test Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). `cnvpytor_to_vcf.py` keeps only the Python copy of that rule, which step 39's couple screen still imports. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (were the commands of `chip_report.py`). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py`. Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|order|sample-bed|chroms|report`. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`) and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`, which keeps the SMN1 rule beside Parascopy). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. -Every verb the 1.0.0 release still has to port is already registered, each in its own module, as a stub that prints `og : not ported yet` on stderr and exits 2: `og report render|index|charts`, `og live serve|results|family|demo`, `og view serve`, `og couple screen|genes`, `og vcf index|shift-back`, `og setup chrm-shift|revel|roh-af`, `og sv survivor-prep|consensus-check`, `og archaic compare|genomes`, `og ancestry-fine freqs|project|paint|simulate`, `og longread clair3-model`, `og map svg`, `og dev demo-sample|demo-events|params-doc|python-gate|everything-check|licences|docs-check|stub-http|steps-check` and `og dev build roh-af|acmg-lof`. A port fills its module's body and does not edit `og/src/main.rs`, the workspace member list or a crate's module list. `rust/og/tests/port_scaffold.rs` lists these verbs and fails when one has no line in `og --help` or reaches a router's "unknown" error. +Every verb the 1.0.0 release still has to port is already registered, each in its own module, as a stub that prints `og : not ported yet` on stderr and exits 2: `og report render|index|charts`, `og live serve|results|family|demo`, `og view serve`, `og couple screen|genes`, `og vcf index|shift-back`, `og setup chrm-shift|revel|roh-af`, `og sv survivor-prep|consensus-check`, `og archaic compare|genomes`, `og ancestry-fine freqs|project|paint`, `og longread clair3-model`, `og map svg`, `og dev demo-sample|demo-events|params-doc|python-gate|everything-check|licences|docs-check|stub-http|steps-check` and `og dev build roh-af|acmg-lof`. A port fills its module's body and does not edit `og/src/main.rs`, the workspace member list or a crate's module list. `rust/og/tests/port_scaffold.rs` lists these verbs and fails when one has no line in `og --help` or reaches a router's "unknown" error. + +`og ancestry-fine simulate` builds an admixed genome with known truth from held-out people of the fine-ancestry panel: an unphased VCF and a BED of each copy's ancestry. The engines that replace ADMIXTURE and RFMix test against it. It and the parts those engines share (the panel reader, the genetic map, `model/ref.geno` and a seeded xoshiro256** generator) live in `og-admix`. A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/family_report.py` is the first one: the pedigree, kinship and PAR rules that `couple_report.py` and `pedigree_svg.py` import. `bin/genome_lookup.py` keeps the query reading and the summary names rule that `results_index.py` imports. diff --git a/rust/README.md b/rust/README.md index 5a834a2e8..2b018bd4f 100644 --- a/rust/README.md +++ b/rust/README.md @@ -25,7 +25,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-render` | `og report render`, `og report index`, `og report charts`: the reports from summary.json (not ported yet) | | `crates/og-live` | `og live serve`, `og live results`, `og live family`, `og live demo`, `og view serve`: the live site (not ported yet) | | `crates/og-couple` | `og couple screen`, `og couple genes`: step 39's couple screen (not ported yet) | -| `crates/og-admix` | what the fine-ancestry engines share: our own painting and projection (not ported yet) | +| `crates/og-admix` | what the fine-ancestry engines share: the panel's haplotypes, the genetic map, `model/ref.geno`, a seeded RNG and the admixed-genome simulator of `og ancestry-fine simulate`; our own painting and projection (not ported yet) | | `og` | the binary: `og [args]`, `og version`, `og --tree-hash` | | `og.sh` | build or download `bin/.og/og-linux-{x86_64,aarch64}`, and check they match this `rust/` | | `rust-toolchain.toml` | the Rust release, the one `OG_BUILDER` (`versions.env`) ships | diff --git a/rust/crates/og-ancestry/src/simulate.rs b/rust/crates/og-ancestry/src/simulate.rs index bccf46ec0..48eea7b51 100644 --- a/rust/crates/og-ancestry/src/simulate.rs +++ b/rust/crates/og-ancestry/src/simulate.rs @@ -1,13 +1,377 @@ -//! `og ancestry-fine simulate`: simulated admixed genomes that check the -//! painting and the projection. -//! -//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so -//! on stderr and exits 2, before reading any argument. +//! `og ancestry-fine simulate`: an admixed genome with known truth, built +//! from held-out people of the panel, to check the painting and the +//! projection against. The model and the files are described in +//! [`og_admix::simulate`]. +use crate::{ArgError, Args, Fail, Result, Spec, repr}; +use og_admix::map::GeneticMap; +use og_admix::simulate::{self, Config}; use std::io::Write; +use std::path::{Path, PathBuf}; -/// `og ancestry-fine simulate ARGS`: not ported yet, so one line on stderr and exit 2. -pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { - let _ = writeln!(err, "og ancestry-fine simulate: not ported yet"); - 2 +pub(crate) const SPEC: Spec = Spec { + verb: "simulate", + usage: "usage: og ancestry-fine simulate --panel DIR --maps DIR --people FILE --mu GROUP=P[,GROUP=P...] --generations T --out OUT.vcf[.gz] --truth OUT.bed [--chroms chrA,chrB] [--switch-rate R] [--seed N] [--sample NAME]\n", + options: &[ + "--panel", + "--maps", + "--people", + "--mu", + "--generations", + "--out", + "--truth", + "--chroms", + "--switch-rate", + "--seed", + "--sample", + ], + required: &[ + "--panel", + "--maps", + "--people", + "--mu", + "--generations", + "--out", + "--truth", + ], +}; + +/// `og ancestry-fine simulate ARGS`: the exit code. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let a = match Args::parse(&SPEC, args).and_then(|a| { + a.float("--generations", 0.0)?; + a.float("--switch-rate", 0.0)?; + a.int("--seed", 0)?; + Ok(a) + }) { + Ok(a) => a, + Err(ArgError::Help) => { + let _ = write!(out, "{}", SPEC.usage); + return 0; + } + Err(ArgError::Usage(msg)) => { + let _ = write!(err, "{}", SPEC.usage); + let _ = writeln!(err, "og ancestry-fine {}: error: {msg}", SPEC.verb); + return 2; + } + }; + match main(&a, out) { + Ok(()) => 0, + Err(Fail(msg)) => { + let _ = out.flush(); + let _ = writeln!(err, "ERROR: {msg}"); + 1 + } + } +} + +/// `GROUP=P,GROUP=P`, in the order given. +fn parse_mu(s: &str) -> Result> { + s.split(',') + .map(|item| { + let (g, p) = item.split_once('=').unwrap_or((item, "")); + let p = pycompat::num::parse_float(p).ok().filter(|_| !g.is_empty()); + p.map(|p| (g.trim().to_owned(), p)) + .ok_or_else(|| Fail(format!("--mu: {} is not GROUP=PROPORTION", repr(item)))) + }) + .collect() +} + +/// The held-out people: sample and group, the first two columns of each +/// line (`paint_map.tsv`'s layout); blank lines and `#` lines skipped. +fn read_people(path: &Path) -> Result> { + let mut out = Vec::new(); + let mut n = 0u64; + crate::for_each_line(path, |line| { + n += 1; + if line.trim().is_empty() || line.starts_with('#') { + return Ok(()); + } + let c: Vec<&str> = crate::split_ws(line).collect(); + if c.len() < 2 { + return Err(Fail(format!( + "{}: line {n}: a sample and its group, not {}", + path.display(), + repr(line) + ))); + } + out.push((c[0].to_owned(), c[1].to_owned())); + Ok(()) + })?; + if out.is_empty() { + return Err(Fail(format!("{}: no held-out people", path.display()))); + } + Ok(out) +} + +/// The chromosomes asked for, else every `.vcf.gz` of the panel +/// that has a `.map`, in chromosome order. +fn chroms(a: &Args, panel: &Path, maps: &Path) -> Result> { + if let Some(list) = a.get("--chroms") { + let c: Vec = list + .split([',', ' ']) + .filter(|c| !c.is_empty()) + .map(str::to_owned) + .collect(); + if c.is_empty() { + return Err(Fail("--chroms names no chromosome".into())); + } + return Ok(c); + } + let entries = std::fs::read_dir(panel).map_err(|e| crate::cannot_read(panel, &e))?; + let mut out = Vec::new(); + for e in entries { + let e = e.map_err(|e| crate::cannot_read(panel, &e))?; + let name = e.file_name().to_string_lossy().into_owned(); + if let Some(c) = name.strip_suffix(".vcf.gz") + && maps.join(format!("{c}.map")).is_file() + { + out.push(c.to_owned()); + } + } + if out.is_empty() { + return Err(Fail(format!( + "no .vcf.gz in {} with a .map in {}", + panel.display(), + maps.display() + ))); + } + out.sort_by(|x, y| crate::chrom_key(x).cmp(&crate::chrom_key(y))); + Ok(out) +} + +pub(crate) fn main(a: &Args, out: &mut dyn Write) -> Result<()> { + let seed = a.int("--seed", 1).unwrap_or(1); + let cfg = Config { + generations: a.float("--generations", 0.0).unwrap_or(0.0), + switch_rate: a.float("--switch-rate", 1.0).unwrap_or(1.0), + mu: parse_mu(a.req("--mu"))?, + seed: u64::try_from(seed) + .map_err(|_| Fail(format!("--seed {seed}: a seed is 0 or more")))?, + }; + cfg.check().map_err(|e| Fail(e.0))?; + let people = read_people(Path::new(a.req("--people")))?; + let (panel, maps) = (Path::new(a.req("--panel")), Path::new(a.req("--maps"))); + let chroms = chroms(a, panel, maps)?; + let sample = a.get("--sample").unwrap_or("SIM"); + if sample.is_empty() || sample.contains(['\t', '\n']) { + return Err(Fail(format!( + "--sample {}: not a sample name", + repr(sample) + ))); + } + let vcf_path = PathBuf::from(a.req("--out")); + let mut pending = crate::Pending::new(); + let file = pending.create(vcf_path.clone())?; + let write_err = |e: std::io::Error| crate::cannot_write(&vcf_path, &e); + let names: Vec<&str> = chroms.iter().map(String::as_str).collect(); + let mut done = Vec::with_capacity(chroms.len()); + let mut run = |vcf: &mut dyn Write| -> Result<()> { + simulate::write_header(vcf, sample, &names, &cfg).map_err(|e| Fail(e.0))?; + for c in &chroms { + let map_path = maps.join(format!("{c}.map")); + let map = GeneticMap::read(&map_path).map_err(|e| Fail(e.0))?; + let panel_path = panel.join(format!("{c}.vcf.gz")); + let chrom = simulate::simulate_chrom(&panel_path, &map, &people, &cfg, vcf) + .map_err(|e| Fail(e.0))?; + if chrom.name != *c { + return Err(Fail(format!( + "{}: its records are on {}, not {c}", + panel_path.display(), + chrom.name + ))); + } + done.push(chrom); + } + Ok(()) + }; + if vcf_path.extension().is_some_and(|e| e == "gz") { + let mut bgzf = + og_io::bgzf::Writer::new(&mut *file, og_io::bgzf::workers()).map_err(write_err)?; + run(&mut bgzf)?; + bgzf.finish().map_err(write_err)?; + } else { + run(file)?; + } + let mut bed = Vec::new(); + simulate::write_truth(&mut bed, &done, &cfg).map_err(|e| Fail(e.0))?; + pending.put(PathBuf::from(a.req("--truth")), &bed)?; + pending.commit_all()?; + let mut bp = vec![0i64; cfg.mu.len()]; + for c in &done { + for r in &c.rows { + bp[r.ancestry] += r.end - r.start; + } + writeln!( + out, + "{}: {} sites, {} panel records left out, {} truth rows", + c.name, + c.sites, + c.skipped, + c.rows.len() + ) + .map_err(|e| Fail(format!("cannot write stdout: {e}")))?; + } + let total: i64 = bp.iter().sum(); + let shares: Vec = cfg + .mu + .iter() + .zip(&bp) + .map(|((g, _), b)| format!("{g} {:.4}", *b as f64 / total.max(1) as f64)) + .collect(); + writeln!( + out, + "ancestry shares of the truth (bp): {}", + shares.join(", ") + ) + .map_err(|e| Fail(format!("cannot write stdout: {e}"))) +} + +#[cfg(test)] +mod tests { + use crate::testutil::{Dir, run}; + + fn setup(d: &Dir) { + let mut a = String::from( + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tY1\tH1\tO1\n", + ); + let mut b = a.clone(); + for i in 1..=300u32 { + a.push_str(&format!( + "chr21\t{}\t.\tA\tG\t.\t.\t.\tGT\t0|0\t1|1\t0|1\n", + i * 1000 + )); + b.push_str(&format!( + "chr2\t{}\t.\tG\tT\t.\t.\t.\tGT\t1|1\t0|0\t0|1\n", + i * 1000 + )); + } + d.put("panel/chr21.vcf.gz", &a); + d.put("panel/chr2.vcf.gz", &b); + d.put("panel/chr9.vcf.gz", &b); + d.put("maps/chr21.map", "chr21 . 0.0 1\nchr21 . 30.0 300000\n"); + d.put("maps/chr2.map", "chr2 . 0.0 1\nchr2 . 30.0 300000\n"); + d.put( + "people.tsv", + "# sample\tgroup\nY1\tAFR\nH1\tEAS\n\nO1\tOCE\n", + ); + } + + fn args<'a>(d: &'a Dir, extra: &[&'a str]) -> Vec { + let mut v: Vec = [ + "simulate", + "--panel", + &d.path("panel"), + "--maps", + &d.path("maps"), + "--people", + &d.path("people.tsv"), + "--mu", + "AFR=0.5,EAS=0.5", + "--generations", + "20", + "--truth", + &d.path("truth.bed"), + ] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + v.extend(extra.iter().map(|s| (*s).to_owned())); + v + } + + fn run_v(v: &[String]) -> (u8, String, String) { + let refs: Vec<&str> = v.iter().map(String::as_str).collect(); + run(&refs) + } + + #[test] + fn writes_the_genome_and_its_truth() { + let d = Dir::new("simulate"); + setup(&d); + let vcf = d.path("sim.vcf"); + let (rc, out, err) = run_v(&args(&d, &["--out", &vcf, "--seed", "3"])); + assert_eq!((rc, err.as_str()), (0, "")); + // chr9 has no map: by default only chr2 and chr21, in that order. + assert!( + out.starts_with("chr2: 300 sites, 0 panel records left out, "), + "{out}" + ); + assert!(out.contains("\nchr21: 300 sites"), "{out}"); + assert!( + out.contains("ancestry shares of the truth (bp): AFR 0."), + "{out}" + ); + let text = d.read("sim.vcf"); + assert!(text.contains("##contig=\n##contig=\n")); + assert!(text.contains("\tFORMAT\tSIM\n")); + assert_eq!(text.lines().filter(|l| !l.starts_with('#')).count(), 600); + let bed = d.read("truth.bed"); + assert!(bed.starts_with("#chrom\tstart\tend\thaplotype\tancestry\nchr2\t999\t")); + assert!( + bed.lines() + .skip(1) + .all(|l| l.ends_with("\tAFR") || l.ends_with("\tEAS")) + ); + // Gzip output is BGZF, and the same seed gives the same genome. + let gz = d.path("sim.vcf.gz"); + let (rc, _, err) = run_v(&args(&d, &["--out", &gz, "--seed", "3"])); + assert_eq!((rc, err.as_str()), (0, "")); + let raw = std::fs::read(&gz).unwrap(); + assert_eq!( + &raw[..4], + &[0x1f, 0x8b, 8, 4], + "a BGZF member with its extra field" + ); + let mut back = String::new(); + std::io::Read::read_to_string(&mut og_io::open(&gz).unwrap(), &mut back).unwrap(); + assert_eq!(back, text); + // One chromosome asked for, another sample name. + let (rc, out, _) = run_v(&args( + &d, + &["--out", &vcf, "--chroms", "chr21", "--sample", "MIX"], + )); + assert_eq!(rc, 0); + assert!(out.starts_with("chr21: "), "{out}"); + assert!(d.read("sim.vcf").contains("\tFORMAT\tMIX\n")); + } + + #[test] + fn refuses_bad_settings_before_writing() { + let d = Dir::new("simulate-bad"); + setup(&d); + let vcf = d.path("sim.vcf"); + let (rc, _, err) = run_v(&args(&d, &["--out", &vcf, "--chroms", "chr9"])); + assert_eq!(rc, 1); + assert!(err.starts_with("ERROR: cannot read "), "{err}"); + let mut v = args(&d, &["--out", &vcf]); + v[8] = "AFR=0.5,EAS".into(); + let (rc, _, err) = run_v(&v); + assert_eq!(rc, 1); + assert!(err.contains("--mu: 'EAS' is not GROUP=PROPORTION"), "{err}"); + v[8] = "AFR=0.5,EAS=0.6".into(); + assert!(run_v(&v).2.contains("sum to 1.1, not 1")); + v[8] = "AFR=0.5,SIB=0.5".into(); + assert!(run_v(&v).2.contains("no held-out person of ancestry 'SIB'")); + let (rc, _, err) = run_v(&args(&d, &["--out", &vcf, "--seed", "-2"])); + assert_eq!(rc, 1); + assert!(err.contains("--seed -2: a seed is 0 or more"), "{err}"); + let (rc, _, err) = run_v(&args(&d, &["--out", &vcf, "--generations", "x"])); + assert_eq!(rc, 2); + assert!( + err.contains("argument --generations: invalid float value: 'x'"), + "{err}" + ); + let (rc, _, err) = run(&["simulate", "--og-scaffold-probe"]); + assert_eq!(rc, 2); + assert!( + err.contains("the following arguments are required"), + "{err}" + ); + assert!(!std::path::Path::new(&vcf).exists(), "nothing written"); + assert!(!std::path::Path::new(&d.path("truth.bed")).exists()); + let (rc, out, _) = run(&["simulate", "--help"]); + assert_eq!(rc, 0); + assert!(out.starts_with("usage: og ancestry-fine simulate")); + } } From b07d1ff11692bbf8a583b35e639c749ba65329cd Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:26:00 +0100 Subject: [PATCH 189/255] test(ancestry): plant every og-admix red control for one CI run Turns the planted-red feature on by default, so Guard's cargo test runs each control on its broken input and must show these tests fail: rng reference vectors, map interpolation (shuffled cM), ref.geno round trip (a flipped byte), the panel reader against bcftools (people in another order), the tract-length law (T doubled), the truth cover (a row dropped) and same seed same bytes (a second seed). The next commit reverts it. Bead: D5 --- rust/crates/og-admix/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/crates/og-admix/Cargo.toml b/rust/crates/og-admix/Cargo.toml index cf646e56e..f9c378653 100644 --- a/rust/crates/og-admix/Cargo.toml +++ b/rust/crates/og-admix/Cargo.toml @@ -7,6 +7,8 @@ license.workspace = true publish.workspace = true [features] +# Red-first run: every control planted, so CI shows each test fail. Reverted in the next commit. +default = ["planted-red"] # Feeds each red control its broken input in place of the good one # (a shuffled map, T doubled, a flipped byte, a swapped sample order, a # second seed), so `cargo test -p og-admix --features planted-red From 21f91eb5c6a3d95c8a68df38830e38f5482473d9 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:27:15 +0100 Subject: [PATCH 190/255] test(bloodtype): a right-shifted ABO deletion must read A2, and a 1/2 at c.803 keep its B copy The e2e case builds a chr9 reference over ABO and runs step 41 on the same deletion written two ways, and on a multi-allelic C>A,G 1/2. og on the right-shifted VCF as written stays in the case as the control: it reads A1/A1, which is what step 41 gave before it left-aligned. Bead: trait-30 --- tests/e2e/bloodtype-1.sh | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/e2e/bloodtype-1.sh b/tests/e2e/bloodtype-1.sh index 984840f2f..2d0482872 100755 --- a/tests/e2e/bloodtype-1.sh +++ b/tests/e2e/bloodtype-1.sh @@ -17,6 +17,9 @@ # - the same reads without any RHD read (both copies deleted) give D-, and # with half of them (one copy) D+ with one copy; # - the VCF without a BAM gives RhD not determined and ABO O at medium; +# - ABO c.1061delC written two bases right (GG>G) gives A2/A2 as the table's +# CG>C does, because step 41 left-aligns first (og on the VCF as written +# reads A1/A1, the control), and a C>A,G 1/2 at c.803 keeps its one B copy; # - the Nextflow processes (BLOODTYPE_DEPTH, BLOODTYPE) on a VCF and BAM # write the same three files as the bash step on the same two files. . "$(dirname "$0")/lib.sh" @@ -143,6 +146,74 @@ check_eq "without a BAM RhD is not determined" "$(cell "$V" RhD phenotype)" "not check "and the step says why" has 'No BAM .*RhD needs read depth' "$(cat "$STEP_LOG")" check_eq "without a BAM ABO is O at medium" "$(cell "$V" ABO phenotype)/$(cell "$V" ABO confidence)" "O/medium" +# --- an indel written further right, and a multi-allelic record ------------------------ +# og matches a marker by position, REF and ALT. A caller can write ABO c.1061delC +# (CG>C at chr9:133255669, in a run of G) as GG>G two bases right; step 41 +# left-aligns the records near the markers against the reference first, so both +# forms give A2. A multi-allelic C>A,G 1/2 at c.803 stays one record and keeps its +# one B copy. The fixture reference has no chr9, so these samples get their own: +# GRCh38's bases over ABO (from the Broad's public copy of GRCh38) after N up to +# there, as contig chr9. Records on other contigs go through unchanged. +HG38="https://storage.googleapis.com/gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.fasta" +fetch_abo() { + rm -f "${T}/abo.fa" + docker run --rm -u "$(id -u):$(id -g)" -e HOME=/tmp -v /etc/ssl/certs:/etc/ssl/certs:ro \ + -e CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt -v "${E2E_WORK}:${E2E_WORK}" -w "$T" "$SAMTOOLS_IMAGE" \ + samtools faidx -o "${T}/abo.fa" "$HG38" chr9:133255001-133258500 +} +check "GRCh38's bases over ABO download" retry fetch_abo +ABO_SEQ=$(grep -v '^>' "${T}/abo.fa" 2>/dev/null | tr -d '\n') +check_eq "bases fetched over ABO" "${#ABO_SEQ}" 3500 +check_eq "GRCh38 at c.1061delC (chr9:133255669-133255673): C, then the run of G" "${ABO_SEQ:668:5}" CGGGT +check_eq "GRCh38 at c.803 (chr9:133255928)" "${ABO_SEQ:927:1}" C +check_eq "GRCh38 at c.261 (chr9:133257521)" "${ABO_SEQ:2520:1}" T +REFBT="${G}/reference-bt/chr9_abo.fa" +mkdir -p "${G}/reference-bt" +{ echo ">chr9"; { head -c 133255000 /dev/zero | tr '\0' N; printf '%s' "$ABO_SEQ"; } | fold -w 60; echo; } > "$REFBT" +dk "$SAMTOOLS_IMAGE" samtools faidx "$REFBT" +check_eq "the ABO reference's chr9" "$(cut -f1,2 "${REFBT}.fai" 2>/dev/null)" "$(printf 'chr9\t133258500')" +# bt_vcf NAME RECORD...: GIAB's records without those over ABO, plus RECORD (fields split by |). +bt_vcf() { + { dk "$BCFTOOLS_IMAGE" bcftools view -t ^chr9:133255000-133258500 "${T}/truth.vcf.gz" + printf '%s\n' "${@:2}" | tr '|' '\t'; } > "${T}/$1.vcf" + dk "$BCFTOOLS_IMAGE" bcftools sort -Oz -o "${T}/$1.vcf.gz" "${T}/$1.vcf" +} +# ev SAMPLE MARKER: the marker's status and ALT copies in the sample's evidence file. +ev() { + awk -F'\t' -v m="$2" 'NR == 1 {for (i = 1; i <= NF; i++) c[$i] = i; next} $2 == m {print $c["status"] " " $c["alt_copies"]}' \ + "${G}/$1/bloodtype/$1_bloodtype_evidence.tsv" 2>/dev/null +} +# The realigned count of bcftools norm's summary in the last step's log. +realigned() { awk -F'\t' '/realigned/ {split($2, n, "/"); print n[4]}' "$STEP_LOG"; } +R="${SAMPLE}btr" L="${SAMPLE}btl" M="${SAMPLE}btm" +bt_vcf "$R" 'chr9|133255671|.|GG|G|50|PASS|.|GT|1/1' 'chr9|133257521|.|T|TC|50|PASS|.|GT|1/1' +bt_vcf "$L" 'chr9|133255669|.|CG|C|50|PASS|.|GT|1/1' 'chr9|133257521|.|T|TC|50|PASS|.|GT|1/1' +bt_vcf "$M" 'chr9|133255928|.|C|A,G|50|PASS|.|GT|1/2' 'chr9|133257521|.|T|TC|50|PASS|.|GT|1/1' +export REF_FASTA="$REFBT" +sample "$R" "${T}/${R}.vcf.gz" "${T}/win.bam" +step "$R" +check_eq "right-shifted GG>G: bcftools norm moved one record" "$(realigned)" 1 +check_eq "right-shifted GG>G at c.1061: ABO A2/A2" "$(cell "$R" ABO phenotype)/$(cell "$R" ABO genotype)" "A2/A2/A2" +check_eq "and c.1061delC reads two copies" "$(ev "$R" ABO_1061)" "called 2" +check "the left-aligned VCF is not left behind" test ! -e "${G}/${R}/bloodtype/${R}_bloodtype_norm.vcf.gz" +sample "$L" "${T}/${L}.vcf.gz" "${T}/win.bam" +step "$L" +check_eq "the same deletion as the table writes it: nothing to move" "$(realigned)" 0 +check_eq "both forms give the same ABO row" "$(grep '^ABO' "${G}/${R}/bloodtype/${R}_bloodtype.tsv" 2>/dev/null)" \ + "$(grep '^ABO' "${G}/${L}/bloodtype/${L}_bloodtype.tsv" 2>/dev/null)" +# Control: og on the right-shifted VCF as written, with the same depth, misses +# the deletion (what step 41 gave before it left-aligned). +"${REPO}/bin/og" bloodtype report --sample "$R" --vcf "${T}/${R}.vcf.gz" --markers "$MARKERS" \ + --depth "${G}/${R}/bloodtype/${R}_bloodtype_depth.tsv" --out "${T}/raw.tsv" --evidence "${T}/raw_evidence.tsv" > /dev/null +check_eq "control: og on the VCF as written reads A1/A1" \ + "$(awk -F'\t' '$1 == "ABO" {print $3}' "${T}/raw.tsv" 2>/dev/null)" "A1/A1" +sample "$M" "${T}/${M}.vcf.gz" "${T}/win.bam" +step "$M" +check_eq "C>A,G 1/2 at c.803 stays one record: one B copy" "$(ev "$M" ABO_803)" "called 1" +check_eq "and ABO A1/B" "$(cell "$M" ABO genotype)" "A1/B" +unset REF_FASTA +rm -f "$REFBT" "${REFBT}.fai" + # --- Nextflow against the bash step ------------------------------------------------------ # The same VCF, and the BAM step 02 wrote from the fixture's reads (its header is the # fixture reference's, which INDEXCOV checks). That BAM has no chr9, chr7, chr17 or From fa035613efe84394d7bb53c1025f8793f358f93d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:27:35 +0100 Subject: [PATCH 191/255] fix(bloodtype): left-align the records near the markers before reading them og bloodtype report matches a marker by position, REF and ALT, and the table writes every indel left-aligned. A caller can write the same indel further right in its repeat, and then the marker missed it: with reads there, the site read as the reference allele at confidence high (ABO c.1061delC as GG>G gave A1/A1, a right-shifted Kell-null deletion gave k/k). Step 41 and the BLOODTYPE process now take the VCF's records within 100 bp of each marker site and left-align them with bcftools norm -c w -f against the reference, keeping multi-allelic records whole. Records on a contig the reference lacks go through unchanged. BLOODTYPE now runs in BCFTOOLS_IMAGE (og is a static binary from bin/), and the step needs the reference and its .fai. The fake-docker case plays the new bcftools call as a copy. Bead: trait-30 --- conf/containers.config | 2 +- main.nf | 7 ++- modules/local/bloodtype/main.nf | 42 ++++++++++++++-- scripts/41-bloodtype.sh | 71 +++++++++++++++++++++++----- scripts/ci/gen-containers-config.sh | 2 +- tests/fake-docker/cn-engine-calls.sh | 2 + 6 files changed, 106 insertions(+), 20 deletions(-) diff --git a/conf/containers.config b/conf/containers.config index f492cf6d2..7e8d7ecc7 100644 --- a/conf/containers.config +++ b/conf/containers.config @@ -16,7 +16,7 @@ process { withName: 'ARCHAIC_GENOTYPES' { container = 'python:3.15.2@sha256:d7a732b20de2f5d3cf977cbcf69aacf30dc928a37908ff9fdb7bb3503024c91a' } withName: 'ARCHAIC_IBDMIX' { container = 'quay.io/biocontainers/ibdmix:1.0.1--h4ac6f70_2' } withName: 'ARCHAIC_REPORT' { container = 'python:3.15.2@sha256:d7a732b20de2f5d3cf977cbcf69aacf30dc928a37908ff9fdb7bb3503024c91a' } - withName: 'BLOODTYPE' { container = 'python:3.15.2@sha256:d7a732b20de2f5d3cf977cbcf69aacf30dc928a37908ff9fdb7bb3503024c91a' } + withName: 'BLOODTYPE' { container = 'staphb/bcftools:1.24' } withName: 'BLOODTYPE_DEPTH' { container = 'staphb/samtools:1.24' } withName: 'BLOODTYPE_REGIONS' { container = 'python:3.15.2@sha256:d7a732b20de2f5d3cf977cbcf69aacf30dc928a37908ff9fdb7bb3503024c91a' } withName: 'CHIP_ANNOTATE' { container = 'staphb/bcftools:1.24' } diff --git a/main.nf b/main.nf index 78939b7c1..181c82b54 100644 --- a/main.nf +++ b/main.nf @@ -930,7 +930,8 @@ workflow { } // ─── Blood groups (opt-in: bloodtype) ─────────────────────────────── - // The antigens of assets/bloodtype_markers.tsv from the VCF, and RhD from + // The antigens of assets/bloodtype_markers.tsv from the VCF, its records + // near the markers left-aligned against the reference first, and RhD from // the BAM's read depth. A sample without a BAM gets [] for the depth: its // RhD is not determined. Equivalent to scripts/41-bloodtype.sh. ch_bloodtype = Channel.empty() @@ -945,7 +946,9 @@ workflow { .join(BLOODTYPE_DEPTH.out.depth.map { meta, f -> [meta.id, f] }, remainder: true) .filter { row -> row[1] != null } .map { row -> [row[1], row[2], row[3], row[4] ?: []] }, - ch_bloodtype_markers + ch_bloodtype_markers, + ch_reference, + ch_reference_fai ) ch_bloodtype = BLOODTYPE.out.table ch_bloodtype_versions = BLOODTYPE_REGIONS.out.versions.mix(BLOODTYPE_DEPTH.out.versions, BLOODTYPE.out.versions) diff --git a/modules/local/bloodtype/main.nf b/modules/local/bloodtype/main.nf index 840d14bf5..f6ca92ec0 100644 --- a/modules/local/bloodtype/main.nf +++ b/modules/local/bloodtype/main.nf @@ -9,8 +9,12 @@ assets/bloodtype_markers.tsv and those regions, for each sample with a BAM: the read depth at every marker site and over RHD and its flanks - BLOODTYPE og bloodtype report on the VCF and that depth, reading - the marker sites through the VCF's index; without a BAM + BLOODTYPE the VCF's records within 100 bp of a marker site, + left-aligned against the reference (bcftools norm -c w, + multi-allelic records kept whole), so an indel a caller + wrote further right in a repeat meets its marker; then + og bloodtype report on that file and the depth, reading + the marker sites through its index; without a BAM ([] in its place) RhD is not determined and sites with no VCF record are read as reference (confidence medium) @@ -103,6 +107,8 @@ process BLOODTYPE { input: tuple val(meta), path(vcf), path(vcf_index), path(depth) // vcf_index: its .tbi or .csi, or []; depth: BLOODTYPE_DEPTH's, or [] without a BAM path(markers) + path(reference) + path(reference_fai) output: tuple val(meta), path("${meta.id}_bloodtype.tsv"), emit: table @@ -114,13 +120,39 @@ process BLOODTYPE { script: def depth_arg = depth ? "--depth ${depth}" : '' + // og bloodtype report matches a marker by position, REF and ALT, so an + // indel must sit where the table writes it: left-aligned. The records + // within 100 bp of a marker site (more than the 38 bp the longest marker + // indel can shift along its repeat) are left-aligned first. A + // multi-allelic record stays whole; -c w keeps a record whose REF differs + // from the reference as it is. Records on a contig the reference lacks go + // through unchanged. The VCF is read through its index when it has a + // current one, else once from the start. The same commands as + // scripts/41-bloodtype.sh; og runs from bin/, on the task's PATH. """ - og bloodtype report --sample ${meta.id} --vcf ${vcf} --markers ${markers} ${depth_arg} \\ + grep -v '^#' ${markers} | tail -n +2 \\ + | awk -F '\\t' -v OFS='\\t' '{s = \$2 - 100; if (s < 0) s = 0; print \$1, s, \$3 + 100; c = \$1; if (sub(/^chr/, "", c)) print c, s, \$3 + 100}' \\ + | sort -k1,1 -k2,2n \\ + | awk -F '\\t' -v OFS='\\t' '\$1 == c && \$2 <= e {if (\$3 > e) e = \$3; next} {if (c != "") print c, s, e; c = \$1; s = \$2; e = \$3} END {if (c != "") print c, s, e}' > bt_windows.bed + cut -f1 ${reference_fai} > bt_ref_contigs.txt + awk -F '\\t' 'NR == FNR {c[\$1] = 1; next} \$1 in c' bt_ref_contigs.txt bt_windows.bed > bt_on_ref.bed + awk -F '\\t' 'NR == FNR {c[\$1] = 1; next} !(\$1 in c)' bt_ref_contigs.txt bt_windows.bed > bt_off_ref.bed + SEL=-T OVERLAP=--targets-overlap + for x in tbi csi; do + if [ -f "${vcf}.\$x" ] && [ ! "${vcf}" -nt "${vcf}.\$x" ]; then SEL=-R OVERLAP=--regions-overlap; fi + done + pick() { if [ -s "\$1" ]; then bcftools view "\$SEL" "\$1" "\$OVERLAP" record -Ou ${vcf}; else bcftools view -h -Ou ${vcf}; fi; } + pick bt_on_ref.bed | bcftools norm -c w -f ${reference} -Ob -o bt_on_ref.bcf + pick bt_off_ref.bed | bcftools view -Ob -o bt_off_ref.bcf + bcftools concat -Ou bt_on_ref.bcf bt_off_ref.bcf | bcftools sort -T ./bt_sort -Oz -o bt_norm.vcf.gz + bcftools index -f -t bt_norm.vcf.gz + + og bloodtype report --sample ${meta.id} --vcf bt_norm.vcf.gz --markers ${markers} ${depth_arg} \\ --out ${meta.id}_bloodtype.tsv --evidence ${meta.id}_bloodtype_evidence.tsv cat <<-END_VERSIONS > versions.yml "${task.process}": - python: ${task.container.replaceFirst(/^[^:@]+[:@]/, '')} + bcftools: ${task.container.replaceFirst(/^[^:@]+[:@]/, '')} END_VERSIONS """ @@ -131,7 +163,7 @@ process BLOODTYPE { cat <<-END_VERSIONS > versions.yml "${task.process}": - python: ${task.container.replaceFirst(/^[^:@]+[:@]/, '')} + bcftools: ${task.container.replaceFirst(/^[^:@]+[:@]/, '')} END_VERSIONS """ } diff --git a/scripts/41-bloodtype.sh b/scripts/41-bloodtype.sh index 1eada4335..a4d26ef9b 100755 --- a/scripts/41-bloodtype.sh +++ b/scripts/41-bloodtype.sh @@ -8,11 +8,15 @@ # neutrophil antigens HNA-1, -3, -4 and -5 (assets/bloodtype_markers.tsv), # and RhD from read depth (the RHD row of bin/cn_loci.tsv): the common # RhD-negative allele deletes the whole RHD gene, which a VCF does not show. -# og bloodtype report applies the rules, reading the marker sites through the -# VCF's .tbi; it is the same code as the BLOODTYPE Nextflow processes. A -# prediction from the DNA, not a serological typing. +# The VCF's records around the marker sites are left-aligned against the +# reference first (bcftools norm, multi-allelic records kept whole), so an +# indel a caller wrote further right in a repeat still meets its marker. og +# bloodtype report then applies the rules, reading the marker sites through +# that file's .tbi; it is the same code as the BLOODTYPE Nextflow processes. +# A prediction from the DNA, not a serological typing. # -# Input: vcf/.vcf.gz; aligned/_sorted.bam with its .bai +# Input: vcf/.vcf.gz; the reference (REF_FASTA) with its .fai; +# aligned/_sorted.bam with its .bai # (ALIGN_DIR as for the other BAM steps), optional: without it RhD is # not determined and sites the VCF has no record for are read as # reference with confidence medium. @@ -26,7 +30,7 @@ GENOME_DIR=${GENOME_DIR:?Set GENOME_DIR to your data directory} # shellcheck source=lib/common.sh . "$(dirname "$0")/lib/common.sh" validate_sample "$SAMPLE" -require_image PYTHON_IMAGE SAMTOOLS_IMAGE +require_image PYTHON_IMAGE SAMTOOLS_IMAGE BCFTOOLS_IMAGE S="${GENOME_DIR}/${SAMPLE}" VCF="${S}/vcf/${SAMPLE}.vcf.gz" @@ -35,17 +39,20 @@ OUTDIR="${S}/bloodtype" TABLE="${OUTDIR}/${SAMPLE}_bloodtype.tsv" EVIDENCE="${OUTDIR}/${SAMPLE}_bloodtype_evidence.tsv" DEPTH="${OUTDIR}/${SAMPLE}_bloodtype_depth.tsv" +NORM="${OUTDIR}/${SAMPLE}_bloodtype_norm.vcf.gz" echo "=== Blood groups (predicted from the genome): ${SAMPLE} ===" -[ -f "$VCF" ] || { echo "ERROR: File not found: ${VCF}" >&2; exit 1; } +for f in "$VCF" "$REF_FASTA" "${REF_FASTA}.fai"; do + [ -f "$f" ] || { echo "ERROR: File not found: ${f}" >&2; exit 1; } +done echo "VCF: ${VCF}" mkdir -p "$OUTDIR" -rm -f "$TABLE" "$EVIDENCE" "$DEPTH" "${OUTDIR}/targets.bed" "${OUTDIR}/contigs.txt" "${OUTDIR}/rhd.bed" +rm -f "$TABLE" "$EVIDENCE" "$DEPTH" "$NORM" "${NORM}.tbi" "${OUTDIR}/targets.bed" "${OUTDIR}/contigs.txt" "${OUTDIR}/rhd.bed" O=$(cpath "$OUTDIR") ARGS=() if [ -f "$BAM" ] && [ -f "${BAM}.bai" ]; then echo "BAM: ${BAM}" - echo "[1/2] Read depth at the marker sites and over RHD (samtools bedcov, MAPQ >= 20)..." + echo "[1/3] Read depth at the marker sites and over RHD (samtools bedcov, MAPQ >= 20)..." # The marker table is a BED with extra columns; bedcov reads it once the # comment and header lines are gone. The RHD regions come from # og cn bed (BLOODTYPE_REGIONS in Nextflow) as a 4-column BED. @@ -66,13 +73,55 @@ if [ -f "$BAM" ] && [ -f "${BAM}.bai" ]; then rm -f "${OUTDIR}/targets.bed" "${OUTDIR}/contigs.txt" "${OUTDIR}/rhd.bed" ARGS+=(--depth "$(cpath "$DEPTH")") else - echo "[1/2] No BAM (${BAM}): RhD needs read depth and is reported as not determined." + echo "[1/3] No BAM (${BAM}): RhD needs read depth and is reported as not determined." fi -echo "[2/2] Blood group rules (og bloodtype report)..." +echo "[2/3] The VCF's records around the marker sites, left-aligned (bcftools norm)..." +# og bloodtype report matches a marker by position, REF and ALT, so an indel +# must sit where the table writes it: left-aligned, as bcftools norm writes it. +# A caller can write it further right in a repeat (DeepVariant does without +# --normalize_reads, and a VCF from any caller is accepted). So the records +# within 100 bp of a marker site (more than the 38 bp the longest marker indel +# can shift along its repeat) are left-aligned against the reference first. +# A multi-allelic record stays whole: split, a 1/2 call becomes 1/0 and 0/1 +# records and a null row would lose one of its two copies. -c w keeps a record +# whose REF differs from the reference as it is (og then reads it as a no +# call). Records on a contig the reference lacks (a VCF named 1, 2, ... against +# a chr-named reference) cannot be aligned and go through unchanged. The VCF is +# read through its index when it has a current one, else once from the start. +# The BLOODTYPE process runs the same commands before og. +# shellcheck disable=SC2016 # the program's $ belong to the inner bash and awk +NORM_PROG=' + MARKERS=$1 VCF=$2 REF=$3 W=$4 OUT=$5 + grep -v "^#" "$MARKERS" | tail -n +2 \ + | awk -F "\t" -v OFS="\t" "$6" | sort -k1,1 -k2,2n | awk -F "\t" -v OFS="\t" "$7" > "$W/bt_windows.bed" + cut -f1 "$REF.fai" > "$W/bt_ref_contigs.txt" + awk -F "\t" "NR == FNR {c[\$1] = 1; next} \$1 in c" "$W/bt_ref_contigs.txt" "$W/bt_windows.bed" > "$W/bt_on_ref.bed" + awk -F "\t" "NR == FNR {c[\$1] = 1; next} !(\$1 in c)" "$W/bt_ref_contigs.txt" "$W/bt_windows.bed" > "$W/bt_off_ref.bed" + SEL=-T OVERLAP=--targets-overlap + for x in tbi csi; do + if [ -f "$VCF.$x" ] && [ ! "$VCF" -nt "$VCF.$x" ]; then SEL=-R OVERLAP=--regions-overlap; fi + done + pick() { if [ -s "$1" ]; then bcftools view "$SEL" "$1" "$OVERLAP" record -Ou "$VCF"; else bcftools view -h -Ou "$VCF"; fi; } + pick "$W/bt_on_ref.bed" | bcftools norm -c w -f "$REF" -Ob -o "$W/bt_on_ref.bcf" + pick "$W/bt_off_ref.bed" | bcftools view -Ob -o "$W/bt_off_ref.bcf" + bcftools concat -Ou "$W/bt_on_ref.bcf" "$W/bt_off_ref.bcf" | bcftools sort -T "$W/bt_sort" -Oz -o "$OUT" + bcftools index -f -t "$OUT" + rm -f "$W/bt_windows.bed" "$W/bt_ref_contigs.txt" "$W/bt_on_ref.bed" "$W/bt_off_ref.bed" "$W/bt_on_ref.bcf" "$W/bt_off_ref.bcf"' +# Each marker row 100 bp each side, under its chr name and without the chr +# (og reads a VCF named either way); then the overlapping windows merged. +# shellcheck disable=SC2016 # awk programs +WINDOWS_AWK='{s = $2 - 100; if (s < 0) s = 0; print $1, s, $3 + 100; c = $1; if (sub(/^chr/, "", c)) print c, s, $3 + 100}' +# shellcheck disable=SC2016 +MERGE_AWK='$1 == c && $2 <= e {if ($3 > e) e = $3; next} {if (c != "") print c, s, e; c = $1; s = $2; e = $3} END {if (c != "") print c, s, e}' +run_in -v "${PGP_ROOT}/assets:/pgp-assets:ro" --cpus 1 --memory 2g "$BCFTOOLS_IMAGE" bash -euo pipefail -c "$NORM_PROG" \ + _ /pgp-assets/bloodtype_markers.tsv "$(cpath "$VCF")" "$REF_FASTA_C" "$O" "$(cpath "$NORM")" "$WINDOWS_AWK" "$MERGE_AWK" +[ -s "$NORM" ] || { echo "ERROR: no left-aligned VCF written (${NORM})" >&2; exit 1; } +echo "[3/3] Blood group rules (og bloodtype report)..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets:/pgp-assets:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ - /pgp-bin/og bloodtype report --sample "$SAMPLE" --vcf "$(cpath "$VCF")" \ + /pgp-bin/og bloodtype report --sample "$SAMPLE" --vcf "$(cpath "$NORM")" \ --markers /pgp-assets/bloodtype_markers.tsv ${ARGS[@]+"${ARGS[@]}"} \ --out "$(cpath "$TABLE")" --evidence "$(cpath "$EVIDENCE")" +rm -f "$NORM" "${NORM}.tbi" [ -s "$TABLE" ] || { echo "ERROR: no table written (${TABLE})" >&2; exit 1; } echo "" diff --git a/scripts/ci/gen-containers-config.sh b/scripts/ci/gen-containers-config.sh index 743f763a7..3f16adb22 100755 --- a/scripts/ci/gen-containers-config.sh +++ b/scripts/ci/gen-containers-config.sh @@ -45,7 +45,7 @@ ANNOTSV ANNOTSV_IMAGE ARCHAIC_GENOTYPES PYTHON_IMAGE ARCHAIC_IBDMIX IBDMIX_IMAGE ARCHAIC_REPORT PYTHON_IMAGE -BLOODTYPE PYTHON_IMAGE +BLOODTYPE BCFTOOLS_IMAGE BLOODTYPE_DEPTH SAMTOOLS_IMAGE BLOODTYPE_REGIONS PYTHON_IMAGE CHIP_ANNOTATE BCFTOOLS_IMAGE diff --git a/tests/fake-docker/cn-engine-calls.sh b/tests/fake-docker/cn-engine-calls.sh index 824e81f2d..9b3714232 100755 --- a/tests/fake-docker/cn-engine-calls.sh +++ b/tests/fake-docker/cn-engine-calls.sh @@ -88,6 +88,8 @@ case "$*" in "samtools bedcov "*) cov "$(map "$3")" ;; "bash -euo pipefail -c "*) case "$5" in + *"bcftools norm"*) # step 41: the records near the markers, left-aligned (here as they are) + cp "$(map "$8")" "$(map "${11}")" ;; *targets.bed*) # step 41: the marker sites, then the RHD regions o=$(map "$8") { grep -v '^#' "$(map "$7")" | tail -n +2; cat "${o}/rhd.bed"; } > "${o}/targets.bed" From 00f48c135b0f71e0fd76c15807b50b61b9035741 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:28:02 +0100 Subject: [PATCH 192/255] refactor(live): retire the Python ops, eta and headroom tests for the cargo ones The three Python test files go: their checks run in og-live/tests/ops*.rs. Guard's rust job runs every planted_* test with the fault-inject feature and requires each to fail and none to pass. The alerts guide, the example rules, the fixture note and the two module comments name the cargo tests. Bead: D1 --- .github/workflows/guard.yml | 20 ++ CHANGELOG.md | 3 + assets/monitoring/pgp-alerts.yml | 6 +- bin/run_eta.py | 2 +- bin/run_headroom.py | 2 +- docs/alerts.md | 2 +- docs/rust.md | 2 +- rust/crates/og-runlog/src/eta.rs | 2 +- tests/fixtures/live/ops/README.md | 4 +- tests/test_owngenome_live_ops.py | 409 --------------------------- tests/test_run_eta.py | 446 ------------------------------ tests/test_run_headroom.py | 406 --------------------------- 12 files changed, 34 insertions(+), 1270 deletions(-) delete mode 100755 tests/test_owngenome_live_ops.py delete mode 100644 tests/test_run_eta.py delete mode 100644 tests/test_run_headroom.py diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index f1a4c1821..afb7cde70 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -348,6 +348,26 @@ jobs: [ "$rc" -ne 0 ] || { echo "ERROR: the planted unreadable-report success passed"; exit 1; } grep -E 'unreadable_pharmcat_is_unassessed.*FAILED' "$log" echo "Control: the unreadable-report success was reported as a failed test." + # The live ops card (og-live/tests/ops*.rs): each planted_* test turns + # on one planted mistake of the fault-inject feature, so every one of + # them must fail, and none may pass. + - name: Control (every planted mistake of the live ops card must fail its check) + run: | + set -o pipefail + tests=(--test ops --test ops_eta --test ops_headroom) + log="${RUNNER_TEMP}/ops-planted.log" + want=$(cargo test --locked -p og-live --features fault-inject "${tests[@]}" planted_ -- --list 2>/dev/null \ + | grep -cE '^planted::planted_[a-z0-9_]+: test$' || true) + [ "$want" -gt 0 ] || { echo "ERROR: no planted test was found"; exit 1; } + rc=0 + cargo test --locked --no-fail-fast -p og-live --features fault-inject "${tests[@]}" planted_ > "$log" 2>&1 || rc=$? + grep -E '^test planted::planted_' "$log" || true + [ "$rc" -ne 0 ] || { cat "$log"; echo "ERROR: every planted mistake passed"; exit 1; } + passed=$(grep -E '^test planted::planted_[a-z0-9_]+ \.\.\. ok$' "$log" || true) + [ -z "$passed" ] || { echo "ERROR: a check passed with its planted mistake on:"; echo "$passed"; exit 1; } + failed=$(grep -cE '^test planted::planted_[a-z0-9_]+ \.\.\. FAILED$' "$log" || true) + [ "$failed" -eq "$want" ] || { cat "$log"; echo "ERROR: ${want} planted tests, ${failed} failed"; exit 1; } + echo "Control: all ${failed} planted mistakes of the live ops card failed their checks." - name: Traits controls (diplotype order, REF check and missing phase rows) env: CARGO_BUILD_JOBS: 3 diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf6c9e6d..3b6b53d81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The variant calling page names the Genome Reference Consortium as the source of the PAR intervals in `assets/par_grch38.bed`. - Every verb the 1.0.0 release still has to port now exists in the `og` binary, each in its own module, as a stub that prints `og : not ported yet` and exits 2. The new crates are og-render, og-live, og-couple and og-admix. Each port then fills one module and never edits `rust/og/src/main.rs` or a crate's module list, so the ports can land in parallel. No step calls these verbs yet, and no output changes. - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. +- The live site's run health card and its `/metrics` text now have a Rust port: og-live's ops module computes the hours left, the memory and disk headroom, the last activity and the five alerts from og-runlog, the same code `og run eta` and `og run headroom` use. It writes the same card and the same metrics text as `bin/owngenome_live_ops.py`, which the Python site keeps running until the Rust site replaces it. A parity test runs both on the same inputs: the card must match by meaning, the metrics text byte for byte. On the made-up run of `tests/fixtures/live/ops`, one card takes 0.14 ms and 4.7 MB peak memory, against 0.55 ms and 15 MB for the Python plugin. + - `tests/test_owngenome_live_ops.py`, `tests/test_run_eta.py` and `tests/test_run_headroom.py` are now cargo tests in `rust/crates/og-live/tests/`. Their planted source mistakes are named faults of a new `fault-inject` cargo feature of og-runlog and og-live, and Guard's rust job requires every one of them to fail its check. The release binary holds none of them. + - One behaviour differs on purpose: a step history the reader cannot use, such as one with no `trace_file` column, no longer turns the card into "run health could not be read". The card still shows; the hours left then come from the catalogue's hours. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. diff --git a/assets/monitoring/pgp-alerts.yml b/assets/monitoring/pgp-alerts.yml index 3b69b3c7e..365ada731 100644 --- a/assets/monitoring/pgp-alerts.yml +++ b/assets/monitoring/pgp-alerts.yml @@ -1,5 +1,7 @@ -# Example Prometheus alert rules for OwnGenome runs, on the metrics of bin/owngenome_live_ops.py -# (GET /metrics of the live site). docs/alerts.md says how to scrape them and what each one means. +# Example Prometheus alert rules for OwnGenome runs, on the metrics of the live site's ops module +# (GET /metrics: bin/owngenome_live_ops.py, and og-live's ops module, rust/crates/og-live/src/ops.rs, +# which writes the same text). docs/alerts.md says how to scrape them and what each one means. +# rust/crates/og-live/tests/ops.rs checks that every metric these rules use is one /metrics writes. # The thresholds the page also shows (disk warn percent, stall minutes) are flags of the site, # so the page and these rules always agree. Each disk is compared with the warn ratio of the site # it came from (on job and instance), so one Prometheus can scrape several sites; the 'for' times and the 98% disk line are yours to tune. diff --git a/bin/run_eta.py b/bin/run_eta.py index bc407fd59..35ea90597 100755 --- a/bin/run_eta.py +++ b/bin/run_eta.py @@ -50,7 +50,7 @@ import run_state # noqa: E402 # Nextflow's default trace columns, in its order, then the ones the estimate needs. -# nextflow.config's trace.fields is the two lists joined; tests/test_run_eta.py checks it. +# nextflow.config's trace.fields is the two lists joined; rust/crates/og-live/tests/ops_eta.rs checks it. DEFAULT_FIELDS = ["task_id", "hash", "native_id", "name", "status", "exit", "submit", "duration", "realtime", "%cpu", "peak_rss", "peak_vmem", "rchar", "wchar"] ADDED_FIELDS = ["process", "tag", "attempt", "cpus", "memory", "start", "complete"] diff --git a/bin/run_headroom.py b/bin/run_headroom.py index 7714f7153..8c0ddc1ce 100755 --- a/bin/run_headroom.py +++ b/bin/run_headroom.py @@ -36,7 +36,7 @@ DEFAULT_WARN_PERCENT = 90 # The exits conf/base.config's errorStrategy retries: usually killed for memory or time. -# tests/test_run_headroom.py checks the two lists agree. +# og-runlog's MEMORY_EXITS is the same list; rust/crates/og-live/tests/ops_headroom.rs checks it against base.config. MEMORY_EXITS = {"143", "137", "104", "134", "139", "247"} DONE = {"COMPLETED", "FAILED", "ABORTED", "CACHED"} diff --git a/docs/alerts.md b/docs/alerts.md index d2dac00fe..7f242948a 100644 --- a/docs/alerts.md +++ b/docs/alerts.md @@ -97,4 +97,4 @@ The hours left, the late steps and the unknown steps are written only while a ru ## How it is tested -`tests/test_owngenome_live_ops.py` runs the plugin on the made-up run of `tests/fixtures/live/ops/` with a fake disk and a fake host, then against planted mistakes that must fail it. The `promtool` job in `.github/workflows/lint.yml` runs `pgp-alerts.test.yml`, then changes one threshold and requires the test to fail. +`rust/crates/og-live/tests/ops.rs` runs og-live's ops module, the Rust port of the plugin, on the made-up run of `tests/fixtures/live/ops/` with a fake disk and a fake host. It also runs the plugin itself on the same inputs: the card must match by meaning and the metrics text byte for byte. `ops_eta.rs` and `ops_headroom.rs` beside it check the hours left and the headroom. Each planted mistake of the `fault-inject` feature must fail its check: Guard's rust job runs them and requires every one to fail. The `promtool` job in `.github/workflows/lint.yml` runs `pgp-alerts.test.yml`, then changes one threshold and requires the test to fail. diff --git a/docs/rust.md b/docs/rust.md index bfe8ead4b..e0a0c084e 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -110,7 +110,7 @@ To add a primitive: write it in `pycompat` with unit tests, add an operation to ## CI -- **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; the four builds through `scripts/setup.sh --og` (both musl ones and the macOS pair); and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. Then the fake-docker case `reports-stale-step` runs with `bin/og` and must check its `summary.json` with `og dev schema`, and every test that carries the line `# og-leg: OG_REQUIRED` (the tests that run `og`, such as the `og sv` and `og pgx` filter tests, the step 05 tests and `tests/test_ancestry_fine_report.py`) runs with `bin/og` and `OG_REQUIRED=1`; in `unit-tests`, which has no `og`, they skip with a note. A new test that needs `og` adds that line; the job has no list to edit. Before the real run the step plants a marked test with a failing assert, which must be picked and fail. The planned-verb test must fail once one verb's arm is dropped from `og/src/main.rs`. `cargo test` includes `og dev contract --static` and `og dev report-only` on the checkout (`rust/crates/og-dev/tests/repo.rs`). It also runs `og dev helpers --static` and `--list` on the checkout (`rust/crates/og-dev/tests/helpers_repo.rs`); the `helper-binaries` job runs the same verb through `scripts/ci/check-container-helpers.sh`, with the og the `og` job builds. +- **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; the planted mistakes of the live ops card (`-p og-live --features fault-inject`, the `planted_*` tests of `crates/og-live/tests/ops*.rs`), every one of which must fail its check; `cargo test`; the differential test at 100,000 cases per operation; the four builds through `scripts/setup.sh --og` (both musl ones and the macOS pair); and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. Then the fake-docker case `reports-stale-step` runs with `bin/og` and must check its `summary.json` with `og dev schema`, and every test that carries the line `# og-leg: OG_REQUIRED` (the tests that run `og`, such as the `og sv` and `og pgx` filter tests, the step 05 tests and `tests/test_ancestry_fine_report.py`) runs with `bin/og` and `OG_REQUIRED=1`; in `unit-tests`, which has no `og`, they skip with a note. A new test that needs `og` adds that line; the job has no list to edit. Before the real run the step plants a marked test with a failing assert, which must be picked and fail. The planned-verb test must fail once one verb's arm is dropped from `og/src/main.rs`. `cargo test` includes `og dev contract --static` and `og dev report-only` on the checkout (`rust/crates/og-dev/tests/repo.rs`). It also runs `og dev helpers --static` and `--list` on the checkout (`rust/crates/og-dev/tests/helpers_repo.rs`); the `helper-binaries` job runs the same verb through `scripts/ci/check-container-helpers.sh`, with the og the `og` job builds. - **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a fresh and a stale macOS pair, a tree hash that cannot be computed, a release download with a bad checksum or attestation, without the macOS pair, with `gh` not logged in, with a network error, without binaries, or whose copy into `bin/.og` fails), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. - **Rust** (`rust.yml`, on a pull request that touches the Rust code, the shim or `versions.env`): times a cold and a cached build, runs `og` by bare name in the four images on amd64 and arm64 (QEMU), runs that check once more with a wrong-arch binary planted, which must fail, runs the fetch smoke with both binaries (arm64 under QEMU) and with pysam in `PYPGX_IMAGE`, which must all match `fetch.pysam.tsv`, and keeps the binaries as the artifact `og-binaries`. Every night it runs the differential test at 10,000,000 cases per operation with a new seed. - **Release** (`release.yml`): a tag builds the four binaries, writes `SHA256SUMS`, attests them and attaches all five files to the release. A pull request that changes `rust/` or the workflow builds them as a dry run. diff --git a/rust/crates/og-runlog/src/eta.rs b/rust/crates/og-runlog/src/eta.rs index 5fe3c2c74..c43f97319 100644 --- a/rust/crates/og-runlog/src/eta.rs +++ b/rust/crates/og-runlog/src/eta.rs @@ -46,7 +46,7 @@ pub const LAST_RUNS: usize = 5; pub const DOCS_THREADS: f64 = 16.0; /// Planted mistakes for the red controls of the live site's tests -/// (`og-live/tests/ops*.rs`, docs/rust.md). With the cargo feature +/// (`og-live/tests/ops*.rs`). With the cargo feature /// `fault-inject` a test turns one on by name for its own thread, and the /// check it plants against must then fail. Without the feature [`on`] is /// always false and the release binary holds none of them. diff --git a/tests/fixtures/live/ops/README.md b/tests/fixtures/live/ops/README.md index 380601229..375bf89b7 100644 --- a/tests/fixtures/live/ops/README.md +++ b/tests/fixtures/live/ops/README.md @@ -1,11 +1,11 @@ # tests/fixtures/live/ops -Made-up inputs for `tests/test_owngenome_live_ops.py`. No real sample: the run is `OPS-001`. +Made-up inputs for `rust/crates/og-live/tests/ops.rs` (the live site's ops card and `/metrics`). No real sample: the run is `OPS-001`. - `events.jsonl`: Nextflow weblog messages as `bin/owngenome_live.py` stores them. A run starts at 2026-01-02 10:00:00 UTC; step 06's task is killed with exit 137 and its second attempt ends at 7.6 of the 8 GB it asked for; step 11's task starts at 10:05 and is still running. `@WORK@` and `@OUT@` are replaced by temporary folders in the test. - `run_status.tsv`: the `logs/run_status.tsv` that `run-all.sh` writes for the same run (steps 06 and 11 planned). -- `step_history.tsv`: a step history of a box called `opsbox` (`bin/run_eta.py`): step 11 took 60 and +- `step_history.tsv`: a step history of a box called `opsbox` (as `og run eta` writes it): step 11 took 60 and 100 seconds in two runs, step 06 took 30. diff --git a/tests/test_owngenome_live_ops.py b/tests/test_owngenome_live_ops.py deleted file mode 100755 index 8921958de..000000000 --- a/tests/test_owngenome_live_ops.py +++ /dev/null @@ -1,409 +0,0 @@ -#!/usr/bin/env python3 -"""bin/owngenome_live_ops.py on the made-up run of tests/fixtures/live/ops, with a fake disk and a fake host. - -Checks: - 1. half a minute into step 11 (usual time 60 to 100 s on this box), the run is not late and not - stalled, and the hours left is a range with both ends; - 2. two hours later, with no message since, step 11 is late and the run is stalled, and stays - stalled when run_state calls it 'stopped' (no end record, quiet past its stale limit); - 3. a run quiet past --stall-minutes whose running step is still inside its usual time is not stalled; - 4. --stall-minutes 300 keeps the quiet run of check 2 from being stalled; - 5. a failed task makes the run failed; a pgp_run_finished with exit 1 does too, exit 0 does not; - 6. the retried kill counts one retry, and 7.6 of 8 GB asked is a memory hint; - 7. max_memory 30 GB on a fake 16 GB host is a memory alert, on a fake 64 GB host it is not; - 8. a disk 95% used is a disk alert at the default 90, not with --disk-warn-percent 96; - 9. a bare 'nextflow run' (no run folder, folders from the weblog) gives the same metric series as - the run-all.sh run; - 10. the result holds none of the run's paths, and the metrics carry only the labels run and fs; - 11. /metrics is open without a password, needs a login or the bearer token with one, and reads - the token file only when a token is needed; - 12. every metric /metrics can write is in docs/alerts.md, and every metric the example rules use - is one /metrics writes; - 13. the step history is read, never written; - 14. with bin/owngenome_live.py present, the server loads the plugin: state.json has plugins.ops and - GET /metrics answers in the Prometheus text format. -Checks 1 to 9 then run against planted mistakes (mutants of owngenome_live_ops.py) and must fail -there: a check that cannot fail is no check. - -Run: python3 tests/test_owngenome_live_ops.py -""" -import contextlib -import importlib.util -import io -import json -import os -import re -import shutil -import socket -import subprocess -import sys -import tempfile -import time -import types -import urllib.request - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -BIN = os.path.join(REPO, "bin") -sys.path.insert(0, BIN) -import run_eta # noqa: E402 -import run_headroom # noqa: E402 -import run_state # noqa: E402 - -FIX = os.path.join(REPO, "tests", "fixtures", "live", "ops") -SAMPLE = "OPS-001" -T0 = 1767348000 # 2026-01-02T10:00:00Z, the fixture's start -G = 1024 ** 3 -FAILS = 0 - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:800]}") - if not ok: - FAILS += 1 - return ok - - -def load(path, name="owngenome_live_ops"): - spec = importlib.util.spec_from_file_location(name, path) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -class FakeFs: - def __init__(self, used_ratio): - self.f_frsize = self.f_bsize = 4096 - self.f_blocks = 1000000 - self.f_bfree = self.f_bavail = int(self.f_blocks * (1 - used_ratio)) - self.f_files, self.f_ffree = 1000, 900 - - -@contextlib.contextmanager -def disk_at(ratio): - keep = run_headroom._statvfs - run_headroom._statvfs = lambda p: FakeFs(ratio) - try: - yield - finally: - run_headroom._statvfs = keep - - -def make_tree(tmp, run_all=True, extra_events=()): - """GENOME_DIR in TMP: the history of box 'opsbox', and for run-all.sh the run folder. Returns - (root, run_dir or None, events, work, out).""" - root = os.path.join(tmp, "genome") - work, out = os.path.join(tmp, "nfwork"), os.path.join(tmp, "nfout") - for d in (root, work, out, os.path.join(root, "pipeline_info", "step_history")): - os.makedirs(d, exist_ok=True) - shutil.copy(os.path.join(FIX, "step_history.tsv"), os.path.join(root, "pipeline_info", "step_history", "opsbox.tsv")) - with open(os.path.join(FIX, "events.jsonl")) as f: - text = f.read().replace("@WORK@", work).replace("@OUT@", out) - events = [json.loads(line) for line in text.splitlines() if line.strip()] + list(extra_events) - run_dir = None - if run_all: - run_dir = os.path.join(root, SAMPLE) - os.makedirs(os.path.join(run_dir, "logs"), exist_ok=True) - os.makedirs(os.path.join(run_dir, "nextflow", "work"), exist_ok=True) - shutil.copy(os.path.join(FIX, "run_status.tsv"), os.path.join(run_dir, "logs", "run_status.tsv")) - os.utime(os.path.join(run_dir, "logs", "run_status.tsv"), (T0, T0)) - return root, run_dir, events, work, out - - -def cfg_for(root, **kw): - c = types.SimpleNamespace(root=root, steps=run_state.STEPS_TSV, eta_box="opsbox", eta_cores=8, - disk_warn_percent=None, mem_warn_percent=None, stall_minutes=60) - for k, v in kw.items(): - setattr(c, k, v) - return c - - -def ops_at(mod, now, run_all=True, extra=(), physical=64 * G, disk=0.50, stopped=False, **kw): - tmp = tempfile.mkdtemp(prefix="pgp-ops-") - try: - root, run_dir, events, work, out = make_tree(tmp, run_all, extra) - steps = run_state.read_steps() - status = run_state.read_run_status(os.path.join(run_dir, "logs", "run_status.tsv")) if run_dir else None - cur = mod.current_run(events) - state = run_state.build(SAMPLE, steps, status, None, [], run_state.event_tasks(cur, SAMPLE), cur, None, now) - if stopped: - state["run"]["state"] = "stopped" - with disk_at(disk): - o = mod.run_ops(run_dir or os.path.join(root, SAMPLE), state, events, cfg_for(root, **kw), - now=now, steps=steps, physical=physical) - return o, {"tmp": tmp, "root": root, "work": work, "out": out, "state": state} - finally: - shutil.rmtree(tmp, ignore_errors=True) - - -def kinds(o): - return {a["kind"]: a["level"] for a in o["alerts"]} - - -def series(text): - """{metric{labels}} names (no values) of a Prometheus text.""" - return {line.rsplit(" ", 1)[0] for line in text.splitlines() if line and not line.startswith("#")} - - -def finished(code): - return {"event": "pgp_run_finished", "exit": code, "utcTime": "2026-01-02T10:06:00Z"} - - -def behaviour_checks(mod, quiet=False): - """Checks 1 to 9; returns how many failed.""" - global FAILS - before = FAILS - out = io.StringIO() - with contextlib.redirect_stdout(out) if quiet else contextlib.nullcontext(): - early, _ = ops_at(mod, T0 + 330) - h = early["hours_left"] or {} - check("1. half a minute into step 11: not late, not stalled, the hours left has both ends", - "late" not in kinds(early) and "stalled" not in kinds(early) and not early["run"]["stalled"] - and h.get("low_s") is not None and h.get("high_s") is not None and h["low_s"] <= h["high_s"], - early) - late, _ = ops_at(mod, T0 + 302 + 7200) - check("2. two hours later and quiet: step 11 is late and the run is stalled", - late["hours_left"]["late_steps"] == ["11"] and late["hours_left"]["high_s"] is None - and kinds(late).get("late") == "warning" and kinds(late).get("stalled") == "warning", late["alerts"]) - dead, _ = ops_at(mod, T0 + 302 + 7200, stopped=True) - check("2b. the same quiet run read 'stopped' (no end record) is still stalled", - dead["run"]["stalled"] and kinds(dead).get("stalled") == "warning", (dead["run"], dead["alerts"])) - inside, _ = ops_at(mod, T0 + 302 + 90, stall_minutes=1) - check("3. quiet past --stall-minutes but step 11 still inside its usual time: not stalled", - inside["run"]["seconds_since_activity"] >= 60 and "stalled" not in kinds(inside) - and "late" not in kinds(inside), (inside["run"], inside["alerts"])) - calm, _ = ops_at(mod, T0 + 302 + 7200, stall_minutes=300) - check("4. --stall-minutes 300 keeps a 2-hour quiet run from being stalled", - "stalled" not in kinds(calm) and "late" in kinds(calm), calm["alerts"]) - failed_task = {"runName": "ops_demo", "event": "process_completed", "utcTime": "2026-01-02T10:06:00Z", - "trace": {"task_id": 4, "hash": "ab/000004", "name": f"CLINICAL:ROH ({SAMPLE})", - "process": "CLINICAL:ROH", "tag": SAMPLE, "status": "FAILED", "exit": 1, - "attempt": 1, "complete": (T0 + 360) * 1000}} - f1, _ = ops_at(mod, T0 + 400, extra=[failed_task]) - f2, _ = ops_at(mod, T0 + 400, extra=[finished(1)]) - f0, _ = ops_at(mod, T0 + 400, extra=[finished(0)]) - check("5. a failed task fails the run; exit 1 fails it; exit 0 does not", - f1["run"]["failed"] and kinds(f1).get("failed") == "critical" and f2["run"]["failed"] - and f2["run"]["exit"] == 1 and not f0["run"]["failed"] and "failed" not in kinds(f0), - (f1["run"], f2["run"], f0["run"])) - m = early["memory"] - check("6. one retry after exit 137, and 7.6 of 8 GB asked is a hint", - m["retries"] == 1 and m["near_request"] == 1 and kinds(early).get("memory") == "warning", - (m, early["alerts"])) - small, _ = ops_at(mod, T0 + 330, physical=16 * G) - check("7. max_memory 30 GB on a 16 GB host warns, on a 64 GB host it does not", - small["memory"]["max_memory_above_ram"] and not early["memory"]["max_memory_above_ram"] - and "max_memory" in [a["text"] for a in small["alerts"] if a["kind"] == "memory"][0], small["memory"]) - full, _ = ops_at(mod, T0 + 330, disk=0.95) - full96, _ = ops_at(mod, T0 + 330, disk=0.95, disk_warn_percent="96") - check("8. a disk 95% used warns at 90, not at --disk-warn-percent 96", - kinds(full).get("disk") == "warning" and "disk" not in kinds(full96) and "disk" not in kinds(early) - and full["disk"] and abs(full["disk"][0]["used_ratio"] - 0.95) < 0.001, (full["disk"], full96["alerts"])) - bare, _ = ops_at(mod, T0 + 330, run_all=False) - a = series(mod.metrics_text([(SAMPLE, early)], 0.9)) - b = series(mod.metrics_text([(SAMPLE, bare)], 0.9)) - check("9. a bare 'nextflow run' gives the same series as the run-all.sh run", a == b and len(a) > 15, - sorted(a ^ b)) - return FAILS - before - - -def path_and_label_checks(mod): - o, env = ops_at(mod, T0 + 330) - blob = json.dumps(o) - leaked = [k for k in ("tmp", "root", "work", "out") if env[k] in blob] + (["/"] if re.search(r'"/[^"]*"', blob) else []) - text = mod.metrics_text([(SAMPLE, o)], 0.9, 64 * G) - labels = {k for m in re.finditer(r"\{([^}]*)\}", text) for k in re.findall(r'(\w+)="', m.group(1))} - bad = [line for line in text.splitlines() if line and not line.startswith("#") - and not re.fullmatch(r'pgp_[a-z_]+(\{[^}]*\})? -?[0-9.e+-]+', line)] - check("10. no path in the result; the metrics carry only the labels run and fs, in a valid format", - not leaked and labels == {"run", "fs"} and not bad, (leaked, labels, bad[:3])) - - -def auth_checks(mod): - def req(password=None, sid=None, header=None): - r = types.SimpleNamespace(live=types.SimpleNamespace(password_hash=password), sid=sid, - headers={"Authorization": header} if header else {}) - return r - ok = (mod.metrics_allowed(req(), None) - and not mod.metrics_allowed(req("h"), None) - and not mod.metrics_allowed(req("h", header="Bearer s3cret"), None) - and mod.metrics_allowed(req("h", header="Bearer s3cret"), "s3cret") - and not mod.metrics_allowed(req("h", header="Bearer wrong"), "s3cret") - and not mod.metrics_allowed(req("h", header="Basic s3cret"), "s3cret") - and mod.metrics_allowed(req("h", sid="abc"), None)) - - def unreadable(): - raise FileNotFoundError("no token file") - try: - mod.metrics_allowed(req("h", header="Bearer s3cret"), unreadable) - raised = False - except OSError: - raised = True - lazy = (mod.metrics_allowed(req("h", sid="abc"), unreadable) and mod.metrics_allowed(req(), unreadable) - and mod.metrics_allowed(req("h", header="Bearer s3cret"), lambda: "s3cret")) - check("11. /metrics: open without a password; with one, a login or the bearer token only; the token " - "file is read only when a token is needed, and an unreadable one is an error", ok and raised and lazy, - (ok, raised, lazy)) - - -def doc_and_rule_checks(mod): - with open(os.path.join(REPO, "docs", "alerts.md")) as f: - doc = f.read() - with open(os.path.join(REPO, "assets", "monitoring", "pgp-alerts.yml")) as f: - rules = f.read() - missing = [n for n in mod.METRICS if f"`{n}`" not in doc] - used = set(re.findall(r"\bpgp_[a-z_]+\b", "\n".join(line.split("#")[0] for line in rules.splitlines() - if re.match(r"\s*expr:", line) or line.startswith(" " * 10)))) - unknown = sorted(n for n in used if n not in mod.METRICS) - check("12. every metric is in docs/alerts.md; every metric the rules use exists", - not missing and not unknown and used, (missing, unknown)) - - -def write_check(mod): - tmp = tempfile.mkdtemp(prefix="pgp-ops-") - try: - root, run_dir, events, _, _ = make_tree(tmp) - hist = os.path.join(root, "pipeline_info", "step_history", "opsbox.tsv") - with open(os.path.join(root, "pipeline_info", "trace_2026-01-02_10-00-00.txt"), "w") as f: - f.write("task_id\thash\tname\tstatus\texit\n1\tzz/1\tROH (OPS-001)\tCOMPLETED\t0\n") - before = {p: os.path.getmtime(os.path.join(dp, p)) for dp, _, fs in os.walk(root) for p in fs} - with open(hist) as f: - text = f.read() - steps = run_state.read_steps() - st = run_state.build(SAMPLE, steps, run_state.read_run_status(os.path.join(run_dir, "logs", "run_status.tsv")), - None, [], run_state.event_tasks(events, SAMPLE), events, None, T0 + 330) - mod.run_ops(run_dir, st, events, cfg_for(root), now=T0 + 330, steps=steps, physical=64 * G) - after = {p: os.path.getmtime(os.path.join(dp, p)) for dp, _, fs in os.walk(root) for p in fs} - with open(hist) as f: - same = f.read() == text - check("13. the step history is read, never written", same and before == after, (sorted(set(after) ^ set(before)))) - finally: - shutil.rmtree(tmp, ignore_errors=True) - - -def hook_check(mod): - """The state hook turns any error into an error field, so the rest of state.json stays.""" - hooks = {} - app = types.SimpleNamespace(add_argument=lambda *a, **k: None, state_hook=lambda n, fn: hooks.__setitem__(n, fn), - route=lambda *a, **k: None, log=lambda *a, **k: None, config=None) - mod.register(app) - tmp = tempfile.mkdtemp(prefix="pgp-ops-") - try: - root, run_dir, events, _, _ = make_tree(tmp) - app.config = cfg_for(root) - hist = os.path.join(root, "pipeline_info", "step_history", "opsbox.tsv") - with open(hist, "w") as f: # a history with no trace_file column - f.write("step\tseconds\nROH\t60\n") - steps = run_state.read_steps() - st = run_state.build(SAMPLE, steps, run_state.read_run_status(os.path.join(run_dir, "logs", "run_status.tsv")), - None, [], run_state.event_tasks(events, SAMPLE), events, None, T0 + 330) - try: - o, raised = hooks["ops"](run_dir, st, events), None - except Exception as e: - o, raised = None, type(e).__name__ - check("15. a state hook error becomes the card's error field, never an exception", - raised is None and isinstance(o, dict) and "could not be read" in str(o.get("error")), (raised, o)) - finally: - shutil.rmtree(tmp, ignore_errors=True) - - -def free_port(): - s = socket.socket() - s.bind(("127.0.0.1", 0)) - p = s.getsockname()[1] - s.close() - return p - - -def server_check(): - if not os.path.isfile(os.path.join(BIN, "owngenome_live.py")): - print("[NOTE] 14. bin/owngenome_live.py is not in this tree yet: the server check runs once it is") - return - tmp = tempfile.mkdtemp(prefix="pgp-ops-") - proc = None - try: - root, run_dir, events, _, _ = make_tree(tmp) - state_dir = os.path.join(tmp, "state") - os.makedirs(os.path.join(state_dir, "runs", SAMPLE)) - with open(os.path.join(state_dir, "runs", SAMPLE, "events.jsonl"), "w") as f: - f.write("".join(json.dumps(e) + "\n" for e in events)) - port, iport = free_port(), free_port() - env = dict(os.environ, PGP_BOX="opsbox") - env.pop("OWNGENOME_LIVE_PASSWORD_HASH", None) - proc = subprocess.Popen([sys.executable, os.path.join(BIN, "owngenome_live.py"), "--root", root, "--state-dir", - state_dir, "--port", str(port), "--ingest-port", str(iport), "--quiet", - "--stall-minutes", "30"], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) - base = f"http://127.0.0.1:{port}" - st = text = None - for _ in range(100): - try: - with urllib.request.urlopen(base + f"/runs/{SAMPLE}/state.json", timeout=5) as r: - st = json.load(r) - with urllib.request.urlopen(base + "/metrics", timeout=5) as r: - text, ctype = r.read().decode(), r.headers.get("Content-Type") - break - except OSError: - time.sleep(0.1) - ops = (st or {}).get("plugins", {}).get("ops") - check("14. owngenome_live.py loads the plugin: state.json has plugins.ops, /metrics answers", - ops is not None and "alerts" in ops and text is not None and "version=0.0.4" in (ctype or "") - and f'pgp_run_running{{run="{SAMPLE}"}} 1' in text and "pgp_runs 1" in text, - (st and st.get("plugins"), (text or "")[:400], proc.poll() is not None and proc.stderr.read())) - finally: - if proc: - proc.terminate() - proc.wait(timeout=10) - shutil.rmtree(tmp, ignore_errors=True) - - -# Planted mistakes: each must make behaviour_checks fail. -MUTANTS = [ - ("stall ignores the quiet time", "quiet is not None and quiet >= stall_s and not covered", - "quiet is not None and not covered"), - ("a stopped run is not stalled", 'stalled = run_st in ("running", "stopped") and', "stalled = running and"), - ("stall ignores the running steps", "quiet >= stall_s and not covered", "quiet >= stall_s"), - ("late steps dropped", 'late = list(rem["late_steps"]) if rem else []', "late = []"), - ("exit code ignored", "(exit_code not in (None, 0)) or error", "error"), - ("disk warn ignored", 'disk_warn = [d for d in snap["disk"] if d["warn"]]', "disk_warn = []"), - ("retries not counted", 'retries = sum(t["oom_retry"] for t in snap["tasks"])', "retries = 0"), - ("bare run measures no folder", "else _abs_dir(m_work)", "else None"), -] - - -def mutant_checks(): - with open(os.path.join(BIN, "owngenome_live_ops.py")) as f: - src = f.read() - tmp = tempfile.mkdtemp(prefix="pgp-ops-mut-") - try: - for name, old, new in MUTANTS: - if old not in src: - check(f"mutant '{name}': its line is in owngenome_live_ops.py", False, old) - continue - p = os.path.join(tmp, "owngenome_live_ops_mut.py") - with open(p, "w") as f: - f.write(src.replace(old, new, 1)) - global FAILS - keep = FAILS - n = behaviour_checks(load(p, "owngenome_live_ops_mut"), quiet=True) - FAILS = keep - check(f"mutant '{name}' fails a check ({n} failed)", n > 0) - finally: - shutil.rmtree(tmp, ignore_errors=True) - - -def main(): - mod = load(os.path.join(BIN, "owngenome_live_ops.py")) - behaviour_checks(mod) - path_and_label_checks(mod) - auth_checks(mod) - doc_and_rule_checks(mod) - write_check(mod) - hook_check(mod) - server_check() - mutant_checks() - print(f"\n{'FAILED: ' + str(FAILS) + ' check(s)' if FAILS else 'All checks passed.'}") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_run_eta.py b/tests/test_run_eta.py deleted file mode 100644 index f3cc0a6ca..000000000 --- a/tests/test_run_eta.py +++ /dev/null @@ -1,446 +0,0 @@ -#!/usr/bin/env python3 -"""bin/run_eta.py (the live alerts' estimate) on synthetic traces (tests/fixtures/run_eta) and synthetic run states. - -Checks: - 1. nextflow.config's trace.fields is Nextflow's 14 default columns in order, then - process, tag, attempt, cpus, memory, start, complete; - 2. an old trace (default columns) and a new one (with the added columns) give the - same step spans; - 3. '1h 2m 3s', '850ms', '18.3 GB' and a trace time parse equal to their raw values; - 4. cached and failed tasks, and a process of no step, never enter the history; - 5. ingesting twice, or a copy of a trace under another name, adds nothing; - 6. two boxes write two history files; - 7. the history holds no sample id and no '/'; - 8. three runs give the min-max range with n=3, and only the last 5 runs count; - 9. with no history, the low end is the catalogue's measured lower end (hours_low) - and the high end its upper end (hours), doubled on an 8-core box and not scaled - below it on a 32-core box; on 16 and on 32 cores every measured step has - low < high, never one figure; - 10. the hours left are the hand-computed longest chain, and the CPU-work bound when - it is larger; - 11. a running step past its high end makes the high end unknown and says so; - 12. a step the catalogue does not time, or only estimates, lands in unknown_steps, never 0; - 13. a step with a task whose last attempt failed or was stopped gives no span; - 14. estimate() gives the estimate and writes the history from an OUTDIR (the - command, og run eta, is held to the same output by og-runlog's parity test); - 15. without --sample, the trace of a run that run-all.sh has not finished is left - out of the step times, and counts once run_status.tsv says it finished; a - run_status.tsv that cannot be read leaves out the newest trace, never a crash. - 16. bin/run_eta.py, run_state.py and run_headroom.py, run as commands, - exit 1 and name their og run command, never a silent exit 0. -Each check is then run against a planted mistake (a mutant of run_eta.py or a -broken config) and must fail there: a check that cannot fail is no check. - -Run: python3 tests/test_run_eta.py -""" -import glob -import importlib.util -import os -import re -import shutil -import subprocess -import sys -import tempfile -import time - -os.environ["TZ"] = "UTC" # trace times are the host's local time -time.tzset() -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import run_eta # noqa: E402 -import run_state # noqa: E402 - -FIX = os.path.join(REPO, "tests", "fixtures", "run_eta") -STEPS = run_state.read_steps() -H = 3600.0 -FAILS = 0 -TMP = None - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:800]}") - if not ok: - FAILS += 1 - - -def mutant(**repl): - """A fresh copy of bin/run_eta.py with the named functions or values replaced.""" - spec = importlib.util.spec_from_file_location("run_eta_mutant", os.path.join(REPO, "bin", "run_eta.py")) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - for k, v in repl.items(): - setattr(mod, k, v(mod) if callable(v) and getattr(v, "wants_mod", False) else v) - return mod - - -def with_mod(f): - f.wants_mod = True - return f - - -def outdir(*traces): - """A fresh OUTDIR whose pipeline_info holds the named fixture traces (name -> fixture).""" - d = tempfile.mkdtemp(dir=TMP) - os.makedirs(os.path.join(d, "pipeline_info")) - for name, fx in traces: - shutil.copy(os.path.join(FIX, fx), os.path.join(d, "pipeline_info", name)) - return d - - -def spans(history): - out = {} - for r in history: - if r["step_span_s"]: - out.setdefault(r["step"], set()).add(float(r["step_span_s"])) - return out - - -def config_fields(text): - block = re.search(r"^trace \{\n(.*?)^\}", text, re.S | re.M) - m = re.search(r"^\s*fields\s*=\s*'([^']*)'", block.group(1), re.M) if block else None - return m.group(1).split(",") if m else None - - -# --- the checks: each returns {name: (ok, detail)} ------------------------------------------ - -def config_checks(text): - got = config_fields(text) - return {"trace.fields: the 14 defaults in order, then the 7 added": ( - got == run_eta.DEFAULT_FIELDS + run_eta.ADDED_FIELDS and len(run_eta.DEFAULT_FIELDS) == 14 - and run_eta.ADDED_FIELDS == ["process", "tag", "attempt", "cpus", "memory", "start", "complete"], got)} - - -def trace_checks(m): - out = {} - old = m.ingest(outdir(("trace_2026-01-02_09-59-00.txt", "trace_old.txt")), "t", STEPS, 16) - new = m.ingest(outdir(("trace_2026-01-02_09-59-00.txt", "trace_new.txt")), "t", STEPS, 16) - want = {"01b": {20 * 60.0}, "02": {2 * 60.0, 4 * H}, "03": {11 * H + 10}, "04": {30 * 60.0}} - out["an old and a new trace give the same spans"] = (spans(old) == spans(new) == want, (spans(old), spans(new))) - - out["human and raw units parse equal"] = ( - m.parse_duration("1h 2m 3s") == m.parse_duration("3723000") == 3723.0 - and m.parse_duration("850ms") == m.parse_duration("850") == 0.85 - and m.parse_memory("18.3 GB") == m.parse_memory(str(int(18.3 * 1024 ** 3))) - and m.parse_memory("40 MB") == m.parse_memory(str(40 * 1024 ** 2)) - and m.parse_time("2026-01-02 10:00:00.000") == m.parse_time("1767348000000") == 1767348000.0, - [m.parse_duration("1h 2m 3s"), m.parse_duration("850ms"), m.parse_memory("18.3 GB"), - m.parse_time("2026-01-02 10:00:00.000")]) - - hashes = [r["hash"] for r in new] - out["cached, failed and unknown-process tasks never enter the history"] = ( - sorted(hashes) == ["aa000001", "aa000002", "aa000003", "aa000006", "aa000007"], hashes) - - d = outdir(("trace_2026-01-02_09-59-00.txt", "trace_new.txt")) - m.ingest(d, "t", STEPS, 16) - path = m.history_path(d, "t") - first = open(path).read() - m.ingest(d, "t", STEPS, 16) - shutil.copy(os.path.join(FIX, "trace_new.txt"), os.path.join(d, "pipeline_info", "trace_2026-01-03_08-00-00.txt")) - m.ingest(d, "t", STEPS, 16) - out["ingesting twice, or a copied trace, adds nothing"] = (open(path).read() == first, open(path).read()) - - m.ingest(d, "box-b", STEPS, 16) - files = sorted(os.listdir(os.path.join(d, "pipeline_info", "step_history"))) - out["two boxes write two files"] = ([f for f in files if f.endswith(".tsv")] == ["box-b.tsv", "t.tsv"], files) - - body = open(path).read() - out["the history holds no sample id and no '/'"] = ( - "demo1" not in body and "/" not in body and body.split("\n", 1)[0].split("\t") == m.HISTORY_COLUMNS, body) - out["the box name keeps only letters, digits, '_' and '-'"] = (m.box_name("my box.local") == "my_box_local", - m.box_name("my box.local")) - return out - - -def hist(step, runs): - """History rows: one run per (trace name, span hours).""" - return [{"trace_file": f"trace_{i}.txt", "hash": f"h{i}", "attempt": "1", "step": step, "process": "X", - "run_level": "no", "cpus": "8", "memory_bytes": "", "peak_rss_bytes": "", "realtime_s": "", - "step_span_s": str(hours * H), "finished_utc": f"2026-01-{i + 1:02d}T00:00:00Z", "box_cpus": "16"} - for i, hours in enumerate(runs)] - - -def range_checks(m): - out = {} - r = m.step_ranges(hist("02", [4, 3, 5]), STEPS, 16)["02"] - out["three runs give min-max with n=3"] = ( - (r["low_s"], r["high_s"], r["n"], r["source"]) == (3 * H, 5 * H, 3, "box"), r) - r = m.step_ranges(hist("02", [9, 3, 4, 5, 6, 7]), STEPS, 16)["02"] - out["only the last 5 runs count"] = ((r["low_s"], r["high_s"], r["n"]) == (3 * H, 7 * H, 5), r) - row = [s for s in STEPS if s["step"] == "03"][0] - dv, dv_low = row["hours"] * H, float(row["hours_low"]) * H - r8 = m.step_ranges([], STEPS, 8)["03"] - r32 = m.step_ranges([], STEPS, 32)["03"] - out["an 8-core box: the measured low end, the high end doubled"] = ( - r8["source"] == "docs" and r8["low_s"] == dv_low and r8["high_s"] == 2 * dv and "16 threads" in r8["label"], r8) - out["a 32-core box is not scaled below the docs hours"] = ( - r32["low_s"] == dv_low and r32["high_s"] == dv, r32) - measured = [s["step"] for s in STEPS if s["hours_from"] == "measured"] - one = {c: [k for k in measured if not (m.step_ranges([], STEPS, c)[k]["source"] == "docs" - and m.step_ranges([], STEPS, c)[k]["low_s"] - < m.step_ranges([], STEPS, c)[k]["high_s"])] for c in (16, 32)} - out["on 16 and 32 cores every measured step has low < high"] = ( - len(measured) >= 25 and not one[16] and not one[32], (len(measured), one)) - - # Steps 14 and 04b have hours in the catalogue, but only the step page's estimate. - est = m.step_ranges([], STEPS, 16) - unk = m.remaining(state([("03", "done", None), ("14", "waiting", None)]), est, NOW, 16) - out["a step the table only estimates gets no number"] = ( - est["14"]["source"] == est["04b"]["source"] == "none" and est["14"]["low_s"] is None - and unk["unknown_steps"] == ["14"] and unk["low_s"] == 0, (est["14"], est["04b"], unk)) - - # Step 07: the preprocessor ended, then PharmCAT was stopped (a killed run). Step 03: a - # failed first attempt that a second one replaced still counts. - def task(h, proc, status, ex, start, end, attempt="1"): - return {"trace_file": "trace_x.txt", "hash": h, "attempt": attempt, "process": proc, "tag": "s1", - "status": status, "exit": ex, "cpus": 1, "memory_bytes": None, "peak_rss_bytes": None, - "realtime_s": end - start, "start": start, "complete": end} - rows = m.history_rows([task("p1", "PHARMCAT_PREPROCESS", "COMPLETED", "0", 0.0, 40.0), - task("p2", "PHARMCAT", "ABORTED", "-", 41.0, 50.0), - task("d1", "DEEPVARIANT", "FAILED", "137", 0.0, 100.0), - task("d2", "DEEPVARIANT", "COMPLETED", "0", 110.0, 300.0, "2")], - m.process_steps(STEPS), 16) - got = {(r["step"], r["process"]): r["step_span_s"] for r in rows} - out["a step with a task that failed or was stopped gives no span"] = ( - got == {("07", "PHARMCAT_PREPROCESS"): None, ("03", "DEEPVARIANT"): 300.0}, got) - return out - - -def state(rows, run="running"): - """A run state with these steps: (step, state, started_utc); upstream from the catalogue.""" - up = {s["step"]: s["upstream"] for s in STEPS} - return {"run": {"state": run}, "steps": [{"step": k, "state": st, "started_utc": t, "upstream": up[k]} - for k, st, t in rows]} - - -NOW = 1767348000.0 # 2026-01-02T10:00:00Z -RANGES = {"02": {"low_s": 4 * H, "high_s": 5 * H, "cpus": 8}, - "03": {"low_s": 10 * H, "high_s": 12 * H, "cpus": 8}, - "13": {"low_s": 4 * H, "high_s": 5 * H, "cpus": 8}, - "23": {"low_s": 0.1 * H, "high_s": 0.2 * H, "cpus": 1}, - "04": {"low_s": 0.5 * H, "high_s": 1 * H, "cpus": 8}, - "33": {"low_s": None, "high_s": None, "cpus": None}} - - -def close(a, b): - return a is not None and abs(a - b) < 1e-6 - - -def remaining_checks(m): - out = {} - st = state([("02", "done", None), ("03", "waiting", None), ("13", "waiting", None), - ("23", "waiting", None), ("04", "waiting", None)]) - # 03 -> 13 -> 23 is the longest chain: low 10 + 4 + 0.1, high 12 + 5 + 0.2. - big = m.remaining(st, RANGES, NOW, 64) - out["the hours left are the longest chain"] = ( - close(big["low_s"], 14.1 * H) and close(big["high_s"], 17.2 * H), big) - # CPU work: 12*8 + 5*8 + 0.2*1 + 1*8 = 144.2 CPU-hours; over 4 cores 36.05 h. - small = m.remaining(st, RANGES, NOW, 4) - out["the CPU-work bound when it is larger"] = (close(small["low_s"], 14.1 * H) and close(small["high_s"], 36.05 * H), - small) - - late = m.remaining(state([("02", "done", None), ("03", "running", "2026-01-01T21:00:00Z"), - ("13", "waiting", None)]), RANGES, NOW, 64) - out["a running step past its high end: high unknown, and said"] = ( - late["high_s"] is None and late["late_steps"] == ["03"] - and late["note"] == "step 03 is taking longer than usual" and close(late["low_s"], 4 * H), late) - going = m.remaining(state([("02", "done", None), ("03", "running", "2026-01-02T06:00:00Z"), - ("13", "waiting", None)]), RANGES, NOW, 64) - out["a running step counts only what is left of it"] = ( - close(going["low_s"], 10 * H) and close(going["high_s"], 13 * H), going) - - ranges = m.step_ranges([], STEPS, 16) - unk = m.remaining(state([("02", "done", None), ("33", "waiting", None)]), ranges, NOW, 16) - out["a step with no time lands in unknown_steps, never 0"] = ( - unk["unknown_steps"] == ["33"] and ranges["33"]["low_s"] is None and ranges["33"]["source"] == "none", unk) - done = m.remaining(state([("03", "waiting", None)], run="finished"), RANGES, NOW, 16) - out["a finished run has nothing left"] = (done["low_s"] == 0 and done["high_s"] == 0, done) - return out - - -def command_checks(m): - d = outdir(("trace_2026-01-02_09-59-00.txt", "trace_new.txt")) - est = m.estimate(d, box="ci", cores=8) - ok = (est.get("box") == "ci" and est["steps"]["02"]["source"] == "box" - and est["steps"]["02"]["high_s"] == 4 * H and est["steps"]["13"]["source"] == "docs" - and est["remaining"] is None and os.path.isfile(m.history_path(d, "ci"))) - out = {"estimate() gives the estimate and writes the history": (ok, est.get("steps", {}).get("02"))} - - # A run-all.sh run still going: its run_status.tsv has a start and no 'meta finished_utc'. - d = outdir(("trace_2026-01-02_09-59-00.txt", "trace_new.txt")) - logs = os.path.join(d, "demo1", "logs") - os.makedirs(logs) - status = os.path.join(logs, "run_status.tsv") - with open(status, "w") as f: - f.write(f"meta\tstarted_epoch\t{int(time.time()) - 60}\nstep\t02\tplanned\n") - going = m.estimate(d, cores=16, box="ci")["steps"]["02"]["source"] - with open(status, "a") as f: - f.write("meta\tfinished_utc\t2026-01-02 12:00:00 UTC\n") - ended = m.estimate(d, cores=16, box="ci")["steps"]["02"]["source"] - out["without --sample, a run not finished yet is left out"] = ((going, ended) == ("docs", "box"), (going, ended)) - - def unreadable(path): - raise PermissionError(13, "Permission denied", path) - keep = run_state.read_run_status - run_state.read_run_status = unreadable - try: - got = m.estimate(d, cores=16, box="ci")["steps"]["02"]["source"] - except OSError as e: - got = repr(e) - finally: - run_state.read_run_status = keep - out["an unreadable run_status.tsv leaves out the newest trace"] = (got == "docs", got) - return out - - -OLD_CLIS = {"run_eta.py": "eta", "run_state.py": "state", "run_headroom.py": "headroom"} - - -def old_cli_refuses(path, cmd): - """An old run_*.py command exits 1 and names its og command, never a silent exit 0.""" - p = subprocess.run([sys.executable, path, "--outdir", TMP], capture_output=True, text=True, timeout=60) - return p.returncode == 1 and f"bin/og run {cmd}" in p.stderr and not p.stdout, (p.returncode, p.stdout, p.stderr) - - -def all_checks(m): - r = {} - r.update(trace_checks(m)) - r.update(range_checks(m)) - r.update(remaining_checks(m)) - r.update(command_checks(m)) - return r - - -def mutants(): - def forced_ok(mod): - orig = mod.read_trace - return lambda p: [dict(t, status="COMPLETED", exit="0") for t in orig(p)] - - def old_start_is_submit(mod): - def f(p): - ts = run_eta.read_trace(p) - if "process" not in open(p).readline(): - for t, line in zip(ts, open(p).read().splitlines()[1:]): - t["start"] = run_eta.parse_time(line.split("\t")[6]) - return ts - return f - - def slash(mod): - orig = mod.history_rows - return lambda *a: [dict(r, trace_file="pipeline_info/" + r["trace_file"]) for r in orig(*a)] - - def sample_in(mod): - orig = mod.history_rows - return lambda *a: [dict(r, process=r["process"] + "_demo1") for r in orig(*a)] - - def none_is_zero(mod): - orig = mod.step_ranges - return lambda *a, **k: {s: (dict(r, low_s=0.0, high_s=0.0) if r["source"] == "none" else r) - for s, r in orig(*a, **k).items()} - - def estimates_count(mod): - orig = mod.step_ranges - return lambda history, steps, *a, **k: orig(history, [dict(s, hours_from="measured") for s in steps], *a, **k) - - def no_low(mod): - orig = mod.step_ranges - return lambda history, steps, *a, **k: orig(history, [dict(s, hours_low="") for s in steps], *a, **k) - - def no_guard(mod): - def f(outdir): - out = set() - for p in glob.glob(os.path.join(outdir, "*", "logs", "run_status.tsv")): - meta = (run_state.read_run_status(p) or {}).get("meta", {}) - if "finished_utc" not in meta and "started_epoch" in meta: - d = os.path.dirname(os.path.dirname(p)) - out |= {os.path.basename(t) for t in run_state.trace_files(d, float(meta["started_epoch"]))} - return out - return f - - def finished_ignored(mod): - orig = mod.remaining - return lambda st, *a: orig(dict(st, run={"state": "running"}), *a) - - for f in (forced_ok, old_start_is_submit, slash, sample_in, none_is_zero, estimates_count, no_low, no_guard, - finished_ignored): - f.wants_mod = True - return { - "an old and a new trace give the same spans": mutant(read_trace=old_start_is_submit), - "human and raw units parse equal": mutant(MEM_UNITS=dict(run_eta.MEM_UNITS, GB=1000 ** 3)), - "cached, failed and unknown-process tasks never enter the history": mutant(read_trace=forced_ok), - "ingesting twice, or a copied trace, adds nothing": mutant(known_tasks=lambda by_file, name: set()), - "two boxes write two files": mutant(box_name=lambda box=None: "same"), - "the history holds no sample id and no '/'": mutant(history_rows=slash), - "three runs give min-max with n=3": mutant(LAST_RUNS=2), - "only the last 5 runs count": mutant(LAST_RUNS=6), - "an 8-core box: the measured low end, the high end doubled": mutant(DOCS_THREADS=8), - "a 32-core box is not scaled below the docs hours": mutant(docs_scale=lambda c: 16.0 / c), - "on 16 and 32 cores every measured step has low < high": mutant(step_ranges=no_low), - "without --sample, a run not finished yet is left out": mutant(live_traces=lambda outdir: set()), - "an unreadable run_status.tsv leaves out the newest trace": mutant(live_traces=no_guard), - "the hours left are the longest chain": mutant(remaining=None), # replaced below - "the CPU-work bound when it is larger": mutant(cpu_work=lambda *a: 0.0), - "a running step past its high end: high unknown, and said": mutant( - left_of=lambda lo, hi, gone: (max(0.0, lo - gone), max(0.0, hi - gone), False)), - "a step with no time lands in unknown_steps, never 0": mutant(step_ranges=none_is_zero), - "a step the table only estimates gets no number": mutant(step_ranges=estimates_count), - "a step with a task that failed or was stopped gives no span": mutant(finished=lambda ts: True), - "a running step counts only what is left of it": mutant(left_of=lambda lo, hi, gone: (lo, hi, False)), - "a finished run has nothing left": mutant(remaining=finished_ignored), - "estimate() gives the estimate and writes the history": mutant(box_name=lambda box=None: "box"), - "the box name keeps only letters, digits, '_' and '-'": mutant(box_name=lambda box=None: box or "box"), - "the history mutant for samples": mutant(history_rows=sample_in), - } - - -def main(): - global TMP - TMP = tempfile.mkdtemp(prefix="test_run_eta.") - try: - cfg = open(os.path.join(REPO, "nextflow.config")).read() - results = config_checks(cfg) - results.update(all_checks(run_eta)) - for name, (ok, detail) in results.items(): - check(name, ok, detail) - - for name, cmd in OLD_CLIS.items(): - ok, detail = old_cli_refuses(os.path.join(REPO, "bin", name), cmd) - check(f"bin/{name} refuses to run and names og run {cmd}", ok, detail) - - print("\nNegative controls: each check against a planted mistake must fail") - for name, cmd in OLD_CLIS.items(): - src = open(os.path.join(REPO, "bin", name)).read() - cut = src.rindex('\nif __name__ == "__main__":') - planted = os.path.join(TMP, name) - with open(planted, "w") as f: - f.write(src[:cut] + "\n") - check(f"fails on a planted mistake: bin/{name} without its refusing entry point", - not old_cli_refuses(planted, cmd)[0], "the check passed on the mutant") - broken = cfg.replace(",memory,start", ",start") - check("fails on a planted mistake: trace.fields without memory", - not config_checks(broken)["trace.fields: the 14 defaults in order, then the 7 added"][0], "passed") - ms = mutants() - # The longest-chain check against a builder that ignores upstream steps. - m = mutant() - orig_remaining = m.remaining - - def no_upstream(st, *a): - return orig_remaining(dict(st, steps=[dict(s, upstream=[]) for s in st["steps"]]), *a) - m.remaining = no_upstream - ms["the hours left are the longest chain"] = m - sample_mutant = ms.pop("the history mutant for samples") - for name, mod in ms.items(): - check(f"fails on a planted mistake: {name}", not all_checks(mod)[name][0], "the check passed on the mutant") - check("fails on a planted mistake: a sample id in the history", - not all_checks(sample_mutant)["the history holds no sample id and no '/'"][0], "passed") - same = all_checks(mutant()) - check("an unchanged copy passes (the controls test the mistake, not the copy)", - all(ok for ok, _ in same.values()), [k for k, (ok, _) in same.items() if not ok]) - finally: - shutil.rmtree(TMP, ignore_errors=True) - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_run_headroom.py b/tests/test_run_headroom.py deleted file mode 100644 index 3faa78ad0..000000000 --- a/tests/test_run_headroom.py +++ /dev/null @@ -1,406 +0,0 @@ -#!/usr/bin/env python3 -"""bin/run_headroom.py (the live alerts' measure) on synthetic disks, synthetic tasks (tests/fixtures/run_headroom) and a fake host. - -Checks: - 1. a disk 91% used warns at the default 90, one 89% used does not; - 2. --disk-warn-percent 80 (and OWNGENOME_DISK_WARN_PERCENT=80) warns at 85%; - 3. a work folder not created yet is measured on its nearest existing parent; - 4. two folders on one filesystem are reported once, on two filesystems twice; - 5. a filesystem with f_files 0 gives no inode ratio; - 6. 7.6 of 8 GB asked warns, 6 of 8 does not; - 7. '7.6 GB' (a trace) and raw bytes (a weblog message) give the same ratio; - 8. a trace with no memory column gives no ratio; - 9. exit 137 then attempt 2 counts one retry, and a task still running shows its request only; - 10. max_memory 64 GB on a fake 32 GB host warns, on a fake 128 GB host does not; - 11. the snapshot holds no '/' in any string value, and takes its folders from the weblog metadata; - 12. MEMORY_EXITS are the exits conf/base.config's errorStrategy retries; - 13. a real statvfs on a temp folder gives a used ratio in 0..1 and the lines start with the disk; - the command, og run headroom, is held to the same lines by og-runlog's parity test. - 14. a running row read after a finished row of the same attempt keeps the finished ratio; - 15. a relative outdir in the metadata is joined to the run's launchDir, an absolute one is kept; - 16. metadata under 'params' (instead of Nextflow's 'parameters') gives the same outdir and max_memory; - 17. a running weblog message read after a finished one of the same task_id keeps the finished ratio. -Each check is then run against a planted mistake (a mutant of run_headroom.py or a -broken config) and must fail there: a check that cannot fail is no check. - -Run: python3 tests/test_run_headroom.py -""" -import contextlib -import importlib.util -import os -import re -import shutil -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import run_eta # noqa: E402 -import run_headroom # noqa: E402 -import run_state # noqa: E402 - -FIX = os.path.join(REPO, "tests", "fixtures", "run_headroom") -G = 1024 ** 3 -FAILS = 0 -TMP = None - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:800]}") - if not ok: - FAILS += 1 - - -def mutant(**repl): - """A fresh copy of bin/run_headroom.py with the named values replaced.""" - spec = importlib.util.spec_from_file_location("run_headroom_mutant", os.path.join(REPO, "bin", "run_headroom.py")) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - for k, v in repl.items(): - setattr(mod, k, v(mod) if getattr(v, "wants_mod", False) else v) - return mod - - -def with_mod(f): - f.wants_mod = True - return f - - -class FakeVfs: - def __init__(self, used_pct, files=1000, ffree=900): - self.f_frsize = self.f_bsize = 4096 - self.f_blocks = 1000 - self.f_bfree = self.f_bavail = 1000 - used_pct * 10 - self.f_files, self.f_ffree = files, ffree - - -@contextlib.contextmanager -def fake_disks(m, vfs, devs=None): - """Every path statvfs-es to VFS (or VFS[path]); st_dev is DEVS[path] or 1.""" - old = m._statvfs, m._dev - m._statvfs = lambda p: vfs[p] if isinstance(vfs, dict) else vfs - m._dev = lambda p: (devs or {}).get(p, 1) - try: - yield - finally: - m._statvfs, m._dev = old - - -def run_main(m, argv, env=None): - """The lines og run headroom prints for ARGV (--work-dir, --outdir, - --disk-warn-percent), from snapshot() and lines(), with ENV set; 2 on a - warn percent snapshot() refuses. og itself is held to the same lines by - og-runlog's parity test.""" - opts = dict(zip(argv[::2], argv[1::2])) - saved = dict(os.environ) - os.environ.update(env or {}) - try: - snap = m.snapshot(work_dir=opts.get("--work-dir"), outdir=opts.get("--outdir"), - disk_warn_percent=opts.get("--disk-warn-percent")) - except ValueError as e: - return 2, str(e) - finally: - os.environ.clear() - os.environ.update(saved) - return 0, "".join(s + "\n" for s in m.lines(snap)) - - -def events(): - return run_state.read_events(os.path.join(FIX, "events.jsonl")) - - -def by_process(tasks): - return {t["process"]: t for t in tasks} - - -def strings(o): - if isinstance(o, str): - yield o - elif isinstance(o, dict): - for v in o.values(): - yield from strings(v) - elif isinstance(o, list): - for v in o: - yield from strings(v) - - -def base_config_exits(text): - m = re.search(r"errorStrategy\s*=\s*\{\s*task\.exitStatus in \[([\d,\s]+)\]", text) - return {x.strip() for x in m.group(1).split(",")} if m else None - - -# --- the checks: each returns {name: (ok, detail)} ------------------------------------------ - -def disk_checks(m): - out = {} - d = tempfile.mkdtemp(dir=TMP) - with fake_disks(m, FakeVfs(91)): - hi = m.disk([("work", d)]) - with fake_disks(m, FakeVfs(89)): - lo = m.disk([("work", d)]) - out["91% warns at the default, 89% does not"] = ( - len(hi) == 1 and hi[0]["warn"] and abs(hi[0]["used_ratio"] - 0.91) < 1e-9 and len(lo) == 1 and not lo[0]["warn"], - (hi, lo)) - - with fake_disks(m, FakeVfs(85)): - _, plain = run_main(m, ["--work-dir", d]) - _, arg = run_main(m, ["--work-dir", d, "--disk-warn-percent", "80"]) - _, env = run_main(m, ["--work-dir", d], {"OWNGENOME_DISK_WARN_PERCENT": "80"}) - out["--disk-warn-percent 80 and OWNGENOME_DISK_WARN_PERCENT=80 warn at 85%"] = ( - "WARN" not in plain.splitlines()[0] and "WARN: at or above 80%" in arg and "WARN: at or above 80%" in env, - (plain, arg, env)) - - seen = [] - with fake_disks(m, FakeVfs(50)): - old = m._statvfs - m._statvfs = lambda p: seen.append(p) or old(p) - miss = m.disk([("work", os.path.join(d, "not", "yet"))]) - out["a missing work folder is measured on its parent"] = ( - len(miss) == 1 and miss[0]["missing"] and seen == [d], (miss, seen)) - - a, b = os.path.join(d, "a"), os.path.join(d, "b") - os.makedirs(a) - os.makedirs(b) - with fake_disks(m, FakeVfs(50), {a: 7, b: 7}): - one = m.disk([("work", a), ("results", b)]) - with fake_disks(m, FakeVfs(50), {a: 7, b: 8}): - two = m.disk([("work", a), ("results", b)]) - out["one st_dev reports once, two report twice"] = ( - [e["fs"] for e in one] == ["work and results"] and [e["fs"] for e in two] == ["work", "results"], (one, two)) - - with fake_disks(m, FakeVfs(50, files=0, ffree=0)): - noi = m.disk([("work", d)]) - with fake_disks(m, FakeVfs(50)): - withi = m.disk([("work", d)]) - out["f_files 0 gives no inode ratio"] = ( - noi[0]["inodes_used_ratio"] is None and abs(withi[0]["inodes_used_ratio"] - 0.1) < 1e-9, (noi, withi)) - - real = m.disk([("work", d)]) - rc, text = run_main(m, ["--work-dir", d, "--outdir", d]) - out["a real statvfs on a temp folder gives a ratio in 0..1 and a disk line"] = ( - len(real) == 1 and real[0]["used_ratio"] is not None and 0 <= real[0]["used_ratio"] <= 1 - and real[0]["size_bytes"] > 0 and rc == 0 and text.startswith("disk work and results: "), (real, rc, text)) - return out - - -def memory_checks(m): - out = {} - trace = run_eta.read_trace(os.path.join(FIX, "trace.txt")) - tt = by_process(m.task_memory(trace)) - out["7.6 of 8 GB warns, 6 of 8 does not"] = ( - tt["FASTP"]["warn"] and abs(tt["FASTP"]["ratio"] - 0.95) < 1e-6 - and not tt["VEP"]["warn"] and abs(tt["VEP"]["ratio"] - 0.75) < 1e-6, tt) - - ev = by_process(m.task_memory(m.event_rows(events()))) - out["'7.6 GB' and raw bytes give the same ratio"] = ( - tt["FASTP"]["ratio"] is not None and abs(tt["FASTP"]["ratio"] - ev["FASTP"]["ratio"]) < 1e-6 - and ev["FASTP"]["step"] == "01b", (tt["FASTP"], ev["FASTP"])) - - nomem = m.task_memory(run_eta.read_trace(os.path.join(FIX, "trace_no_memory.txt"))) - out["no memory column gives no ratio"] = ( - len(nomem) == 1 and nomem[0]["ratio"] is None and not nomem[0]["warn"], nomem) - - out["exit 137 then attempt 2 counts one retry; a running task shows its request only"] = ( - ev["VEP"]["oom_retry"] == 1 and ev["VEP"]["attempt"] == 2 and abs(ev["VEP"]["ratio"] - 9 / 16) < 1e-6 - and ev["FASTP"]["oom_retry"] == 0 - and ev["DEEPVARIANT"]["state"] == "running" and ev["DEEPVARIANT"]["ratio"] is None - and ev["DEEPVARIANT"]["request_bytes"] == 16 * G and ev["DEEPVARIANT"]["peak_rss_bytes"] is None, ev) - - done = {"process": "FASTP", "tag": "demo", "hash": "ab/1", "attempt": "1", "status": "COMPLETED", - "exit": "0", "memory_bytes": 8 * G, "peak_rss_bytes": 6 * G} - late = {"process": "FASTP", "tag": "demo", "hash": "ab/2", "attempt": "1", "status": "RUNNING", - "exit": "", "memory_bytes": 8 * G, "peak_rss_bytes": None} - mixed = m.task_memory([done, late]) - out["a running row read after a finished one of the same attempt keeps the finished ratio"] = ( - len(mixed) == 1 and mixed[0]["state"] == "completed" and abs((mixed[0]["ratio"] or 0) - 0.75) < 1e-6, mixed) - - fin = {"task_id": 7, "process": "VEP", "name": "VEP (demo)", "attempt": 1, "status": "COMPLETED", - "exit": 0, "memory": 8 * G, "peak_rss": 6 * G} - run = {"task_id": 7, "process": "VEP", "name": "VEP (demo)", "attempt": 1, "status": "RUNNING", - "memory": 8 * G} - same = m.task_memory([fin, run]) - out["a running message after a finished one of the same task_id keeps the finished ratio"] = ( - len(same) == 1 and same[0]["state"] == "completed" and abs((same[0]["ratio"] or 0) - 0.75) < 1e-6, same) - - small = m.host_memory("64.GB", physical=32 * G) - big = m.host_memory("64.GB", physical=128 * G) - out["64 GB max on a 32 GB host warns, on 128 GB not"] = ( - small["warn"] and small["max_memory_bytes"] == 64 * G and not big["warn"], (small, big)) - return out - - -def snapshot_checks(m): - out = {} - d = tempfile.mkdtemp(dir=TMP) - work, res = os.path.join(d, "work"), os.path.join(d, "results") - os.makedirs(work) - os.makedirs(res) - meta = {"event": "started", "metadata": {"workflow": {"workDir": work}, - "parameters": {"outdir": res, "max_memory": "64.GB"}}} - with fake_disks(m, FakeVfs(50), {work: 1, res: 2}): - snap = m.snapshot([meta] + events(), physical=32 * G, - trace_rows=run_eta.read_trace(os.path.join(FIX, "trace.txt"))) - bad = [s for s in strings(snap) if "/" in s] - out["the snapshot holds no '/' and takes its folders from the metadata"] = ( - not bad and [e["fs"] for e in snap["disk"]] == ["work", "results"] and snap["host"]["warn"] - and snap["warn_used_ratio"] == 0.9 and snap["mem_warn_ratio"] == 0.9 - and {t["process"] for t in snap["tasks"]} == {"FASTP", "VEP", "DEEPVARIANT"}, (bad, snap)) - - rel = m.metadata([{"metadata": {"workflow": {"workDir": "/x/work", "launchDir": "/x"}, - "parameters": {"outdir": "./results"}}}]) - ab = m.metadata([{"metadata": {"workflow": {"workDir": "/x/work", "launchDir": "/x"}, - "parameters": {"outdir": "/y/results"}}}]) - out["a relative outdir is joined to launchDir, an absolute one is kept"] = ( - rel == ("/x/work", os.path.join("/x", "./results"), None) and ab == ("/x/work", "/y/results", None), - (rel, ab)) - - alt = m.metadata([{"metadata": {"workflow": {"workDir": "/x/work"}, - "params": {"outdir": "/y/r", "max_memory": "8.GB"}}}]) - out["metadata under 'params' gives the outdir and max_memory"] = (alt == ("/x/work", "/y/r", "8.GB"), alt) - return out - - -def all_checks(m): - out = {} - for f in (disk_checks, memory_checks, snapshot_checks): - out.update(f(m)) - return out - - -def mutants(): - """{check name: a module with a planted mistake that check must catch}.""" - ms = {} - ms["91% warns at the default, 89% does not"] = mutant(DEFAULT_WARN_PERCENT=95) - - @with_mod - def ignore_percent(mod): - orig = mod.warn_percent - return lambda value=None, env=None: orig(None, None) - ms["--disk-warn-percent 80 and OWNGENOME_DISK_WARN_PERCENT=80 warn at 85%"] = mutant(warn_percent=ignore_percent) - ms["a missing work folder is measured on its parent"] = mutant(nearest_existing=os.path.abspath) - - @with_mod - def no_merge(mod): - orig = mod.disk - return lambda paths, warn=None: [e for p in paths for e in orig([p], warn)] - ms["one st_dev reports once, two report twice"] = mutant(disk=no_merge) - - @with_mod - def inodes_always(mod): - orig = mod._measure - - def measure(path): - used, free, size, inodes = orig(path) - return used, free, size, inodes if inodes is not None else 0.0 - return measure - ms["f_files 0 gives no inode ratio"] = mutant(_measure=inodes_always) - ms["a real statvfs on a temp folder gives a ratio in 0..1 and a disk line"] = mutant( - _statvfs=lambda p: FakeVfs(150)) - - @with_mod - def strict_above(mod): - orig = mod.task_memory - - def tm(rows, warn=None, steps=None): - out = orig(rows, warn, steps) - for t in out: - t["warn"] = t["ratio"] is not None and t["ratio"] > 0.99 - return out - return tm - ms["7.6 of 8 GB warns, 6 of 8 does not"] = mutant(task_memory=strict_above) - - @with_mod - def decimal_peak(mod): - orig = mod._row - - def row(r, i): - t = orig(r, i) - if "memory_bytes" not in r and t["peak"] is not None: - t["peak"] = t["peak"] * 1000 ** 3 // 1024 ** 3 # weblog bytes read as decimal units - return t - return row - ms["'7.6 GB' and raw bytes give the same ratio"] = mutant(_row=decimal_peak) - - @with_mod - def zero_request_is_8g(mod): - orig = mod._row - - def row(r, i): - t = orig(r, i) - t["memory"] = t["memory"] or 8 * G - return t - return row - ms["no memory column gives no ratio"] = mutant(_row=zero_request_is_8g) - ms["exit 137 then attempt 2 counts one retry; a running task shows its request only"] = mutant( - MEMORY_EXITS={"143"}) - ms["a running row read after a finished one of the same attempt keeps the finished ratio"] = mutant( - _precedence=lambda t: (t["attempt"], t["order"])) - ms["64 GB max on a 32 GB host warns, on 128 GB not"] = mutant( - host_memory=lambda max_memory=None, physical=None: {"memory_bytes": physical, "max_memory_bytes": None, - "warn": False}) - - @with_mod - def path_label(mod): - orig = mod.disk - return lambda paths, warn=None: orig([(p or label, p) for label, p in paths], warn) - ms["the snapshot holds no '/' and takes its folders from the metadata"] = mutant(disk=path_label) - - @with_mod - def no_launch_dir(mod): - orig = mod.metadata - - def md(events): - evs = [{**ev, "metadata": {**ev["metadata"], "workflow": { - k: v for k, v in (ev["metadata"].get("workflow") or {}).items() if k != "launchDir"}}} - if isinstance(ev, dict) and isinstance(ev.get("metadata"), dict) else ev for ev in events] - return orig(evs) - return md - ms["a relative outdir is joined to launchDir, an absolute one is kept"] = mutant(metadata=no_launch_dir) - - @with_mod - def parameters_only(mod): - orig = mod.metadata - return lambda events: orig([{**ev, "metadata": {k: v for k, v in ev["metadata"].items() if k != "params"}} - if isinstance(ev, dict) and isinstance(ev.get("metadata"), dict) else ev - for ev in events]) - ms["metadata under 'params' gives the outdir and max_memory"] = mutant(metadata=parameters_only) - ms["a running message after a finished one of the same task_id keeps the finished ratio"] = mutant( - _replaces=lambda new, old: True) - return ms - - -def main(): - global TMP - TMP = tempfile.mkdtemp(prefix="test_run_headroom.") - # the checks assume the default warn percents: a threshold set in the caller's shell must not leak in - env_saved = {k: os.environ.pop(k) for k in ("OWNGENOME_DISK_WARN_PERCENT", "OWNGENOME_MEM_WARN_PERCENT") if k in os.environ} - try: - base = open(os.path.join(REPO, "conf", "base.config")).read() - got = base_config_exits(base) - check("MEMORY_EXITS are the exits conf/base.config retries", got == run_headroom.MEMORY_EXITS, got) - results = all_checks(run_headroom) - for name, (ok, detail) in results.items(): - check(name, ok, detail) - - print("\nNegative controls: each check against a planted mistake must fail") - broken = base.replace("[143,137,", "[143,") - check("fails on a planted mistake: base.config without exit 137", - base_config_exits(broken) != run_headroom.MEMORY_EXITS, "passed") - ms = mutants() - check("every check has a planted mistake", set(ms) == set(results), set(results) ^ set(ms)) - for name, mod in ms.items(): - res = all_checks(mod) - check(f"fails on a planted mistake: {name}", not res[name][0], "the check passed on the mutant") - same = all_checks(mutant()) - check("an unchanged copy passes (the controls test the mistake, not the copy)", - all(ok for ok, _ in same.values()), [k for k, (ok, _) in same.items() if not ok]) - finally: - shutil.rmtree(TMP, ignore_errors=True) - os.environ.update(env_saved) - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) From e0723a7974b6142d91b45ab23afb2fd2b61f08cf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:28:04 +0100 Subject: [PATCH 193/255] docs(bloodtype): say that step 41 left-aligns the records near the markers The input contract said DeepVariant writes left-aligned records. It does not always do so without --normalize_reads, and a VCF from another caller is accepted too. The page now says how the step makes the records left-aligned itself, what stays whole and what goes through unchanged. Bead: trait-30 --- CHANGELOG.md | 1 + docs/41-bloodtype.md | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e6e8b1d8..11be91729 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - A 0/0 record over a REF shorter than the marker's (an all-sites `C . 0/0`, or a gVCF block that starts at the position) now counts as no record, so the reads decide. Before it was a no call: ABO read "A, A1 or A2 not determined", and Vel, HPA-14 and every system with a deletion marker lost confidence. - A record that starts upstream and spans a marker with an allele that changes the marker's base (a deletion over it, or a padded `CG CA`, as DeepVariant writes `CG C,CA` when a deletion and an SNV overlap) now makes the site a no call, with a note that names the record. Before, the site had no record at its own position and 8 reads made it a confident reference: a K/K person read k/k at high confidence. - A VCF with one record per position and no overlapping records, as DeepVariant writes, gives the same output as before: the 34 parity cases still agree with the Python oracle byte for byte. +- Step 41 (blood groups) now matches an indel however the caller wrote it. The marker table writes each indel left-aligned, and the step matched records by position, so the same deletion written further right in its repeat missed the marker. With reads there the site then read as the reference allele at confidence high: ABO c.1061delC written as `GG G` at chr9:133255671 gave A1/A1 instead of A2/A2, and a Kell-null deletion written that way gave k/k instead of null/null. The step now left-aligns the VCF's records within 100 bp of each marker site against the reference first (`bcftools norm -c w -f`; in the pipeline the `BLOODTYPE` process does it, and now runs in `BCFTOOLS_IMAGE`), and reads that file. Multi-allelic records stay whole, so a `C A,G 1/2` at ABO c.803 keeps its one B copy. Records on a contig the reference lacks go through unchanged. Step 41 now needs the reference and its `.fai` (`REF_FASTA`). The output files and their columns do not change; a sample whose VCF had such a right-shifted marker indel now gets that allele called. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/docs/41-bloodtype.md b/docs/41-bloodtype.md index 9f1941391..7f1f0a919 100644 --- a/docs/41-bloodtype.md +++ b/docs/41-bloodtype.md @@ -15,6 +15,7 @@ Most blood group antigens differ by one known variant, so the genome predicts th ## Tool - `og bloodtype report` (OwnGenome's Rust binary, `rust/crates/og-bloodtype`), applying the marker table `assets/bloodtype_markers.tsv`. It reads the marker sites through the VCF's `.tbi` or `.csi`; without a usable index (none, or one older than the VCF) it reads the VCF once, line by line, with the same result. `og bloodtype check-markers` checks that the table and the rules agree +- bcftools `norm` to left-align the VCF's records near the marker sites against the reference first (see [Indels written another way](#indels-written-another-way)) - samtools `bedcov` for the read depth over the marker sites and RHD; the RHD regions are the RHD row of `bin/cn_loci.tsv`, and og cn's engine turns their depth into the copy number (see [Copy number from depth](cn-loci.md)) The variants and the antigens they define are the ISBT allele definitions, as RBCeq2 2.4.4 packages them (database v2.5.1, MIT licence, Australian Red Cross Lifeblood). The table's header names the release and the checksum of the file they were read from. Each position and REF base was checked against Ensembl GRCh38 and its dbSNP id. The HPA positions come from dbSNP through Ensembl, because that RBCeq2 database lists them at their GRCh37 positions. @@ -27,13 +28,14 @@ Why not run RBCeq2 or bloodTyper itself: RBCeq2 is on PyPI only, with no Biocond ## Docker Image -- `PYTHON_IMAGE` for the rules, `SAMTOOLS_IMAGE` for the depth +- `PYTHON_IMAGE` for the rules, `SAMTOOLS_IMAGE` for the depth, `BCFTOOLS_IMAGE` for the left-alignment. In the Nextflow pipeline the `BLOODTYPE` process runs both the left-alignment and the rules in `BCFTOOLS_IMAGE` (`og` is a static binary) Pinned in `versions.env`; [Image versions](versions.md) lists the current tags. ## Input -- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample, left-aligned records (DeepVariant writes them so). Split multi-allelic records, all-sites VCFs (a reference record and then the variant at one position) and gVCF reference blocks are read too: every record that covers a site counts, whatever their order (see below) +- `${SAMPLE}/vcf/${SAMPLE}.vcf.gz`: one sample, from any caller. Its records near the marker sites are left-aligned first, so an indel need not be written the way the table writes it. Split and whole multi-allelic records, all-sites VCFs (a reference record and then the variant at one position) and gVCF reference blocks are read too: every record that covers a site counts, whatever their order (see below) +- the reference (`REF_FASTA`, `reference/GRCh38_no_alt_analysis_set.fasta` by default) with its `.fai`, for the left-alignment; `--reference` in the Nextflow pipeline - `${SAMPLE}/aligned/${SAMPLE}_sorted.bam` with its `.bai` (`ALIGN_DIR` as for the other BAM steps). Optional: without it RhD is not determined, and a site the VCF has no record for is read as the reference allele at confidence medium. ## Command @@ -64,6 +66,18 @@ With `run-all.sh`: `TOOLS=...,bloodtype`. It is opt-in and needs no data, so a r A site the VCF has no record for is the reference allele when the BAM has at least 8 reads there at MAPQ 20, and not determined below that. Without the BAM it is read as reference, and the confidence says so. The step leaves out of `samtools bedcov` the rows on a contig the BAM has no `@SQ` line for (a BAM cut to a few chromosomes), because `bedcov` would stop on them. Those sites then have no depth row: the ones with no VCF record are not determined, and RhD is not determined without chr1. A record at a site whose REF is not the table's REF is a no call, because it says nothing about the table's ALT. There are two exceptions. A 0/0 call over a longer REF that starts with the table's REF is the reference allele. A 0/0 record over a shorter REF (an all-sites VCF's `C . 0/0`, or a gVCF block that starts at a deletion marker such as ABO c.1061delC) says nothing past its own base, so the site reads as one with no record: the reads decide. +### Indels written another way + +The table writes each indel left-aligned, the way `bcftools norm` writes it: ABO c.1061delC is `CG C` at chr9:133255669, at the start of a run of G. A caller can write the same deletion further right in the run, as `GG G` at chr9:133255671 (DeepVariant does so in rare cases without `--normalize_reads`, which the pipeline does not pass, and a VCF from another caller may do so too). Matched by position as written, that record would miss the marker, and with reads there the site would read as the reference allele at confidence high. So the step first takes the VCF's records within 100 bp of each marker site and left-aligns them against the reference (`bcftools norm -c w -f`). 100 bp is more than the 38 bp the longest marker indel can shift along its repeat on GRCh38. The rules then read that file. + +- A multi-allelic record stays one record (no `-m -any`). A `C A,G 1/2` at ABO c.803 keeps its one B copy, and a null row's `1/2` of two of its ALTs keeps both copies. +- A multi-allelic indel is left-aligned as a whole, so its REF can stay longer than the marker's (`TCC TC,T` against `TC T`). Its REF then differs from the table's, and the site is a no call, as the paragraph above says. +- A record whose REF differs from the reference is kept as it is (`-c w`), with a warning in the log. +- A record on a contig the reference lacks (a VCF named `1`, `2`, ... against the chr-named reference) cannot be aligned. It goes through unchanged. +- The left-aligned file is a work file: the bash step deletes it after the rules run, and the pipeline does not publish it. + +### Several records at one site + When several records cover one site, each one is read and their order does not matter: - A call of the table's ALT wins over the other records at the site's position. An all-sites VCF writes `T . 0/0` and then `T TC 0/1` at ABO c.261, so the insertion counts. Another allele that a second record calls there (a split `T G 1/0`) is named in the note. @@ -102,7 +116,7 @@ Lacking an antigen is common (most people lack HPA-1b, for example). It means a ## Runtime -Not measured on a whole genome. The step reads the marker positions through the VCF's index (the VCF once, line by line, without one), and samtools reads the BAM at 284 small regions. +Not measured on a whole genome. bcftools reads the VCF's records near the marker sites through its index (the VCF once from the start without one) and left-aligns them; og reads the marker positions of that small file; samtools reads the BAM at 284 small regions. ## Notes From 9a9d1954e9be27cc03b2eff086f525677d8c4d0a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:29:00 +0100 Subject: [PATCH 194/255] fix(traits): read low-quality calls at trait sites as not read Step 42 kept each site's GQ but never compared it, so a GQ 0 call at APOE, HFE, SERPINA1 or any traits-table row gave a confident reading, and a call the caller filtered (LowQual) was read too. A call below GQ 20, or with a FILTER other than PASS, . or RefCall, now reads low_gq with no reading and names the GQ or the filter; such a site is not phased. The genotypes table gains the record's FILTER as a seventh column, and the summary's traits section counts low_gq. Bead: trait-01 Bead: trait-67 Bead: D10 --- CHANGELOG.md | 1 + docs/42-traits.md | 4 +- modules/local/traits/main.nf | 4 +- rust/crates/og-steps/src/traits.rs | 104 ++++++++++++++---- .../og-summary/src/collect/group_c/traits.rs | 2 +- scripts/42-traits.sh | 7 +- 6 files changed, 95 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85edb5eae..2e255a7dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- Step 42 no longer reads a trait from a low-quality call. It kept each site's GQ but never checked it, so a call of one or two reads (GQ 0) at APOE gave a confident e2/e4 or e4/e4, and the same held for HFE, SERPINA1 and every row of the traits table; a call the caller had filtered (FILTER `LowQual`) was read as well. A call with a GQ below 20, or with a FILTER other than `PASS`, `.` or DeepVariant's `RefCall`, now has the new status `low_gq`: the row keeps its genotype, gives no reading, and says "not read; check the reads" with the GQ or the filter. A rule with such a site reads `low_gq` with no result, and such a site is not sent to WhatsHap. GQ 20 and a call without a GQ are read as before. The floor of 20 is the one `og pgx refblock-fill` and the Y private SNPs use. What changes in the outputs: `traits/_traits_genotypes.tsv` gains a seventh column, the record's FILTER; `low_gq` is a new value of `status` in `traits/_traits.tsv` and `traits/_genotype_rules.tsv`; summary.json's traits section gains a `low_gq` count beside `called`, `not_covered` and `other_allele`. No column is renamed or removed. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/42-traits.md b/docs/42-traits.md index b43df3ed9..f1ca6f5a6 100644 --- a/docs/42-traits.md +++ b/docs/42-traits.md @@ -38,9 +38,9 @@ In the Nextflow pipeline (`traits` in `--tools`, part of the default list) the p |---|---| | `traits/${SAMPLE}_traits.tsv` | one row per table row, then the APOE line: `rsid`, `gene`, `chrom`, `pos`, `trait`, `genotype` (reference allele first), `status`, `reading`, `ancestry`, `note`, `pmids` | | `traits/${SAMPLE}_genotype_rules.tsv` | one line per rule: `rule`, `gene`, `trait`, `rsids`, `chrom`, `pos`, `genotypes` (one per site, reference allele first), `gq` (the GQ of each site's record, a reference block's own GQ for a homozygous reference site, `.` when the gVCF header has no FORMAT/GQ), `phase`, `result`, `status`, `reading`, `ancestry`, `note`, `pmids` | -| `traits/${SAMPLE}_traits_genotypes.tsv` | what bcftools read at the table's positions and the rule sites: chromosome, position, REF, ALT, the genotype as bases and its GQ | +| `traits/${SAMPLE}_traits_genotypes.tsv` | what bcftools read at the table's positions and the rule sites: chromosome, position, REF, ALT, the genotype as bases, its GQ and the record's FILTER | -`status` is `called`, `not_covered` (no genotype at the position, from no reads or a no-call: not read) or `other_allele` (an allele the table does not describe, or an indel starting at the position: not read). A record with a missing allele (a no-call or a half call such as `./T`) is dropped by the genotype filter, so its position reads as `not_covered`. A gVCF whose reference base differs from the table's GRCh38 base stops the step: the gVCF is on another build. +`status` is `called`, `not_covered` (no genotype at the position, from no reads or a no-call: not read), `low_gq` (a low-quality call: not read) or `other_allele` (an allele the table does not describe, or an indel starting at the position: not read). A call is low quality when its GQ is below 20, or when the caller filtered it (a FILTER other than `PASS`, `.` or DeepVariant's `RefCall`, such as `LowQual`). The row keeps its genotype, and the reading names the GQ or the filter. GQ 20 is read, and so is a call without a GQ (`.`, a gVCF with no FORMAT/GQ). A reference block counts the same way: a homozygous reference site in a block with GQ 5 reads `low_gq`, not reference. A rule with one such site reads `low_gq` with no result, unless another site has no genotype (`not_covered`). The floor of 20 is the one `og pgx refblock-fill` and the Y private SNPs use; a call of one or two reads gets a GQ near 0, so an APOE e4/e4 or an HFE C282Y/C282Y is never read from it. A record with a missing allele (a no-call or a half call such as `./T`) is dropped by the genotype filter, so its position reads as `not_covered`. A gVCF whose reference base differs from the table's GRCh38 base stops the step: the gVCF is on another build. Both reports have a Traits section: every row with its genotype, reading, note and PubMed id, and a mark on the rows whose reading depends on ancestry. The APOE lines come last, after a warning, and in the HTML report they are folded away until you open them. Both reports also show the genotype rules (`traits/${SAMPLE}_genotype_rules.tsv`) under the traits rows: each rule's sites, genotypes, result or status, how it was phased, and its reading. The APOE rule is left out there, because its line is already in the traits table and stays folded away. The three TAS2R38 rows of the traits table are each read alone, and their note points at the rule line. diff --git a/modules/local/traits/main.nf b/modules/local/traits/main.nf index b6087ab75..23f37710f 100644 --- a/modules/local/traits/main.nf +++ b/modules/local/traits/main.nf @@ -5,7 +5,7 @@ TRAITS_SITES genotypes the positions of assets/traits.tsv and of the rule sites of assets/genotype_rules.tsv from the gVCF: each reference block over a position becomes a record with - the reference base and the block's GQ, and a position + the reference base and the block's GQ and FILTER, and a position without reads has no line (it is "not covered", never the reference) TRAITS_PHASE for a sample with a BAM: WhatsHap phases the heterozygous @@ -56,7 +56,7 @@ process TRAITS_SITES { awk -F'\\t' '\$1 == "site" { print \$4 "\\t" \$5 }' ${rules}; } | LC_ALL=C sort -k1,1 -k2,2n -u > sites.tsv bcftools convert --gvcf2vcf -f ${reference} -R sites.tsv -Ou ${gvcf} \\ | bcftools view -T sites.tsv --trim-alt-alleles -i 'GT!="mis"' -Ou \\ - | bcftools query -u -f '%CHROM\\t%POS\\t%REF\\t%ALT\\t[%TGT]\\t[%GQ]\\n' > ${meta.id}_traits_genotypes.tsv + | bcftools query -u -f '%CHROM\\t%POS\\t%REF\\t%ALT\\t[%TGT]\\t[%GQ]\\t%FILTER\\n' > ${meta.id}_traits_genotypes.tsv rm -f sites.tsv printf '"%s":\\n bcftools: %s\\n' "${task.process}" "${task.container.replaceFirst(/^[^:@]+[:@]/, '')}" > versions.yml diff --git a/rust/crates/og-steps/src/traits.rs b/rust/crates/og-steps/src/traits.rs index b714cda42..0bf6a38b0 100644 --- a/rust/crates/og-steps/src/traits.rs +++ b/rust/crates/og-steps/src/traits.rs @@ -67,11 +67,18 @@ struct Rule { haps: Vec, combos: Vec, } +/// A call whose GQ is below this is not read (decision D10, the floor of +/// `og pgx refblock-fill` and the Y private SNPs). A missing GQ stays readable. +const MIN_GQ: u32 = 20; +/// The FILTER values that keep a call readable: a pass, no filter (a +/// reference block), and DeepVariant's RefCall (a candidate it called 0/0). +const READABLE_FILTERS: &[&str] = &["", ".", "PASS", "RefCall"]; #[derive(Clone, Debug, PartialEq, Eq)] enum Status { Called, NotCovered, OtherAllele, + LowGq, Unphased, OtherHaplotype, } @@ -81,16 +88,44 @@ impl Status { Self::Called => "called", Self::NotCovered => "not_covered", Self::OtherAllele => "other_allele", + Self::LowGq => "low_gq", Self::Unphased => "unphased", Self::OtherHaplotype => "other_haplotype", } } } +/// Why a call is too weak to read. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Weak { + /// Its GQ, below [`MIN_GQ`]. + Gq(String), + /// The caller's FILTER value. + Filtered(String), +} +impl Weak { + /// The record's GQ and FILTER columns: `None` when the call is readable. + fn of(gq: &str, filter: &str) -> Option { + if !READABLE_FILTERS.contains(&filter) { + return Some(Self::Filtered(filter.to_owned())); + } + gq.parse::() + .is_ok_and(|q| q.is_finite() && q < f64::from(MIN_GQ)) + .then(|| Self::Gq(gq.to_owned())) + } + fn why(&self) -> String { + match self { + Self::Gq(q) => format!("genotype quality {q}, below {MIN_GQ}"), + Self::Filtered(f) => format!("the caller filtered the call: {f}"), + } + } +} #[derive(Clone, Debug)] struct SiteGt { alleles: Vec, status: Status, gq: String, + /// Set when the call is too weak to read: the status is then LowGq. + weak: Option, snv: bool, } impl Default for SiteGt { @@ -99,6 +134,7 @@ impl Default for SiteGt { alleles: vec![], status: Status::NotCovered, gq: ".".into(), + weak: None, snv: false, } } @@ -552,7 +588,9 @@ fn genotype(site: &Site, records: &Genotypes) -> SiteGt { a.clone(), ) }); - gt.status = if gt.alleles.len() == 2 + gt.status = if gt.weak.is_some() { + Status::LowGq + } else if gt.alleles.len() == 2 && gt .alleles .iter() @@ -586,7 +624,7 @@ fn read_genotypes(path: &str, sites: &[&Site]) -> Result { let c: Vec<_> = line.split('\t').collect(); if c.len() < 5 { return Err(format!( - "{path}:{n}: {} columns, expected CHROM POS REF ALT GENOTYPE [GQ]", + "{path}:{n}: {} columns, expected CHROM POS REF ALT GENOTYPE [GQ [FILTER]]", c.len() )); } @@ -620,7 +658,15 @@ fn read_genotypes(path: &str, sites: &[&Site]) -> Result { a.clone(), ) }); - let status = if al.len() == 2 + let gq = c + .get(5) + .filter(|g| !g.is_empty()) + .unwrap_or(&".") + .to_string(); + let weak = Weak::of(&gq, c.get(6).copied().unwrap_or(".")); + let status = if weak.is_some() { + Status::LowGq + } else if al.len() == 2 && al .iter() .all(|a| a == &site.row["ref"] || a == &site.row["alt"]) @@ -629,17 +675,13 @@ fn read_genotypes(path: &str, sites: &[&Site]) -> Result { } else { Status::OtherAllele }; - let gq = c - .get(5) - .filter(|g| !g.is_empty()) - .unwrap_or(&".") - .to_string(); result.insert( key, SiteGt { alleles: al, status, gq, + weak, snv, }, ); @@ -753,18 +795,35 @@ fn rule_line(r: &Rule, gt: &Genotypes, phased: Option<&Phased>) -> Row { rs[rs.len() - 1] ) }; - let status = if g.iter().any(|g| g.status == Status::NotCovered) { - Status::NotCovered + let weak: Vec<_> = r + .sites + .iter() + .zip(&g) + .filter_map(|(s, g)| g.weak.as_ref().map(|w| (s, w))) + .collect(); + let (status, reading) = if g.iter().any(|g| g.status == Status::NotCovered) { + ( + Status::NotCovered, + format!("Not read: needs a genotype at {need}."), + ) + } else if !weak.is_empty() { + ( + Status::LowGq, + format!( + "A low-quality call at {}: not read; check the reads.", + weak.iter() + .map(|(s, w)| format!("{} ({})", s.row["rsid"], w.why())) + .collect::>() + .join(" and ") + ), + ) } else { - Status::OtherAllele + ( + Status::OtherAllele, + format!("Not read: needs a genotype at {need}."), + ) }; - return finish( - out, - ".", - ".", - status, - format!("Not read: needs a genotype at {need}."), - ); + return finish(out, ".", ".", status, reading); } let mut phase = if r.meta["needs_phase"] == "no" { "not_used" @@ -1017,6 +1076,10 @@ fn run(verb: &str, opts: &HashMap) -> Result { "No genotype at this position in the gVCF (no reads, or a no-call): not read." .into() } + Status::LowGq => format!( + "A low-quality call at this position ({}): not read; check the reads.", + g.weak.as_ref().map(Weak::why).unwrap_or_default() + ), Status::Called => s.row[["hom_ref", "het", "hom_alt"] [g.alleles.iter().filter(|a| *a == &s.row["alt"]).count()]] .clone(), @@ -1198,6 +1261,7 @@ mod tests { alleles: al, status: Status::Called, gq: "40".into(), + weak: None, snv: true, }, ) @@ -1448,7 +1512,9 @@ mod tests { // HFE C282Y/H63D: a LowQual call at C282Y is not read; RefCall and // a pass are. let hfe = |filter: &str| { - format!("chr6\t26092913\tG\tA\tG/A\t50\t{filter}\nchr6\t26090951\tC\tG\tC/G\t50\tRefCall\n") + format!( + "chr6\t26092913\tG\tA\tG/A\t50\t{filter}\nchr6\t26090951\tC\tG\tC/G\t50\tRefCall\n" + ) }; let line = read(&hfe("LowQual"), "HFE"); assert_eq!((&line["status"][..], &line["result"][..]), ("low_gq", ".")); diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs index 801489036..d051b1dcc 100644 --- a/rust/crates/og-summary/src/collect/group_c/traits.rs +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -66,7 +66,7 @@ pub fn traits(c: &Ctx<'_>) -> Out { "apoe", Value::Array(t.rows.iter().filter(apoe).map(row).collect()), ); - for st in ["called", "not_covered", "other_allele"] { + for st in ["called", "not_covered", "other_allele", "low_gq"] { out.put( st, count( diff --git a/scripts/42-traits.sh b/scripts/42-traits.sh index ea3023328..68dbc3dc4 100755 --- a/scripts/42-traits.sh +++ b/scripts/42-traits.sh @@ -21,7 +21,8 @@ # processes. # # Output: traits/_traits.tsv (rsid, gene, chrom, pos, trait, genotype, -# status, reading, ancestry, note, pmids). status: called, not_covered, or +# status, reading, ancestry, note, pmids). status: called, not_covered, +# low_gq (GQ below 20, or a FILTER other than PASS or RefCall: not read), or # other_allele (an allele the table does not describe). # traits/_genotype_rules.tsv: one line per rule (docs/42-traits.md). set -euo pipefail @@ -68,7 +69,7 @@ rm -f "$OUT" "$RULES_OUT" "$PHASE_IN" "$PHASED" awk -F'\t' '$1 == "site" { print $4 "\t" $5 }' "$RULES"; } | LC_ALL=C sort -k1,1 -k2,2n -u > "$SITES" # gvcf2vcf expands each reference block over a position into a record with -# the reference base (with the block's GQ), -T keeps the table's positions, +# the reference base (with the block's GQ and FILTER), -T keeps the table's positions, # --trim-alt-alleles drops <*>, and a no-call is dropped, so a position # without reads has no line. echo "[1/3] Genotyping the table's positions from the gVCF..." @@ -79,7 +80,7 @@ run_in --cpus 1 --memory 2g "$BCFTOOLS_IMAGE" \ [ "$n" -eq 1 ] || { echo "ERROR: $3 holds $n samples; step 42 reads a one-sample gVCF (keep one with bcftools view -s )." >&2; exit 1; } bcftools convert --gvcf2vcf -f "$1" -R "$2" -Ou "$3" \ | bcftools view -T "$2" --trim-alt-alleles -i "GT!=\"mis\"" -Ou \ - | bcftools query -u -f "%CHROM\t%POS\t%REF\t%ALT\t[%TGT]\t[%GQ]\n"' \ + | bcftools query -u -f "%CHROM\t%POS\t%REF\t%ALT\t[%TGT]\t[%GQ]\t%FILTER\n"' \ _ "${REF_FASTA_C}" "$(cpath "$SITES")" "$(cpath "$GVCF")" > "${GENOTYPES}.tmp" \ || { rm -f "${GENOTYPES}.tmp" "$SITES"; echo "ERROR: step 42 could not genotype ${GVCF} at the table's positions (the line above says why)." >&2; exit 1; } mv -f "${GENOTYPES}.tmp" "$GENOTYPES" From 082824d97073e1a384f9afc43fcafdbd0a9d19f5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:29:00 +0100 Subject: [PATCH 195/255] test(e2e): traits-panel-1 checks the call floor on a planted GQ 5 and LowQual call Bead: trait-01 Bead: trait-67 --- tests/e2e/traits-panel-1.sh | 51 ++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/tests/e2e/traits-panel-1.sh b/tests/e2e/traits-panel-1.sh index e6cb56765..81be07124 100644 --- a/tests/e2e/traits-panel-1.sh +++ b/tests/e2e/traits-panel-1.sh @@ -7,11 +7,14 @@ # --online); control: a copy with one wrong reference base and one PubMed # id that does not exist must fail, naming both; # - scripts/42-traits.sh: one row per table row plus the APOE line; the -# table positions inside the fixture's slices are read (called) with the -# genotype of the GIAB v4.2.1 truth, every position outside them is -# not_covered (no reads, never taken for the reference); +# table positions inside the fixture's slices carry the genotype of the +# GIAB v4.2.1 truth and are read (called), or low_gq when the gVCF's GQ +# there is below 20 or its FILTER is not PASS or RefCall; every position +# outside them is not_covered (no reads, never taken for the reference); # - a planted alternate allele: a copy of the gVCF with CYP2A6*2 (rs1801272) # homozygous gives that row the table's hom_alt reading, not HG002's; +# - the call floor: the same planted call at GQ 5, and at GQ 40 with FILTER +# LowQual, reads low_gq ("not read"), and every other row is as in HG002; # - both reports (bin/render_report.py on the sample folder) carry the # section, the HTML one with the APOE lines folded away; # - nextflow run main.nf --tools traits,html_report on the same VCF and @@ -64,6 +67,16 @@ IN=$(awk -F'\t' '$6 == "in"' "${CASE_TMP}/where.tsv" | wc -l | tr -d ' ') check_ge "table positions inside the fixture's slices" "$IN" 2 status_of() { awk -F'\t' -v r="$1" '$1 == r { print $7 }' "$OUT"; } geno_of() { awk -F'\t' -v r="$1" '$1 == r { print $6 }' "$OUT"; } +# What step 42 should make of the gVCF's call at a position: low_gq below GQ +# 20 or with a FILTER other than PASS or RefCall, else called. +GENOS="${G}/${SAMPLE}/traits/${SAMPLE}_traits_genotypes.tsv" +check "the genotypes table has the GQ and FILTER columns" \ + test "$(awk -F'\t' 'NF != 7' "$GENOS" | wc -l | tr -d ' ')" -eq 0 -a -s "$GENOS" +want_status() { + awk -F'\t' -v c="$1" -v p="$2" '$1 == c && $2 == p && length($4) <= 1 { + weak = ($7 != "." && $7 != "PASS" && $7 != "RefCall") || ($6 ~ /^[0-9.]+$/ && $6 + 0 < 20) + print (weak ? "low_gq" : "called") " (GQ " $6 ", FILTER " $7 ")"; exit }' "$GENOS" +} OUTSIDE_NC=0 OUTSIDE=0 while IFS=$'\t' read -r rs c p ref _alt where; do if [ "$where" = out ]; then @@ -79,7 +92,8 @@ while IFS=$'\t' read -r rs c p ref _alt where; do WANT=$(tr '|' '/' <<< "$T" | awk -F/ -v r="$ref" '{ if ($2 == r) print $2 "/" $1; else print $1 "/" $2 }') elif [ -n "$INBED" ]; then WANT="${ref}/${ref}" else WANT="outside the GIAB benchmark regions"; fi - check_eq "${rs} (${c}:${p}) is read" "$(status_of "$rs")" called + W=$(want_status "$c" "$p") + check_eq "${rs} (${c}:${p}) reads ${W}" "$(status_of "$rs")" "${W%% *}" check_eq "${rs} genotype is the GIAB truth's" "$(geno_of "$rs")" "$WANT" done < "${CASE_TMP}/where.tsv" check_eq "table positions outside the slices that are not_covered" "$OUTSIDE_NC" "$OUTSIDE" @@ -135,6 +149,35 @@ check "planted: the reading differs from HG002's (${R1})" test "$R1" != "$R2" check_eq "planted: the other rows read as in HG002" \ "$(awk -F'\t' '$1 != "rs1801272"' "$OUT2" 2>/dev/null | md5sum)" "$(awk -F'\t' '$1 != "rs1801272"' "$OUT" | md5sum)" cp "$OUT2" "${CASE_TMP}/planted_traits.tsv" 2>/dev/null + +# --- the call floor: the planted call at GQ 5, and with FILTER LowQual -------------- +for weak in gq lowqual; do + P3="${SAMPLE}W${weak}" + rm -rf "${G:?}/${P3}" + mkdir -p "${G}/${P3}/vcf" + awk -F'\t' -v OFS='\t' -v p="$PL_POS" -v w="$weak" '$1 == "chr19" && $2 == p { + if (w == "gq") sub(/^1\/1:40:/, "1/1:5:", $10); else $7 = "LowQual" } { print }' \ + "${G}/${P2}/vcf/planted.vcf" > "${G}/${P3}/vcf/planted.vcf" + check_eq "floor (${weak}): the copy differs from the planted gVCF in one line" \ + "$(diff "${G}/${P2}/vcf/planted.vcf" "${G}/${P3}/vcf/planted.vcf" | grep -c '^>' || true)" 1 + in_genome "$BCFTOOLS_IMAGE" bash -c "bcftools view -Oz -o '${P3}/vcf/${P3}.g.vcf.gz' '${P3}/vcf/planted.vcf' && bcftools index -t '${P3}/vcf/${P3}.g.vcf.gz'" + run_step 42-traits.sh "$P3" + check_step_exit 42-traits.sh + OUT3="${G}/${P3}/traits/${P3}_traits.tsv" + check_eq "floor (${weak}): rs1801272 status" "$(awk -F'\t' '$1 == "rs1801272" { print $7 }' "$OUT3" 2>/dev/null)" low_gq + check_eq "floor (${weak}): rs1801272 genotype is still shown" "$(awk -F'\t' '$1 == "rs1801272" { print $6 }' "$OUT3" 2>/dev/null)" T/T + R3=$(awk -F'\t' '$1 == "rs1801272" { print $8 }' "$OUT3" 2>/dev/null) + check "floor (${weak}): the reading says not read (${R3})" has 'not read; check the reads' "$R3" + if [ "$weak" = gq ]; then + check "floor (gq): the reading names the GQ" has 'genotype quality 5, below 20' "$R3" + else + check "floor (lowqual): the reading names the filter" has 'filtered the call: LowQual' "$R3" + fi + check "floor (${weak}): the hom_alt reading is not given" test "$R3" != "$HOMALT" + check_eq "floor (${weak}): the other rows read as in HG002" \ + "$(awk -F'\t' '$1 != "rs1801272"' "$OUT3" 2>/dev/null | md5sum)" "$(awk -F'\t' '$1 != "rs1801272"' "$OUT" | md5sum)" + in_genome "$BCFTOOLS_IMAGE" rm -rf "$P3" +done in_genome "$BCFTOOLS_IMAGE" rm -rf "$P2" # --- both reports --------------------------------------------------------------------- From d7e21ee7f75a28bb7a732aabb4d3ed176cb65b19 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:29:01 +0100 Subject: [PATCH 196/255] test(parity): summary goldens carry the traits low_gq count The float cases now keep their fixture file times, so a golden's file_date does not depend on the minute the case ran. Bead: trait-01 --- tests/parity/gen/summary-collect-c.sh | 6 +- .../all-history/all-history.json | 15550 ++++++++++++++++ .../golden/summary-collect/all-nf/all-nf.json | 14035 ++++++++++++++ .../golden/summary-collect/all/all.json | 15476 +++++++++++++++ .../c-float--inf/float--inf.json | 1993 ++ .../c-float-inf/float-inf.json | 2005 ++ .../c-float-nan/float-nan.json | 1957 ++ .../golden/summary-collect/c-full/full.json | 2188 +++ .../summary-collect/c-g2-gvcf/g2-gvcf.json | 353 + .../summary-collect/c-g2-left/g2-left.json | 353 + .../c-g2-nocall/g2-nocall.json | 353 + .../summary-collect/c-g2-none/g2-none.json | 353 + .../summary-collect/c-g2-novcf/g2-novcf.json | 353 + .../summary-collect/c-g2-other/g2-other.json | 353 + .../c-g2-shifted/g2-shifted.json | 353 + .../summary-collect/c-g2-two/g2-two.json | 353 + .../golden/summary-collect/c-nf/nf.json | 1005 + 17 files changed, 57037 insertions(+), 2 deletions(-) create mode 100644 tests/parity/golden/summary-collect/all-history/all-history.json create mode 100644 tests/parity/golden/summary-collect/all-nf/all-nf.json create mode 100644 tests/parity/golden/summary-collect/all/all.json create mode 100644 tests/parity/golden/summary-collect/c-float--inf/float--inf.json create mode 100644 tests/parity/golden/summary-collect/c-float-inf/float-inf.json create mode 100644 tests/parity/golden/summary-collect/c-float-nan/float-nan.json create mode 100644 tests/parity/golden/summary-collect/c-full/full.json create mode 100644 tests/parity/golden/summary-collect/c-g2-gvcf/g2-gvcf.json create mode 100644 tests/parity/golden/summary-collect/c-g2-left/g2-left.json create mode 100644 tests/parity/golden/summary-collect/c-g2-nocall/g2-nocall.json create mode 100644 tests/parity/golden/summary-collect/c-g2-none/g2-none.json create mode 100644 tests/parity/golden/summary-collect/c-g2-novcf/g2-novcf.json create mode 100644 tests/parity/golden/summary-collect/c-g2-other/g2-other.json create mode 100644 tests/parity/golden/summary-collect/c-g2-shifted/g2-shifted.json create mode 100644 tests/parity/golden/summary-collect/c-g2-two/g2-two.json create mode 100644 tests/parity/golden/summary-collect/c-nf/nf.json diff --git a/tests/parity/gen/summary-collect-c.sh b/tests/parity/gen/summary-collect-c.sh index 9c853a9e4..b3dc8eac4 100644 --- a/tests/parity/gen/summary-collect-c.sh +++ b/tests/parity/gen/summary-collect-c.sh @@ -181,7 +181,8 @@ rs3798220\tLPA\tchr6\t2\tLp(a)\tT|C\tcalled\tone copy\tall\t.\t1 rs78655421\tCFTR\tchr7\t3\tR117H\tG/A\tcalled\tone copy\tall\t.\t1 rs429358\tAPOE\tchr19\t4\tAPOE\tT/C\tcalled\te3/e4\tall\t.\t1 rs1\tX\tchr1\t5\tx\t.\tnot_covered\tno reads\tall\t.\t1 -rs2\tX\tchr1\t6\tx\tC/C\tother_allele\tother\tall\t.\t1\n" +rs2\tX\tchr1\t6\tx\tC/C\tother_allele\tother\tall\t.\t1 +rs3\tX\tchr1\t7\tx\tC/T\tlow_gq\tA low-quality call at this position (genotype quality 5, below 20): not read; check the reads.\tall\t.\t1\n" put $f/traits/S_genotype_rules.tsv "rule\tgene\ttrait\trsids\tchrom\tpos\tgenotypes\tgq\tphase\tresult\tstatus\treading\tancestry\tnote\tpmids APOE\tAPOE\tAPOE\trs429358,rs7412\tchr19\t1,2\tT/C,C/C\t40\t.\te3/e4\tcalled\tr\tall\t.\t1 TAS2R38\tTAS2R38\tbitter\trs1\tchr7\t1\tC/G\t40\tunphased\tPAV/AVI\tcalled\ttaster\tall\t.\t1 @@ -357,7 +358,7 @@ put $f/expansion_hunter/S_eh_stranger.vcf "${VCF_HEAD}chr4\t39348425\t.\tA\t "$f/cells.tmp" mv -f "$f/cells.tmp" "$f/$file" + at "$NEW" "$f/$file" done put "$f/hla_t1k/S_hla_genotype.tsv" "HLA-A\t2\tA*02:01\t1\t${special}\tA*01:01\t1\t${special}\n" # Python raises an uncaught OverflowError if an infinite read length diff --git a/tests/parity/golden/summary-collect/all-history/all-history.json b/tests/parity/golden/summary-collect/all-history/all-history.json new file mode 100644 index 000000000..676dee776 --- /dev/null +++ b/tests/parity/golden/summary-collect/all-history/all-history.json @@ -0,0 +1,15550 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:15 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "male" + }, + "databases": { + "clinvar_release": "2021-03-01", + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [ + [ + "run", + "written_utc", + "2021-03-04 05:06:07 UTC" + ], + [ + "image", + "DEEPVARIANT_IMAGE", + "v1.10.0" + ], + [ + "data", + "clinvar_file_date", + "2021-03-01" + ] + ], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": 31.25 + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S_callability.bed.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "min_depth": 10, + "min_mapq": 20, + "min_share": 0.9, + "catalog_genes": 1572, + "read": 217, + "not_read": [ + { + "gene": "OR4F5", + "hgnc_id": "HGNC:14825", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR4F29", + "hgnc_id": "HGNC:31275", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F16", + "hgnc_id": "HGNC:15079", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "AGRN", + "hgnc_id": "HGNC:329", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ATAD3B", + "hgnc_id": "HGNC:24007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MMP23B", + "hgnc_id": "HGNC:7171", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CDK11B", + "hgnc_id": "HGNC:1729", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35E2B", + "hgnc_id": "HGNC:33941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDK11A", + "hgnc_id": "HGNC:1730", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TTC34", + "hgnc_id": "HGNC:34297", + "reasons": [ + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPL22", + "hgnc_id": "HGNC:10315", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MASP2", + "hgnc_id": "HGNC:6902", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPPA", + "hgnc_id": "HGNC:7939", + "reasons": [ + "CMRG" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KIAA2013", + "hgnc_id": "HGNC:28513", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF1", + "hgnc_id": "HGNC:28840", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF11", + "hgnc_id": "HGNC:14086", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL1", + "hgnc_id": "HGNC:29295", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRAMEF4", + "hgnc_id": "HGNC:31971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF10", + "hgnc_id": "HGNC:27997", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRAMEF7", + "hgnc_id": "HGNC:28415", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF6", + "hgnc_id": "HGNC:30583", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF27", + "hgnc_id": "HGNC:51234", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL3", + "hgnc_id": "HGNC:51235", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HNRNPCL2", + "hgnc_id": "HGNC:48813", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF26", + "hgnc_id": "HGNC:49178", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HNRNPCL4", + "hgnc_id": "HGNC:51333", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "PRAMEF9", + "hgnc_id": "HGNC:27996", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF13", + "hgnc_id": "HGNC:13262", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRAMEF18", + "hgnc_id": "HGNC:30693", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRAMEF8", + "hgnc_id": "HGNC:24074", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF33", + "hgnc_id": "HGNC:49193", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PRAMEF15", + "hgnc_id": "HGNC:26764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRAMEF14", + "hgnc_id": "HGNC:13576", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF19", + "hgnc_id": "HGNC:24908", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "PRAMEF17", + "hgnc_id": "HGNC:29485", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CELA2B", + "hgnc_id": "HGNC:29995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CLCNKA", + "hgnc_id": "HGNC:2026", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CLCNKB", + "hgnc_id": "HGNC:2027", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NBPF1", + "hgnc_id": "HGNC:26088", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CROCC", + "hgnc_id": "HGNC:21299", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AKR7L", + "hgnc_id": "HGNC:24056", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF3", + "hgnc_id": "HGNC:25076", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CELA3B", + "hgnc_id": "HGNC:15945", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CELA3A", + "hgnc_id": "HGNC:15944", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HMGCL", + "hgnc_id": "HGNC:5005", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNR2", + "hgnc_id": "HGNC:2160", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RSRP1", + "hgnc_id": "HGNC:25234", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGV", + "hgnc_id": "HGNC:26031", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "XKR8", + "hgnc_id": "HGNC:25508", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BMP8A", + "hgnc_id": "HGNC:21650", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BMP8B", + "hgnc_id": "HGNC:1075", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OXCT2", + "hgnc_id": "HGNC:18606", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CYP4Z1", + "hgnc_id": "HGNC:20583", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4A22", + "hgnc_id": "HGNC:20575", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GBP1", + "hgnc_id": "HGNC:4182", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SETSIP", + "hgnc_id": "HGNC:42937", + "reasons": [ + "segdup" + ], + "share": 0.4994, + "status": "not read" + }, + { + "gene": "AGL", + "hgnc_id": "HGNC:321", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "EXTL2", + "hgnc_id": "HGNC:3516", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AMY2A", + "hgnc_id": "HGNC:477", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF4", + "hgnc_id": "HGNC:26550", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NBPF6", + "hgnc_id": "HGNC:31988", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LRIG2", + "hgnc_id": "HGNC:20889", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "VANGL1", + "hgnc_id": "HGNC:15512", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IGSF3", + "hgnc_id": "HGNC:5950", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SEC22B", + "hgnc_id": "HGNC:10700", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF26", + "hgnc_id": "HGNC:49571", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2NLR", + "hgnc_id": "HGNC:53925", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4A", + "hgnc_id": "HGNC:24369", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM72B", + "hgnc_id": "HGNC:24805", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SRGAP2C", + "hgnc_id": "HGNC:30584", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3-7", + "hgnc_id": "HGNC:32060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPIAL4E", + "hgnc_id": "HGNC:33997", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF15", + "hgnc_id": "HGNC:28791", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4F", + "hgnc_id": "HGNC:33999", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "SRGAP2B", + "hgnc_id": "HGNC:35237", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM72D", + "hgnc_id": "HGNC:33593", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PPIAL4D", + "hgnc_id": "HGNC:33998", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "GPR89A", + "hgnc_id": "HGNC:31984", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBM8A", + "hgnc_id": "HGNC:9905", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "NBPF10", + "hgnc_id": "HGNC:31992", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLA", + "hgnc_id": "HGNC:31862", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPIAL4H", + "hgnc_id": "HGNC:53889", + "reasons": [ + "segdup" + ], + "share": 0.6658, + "status": "not read" + }, + { + "gene": "NBPF12", + "hgnc_id": "HGNC:24297", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF11", + "hgnc_id": "HGNC:31993", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PPIAL4G", + "hgnc_id": "HGNC:33996", + "reasons": [ + "segdup" + ], + "share": 0.1656, + "status": "not read" + }, + { + "gene": "NBPF14", + "hgnc_id": "HGNC:25232", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLB", + "hgnc_id": "HGNC:53923", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NUDT4B", + "hgnc_id": "HGNC:18012", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PDE4DIP", + "hgnc_id": "HGNC:15580", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NOTCH2NLC", + "hgnc_id": "HGNC:53924", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF19", + "hgnc_id": "HGNC:31999", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4C", + "hgnc_id": "HGNC:33995", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "H2BC18", + "hgnc_id": "HGNC:24700", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3C13", + "hgnc_id": "HGNC:25311", + "reasons": [ + "segdup" + ], + "share": 0.8316, + "status": "not read" + }, + { + "gene": "H3C14", + "hgnc_id": "HGNC:20503", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H2AC18", + "hgnc_id": "HGNC:4736", + "reasons": [ + "segdup" + ], + "share": 0.1651, + "status": "not read" + }, + { + "gene": "H2AC19", + "hgnc_id": "HGNC:29668", + "reasons": [ + "segdup" + ], + "share": 0.3328, + "status": "not read" + }, + { + "gene": "H3C15", + "hgnc_id": "HGNC:20505", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H4C15", + "hgnc_id": "HGNC:29607", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HRNR", + "hgnc_id": "HGNC:20846", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LCE3E", + "hgnc_id": "HGNC:29463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCE3D", + "hgnc_id": "HGNC:16615", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "S100A8", + "hgnc_id": "HGNC:10498", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "S100A7A", + "hgnc_id": "HGNC:21657", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "S100A7", + "hgnc_id": "HGNC:10497", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FLAD1", + "hgnc_id": "HGNC:24671", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MTX1", + "hgnc_id": "HGNC:7504", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN3", + "hgnc_id": "HGNC:19183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FCRL5", + "hgnc_id": "HGNC:18508", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA6", + "hgnc_id": "HGNC:5239", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FCGR2B", + "hgnc_id": "HGNC:3618", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD17B7", + "hgnc_id": "HGNC:5215", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD247", + "hgnc_id": "HGNC:1677", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "XCL2", + "hgnc_id": "HGNC:10646", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "PRG4", + "hgnc_id": "HGNC:9364", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PTPRC", + "hgnc_id": "HGNC:9666", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPP1R12B", + "hgnc_id": "HGNC:7619", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KISS1", + "hgnc_id": "HGNC:6341", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SRGAP2", + "hgnc_id": "HGNC:19751", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CR1", + "hgnc_id": "HGNC:2334", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CR1L", + "hgnc_id": "HGNC:2335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZC3H11B", + "hgnc_id": "HGNC:25659", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LBR", + "hgnc_id": "HGNC:6518", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "LEFTY2", + "hgnc_id": "HGNC:3122", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "OR2T33", + "hgnc_id": "HGNC:31255", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T7", + "hgnc_id": "HGNC:15019", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T2", + "hgnc_id": "HGNC:14725", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T3", + "hgnc_id": "HGNC:14727", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR2T5", + "hgnc_id": "HGNC:15017", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "OR2G6", + "hgnc_id": "HGNC:27019", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR2T34", + "hgnc_id": "HGNC:31256", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T11", + "hgnc_id": "HGNC:19574", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T35", + "hgnc_id": "HGNC:31257", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T27", + "hgnc_id": "HGNC:31252", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SNTG2", + "hgnc_id": "HGNC:13741", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TPO", + "hgnc_id": "HGNC:12015", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KLF11", + "hgnc_id": "HGNC:11811", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CCDC121", + "hgnc_id": "HGNC:25833", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "ABCG8", + "hgnc_id": "HGNC:13887", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PCBP1", + "hgnc_id": "HGNC:8647", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "NAT8", + "hgnc_id": "HGNC:18069", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "C2orf78", + "hgnc_id": "HGNC:34349", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CD8B", + "hgnc_id": "HGNC:1707", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RGPD1", + "hgnc_id": "HGNC:32414", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLGLB1", + "hgnc_id": "HGNC:9072", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLGLB2", + "hgnc_id": "HGNC:9073", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RGPD2", + "hgnc_id": "HGNC:32415", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPIA", + "hgnc_id": "HGNC:10297", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEKT4", + "hgnc_id": "HGNC:31012", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAHD2A", + "hgnc_id": "HGNC:24252", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TRIM43B", + "hgnc_id": "HGNC:37146", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM43", + "hgnc_id": "HGNC:19015", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANKRD36C", + "hgnc_id": "HGNC:32946", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPAT2", + "hgnc_id": "HGNC:27168", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD36", + "hgnc_id": "HGNC:24079", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANKRD36B", + "hgnc_id": "HGNC:29333", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "RGPD3", + "hgnc_id": "HGNC:32416", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD4", + "hgnc_id": "HGNC:32417", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RANBP2", + "hgnc_id": "HGNC:9848", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD5", + "hgnc_id": "HGNC:32418", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MALL", + "hgnc_id": "HGNC:6818", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MTLN", + "hgnc_id": "HGNC:27339", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LIMS4", + "hgnc_id": "HGNC:39941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD6", + "hgnc_id": "HGNC:32419", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANAPC1", + "hgnc_id": "HGNC:19988", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD8", + "hgnc_id": "HGNC:9849", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FOXD4L1", + "hgnc_id": "HGNC:18521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RABL2A", + "hgnc_id": "HGNC:9799", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RAB6C", + "hgnc_id": "HGNC:16525", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "POTEF", + "hgnc_id": "HGNC:33905", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CCDC74B", + "hgnc_id": "HGNC:25267", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SMPD4", + "hgnc_id": "HGNC:32949", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3E", + "hgnc_id": "HGNC:20765", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEI", + "hgnc_id": "HGNC:37093", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CFC1B", + "hgnc_id": "HGNC:33983", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CFC1", + "hgnc_id": "HGNC:18292", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "POTEJ", + "hgnc_id": "HGNC:37094", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PLEKHB2", + "hgnc_id": "HGNC:19236", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB6D", + "hgnc_id": "HGNC:30272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MZT2A", + "hgnc_id": "HGNC:33187", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TUBA3D", + "hgnc_id": "HGNC:24071", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CCDC74A", + "hgnc_id": "HGNC:25197", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "ANKRD30BL", + "hgnc_id": "HGNC:35167", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NOSTRIN", + "hgnc_id": "HGNC:20203", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IFT70B", + "hgnc_id": "HGNC:26425", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CASP10", + "hgnc_id": "HGNC:1500", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ALPP", + "hgnc_id": "HGNC:439", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ALPG", + "hgnc_id": "HGNC:441", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT1A6", + "hgnc_id": "HGNC:12538", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UGT1A5", + "hgnc_id": "HGNC:12537", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT1A3", + "hgnc_id": "HGNC:12535", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MLPH", + "hgnc_id": "HGNC:29643", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR6B2", + "hgnc_id": "HGNC:15041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AQP12B", + "hgnc_id": "HGNC:6096", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "AQP12A", + "hgnc_id": "HGNC:19941", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ANO7", + "hgnc_id": "HGNC:31677", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CHL1", + "hgnc_id": "HGNC:1939", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMC3", + "hgnc_id": "HGNC:23999", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANCD2", + "hgnc_id": "HGNC:3585", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DAZL", + "hgnc_id": "HGNC:2685", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF35", + "hgnc_id": "HGNC:13099", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LZTFL1", + "hgnc_id": "HGNC:6741", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MST1", + "hgnc_id": "HGNC:7380", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MST1R", + "hgnc_id": "HGNC:7381", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HYAL1", + "hgnc_id": "HGNC:5320", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FRG2C", + "hgnc_id": "HGNC:33626", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "ZNF717", + "hgnc_id": "HGNC:29448", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR5H6", + "hgnc_id": "HGNC:14767", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ROPN1", + "hgnc_id": "HGNC:17692", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ROPN1B", + "hgnc_id": "HGNC:31927", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RPN1", + "hgnc_id": "HGNC:10381", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ALG1L2", + "hgnc_id": "HGNC:37258", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BFSP2", + "hgnc_id": "HGNC:1041", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRR23B", + "hgnc_id": "HGNC:33764", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SOX2", + "hgnc_id": "HGNC:11195", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "EIF2B5", + "hgnc_id": "HGNC:3261", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KNG1", + "hgnc_id": "HGNC:6383", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TM4SF19", + "hgnc_id": "HGNC:25167", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF595", + "hgnc_id": "HGNC:27196", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDE6B", + "hgnc_id": "HGNC:8786", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAK", + "hgnc_id": "HGNC:4113", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FGFRL1", + "hgnc_id": "HGNC:3693", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RNF212", + "hgnc_id": "HGNC:27729", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UVSSA", + "hgnc_id": "HGNC:29304", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DOK7", + "hgnc_id": "HGNC:26594", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HMX1", + "hgnc_id": "HGNC:5017", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A26", + "hgnc_id": "HGNC:43746", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "USP17L10", + "hgnc_id": "HGNC:44438", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L11", + "hgnc_id": "HGNC:44439", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L12", + "hgnc_id": "HGNC:44440", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L13", + "hgnc_id": "HGNC:44441", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L17", + "hgnc_id": "HGNC:44445", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L18", + "hgnc_id": "HGNC:44446", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L19", + "hgnc_id": "HGNC:44447", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L20", + "hgnc_id": "HGNC:44448", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L21", + "hgnc_id": "HGNC:44449", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L22", + "hgnc_id": "HGNC:44450", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L24", + "hgnc_id": "HGNC:44453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L25", + "hgnc_id": "HGNC:44452", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L26", + "hgnc_id": "HGNC:44454", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L5", + "hgnc_id": "HGNC:37177", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L27", + "hgnc_id": "HGNC:44455", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L28", + "hgnc_id": "HGNC:44456", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L30", + "hgnc_id": "HGNC:44458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB131A", + "hgnc_id": "HGNC:18108", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRD5", + "hgnc_id": "HGNC:3026", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "UGT2B10", + "hgnc_id": "HGNC:12544", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B28", + "hgnc_id": "HGNC:13479", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UGT2A1", + "hgnc_id": "HGNC:12542", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UGT2A2", + "hgnc_id": "HGNC:28183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HTN3", + "hgnc_id": "HGNC:5284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HTN1", + "hgnc_id": "HGNC:5283", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "AFP", + "hgnc_id": "HGNC:317", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGY", + "hgnc_id": "HGNC:28213", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PYURF", + "hgnc_id": "HGNC:44317", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CISD2", + "hgnc_id": "HGNC:24212", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC9B1", + "hgnc_id": "HGNC:24244", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KLHL2", + "hgnc_id": "HGNC:6353", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GK3", + "hgnc_id": "HGNC:4292", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "HPF1", + "hgnc_id": "HGNC:26051", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDLIM3", + "hgnc_id": "HGNC:20767", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAT1", + "hgnc_id": "HGNC:3595", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG1", + "hgnc_id": "HGNC:3954", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FRG2", + "hgnc_id": "HGNC:19136", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DUX4", + "hgnc_id": "HGNC:50800", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SDHA", + "hgnc_id": "HGNC:10680", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDCD6", + "hgnc_id": "HGNC:8765", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZDHHC11B", + "hgnc_id": "HGNC:32962", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZDHHC11", + "hgnc_id": "HGNC:19158", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TERT", + "hgnc_id": "HGNC:11730", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A3", + "hgnc_id": "HGNC:11049", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAF11L2", + "hgnc_id": "HGNC:53845", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L3", + "hgnc_id": "HGNC:53846", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L4", + "hgnc_id": "HGNC:53847", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L5", + "hgnc_id": "HGNC:53848", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "TAF11L7", + "hgnc_id": "HGNC:53850", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAF11L8", + "hgnc_id": "HGNC:53851", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "TAF11L9", + "hgnc_id": "HGNC:53852", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L10", + "hgnc_id": "HGNC:53853", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L11", + "hgnc_id": "HGNC:53854", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L12", + "hgnc_id": "HGNC:53855", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "TAF11L14", + "hgnc_id": "HGNC:53857", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMB", + "hgnc_id": "HGNC:30465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MARVELD2", + "hgnc_id": "HGNC:26401", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERF1B", + "hgnc_id": "HGNC:10756", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NAIP", + "hgnc_id": "HGNC:7634", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SKIC3", + "hgnc_id": "HGNC:23639", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIX1", + "hgnc_id": "HGNC:18581", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SAR1B", + "hgnc_id": "HGNC:10535", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DND1", + "hgnc_id": "HGNC:23799", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDHA10", + "hgnc_id": "HGNC:8664", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PCDHB7", + "hgnc_id": "HGNC:8692", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PCDHB8", + "hgnc_id": "HGNC:8693", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PCDHB11", + "hgnc_id": "HGNC:8682", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PCDHB12", + "hgnc_id": "HGNC:8683", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPP1R2B", + "hgnc_id": "HGNC:16318", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPM1", + "hgnc_id": "HGNC:7910", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "THOC3", + "hgnc_id": "HGNC:19072", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SIMC1", + "hgnc_id": "HGNC:24779", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM153A", + "hgnc_id": "HGNC:29940", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FAM153CP", + "hgnc_id": "HGNC:33936", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FLT4", + "hgnc_id": "HGNC:3767", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR4F3", + "hgnc_id": "HGNC:8300", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "DUSP22", + "hgnc_id": "HGNC:16077", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TUBB2A", + "hgnc_id": "HGNC:12412", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "H2AC13", + "hgnc_id": "HGNC:4725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H2AC14", + "hgnc_id": "HGNC:4727", + "reasons": [ + "segdup" + ], + "share": 0.8319, + "status": "not read" + }, + { + "gene": "OR2J1", + "hgnc_id": "HGNC:8259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2J2", + "hgnc_id": "HGNC:8260", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HLA-G", + "hgnc_id": "HGNC:4964", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HLA-A", + "hgnc_id": "HGNC:4931", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "MICA", + "hgnc_id": "HGNC:7090", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MICB", + "hgnc_id": "HGNC:7091", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZBTB12", + "hgnc_id": "HGNC:19066", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PBX2", + "hgnc_id": "HGNC:8633", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HLA-DRB1", + "hgnc_id": "HGNC:4948", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GSTA2", + "hgnc_id": "HGNC:4627", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GSTA1", + "hgnc_id": "HGNC:4626", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTA5", + "hgnc_id": "HGNC:19662", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRIM2", + "hgnc_id": "HGNC:9370", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FKBP1C", + "hgnc_id": "HGNC:21376", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SLC17A5", + "hgnc_id": "HGNC:10933", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEC63", + "hgnc_id": "HGNC:21082", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR2A4", + "hgnc_id": "HGNC:14729", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE9", + "hgnc_id": "HGNC:37275", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "PCMT1", + "hgnc_id": "HGNC:8728", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RAET1G", + "hgnc_id": "HGNC:16795", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ULBP1", + "hgnc_id": "HGNC:14893", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RAET1L", + "hgnc_id": "HGNC:16798", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ULBP3", + "hgnc_id": "HGNC:14895", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC22A1", + "hgnc_id": "HGNC:10963", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMOC2", + "hgnc_id": "HGNC:20323", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RBAK-RBAKDN", + "hgnc_id": "HGNC:42971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBAK", + "hgnc_id": "HGNC:17680", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC29A4", + "hgnc_id": "HGNC:23097", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FSCN1", + "hgnc_id": "HGNC:11148", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM", + "hgnc_id": "HGNC:8105", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCZ1", + "hgnc_id": "HGNC:21691", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RSPH10B2", + "hgnc_id": "HGNC:34385", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CCZ1B", + "hgnc_id": "HGNC:21717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOD1", + "hgnc_id": "HGNC:16390", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FKBP9", + "hgnc_id": "HGNC:3725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DPY19L1", + "hgnc_id": "HGNC:22205", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SEPTIN7", + "hgnc_id": "HGNC:1717", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE1", + "hgnc_id": "HGNC:16408", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIA", + "hgnc_id": "HGNC:9253", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDC14C", + "hgnc_id": "HGNC:22427", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSPH", + "hgnc_id": "HGNC:9577", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCT6A", + "hgnc_id": "HGNC:1620", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF716", + "hgnc_id": "HGNC:32458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF727", + "hgnc_id": "HGNC:22785", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF735", + "hgnc_id": "HGNC:32466", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF736", + "hgnc_id": "HGNC:32467", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GUSB", + "hgnc_id": "HGNC:4696", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RABGEF1", + "hgnc_id": "HGNC:17676", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TYW1", + "hgnc_id": "HGNC:25598", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE21", + "hgnc_id": "HGNC:51517", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYW1B", + "hgnc_id": "HGNC:33908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POM121", + "hgnc_id": "HGNC:19702", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM74", + "hgnc_id": "HGNC:17453", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPDYE8", + "hgnc_id": "HGNC:33771", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE9", + "hgnc_id": "HGNC:45034", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE10", + "hgnc_id": "HGNC:51506", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NSUN5", + "hgnc_id": "HGNC:16385", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM50", + "hgnc_id": "HGNC:19017", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLIP2", + "hgnc_id": "HGNC:2586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GTF2IRD2", + "hgnc_id": "HGNC:30775", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE12", + "hgnc_id": "HGNC:51508", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CASTOR2", + "hgnc_id": "HGNC:37073", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GTF2IRD2B", + "hgnc_id": "HGNC:33125", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPDYE13", + "hgnc_id": "HGNC:51509", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE14", + "hgnc_id": "HGNC:51510", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM73", + "hgnc_id": "HGNC:18162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POM121C", + "hgnc_id": "HGNC:34005", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPDYE5", + "hgnc_id": "HGNC:35464", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DTX2", + "hgnc_id": "HGNC:15973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UPK3B", + "hgnc_id": "HGNC:21444", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE16", + "hgnc_id": "HGNC:51512", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE17", + "hgnc_id": "HGNC:51513", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE18", + "hgnc_id": "HGNC:51514", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "STEAP1", + "hgnc_id": "HGNC:11378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ASNS", + "hgnc_id": "HGNC:753", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM2", + "hgnc_id": "HGNC:34396", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP3A7-CYP3A51P", + "hgnc_id": "HGNC:51504", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP3A4", + "hgnc_id": "HGNC:2637", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "STAG3", + "hgnc_id": "HGNC:11356", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PVRIG", + "hgnc_id": "HGNC:32190", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SPDYE3", + "hgnc_id": "HGNC:35462", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SAP25", + "hgnc_id": "HGNC:41908", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZAN", + "hgnc_id": "HGNC:12857", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC12", + "hgnc_id": "HGNC:7510", + "reasons": [ + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE6", + "hgnc_id": "HGNC:35465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "POLR2J", + "hgnc_id": "HGNC:9197", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RASA4B", + "hgnc_id": "HGNC:35202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J3", + "hgnc_id": "HGNC:33853", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UPK3BL2", + "hgnc_id": "HGNC:53444", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "RASA4", + "hgnc_id": "HGNC:23181", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UPK3BL1", + "hgnc_id": "HGNC:37278", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "POLR2J3-UPK3BL2", + "hgnc_id": "HGNC:58364", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SPDYE2B", + "hgnc_id": "HGNC:48334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J2", + "hgnc_id": "HGNC:23208", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAMB1", + "hgnc_id": "HGNC:6486", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2B", + "hgnc_id": "HGNC:18272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KLF14", + "hgnc_id": "HGNC:23025", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AKR1B10", + "hgnc_id": "HGNC:382", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1B15", + "hgnc_id": "HGNC:37281", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BRAF", + "hgnc_id": "HGNC:1097", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRBV9", + "hgnc_id": "HGNC:12246", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRSS2", + "hgnc_id": "HGNC:9483", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAGE15", + "hgnc_id": "HGNC:37295", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TCAF2", + "hgnc_id": "HGNC:26878", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TCAF2C", + "hgnc_id": "HGNC:53641", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CTAGE6", + "hgnc_id": "HGNC:28644", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "OR2F2", + "hgnc_id": "HGNC:8247", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CTAGE4", + "hgnc_id": "HGNC:24772", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2A42", + "hgnc_id": "HGNC:31230", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2A7", + "hgnc_id": "HGNC:8234", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE8", + "hgnc_id": "HGNC:37294", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "OR2A1", + "hgnc_id": "HGNC:8229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARHGEF5", + "hgnc_id": "HGNC:13209", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACTR3C", + "hgnc_id": "HGNC:37282", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KMT2C", + "hgnc_id": "HGNC:13726", + "reasons": [ + "CMRG", + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ACTR3B", + "hgnc_id": "HGNC:17256", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F21", + "hgnc_id": "HGNC:19583", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "FBXO25", + "hgnc_id": "HGNC:13596", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARHGEF10", + "hgnc_id": "HGNC:14103", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A15", + "hgnc_id": "HGNC:32263", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A3", + "hgnc_id": "HGNC:32251", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A13", + "hgnc_id": "HGNC:32261", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A5", + "hgnc_id": "HGNC:32253", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM90A20", + "hgnc_id": "HGNC:32268", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L1", + "hgnc_id": "HGNC:37182", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L4", + "hgnc_id": "HGNC:37176", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNF705G", + "hgnc_id": "HGNC:37134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB108C", + "hgnc_id": "HGNC:30847", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB4B", + "hgnc_id": "HGNC:30193", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "DEFB103B", + "hgnc_id": "HGNC:31702", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "DEFB104B", + "hgnc_id": "HGNC:26165", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB106B", + "hgnc_id": "HGNC:28879", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "DEFB105B", + "hgnc_id": "HGNC:29930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "DEFB107B", + "hgnc_id": "HGNC:31918", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRR23D1", + "hgnc_id": "HGNC:49420", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A7", + "hgnc_id": "HGNC:32255", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FAM90A23", + "hgnc_id": "HGNC:32271", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A14", + "hgnc_id": "HGNC:32262", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A18", + "hgnc_id": "HGNC:32266", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A16", + "hgnc_id": "HGNC:32264", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A8", + "hgnc_id": "HGNC:32256", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A17", + "hgnc_id": "HGNC:32265", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A9", + "hgnc_id": "HGNC:32257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A10", + "hgnc_id": "HGNC:32258", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PRR23D2", + "hgnc_id": "HGNC:49396", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB107A", + "hgnc_id": "HGNC:18086", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB105A", + "hgnc_id": "HGNC:18087", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DEFB106A", + "hgnc_id": "HGNC:18088", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPAG11A", + "hgnc_id": "HGNC:33342", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB103A", + "hgnc_id": "HGNC:15967", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DEFB4A", + "hgnc_id": "HGNC:2767", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNF705B", + "hgnc_id": "HGNC:32284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "USP17L8", + "hgnc_id": "HGNC:37181", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L3", + "hgnc_id": "HGNC:37175", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "FAM90A11", + "hgnc_id": "HGNC:32259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A24", + "hgnc_id": "HGNC:32272", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A12", + "hgnc_id": "HGNC:32260", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLC35G5", + "hgnc_id": "HGNC:15546", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "DEFB130B", + "hgnc_id": "HGNC:39814", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF705D", + "hgnc_id": "HGNC:33202", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L2", + "hgnc_id": "HGNC:34434", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB109D", + "hgnc_id": "HGNC:30838", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM86B1", + "hgnc_id": "HGNC:28268", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB130A", + "hgnc_id": "HGNC:18107", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM86B2", + "hgnc_id": "HGNC:32222", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RBPMS", + "hgnc_id": "HGNC:19097", + "reasons": [ + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IMPA1", + "hgnc_id": "HGNC:6050", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC10A5", + "hgnc_id": "HGNC:22981", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH17", + "hgnc_id": "HGNC:1756", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP11B1", + "hgnc_id": "HGNC:2591", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP11B2", + "hgnc_id": "HGNC:2592", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GPIHBP1", + "hgnc_id": "HGNC:24945", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "WASHC1", + "hgnc_id": "HGNC:24361", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOXD4", + "hgnc_id": "HGNC:3805", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNG1A", + "hgnc_id": "HGNC:17134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KCNV2", + "hgnc_id": "HGNC:19698", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RLN2", + "hgnc_id": "HGNC:10027", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RLN1", + "hgnc_id": "HGNC:10026", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "IFNA21", + "hgnc_id": "HGNC:5424", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA4", + "hgnc_id": "HGNC:5425", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "IFNA7", + "hgnc_id": "HGNC:5428", + "reasons": [ + "segdup" + ], + "share": 0.3324, + "status": "not read" + }, + { + "gene": "IFNA10", + "hgnc_id": "HGNC:5418", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "IFNA16", + "hgnc_id": "HGNC:5421", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "IFNA17", + "hgnc_id": "HGNC:5422", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "IFNA13", + "hgnc_id": "HGNC:5419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA1", + "hgnc_id": "HGNC:5417", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "ANKRD18B", + "hgnc_id": "HGNC:23644", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRSS3", + "hgnc_id": "HGNC:9486", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALT", + "hgnc_id": "HGNC:4135", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31F1", + "hgnc_id": "HGNC:41911", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CNTNAP3", + "hgnc_id": "HGNC:13834", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31A1", + "hgnc_id": "HGNC:23394", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L6", + "hgnc_id": "HGNC:31986", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNG1F", + "hgnc_id": "HGNC:31978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNTNAP3B", + "hgnc_id": "HGNC:32035", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31A6", + "hgnc_id": "HGNC:32006", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPATA31A7", + "hgnc_id": "HGNC:32007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CNTNAP3C", + "hgnc_id": "HGNC:53878", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L5", + "hgnc_id": "HGNC:18522", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNG1E", + "hgnc_id": "HGNC:24584", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXD4L4", + "hgnc_id": "HGNC:23762", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF658", + "hgnc_id": "HGNC:25226", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD20A1", + "hgnc_id": "HGNC:23665", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNG1C", + "hgnc_id": "HGNC:18519", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L3", + "hgnc_id": "HGNC:18523", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PGM5", + "hgnc_id": "HGNC:8908", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FXN", + "hgnc_id": "HGNC:3951", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TJP2", + "hgnc_id": "HGNC:11828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPATA31D4", + "hgnc_id": "HGNC:38601", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31D3", + "hgnc_id": "HGNC:38603", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPATA31D1", + "hgnc_id": "HGNC:37283", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "KIF27", + "hgnc_id": "HGNC:18632", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPATA31C1", + "hgnc_id": "HGNC:27846", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPATA31C2", + "hgnc_id": "HGNC:24508", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC71A2", + "hgnc_id": "HGNC:23376", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2G", + "hgnc_id": "HGNC:23449", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CTSV", + "hgnc_id": "HGNC:2538", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR13C5", + "hgnc_id": "HGNC:15100", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR13C2", + "hgnc_id": "HGNC:14701", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR13C9", + "hgnc_id": "HGNC:15104", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "POLE3", + "hgnc_id": "HGNC:13546", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ORM1", + "hgnc_id": "HGNC:8498", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ORM2", + "hgnc_id": "HGNC:8499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SLC27A4", + "hgnc_id": "HGNC:10998", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PKN3", + "hgnc_id": "HGNC:17999", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ABO", + "hgnc_id": "HGNC:79", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCN1", + "hgnc_id": "HGNC:6525", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OBP2A", + "hgnc_id": "HGNC:23380", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SOHLH1", + "hgnc_id": "HGNC:27845", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "INPP5E", + "hgnc_id": "HGNC:21474", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AJM1", + "hgnc_id": "HGNC:37284", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRIN1", + "hgnc_id": "HGNC:4584", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8", + "hgnc_id": "HGNC:20773", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "AKR1C1", + "hgnc_id": "HGNC:384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1C2", + "hgnc_id": "HGNC:385", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DCLRE1C", + "hgnc_id": "HGNC:17642", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRC1", + "hgnc_id": "HGNC:7228", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD30A", + "hgnc_id": "HGNC:17234", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF33A", + "hgnc_id": "HGNC:13096", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF37A", + "hgnc_id": "HGNC:13102", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLD5P1", + "hgnc_id": "HGNC:55072", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF33B", + "hgnc_id": "HGNC:13097", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BMS1", + "hgnc_id": "HGNC:23505", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AGAP4", + "hgnc_id": "HGNC:23459", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23", + "hgnc_id": "HGNC:17312", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ANTXRL", + "hgnc_id": "HGNC:27277", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANXA8L1", + "hgnc_id": "HGNC:23334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPY4R", + "hgnc_id": "HGNC:9329", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPRIN2", + "hgnc_id": "HGNC:23730", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PTPN20", + "hgnc_id": "HGNC:23423", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANXA8", + "hgnc_id": "HGNC:546", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM25G", + "hgnc_id": "HGNC:23590", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP9", + "hgnc_id": "HGNC:23463", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SYT15B", + "hgnc_id": "HGNC:51487", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPY4R2", + "hgnc_id": "HGNC:52383", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PARG", + "hgnc_id": "HGNC:8605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23B", + "hgnc_id": "HGNC:23581", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TIMM23B-AGAP6", + "hgnc_id": "HGNC:45009", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP6", + "hgnc_id": "HGNC:23466", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASHC2A", + "hgnc_id": "HGNC:23416", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ASAH2B", + "hgnc_id": "HGNC:23456", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EIF5AL1", + "hgnc_id": "HGNC:17419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2B", + "hgnc_id": "HGNC:23445", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NUTM2E", + "hgnc_id": "HGNC:23448", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMEM254", + "hgnc_id": "HGNC:25804", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PLAC9", + "hgnc_id": "HGNC:19255", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ADIRF", + "hgnc_id": "HGNC:24043", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NUTM2A", + "hgnc_id": "HGNC:23438", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2D", + "hgnc_id": "HGNC:23447", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PAPSS2", + "hgnc_id": "HGNC:8604", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PTEN", + "hgnc_id": "HGNC:9588", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIPN", + "hgnc_id": "HGNC:23452", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "BTRC", + "hgnc_id": "HGNC:1144", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SFR1", + "hgnc_id": "HGNC:29574", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DHX32", + "hgnc_id": "HGNC:16717", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANK1", + "hgnc_id": "HGNC:23527", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ECHS1", + "hgnc_id": "HGNC:3151", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPRN", + "hgnc_id": "HGNC:16871", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG2B", + "hgnc_id": "HGNC:33518", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SCGB1C1", + "hgnc_id": "HGNC:18394", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFITM3", + "hgnc_id": "HGNC:5414", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DRD4", + "hgnc_id": "HGNC:3025", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEAF1", + "hgnc_id": "HGNC:14677", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP5-2", + "hgnc_id": "HGNC:23597", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP5-4", + "hgnc_id": "HGNC:23599", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MRPL23", + "hgnc_id": "HGNC:10322", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H19", + "hgnc_id": "HGNC:4713", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSU72L5", + "hgnc_id": "HGNC:43624", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "SSU72L2", + "hgnc_id": "HGNC:43621", + "reasons": [ + "segdup" + ], + "share": 0.666, + "status": "not read" + }, + { + "gene": "SSU72L4", + "hgnc_id": "HGNC:43623", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "SSU72L1", + "hgnc_id": "HGNC:43626", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR51A4", + "hgnc_id": "HGNC:14795", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR51A2", + "hgnc_id": "HGNC:14764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HBG1", + "hgnc_id": "HGNC:4831", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR10A5", + "hgnc_id": "HGNC:15131", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRGPRX4", + "hgnc_id": "HGNC:17617", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MDK", + "hgnc_id": "HGNC:6972", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPI1", + "hgnc_id": "HGNC:11241", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C3", + "hgnc_id": "HGNC:14697", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4C5", + "hgnc_id": "HGNC:14702", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4A47", + "hgnc_id": "HGNC:31266", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM51G", + "hgnc_id": "HGNC:43972", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64C", + "hgnc_id": "HGNC:37148", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOLH1", + "hgnc_id": "HGNC:3788", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C13", + "hgnc_id": "HGNC:15169", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4C12", + "hgnc_id": "HGNC:15168", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4C45", + "hgnc_id": "HGNC:31270", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4C46", + "hgnc_id": "HGNC:31271", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4A5", + "hgnc_id": "HGNC:15162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM48", + "hgnc_id": "HGNC:19021", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR4A15", + "hgnc_id": "HGNC:15152", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM51", + "hgnc_id": "HGNC:19023", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR5F1", + "hgnc_id": "HGNC:8343", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR8H2", + "hgnc_id": "HGNC:15308", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR5M8", + "hgnc_id": "HGNC:14846", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR9G1", + "hgnc_id": "HGNC:15319", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR1S2", + "hgnc_id": "HGNC:15141", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1S1", + "hgnc_id": "HGNC:8227", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GLYATL1B", + "hgnc_id": "HGNC:37865", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PGA3", + "hgnc_id": "HGNC:8885", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "B3GAT3", + "hgnc_id": "HGNC:923", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ESRRA", + "hgnc_id": "HGNC:3471", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC22A12", + "hgnc_id": "HGNC:17989", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RBM4B", + "hgnc_id": "HGNC:28842", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TPCN2", + "hgnc_id": "HGNC:20820", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FGF3", + "hgnc_id": "HGNC:3681", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KRTAP5-8", + "hgnc_id": "HGNC:23603", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP5-10", + "hgnc_id": "HGNC:23605", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "KRTAP5-11", + "hgnc_id": "HGNC:23606", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "XNDC1N-ZNF705EP-ALG1L9P", + "hgnc_id": "HGNC:55871", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DEFB108B", + "hgnc_id": "HGNC:29966", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "XNDC1N", + "hgnc_id": "HGNC:54661", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "INTS4", + "hgnc_id": "HGNC:25048", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOX4", + "hgnc_id": "HGNC:7891", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TRIM77", + "hgnc_id": "HGNC:34228", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "TRIM49", + "hgnc_id": "HGNC:13431", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM64B", + "hgnc_id": "HGNC:37147", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM49D1", + "hgnc_id": "HGNC:43973", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64", + "hgnc_id": "HGNC:14663", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM49C", + "hgnc_id": "HGNC:38877", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UBTFL1", + "hgnc_id": "HGNC:14533", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CARD16", + "hgnc_id": "HGNC:33701", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA8", + "hgnc_id": "HGNC:5241", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR8B2", + "hgnc_id": "HGNC:8471", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TMEM45B", + "hgnc_id": "HGNC:25194", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CD4", + "hgnc_id": "HGNC:1678", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PEX5", + "hgnc_id": "HGNC:9719", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC1", + "hgnc_id": "HGNC:604", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GDF3", + "hgnc_id": "HGNC:4218", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NANOG", + "hgnc_id": "HGNC:20857", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A1", + "hgnc_id": "HGNC:25526", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRH1", + "hgnc_id": "HGNC:9366", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRR4", + "hgnc_id": "HGNC:18020", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRH2", + "hgnc_id": "HGNC:9367", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAS2R14", + "hgnc_id": "HGNC:14920", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TAS2R31", + "hgnc_id": "HGNC:19113", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAS2R46", + "hgnc_id": "HGNC:18877", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAS2R43", + "hgnc_id": "HGNC:18875", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "TAS2R30", + "hgnc_id": "HGNC:19112", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "PRB4", + "hgnc_id": "HGNC:9340", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRB1", + "hgnc_id": "HGNC:9337", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "DDX11", + "hgnc_id": "HGNC:2736", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H3-5", + "hgnc_id": "HGNC:33164", + "reasons": [ + "segdup" + ], + "share": 0.1661, + "status": "not read" + }, + { + "gene": "ALG10", + "hgnc_id": "HGNC:23162", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ALG10B", + "hgnc_id": "HGNC:31088", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBA1B", + "hgnc_id": "HGNC:18809", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "COX14", + "hgnc_id": "HGNC:28216", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NACA", + "hgnc_id": "HGNC:7629", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DPY19L2", + "hgnc_id": "HGNC:19414", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNF1A", + "hgnc_id": "HGNC:11621", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HPD", + "hgnc_id": "HGNC:5147", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCAR2", + "hgnc_id": "HGNC:24827", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HCAR3", + "hgnc_id": "HGNC:16824", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "P2RX2", + "hgnc_id": "HGNC:15459", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PGAM5", + "hgnc_id": "HGNC:28763", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA3", + "hgnc_id": "HGNC:4426", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "C1QTNF9B", + "hgnc_id": "HGNC:34072", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C1QTNF9", + "hgnc_id": "HGNC:28732", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PARP4", + "hgnc_id": "HGNC:271", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PABPC3", + "hgnc_id": "HGNC:8556", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "MRPS31", + "hgnc_id": "HGNC:16632", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNRNPA1L2", + "hgnc_id": "HGNC:27067", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRR20C", + "hgnc_id": "HGNC:37221", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRR20B", + "hgnc_id": "HGNC:37220", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "PRR20D", + "hgnc_id": "HGNC:37222", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRR20E", + "hgnc_id": "HGNC:37223", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "F7", + "hgnc_id": "HGNC:3544", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PROZ", + "hgnc_id": "HGNC:9460", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEM", + "hgnc_id": "HGNC:37096", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEG", + "hgnc_id": "HGNC:33896", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR11H2", + "hgnc_id": "HGNC:14716", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4N2", + "hgnc_id": "HGNC:14742", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR4Q3", + "hgnc_id": "HGNC:15426", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR4M1", + "hgnc_id": "HGNC:14735", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RNASE3", + "hgnc_id": "HGNC:10046", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RNASE2", + "hgnc_id": "HGNC:10045", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10G2", + "hgnc_id": "HGNC:8170", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "DHRS4", + "hgnc_id": "HGNC:16985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHRS4L2", + "hgnc_id": "HGNC:19731", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARF6", + "hgnc_id": "HGNC:659", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACOT2", + "hgnc_id": "HGNC:18431", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NDUFB1", + "hgnc_id": "HGNC:7695", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "JAG2", + "hgnc_id": "HGNC:6189", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BTBD6", + "hgnc_id": "HGNC:19897", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "IGHV3-21", + "hgnc_id": "HGNC:5586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA6L6", + "hgnc_id": "HGNC:37225", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEB3", + "hgnc_id": "HGNC:51240", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LINC02203", + "hgnc_id": "HGNC:53069", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4M2B", + "hgnc_id": "HGNC:55109", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4N4C", + "hgnc_id": "HGNC:55110", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "POTEB", + "hgnc_id": "HGNC:33734", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4M2-OT1", + "hgnc_id": "HGNC:56199", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4N4", + "hgnc_id": "HGNC:15375", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L22", + "hgnc_id": "HGNC:50289", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6L1", + "hgnc_id": "HGNC:37444", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L26", + "hgnc_id": "HGNC:56306", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8S", + "hgnc_id": "HGNC:44409", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6L2", + "hgnc_id": "HGNC:26695", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HERC2", + "hgnc_id": "HGNC:4868", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L24", + "hgnc_id": "HGNC:55710", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA8F", + "hgnc_id": "HGNC:32378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8G", + "hgnc_id": "HGNC:25328", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6L25", + "hgnc_id": "HGNC:55711", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8M", + "hgnc_id": "HGNC:44404", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8J", + "hgnc_id": "HGNC:38650", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8T", + "hgnc_id": "HGNC:44410", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8R", + "hgnc_id": "HGNC:44407", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8Q", + "hgnc_id": "HGNC:44408", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8H", + "hgnc_id": "HGNC:37443", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8O", + "hgnc_id": "HGNC:44406", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8N", + "hgnc_id": "HGNC:44405", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8A", + "hgnc_id": "HGNC:31972", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8B", + "hgnc_id": "HGNC:31973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NANOGP8", + "hgnc_id": "HGNC:23106", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "NDUFAF1", + "hgnc_id": "HGNC:18828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CKMT1B", + "hgnc_id": "HGNC:1995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CATSPER2", + "hgnc_id": "HGNC:18810", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CKMT1A", + "hgnc_id": "HGNC:31736", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TERB2", + "hgnc_id": "HGNC:28520", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LDHAL6B", + "hgnc_id": "HGNC:21481", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6B", + "hgnc_id": "HGNC:32205", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6A", + "hgnc_id": "HGNC:13567", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ISLR2", + "hgnc_id": "HGNC:29286", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6C", + "hgnc_id": "HGNC:32206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6D", + "hgnc_id": "HGNC:32204", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ADAMTS7", + "hgnc_id": "HGNC:223", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EFL1", + "hgnc_id": "HGNC:25789", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SAXO2", + "hgnc_id": "HGNC:33727", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L10", + "hgnc_id": "HGNC:37228", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GOLGA6L9", + "hgnc_id": "HGNC:37229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "WHAMM", + "hgnc_id": "HGNC:30493", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4F4", + "hgnc_id": "HGNC:8301", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MPG", + "hgnc_id": "HGNC:7211", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HBM", + "hgnc_id": "HGNC:4826", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AXIN1", + "hgnc_id": "HGNC:903", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSTR5", + "hgnc_id": "HGNC:11334", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TPSD1", + "hgnc_id": "HGNC:14118", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GNPTG", + "hgnc_id": "HGNC:23026", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UNKL", + "hgnc_id": "HGNC:14184", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PKD1", + "hgnc_id": "HGNC:9008", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PDPK1", + "hgnc_id": "HGNC:8816", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "EEF2KMT", + "hgnc_id": "HGNC:32221", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NPIPA3", + "hgnc_id": "HGNC:41978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPA2", + "hgnc_id": "HGNC:41979", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NOMO1", + "hgnc_id": "HGNC:30060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDXDC1", + "hgnc_id": "HGNC:28995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RRN3", + "hgnc_id": "HGNC:30346", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPA5", + "hgnc_id": "HGNC:41980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MPV17L", + "hgnc_id": "HGNC:26827", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NOMO3", + "hgnc_id": "HGNC:25242", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPA6", + "hgnc_id": "HGNC:27893", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPA8", + "hgnc_id": "HGNC:41983", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPA9", + "hgnc_id": "HGNC:41984", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NOMO2", + "hgnc_id": "HGNC:22652", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SMG1", + "hgnc_id": "HGNC:30045", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IQCK", + "hgnc_id": "HGNC:28556", + "reasons": [ + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACSM5", + "hgnc_id": "HGNC:26060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACSM2B", + "hgnc_id": "HGNC:30931", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPB3", + "hgnc_id": "HGNC:28989", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB4", + "hgnc_id": "HGNC:41985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB5", + "hgnc_id": "HGNC:37233", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB6", + "hgnc_id": "HGNC:37454", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB7", + "hgnc_id": "HGNC:33832", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A2", + "hgnc_id": "HGNC:11454", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB8", + "hgnc_id": "HGNC:37490", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EIF3C", + "hgnc_id": "HGNC:3279", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB9", + "hgnc_id": "HGNC:41987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB11", + "hgnc_id": "HGNC:37453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BOLA2-SMG1P6", + "hgnc_id": "HGNC:53563", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BOLA2", + "hgnc_id": "HGNC:29488", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLX1B", + "hgnc_id": "HGNC:28748", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SULT1A4", + "hgnc_id": "HGNC:30004", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NPIPB12", + "hgnc_id": "HGNC:37491", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLX1A", + "hgnc_id": "HGNC:20922", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A3", + "hgnc_id": "HGNC:11455", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB13", + "hgnc_id": "HGNC:41989", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PHKG2", + "hgnc_id": "HGNC:8931", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "VKORC1", + "hgnc_id": "HGNC:23663", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRABD5", + "hgnc_id": "HGNC:26987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TP53TG3", + "hgnc_id": "HGNC:30759", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TP53TG3C", + "hgnc_id": "HGNC:42962", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "TP53TG3E", + "hgnc_id": "HGNC:51816", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TP53TG3B", + "hgnc_id": "HGNC:37202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TP53TG3F", + "hgnc_id": "HGNC:51817", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "CCNYL1B", + "hgnc_id": "HGNC:56313", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CES1", + "hgnc_id": "HGNC:1863", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MT1M", + "hgnc_id": "HGNC:14296", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MATCAP1", + "hgnc_id": "HGNC:34408", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD11B2", + "hgnc_id": "HGNC:5209", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLEC18A", + "hgnc_id": "HGNC:30388", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PDPR", + "hgnc_id": "HGNC:30264", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EXOSC6", + "hgnc_id": "HGNC:19055", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HPR", + "hgnc_id": "HGNC:5156", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB15", + "hgnc_id": "HGNC:34409", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CLEC18B", + "hgnc_id": "HGNC:33849", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CA5A", + "hgnc_id": "HGNC:1377", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF469", + "hgnc_id": "HGNC:23216", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH15", + "hgnc_id": "HGNC:1754", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANKRD11", + "hgnc_id": "HGNC:21316", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MC1R", + "hgnc_id": "HGNC:6929", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PRDM7", + "hgnc_id": "HGNC:9351", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ABR", + "hgnc_id": "HGNC:81", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SERPINF2", + "hgnc_id": "HGNC:9075", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SRR", + "hgnc_id": "HGNC:14398", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1D5", + "hgnc_id": "HGNC:8186", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "P2RX5", + "hgnc_id": "HGNC:8536", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ALOX15", + "hgnc_id": "HGNC:433", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ENO3", + "hgnc_id": "HGNC:3354", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP6", + "hgnc_id": "HGNC:12629", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35G6", + "hgnc_id": "HGNC:31351", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "ALOXE3", + "hgnc_id": "HGNC:13743", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HES7", + "hgnc_id": "HGNC:15977", + "reasons": [ + "CMRG" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CDRT15", + "hgnc_id": "HGNC:14395", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "FBXW10B", + "hgnc_id": "HGNC:14379", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM16", + "hgnc_id": "HGNC:17241", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF286A-TBC1D26", + "hgnc_id": "HGNC:55384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF286A", + "hgnc_id": "HGNC:13501", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D26", + "hgnc_id": "HGNC:28745", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UBB", + "hgnc_id": "HGNC:12463", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "ATPAF2", + "hgnc_id": "HGNC:18802", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EVPLL", + "hgnc_id": "HGNC:35236", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS9C", + "hgnc_id": "HGNC:33874", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D28", + "hgnc_id": "HGNC:26858", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXO3B", + "hgnc_id": "HGNC:3822", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FBXW10", + "hgnc_id": "HGNC:1211", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRAP", + "hgnc_id": "HGNC:4562", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GRAPL", + "hgnc_id": "HGNC:37240", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LGALS9B", + "hgnc_id": "HGNC:24842", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDRT15L2", + "hgnc_id": "HGNC:34075", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MEIS3P2", + "hgnc_id": "HGNC:17638", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAP2K3", + "hgnc_id": "HGNC:6843", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KCNJ18", + "hgnc_id": "HGNC:39080", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS9", + "hgnc_id": "HGNC:6570", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXN1", + "hgnc_id": "HGNC:12765", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SUZ12", + "hgnc_id": "HGNC:17101", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37B", + "hgnc_id": "HGNC:29070", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SLC35G3", + "hgnc_id": "HGNC:26848", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CCL4", + "hgnc_id": "HGNC:10630", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3B", + "hgnc_id": "HGNC:27011", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCL4L2", + "hgnc_id": "HGNC:24066", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "TBC1D3I", + "hgnc_id": "HGNC:32709", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TBC1D3G", + "hgnc_id": "HGNC:29860", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TBC1D3F", + "hgnc_id": "HGNC:18257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3K", + "hgnc_id": "HGNC:51245", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TBC1D3L", + "hgnc_id": "HGNC:51246", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D3D", + "hgnc_id": "HGNC:28944", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D3C", + "hgnc_id": "HGNC:24889", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TBC1D3E", + "hgnc_id": "HGNC:27071", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MRPL45", + "hgnc_id": "HGNC:16651", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP1-1", + "hgnc_id": "HGNC:16772", + "reasons": [ + "CMRG" + ], + "share": 0.1659, + "status": "not read" + }, + { + "gene": "KRTAP4-7", + "hgnc_id": "HGNC:18898", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP4-8", + "hgnc_id": "HGNC:17230", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP4-16", + "hgnc_id": "HGNC:18921", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRTAP4-9", + "hgnc_id": "HGNC:18910", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP4-12", + "hgnc_id": "HGNC:16776", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP9-7", + "hgnc_id": "HGNC:18915", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "KRT14", + "hgnc_id": "HGNC:6416", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRT16", + "hgnc_id": "HGNC:6423", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRT17", + "hgnc_id": "HGNC:6427", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HSD17B1", + "hgnc_id": "HGNC:5210", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "G6PC3", + "hgnc_id": "HGNC:24861", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ARHGAP27", + "hgnc_id": "HGNC:31813", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLEKHM1", + "hgnc_id": "HGNC:29017", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ARL17B", + "hgnc_id": "HGNC:32387", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37A", + "hgnc_id": "HGNC:29069", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARL17A", + "hgnc_id": "HGNC:24096", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NSF", + "hgnc_id": "HGNC:8016", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPEPPS", + "hgnc_id": "HGNC:7900", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HOXB8", + "hgnc_id": "HGNC:5119", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GIP", + "hgnc_id": "HGNC:4270", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHX40", + "hgnc_id": "HGNC:18018", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPS6KB1", + "hgnc_id": "HGNC:10436", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2A", + "hgnc_id": "HGNC:25755", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TLK2", + "hgnc_id": "HGNC:11842", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CSH2", + "hgnc_id": "HGNC:2441", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "GH2", + "hgnc_id": "HGNC:4262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CSH1", + "hgnc_id": "HGNC:2440", + "reasons": [ + "segdup" + ], + "share": 0.6663, + "status": "not read" + }, + { + "gene": "CSHL1", + "hgnc_id": "HGNC:2442", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "LRRC37A3", + "hgnc_id": "HGNC:32427", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C17orf58", + "hgnc_id": "HGNC:27568", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "AMZ2", + "hgnc_id": "HGNC:28041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMC6", + "hgnc_id": "HGNC:18021", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CANT1", + "hgnc_id": "HGNC:19721", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RNF213", + "hgnc_id": "HGNC:14539", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ARL16", + "hgnc_id": "HGNC:27902", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8B", + "hgnc_id": "HGNC:24983", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYMS", + "hgnc_id": "HGNC:12441", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC35G4", + "hgnc_id": "HGNC:31043", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "POTEC", + "hgnc_id": "HGNC:33894", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ANKRD30B", + "hgnc_id": "HGNC:24165", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC25A52", + "hgnc_id": "HGNC:23324", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KATNAL2", + "hgnc_id": "HGNC:25387", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SERPINB4", + "hgnc_id": "HGNC:10570", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERPINB3", + "hgnc_id": "HGNC:10569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALR1", + "hgnc_id": "HGNC:4132", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CTDP1", + "hgnc_id": "HGNC:2498", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "BSG", + "hgnc_id": "HGNC:1116", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN2", + "hgnc_id": "HGNC:4846", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KISS1R", + "hgnc_id": "HGNC:4510", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "STK11", + "hgnc_id": "HGNC:11389", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TCF3", + "hgnc_id": "HGNC:11633", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ABHD17A", + "hgnc_id": "HGNC:28756", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "GIPC3", + "hgnc_id": "HGNC:18183", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBXA2R", + "hgnc_id": "HGNC:11608", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CREB3L3", + "hgnc_id": "HGNC:18855", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RFX2", + "hgnc_id": "HGNC:9983", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "C3", + "hgnc_id": "HGNC:1318", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MBD3L2B", + "hgnc_id": "HGNC:53435", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MBD3L4", + "hgnc_id": "HGNC:37206", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBD3L2", + "hgnc_id": "HGNC:18532", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "STXBP2", + "hgnc_id": "HGNC:11445", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TYK2", + "hgnc_id": "HGNC:12440", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF799", + "hgnc_id": "HGNC:28071", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF443", + "hgnc_id": "HGNC:20878", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP4F3", + "hgnc_id": "HGNC:2646", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4F12", + "hgnc_id": "HGNC:18857", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10H2", + "hgnc_id": "HGNC:8173", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CALR3", + "hgnc_id": "HGNC:20407", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MYO9B", + "hgnc_id": "HGNC:7609", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "INSL3", + "hgnc_id": "HGNC:6086", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF738", + "hgnc_id": "HGNC:32469", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF100", + "hgnc_id": "HGNC:12880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF208", + "hgnc_id": "HGNC:12999", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF676", + "hgnc_id": "HGNC:20429", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF729", + "hgnc_id": "HGNC:32464", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF492", + "hgnc_id": "HGNC:23707", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF728", + "hgnc_id": "HGNC:32463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHPN2", + "hgnc_id": "HGNC:19974", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GPI", + "hgnc_id": "HGNC:4458", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF302", + "hgnc_id": "HGNC:13848", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF181", + "hgnc_id": "HGNC:12971", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FFAR3", + "hgnc_id": "HGNC:4499", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "COX6B1", + "hgnc_id": "HGNC:2280", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS7", + "hgnc_id": "HGNC:6568", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS7B", + "hgnc_id": "HGNC:34447", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "IFNL3", + "hgnc_id": "HGNC:18365", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IFNL4", + "hgnc_id": "HGNC:44480", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "IFNL2", + "hgnc_id": "HGNC:18364", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CYP2A7", + "hgnc_id": "HGNC:2611", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CYP2G1P", + "hgnc_id": "HGNC:2633", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A13", + "hgnc_id": "HGNC:2608", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CYP2F1", + "hgnc_id": "HGNC:2632", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG8", + "hgnc_id": "HGNC:9525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG1", + "hgnc_id": "HGNC:9514", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PSG6", + "hgnc_id": "HGNC:9523", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PSG7", + "hgnc_id": "HGNC:9524", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSG11", + "hgnc_id": "HGNC:9516", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PSG2", + "hgnc_id": "HGNC:9519", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG4", + "hgnc_id": "HGNC:9521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG9", + "hgnc_id": "HGNC:9526", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ETHE1", + "hgnc_id": "HGNC:23287", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF285", + "hgnc_id": "HGNC:13079", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "APOC1", + "hgnc_id": "HGNC:607", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOC4", + "hgnc_id": "HGNC:611", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BLOC1S3", + "hgnc_id": "HGNC:20914", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DMPK", + "hgnc_id": "HGNC:2933", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FUT1", + "hgnc_id": "HGNC:4012", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BAX", + "hgnc_id": "HGNC:959", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LHB", + "hgnc_id": "HGNC:6584", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "CGB3", + "hgnc_id": "HGNC:1886", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CGB2", + "hgnc_id": "HGNC:16722", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "CGB1", + "hgnc_id": "HGNC:16721", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "CGB5", + "hgnc_id": "HGNC:16452", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CGB8", + "hgnc_id": "HGNC:16453", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CGB7", + "hgnc_id": "HGNC:16451", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PNKP", + "hgnc_id": "HGNC:9154", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SIGLEC16", + "hgnc_id": "HGNC:24851", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KLK4", + "hgnc_id": "HGNC:6365", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ETFB", + "hgnc_id": "HGNC:3482", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF808", + "hgnc_id": "HGNC:33230", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF600", + "hgnc_id": "HGNC:30951", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF468", + "hgnc_id": "HGNC:33105", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VN1R4", + "hgnc_id": "HGNC:19871", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "FAM90A27P", + "hgnc_id": "HGNC:43617", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF765", + "hgnc_id": "HGNC:25092", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF761", + "hgnc_id": "HGNC:23179", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF813", + "hgnc_id": "HGNC:33257", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRKCG", + "hgnc_id": "HGNC:9402", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBOAT7", + "hgnc_id": "HGNC:15505", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LILRA6", + "hgnc_id": "HGNC:15495", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "LILRB2", + "hgnc_id": "HGNC:6606", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "LAIR1", + "hgnc_id": "HGNC:6477", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAIR2", + "hgnc_id": "HGNC:6478", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LILRB1", + "hgnc_id": "HGNC:6605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KIR3DL3", + "hgnc_id": "HGNC:16312", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KIR2DL3", + "hgnc_id": "HGNC:6331", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KIR2DL1", + "hgnc_id": "HGNC:6329", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KIR2DL4", + "hgnc_id": "HGNC:6332", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KIR3DL1", + "hgnc_id": "HGNC:6338", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KIR3DL2", + "hgnc_id": "HGNC:6339", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NLRP7", + "hgnc_id": "HGNC:22947", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NLRP2", + "hgnc_id": "HGNC:22948", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GP6", + "hgnc_id": "HGNC:14388", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TNNT1", + "hgnc_id": "HGNC:11948", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TNNI3", + "hgnc_id": "HGNC:11947", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL4A", + "hgnc_id": "HGNC:16449", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RFPL4AL1", + "hgnc_id": "HGNC:45147", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZNF419", + "hgnc_id": "HGNC:20648", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF773", + "hgnc_id": "HGNC:30487", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF587B", + "hgnc_id": "HGNC:37142", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF814", + "hgnc_id": "HGNC:33258", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF417", + "hgnc_id": "HGNC:20646", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC27A5", + "hgnc_id": "HGNC:10999", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SIRPB1", + "hgnc_id": "HGNC:15928", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CST4", + "hgnc_id": "HGNC:2476", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CST1", + "hgnc_id": "HGNC:2473", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CST2", + "hgnc_id": "HGNC:2474", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PLAGL2", + "hgnc_id": "HGNC:9047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEMG1", + "hgnc_id": "HGNC:10742", + "reasons": [ + "CMRG" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PLTP", + "hgnc_id": "HGNC:9093", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRNA4", + "hgnc_id": "HGNC:1958", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EEF1A2", + "hgnc_id": "HGNC:3192", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PTK6", + "hgnc_id": "HGNC:9617", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MYT1", + "hgnc_id": "HGNC:7622", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCMTD2", + "hgnc_id": "HGNC:15882", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TPTE", + "hgnc_id": "HGNC:12023", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POTED", + "hgnc_id": "HGNC:23822", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP21-1", + "hgnc_id": "HGNC:18945", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SMIM11", + "hgnc_id": "HGNC:1293", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMIM34", + "hgnc_id": "HGNC:39601", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KCNE1", + "hgnc_id": "HGNC:6240", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CBR3", + "hgnc_id": "HGNC:1549", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NDUFV3", + "hgnc_id": "HGNC:7719", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CBS", + "hgnc_id": "HGNC:1550", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "U2AF1", + "hgnc_id": "HGNC:12453", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SIK1", + "hgnc_id": "HGNC:11142", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSF2BP", + "hgnc_id": "HGNC:5226", + "reasons": [ + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BC12L", + "hgnc_id": "HGNC:4762", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3319, + "status": "not read" + }, + { + "gene": "TRAPPC10", + "hgnc_id": "HGNC:11868", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PWP2", + "hgnc_id": "HGNC:9711", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GATD3", + "hgnc_id": "HGNC:1273", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DNMT3L", + "hgnc_id": "HGNC:2980", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-1", + "hgnc_id": "HGNC:22966", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "KRTAP10-2", + "hgnc_id": "HGNC:22967", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRTAP10-4", + "hgnc_id": "HGNC:20521", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "KRTAP10-5", + "hgnc_id": "HGNC:22969", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "KRTAP10-6", + "hgnc_id": "HGNC:20523", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "KRTAP10-8", + "hgnc_id": "HGNC:20525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-9", + "hgnc_id": "HGNC:22971", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KRTAP10-10", + "hgnc_id": "HGNC:22972", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP12-2", + "hgnc_id": "HGNC:20530", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "KRTAP12-1", + "hgnc_id": "HGNC:20529", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "COL18A1", + "hgnc_id": "HGNC:2195", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR11H1", + "hgnc_id": "HGNC:15404", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEH", + "hgnc_id": "HGNC:133", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCT8L2", + "hgnc_id": "HGNC:15553", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "USP18", + "hgnc_id": "HGNC:12616", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GGTLC3", + "hgnc_id": "HGNC:33426", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TMEM191B", + "hgnc_id": "HGNC:33600", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM246B", + "hgnc_id": "HGNC:54843", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DGCR6", + "hgnc_id": "HGNC:2846", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRODH", + "hgnc_id": "HGNC:9453", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM246C", + "hgnc_id": "HGNC:54842", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "CCDC188", + "hgnc_id": "HGNC:51899", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RTN4R", + "hgnc_id": "HGNC:18601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3FP", + "hgnc_id": "HGNC:24067", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LRRC74B", + "hgnc_id": "HGNC:34301", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM246A", + "hgnc_id": "HGNC:54844", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RIMBP3B", + "hgnc_id": "HGNC:33891", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HIC2", + "hgnc_id": "HGNC:18595", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TMEM191C", + "hgnc_id": "HGNC:33601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GGTLC2", + "hgnc_id": "HGNC:18596", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IGLL1", + "hgnc_id": "HGNC:5870", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRICH1", + "hgnc_id": "HGNC:28031", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTT2B", + "hgnc_id": "HGNC:33437", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DDTL", + "hgnc_id": "HGNC:33446", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GSTT2", + "hgnc_id": "HGNC:4642", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL1", + "hgnc_id": "HGNC:9977", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C22orf42", + "hgnc_id": "HGNC:27160", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RFPL3", + "hgnc_id": "HGNC:9980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3D", + "hgnc_id": "HGNC:17354", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOBEC3H", + "hgnc_id": "HGNC:24100", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ATF4", + "hgnc_id": "HGNC:786", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NDUFA6", + "hgnc_id": "HGNC:7690", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2D7", + "hgnc_id": "HGNC:2624", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RRP7A", + "hgnc_id": "HGNC:24286", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYB5R3", + "hgnc_id": "HGNC:2873", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "A4GALT", + "hgnc_id": "HGNC:18149", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TTLL1", + "hgnc_id": "HGNC:1312", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MLC1", + "hgnc_id": "HGNC:17082", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBGCP6", + "hgnc_id": "HGNC:18127", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACR", + "hgnc_id": "HGNC:126", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "XG", + "hgnc_id": "HGNC:12806", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GYG2", + "hgnc_id": "HGNC:4700", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ARSD", + "hgnc_id": "HGNC:717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MXRA5", + "hgnc_id": "HGNC:7539", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRKX", + "hgnc_id": "HGNC:9441", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "VCX3A", + "hgnc_id": "HGNC:18159", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "VCX2", + "hgnc_id": "HGNC:18158", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VCX3B", + "hgnc_id": "HGNC:31838", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM9A", + "hgnc_id": "HGNC:18403", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM9C", + "hgnc_id": "HGNC:18405", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SUPT20HL2", + "hgnc_id": "HGNC:31797", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SUPT20HL1", + "hgnc_id": "HGNC:30773", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "MAGEB6", + "hgnc_id": "HGNC:23796", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DCAF8L1", + "hgnc_id": "HGNC:31810", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM47B", + "hgnc_id": "HGNC:26659", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "H2AL1Q", + "hgnc_id": "HGNC:53959", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "FAM47C", + "hgnc_id": "HGNC:25301", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "H2AL3", + "hgnc_id": "HGNC:53960", + "reasons": [ + "segdup" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "SPACA5B", + "hgnc_id": "HGNC:19142", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SSX5", + "hgnc_id": "HGNC:11339", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SSX1", + "hgnc_id": "HGNC:11335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSX3", + "hgnc_id": "HGNC:11337", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SSX4", + "hgnc_id": "HGNC:11338", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX4B", + "hgnc_id": "HGNC:16880", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12J", + "hgnc_id": "HGNC:17778", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE13", + "hgnc_id": "HGNC:29081", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE2E", + "hgnc_id": "HGNC:31960", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE12B", + "hgnc_id": "HGNC:26779", + "reasons": [ + "segdup" + ], + "share": 0.4957, + "status": "not read" + }, + { + "gene": "GAGE12C", + "hgnc_id": "HGNC:28402", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GAGE12D", + "hgnc_id": "HGNC:31904", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12F", + "hgnc_id": "HGNC:31906", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE12G", + "hgnc_id": "HGNC:31907", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE12H", + "hgnc_id": "HGNC:31908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE2A", + "hgnc_id": "HGNC:4099", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GAGE1", + "hgnc_id": "HGNC:4098", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CENPVL3", + "hgnc_id": "HGNC:43880", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CENPVL1", + "hgnc_id": "HGNC:31851", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MAGED4B", + "hgnc_id": "HGNC:22880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MAGED4", + "hgnc_id": "HGNC:23793", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE1A", + "hgnc_id": "HGNC:4111", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "XAGE1B", + "hgnc_id": "HGNC:25400", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX7", + "hgnc_id": "HGNC:19653", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSX2B", + "hgnc_id": "HGNC:22263", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN5", + "hgnc_id": "HGNC:33178", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "XAGE5", + "hgnc_id": "HGNC:30930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE3", + "hgnc_id": "HGNC:14618", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM156B", + "hgnc_id": "HGNC:31962", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM156A", + "hgnc_id": "HGNC:30114", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPIN2B", + "hgnc_id": "HGNC:33147", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPIN2A", + "hgnc_id": "HGNC:20694", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZXDA", + "hgnc_id": "HGNC:13198", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CXorf49", + "hgnc_id": "HGNC:30891", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CXorf49B", + "hgnc_id": "HGNC:34229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "DMRTC1B", + "hgnc_id": "HGNC:31686", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM236D", + "hgnc_id": "HGNC:52642", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DMRTC1", + "hgnc_id": "HGNC:13910", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM236C", + "hgnc_id": "HGNC:52641", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM236A", + "hgnc_id": "HGNC:44268", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PABPC1L2B", + "hgnc_id": "HGNC:31852", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PABPC1L2A", + "hgnc_id": "HGNC:27989", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TGIF2LX", + "hgnc_id": "HGNC:18570", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDH11X", + "hgnc_id": "HGNC:8656", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TCEAL6", + "hgnc_id": "HGNC:24553", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "TCP11X1", + "hgnc_id": "HGNC:48369", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NXF2", + "hgnc_id": "HGNC:8072", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NXF2B", + "hgnc_id": "HGNC:23984", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB40AL", + "hgnc_id": "HGNC:25410", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TMSB15B", + "hgnc_id": "HGNC:28612", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BW1", + "hgnc_id": "HGNC:27252", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "H2BW2", + "hgnc_id": "HGNC:27867", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "TMSB15C", + "hgnc_id": "HGNC:55173", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PSMD10", + "hgnc_id": "HGNC:9555", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CT47C1", + "hgnc_id": "HGNC:53820", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHOXF2B", + "hgnc_id": "HGNC:33519", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RHOXF2", + "hgnc_id": "HGNC:30011", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT47B1", + "hgnc_id": "HGNC:33293", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A12", + "hgnc_id": "HGNC:33292", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT47A11", + "hgnc_id": "HGNC:27397", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CT47A9", + "hgnc_id": "HGNC:33290", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A8", + "hgnc_id": "HGNC:33289", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "CT47A7", + "hgnc_id": "HGNC:33288", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT47A6", + "hgnc_id": "HGNC:33287", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A5", + "hgnc_id": "HGNC:33286", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CT47A4", + "hgnc_id": "HGNC:33285", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT47A2", + "hgnc_id": "HGNC:33283", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A1", + "hgnc_id": "HGNC:33282", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "RTL8B", + "hgnc_id": "HGNC:33156", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "RTL8C", + "hgnc_id": "HGNC:2569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RTL8A", + "hgnc_id": "HGNC:24514", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "SMIM10L2B", + "hgnc_id": "HGNC:34500", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT55", + "hgnc_id": "HGNC:26047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ETDA", + "hgnc_id": "HGNC:53449", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "SMIM10L2A", + "hgnc_id": "HGNC:34499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT45A1", + "hgnc_id": "HGNC:33267", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A3", + "hgnc_id": "HGNC:33268", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT45A5", + "hgnc_id": "HGNC:33270", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT45A2", + "hgnc_id": "HGNC:28400", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT45A7", + "hgnc_id": "HGNC:51260", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CT45A8", + "hgnc_id": "HGNC:51261", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT45A9", + "hgnc_id": "HGNC:51262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A10", + "hgnc_id": "HGNC:51263", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SAGE1", + "hgnc_id": "HGNC:30369", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPANXC", + "hgnc_id": "HGNC:14331", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXA1", + "hgnc_id": "HGNC:11218", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "SPANXA2", + "hgnc_id": "HGNC:14328", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "SPANXD", + "hgnc_id": "HGNC:14332", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "MAGEC1", + "hgnc_id": "HGNC:6812", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPANXN3", + "hgnc_id": "HGNC:33176", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UBE2NL", + "hgnc_id": "HGNC:31710", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN1", + "hgnc_id": "HGNC:33174", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CXorf51B", + "hgnc_id": "HGNC:42787", + "reasons": [ + "segdup" + ], + "share": 0.3323, + "status": "not read" + }, + { + "gene": "CXorf51A", + "hgnc_id": "HGNC:30533", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "EOLA1", + "hgnc_id": "HGNC:28089", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HSFX3", + "hgnc_id": "HGNC:52395", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HSFX2", + "hgnc_id": "HGNC:32701", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSFX1", + "hgnc_id": "HGNC:29603", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MAGEA9", + "hgnc_id": "HGNC:6807", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "EOLA2", + "hgnc_id": "HGNC:17402", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSFX4", + "hgnc_id": "HGNC:52398", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAGEA3", + "hgnc_id": "HGNC:6801", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MAGEA2B", + "hgnc_id": "HGNC:19340", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CSAG1", + "hgnc_id": "HGNC:24294", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "MAGEA12", + "hgnc_id": "HGNC:6799", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "MAGEA2", + "hgnc_id": "HGNC:6800", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CSAG3", + "hgnc_id": "HGNC:26237", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MAGEA6", + "hgnc_id": "HGNC:6804", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PWWP4", + "hgnc_id": "HGNC:55197", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A8", + "hgnc_id": "HGNC:11055", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BCAP31", + "hgnc_id": "HGNC:16695", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OPN1LW", + "hgnc_id": "HGNC:9936", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OPN1MW", + "hgnc_id": "HGNC:4206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OPN1MW2", + "hgnc_id": "HGNC:26952", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEX28", + "hgnc_id": "HGNC:2563", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAG1A", + "hgnc_id": "HGNC:24198", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "CTAG1B", + "hgnc_id": "HGNC:2491", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAG2", + "hgnc_id": "HGNC:2492", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H2AB1", + "hgnc_id": "HGNC:22516", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "F8A1", + "hgnc_id": "HGNC:3547", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "F8A2", + "hgnc_id": "HGNC:31849", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "F8A3", + "hgnc_id": "HGNC:31850", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "H2AB3", + "hgnc_id": "HGNC:14455", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IL9R", + "hgnc_id": "HGNC:6030", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASH6P", + "hgnc_id": "HGNC:31685", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "TGIF2LY", + "hgnc_id": "HGNC:18569", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "PCDH11Y", + "hgnc_id": "HGNC:15813", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY2", + "hgnc_id": "HGNC:23924", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY4", + "hgnc_id": "HGNC:37287", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY8", + "hgnc_id": "HGNC:37471", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY3", + "hgnc_id": "HGNC:33876", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY1", + "hgnc_id": "HGNC:12381", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY9", + "hgnc_id": "HGNC:37472", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY10", + "hgnc_id": "HGNC:37473", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY", + "hgnc_id": "HGNC:12668", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY1B", + "hgnc_id": "HGNC:31751", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2B", + "hgnc_id": "HGNC:23921", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2A", + "hgnc_id": "HGNC:1810", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY1", + "hgnc_id": "HGNC:18568", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY2", + "hgnc_id": "HGNC:23950", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1B", + "hgnc_id": "HGNC:23914", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1A1", + "hgnc_id": "HGNC:9912", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1D", + "hgnc_id": "HGNC:23915", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1E", + "hgnc_id": "HGNC:23916", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1F", + "hgnc_id": "HGNC:23974", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1J", + "hgnc_id": "HGNC:23917", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2", + "hgnc_id": "HGNC:13508", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ1", + "hgnc_id": "HGNC:2682", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ2", + "hgnc_id": "HGNC:15964", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1B", + "hgnc_id": "HGNC:23920", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2B", + "hgnc_id": "HGNC:25449", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ3", + "hgnc_id": "HGNC:15965", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ4", + "hgnc_id": "HGNC:15966", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2C", + "hgnc_id": "HGNC:18225", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1", + "hgnc_id": "HGNC:1809", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "C4B_2", + "hgnc_id": "HGNC:42398", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTT1", + "hgnc_id": "HGNC:4641", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTTP2", + "hgnc_id": "HGNC:33606", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GTF2H2C_2", + "hgnc_id": "HGNC:35418", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB3", + "hgnc_id": "HGNC:4951", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB4", + "hgnc_id": "HGNC:4952", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL2", + "hgnc_id": "HGNC:6330", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5A", + "hgnc_id": "HGNC:16345", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5B", + "hgnc_id": "HGNC:16346", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS1", + "hgnc_id": "HGNC:6333", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS2", + "hgnc_id": "HGNC:6334", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS3", + "hgnc_id": "HGNC:6335", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS5", + "hgnc_id": "HGNC:6337", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR3DS1", + "hgnc_id": "HGNC:6340", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "LILRA3", + "hgnc_id": "HGNC:6604", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U8", + "hgnc_id": "HGNC:27538", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U9", + "hgnc_id": "HGNC:29166", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR9G9", + "hgnc_id": "HGNC:31940", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TAS2R45", + "hgnc_id": "HGNC:18876", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + } + ], + "not_read_owned": [ + { + "gene": "RHCE", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "AMY1A", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AMY1B", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AMY1C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GSTM1", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FCGR1A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GBA1", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FCGR2A", + "owner": "parascopy", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FCGR2C", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "FCGR3B", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR3", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR1", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PROS1", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B17", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT2B15", + "owner": "pypgx", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPE", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPB", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GTF2H2C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMN2", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMN1", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2H2", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSPA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "C4A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "C4B", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP21A2", + "owner": "cn_loci:CYP21A2", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LPA", + "owner": "cn_loci:LPA_KIV2", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PMS2", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2I", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DEFA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CEL", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC5B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KLRC2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KRT6B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRT6C", + "owner": "parascopy", + "share": 0.1665, + "status": "not read" + }, + { + "gene": "KRT6A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRFAM7A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "STRC", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SORD", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HBA2", + "owner": "cn_loci:HBA", + "share": 0.5, + "status": "not read" + }, + { + "gene": "HBA1", + "owner": "cn_loci:HBA", + "share": 0.6659, + "status": "not read" + }, + { + "gene": "TPSB2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TPSAB1", + "owner": "parascopy", + "share": 0.3331, + "status": "not read" + }, + { + "gene": "OTOA", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SULT1A1", + "owner": "pypgx", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HYDIN", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HP", + "owner": "cn_loci:HP", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CCL3L3", + "owner": "cn_loci:CCL3L1", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A6", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP2B6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD177", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "APOBEC3B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP2D6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + } + ], + "not_on_reference": [ + "TGIF2LY", + "PCDH11Y", + "TSPY2", + "TSPY4", + "TSPY8", + "TSPY3", + "TSPY1", + "TSPY9", + "TSPY10", + "VCY", + "VCY1B", + "CDY2B", + "CDY2A", + "HSFY1", + "HSFY2", + "RBMY1B", + "RBMY1A1", + "RBMY1D", + "RBMY1E", + "RBMY1F", + "RBMY1J", + "BPY2", + "DAZ1", + "DAZ2", + "CDY1B", + "BPY2B", + "DAZ3", + "DAZ4", + "BPY2C", + "CDY1", + "C4B_2", + "GSTT1", + "GSTTP2", + "GTF2H2C_2", + "HLA-DRB3", + "HLA-DRB4", + "KIR2DL2", + "KIR2DL5A", + "KIR2DL5B", + "KIR2DS1", + "KIR2DS2", + "KIR2DS3", + "KIR2DS5", + "KIR3DS1", + "LILRA3", + "OR8U8", + "OR8U9", + "OR9G9", + "TAS2R45" + ] + } + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "ok", + "source": "indexcov/indexcov-indexcov.ped", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "inferred_sex": "male", + "cn_chrX": "1.01", + "cn_chrY": "0.98", + "karyotype": null + } + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "declared_sex": "male", + "mosaic_call": "not_detected", + "mosaic_depth_ratio": "0.98", + "mosaic_depth_ratio_expected": "1", + "mosaic_loss_depth": "0.02", + "mosaic_loss_depth_low": "-0.01", + "mosaic_loss_depth_high": "0.05", + "mosaic_loss_af": "NA", + "mosaic_floor": "0.1", + "mosaic_x_het_sites": "0", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "call": "not_detected", + "depth_ratio": 0.98, + "depth_ratio_expected": 1.0, + "loss_depth": 0.02, + "loss_depth_low": -0.01, + "loss_depth_high": 0.05, + "loss_af": null, + "floor": 0.1, + "x_het_sites": 0, + "loss_af_low": null, + "loss_af_high": null + } + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "ok", + "source": "qc/S_read_count.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "input_reads": 1000, + "primary_records": 1000, + "missing_reads": 0, + "read_count_check": "ok" + } + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "ok", + "source": "vcf/S.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1, + "snps": 0, + "indels": 1 + } + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 8, + "by_stars": { + "4": 0, + "3": 2, + "2": 3, + "1": 1, + "0": 2 + }, + "hits": [ + { + "stars": 4, + "chrom": "chr7", + "pos": 117559590, + "ref": "ATCT", + "alt": "A", + "genotype": "het", + "gene": "CFTR", + "significance": "Pathogenic", + "review_status": "practice guideline", + "combined_allele": ".", + "dp": 30, + "gq": 15, + "low_confidence": true, + "gnomad_exome_af": 0.008, + "gnomad_genome_af": 0.007, + "rsids": [ + "rs113993960" + ], + "af": 0.008, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr17", + "pos": 43057100, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "BRCA1,NBR2", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": "part of a combined allele: chr17:43057100 C>T, chr17:43057102 A>G; unphased, so the parts may sit on different haplotypes", + "dp": 35, + "gq": 60, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr6", + "pos": 26092913, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HFE", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": ".", + "dp": 40, + "gq": 99, + "low_confidence": false, + "gnomad_exome_af": 0.06, + "gnomad_genome_af": 0.05, + "rsids": [ + "rs99", + "rs1800562" + ], + "af": 0.06, + "common": true, + "cdna": { + "check": "clean", + "note": "", + "alt_reads": 20, + "alt_clipped": 0, + "ref_reads": 20, + "ref_clipped": 0, + "construct": null + } + }, + { + "stars": 2, + "chrom": "chr1", + "pos": 45331556, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "MUTYH", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 30, + "gq": 50, + "low_confidence": false, + "gnomad_exome_af": 0.002, + "gnomad_genome_af": 0.0005, + "rsids": [ + "rs36053993" + ], + "af": 0.002, + "common": false, + "cdna": { + "check": "flagged", + "note": "possible cDNA/transgene contamination: check the reads", + "alt_reads": 12, + "alt_clipped": 6, + "ref_reads": 10, + "ref_clipped": 0, + "construct": "yes" + } + }, + { + "stars": 2, + "chrom": "chr16", + "pos": 176700, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HBA1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 2, + "chrom": "chr5", + "pos": 70951946, + "ref": "C", + "alt": "T", + "genotype": "hom", + "gene": "SMN1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr13", + "pos": 20189473, + "ref": "C", + "alt": "T", + "genotype": "one copy, other copy not called ref (1/.)", + "gene": "GJB2", + "significance": "Likely pathogenic", + "review_status": "criteria provided, single submitter", + "combined_allele": "combined allele of chr13:20189473 C>T + chr13:20189475 G>A", + "dp": 5, + "gq": 40, + "low_confidence": true, + "gnomad_exome_af": 0.03, + "gnomad_genome_af": null, + "rsids": [ + "rs80338939" + ], + "af": 0.03, + "common": true, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr3", + "pos": 15645186, + "ref": "G", + "alt": "C", + "genotype": "het", + "gene": "BTD", + "significance": "Pathogenic/Likely pathogenic", + "review_status": "criteria provided, conflicting classifications", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chr2", + "pos": 100, + "ref": "A", + "alt": "G", + "genotype": "het", + "gene": ".", + "significance": ".", + "review_status": ".", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chrX", + "pos": 101398000, + "ref": "C", + "alt": "T", + "genotype": "1", + "gene": "GLA", + "significance": "Pathogenic", + "review_status": "no assertion criteria provided", + "combined_allele": "possibly one allele with chrX:101398004 G>A: check ClinVar for a combined record", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "low_confidence": 2, + "call_floor": { + "GQ": 20, + "DP": 8 + }, + "frequency": { + "source": "vep/S_vep.vcf.gz", + "note": null, + "common_af": 0.01 + }, + "cdna_flagged": 1, + "construct_sites": 1, + "cdna_scan": { + "note": "2 genes with spliced reads", + "loci": [ + { + "chrom": "chr1", + "start": 100, + "end": 200, + "junctions": 2, + "reads": 5, + "junction_list": "A:1-2,A:3-4" + } + ] + }, + "deep_intronic": [ + { + "chrom": "chr7", + "pos": 117639961, + "ref": "C", + "alt": "T", + "gene": "CFTR", + "hgvs": "c.3718-2477C>T", + "clinvar_id": "7129", + "call": "no call", + "detail": "" + } + ] + } + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_on_list": 3292, + "list": "assets/carrier_genes.tsv", + "carriers": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "MUTYH-related polyposis" + }, + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Symptomatic form of HFE-related hemochromatosis" + }, + { + "gene": "CFTR", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 4, + "inheritance": "AR", + "condition": "Cystic fibrosis" + }, + { + "gene": "GJB2", + "variant": "chr13:20189473 C>T", + "genotype": "one copy, other copy not called ref (1/.)", + "significance": "Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Nonsyndromic hearing loss, recessive (DFNB1A)" + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Fanconi anemia" + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 0, + "inheritance": "XL", + "condition": "Fabry disease" + }, + { + "gene": "HBA1", + "variant": "chr16:176700 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Alpha-thalassemia" + }, + { + "gene": "SMN1", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Spinal muscular atrophy", + "hemizygous": "called homozygous inside one SMN1 copy (Parascopy, step 35)" + } + ], + "set_aside": [ + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "significance": "Pathogenic/Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Biotinidase deficiency", + "kind": "hypomorph", + "name": "p.Asp444His (p.Asp424His on NM_001370658.1), partial biotinidase deficiency" + } + ], + "two_hits": { + "genes": [ + { + "gene": "SMN1", + "verdict": "biallelic", + "why": "the variant is called homozygous inside the deletion: it is hemizygous, on the one copy the deletion leaves", + "deletion": "one SMN1 copy (Parascopy, step 35)", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "allele": null, + "condition": "Spinal muscular atrophy", + "inheritance": "AR" + } + ], + "deletions_read": [ + "AnnotSV class 4 and 5 deletions (step 05)", + "step 35's interval table (read depth)", + "Parascopy's SMN1 count (step 35)" + ], + "deletions_not_read": [ + "step 35's HBA1/HBA2 count: not called" + ] + } + } + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "ok", + "source": "pharmcat/S.report.json", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "version": "3.4.0", + "html_report": true + } + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "ok", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "gene": "UGT1A1", + "diplotype": "*1/*28", + "phenotype": "Intermediate", + "status": "non-normal" + } + ], + "non_normal": 1, + "ambiguous": 0, + "not_called": 0, + "unclassified": 0, + "parse_failed": false, + "warnings": [ + "CYP2C19 has two possible phenotypes", + "last" + ] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "ok", + "source": "pypgx/S_pypgx_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_total": 6, + "genes_called": 3, + "cyp2d6": "*1/*2", + "cnv_call": { + "CYP2D6": "Normal", + "CYP2A6": "not called: pypgx failed", + "CYP2B6": "not called: the CYP2D6 depth check failed", + "SLCO1B1": "not called: pypgx gave no copy-number call" + }, + "not_callable": [ + "CYP2E1" + ], + "low_gq": [ + { + "gene": "UGT1A1", + "diplotype": "Indeterminate", + "min_gq": 12, + "weakest": "chr2:1:A:G" + }, + { + "gene": "SLCO1B1", + "diplotype": "*1/*5", + "min_gq": null, + "weakest": "" + } + ], + "cyp2d6_evidence": { + "phenotype": "Normal Metabolizer", + "defining_filter": "PASS", + "quality": "ok" + }, + "depth_check": { + "status": "ok", + "gene_mapq1_share": "0.99", + "body_mapq20_share": "0.95" + }, + "comparison": { + "conflicts": 1, + "one_tool_only": 2, + "concordant": 1 + } + } + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/S_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "*2/*1", + "filter": "PASS" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": 1, + "text": "1 (quality 31, filter PASS)", + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": 3, + "text": "3 (quality 25, filter PASS)" + }, + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "one SMN1 copy: the pattern of an SMA carrier", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "called", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality 50, filter PASS); chr7:5980001-5990000 3 (quality 12, filter LowQual)" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": 1, + "text": "1 (quality 31, filter PASS)" + }, + "agree": "disagree" + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01:01", + "A*01:01:01" + ], + "quality": [ + "60", + "50" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-B", + "alleles": [ + "B*57:01:01:02", + "B*08:01:01,B*08:01:02,B*08:01:03" + ], + "quality": [ + "0", + "0" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1, + 5 + ], + "one_allele_type": false + }, + { + "gene": "HLA-C", + "alleles": [ + "C*07:01:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*05:01:01", + "DQA1*02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01:01", + "DQB1*02:02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*03:01:01", + "DRB1*07:01:01" + ], + "quality": [ + "40", + "nan" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HFE", + "alleles": [ + "HFE*001" + ], + "quality": [ + "20" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true, + "label": "T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the variant calls (ClinVar section)" + } + ], + "derived": { + "coeliac": { + "status": "derived", + "reason": "", + "dq2_5": "present", + "dq2_5_phase": "likely cis (DRB1*03 present)", + "dq2_2": "present", + "dq8": "absent", + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": 0.95, + "low": 0.93, + "high": 0.97 + }, + { + "code": "AFR", + "group": "Africa", + "fraction": 0.05, + "low": null, + "high": null + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": 0.6, + "low": 0.5, + "high": 0.7 + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": null, + "low": null, + "high": null + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": null, + "low": null, + "high": null + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 8, + "deletions": 5, + "duplications": 3, + "significant": 7, + "significant_filtered": 4, + "filtered_chry": 1, + "dropped": { + "q0": 1, + "gap": 1, + "band": 1 + }, + "declared_sex": "male" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "depth": 30.0, + "read_floor": 2000, + "low_reads": true + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 3, + "total_mb": 37.5, + "largest_mb": 27.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + }, + { + "region": "chr15:23000001-50000000", + "mb": 27.0 + } + ], + "masked_mb": 46.5, + "sex_chromosome_segments": 1, + "long_mb": 34.0, + "f_roh": 0.0128, + "relationship": null, + "upd_flag": { + "chrom": "chr15", + "mb": 27.0, + "share": 0.79, + "imprinted": [ + { + "region": "15q11-q13 (SNRPN, UBE3A)", + "maternal_upd": "Prader-Willi syndrome", + "paternal_upd": "Angelman syndrome", + "pmid": "22237428;27615419" + } + ] + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*03:01:01/DRB1*07:01:01" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32): *1/*2; Cyrius (step 21): *2/*1" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + }, + { + "hgvs": "m.8344A>G", + "gene": "MT-TK", + "disease": "MERRF (myoclonic epilepsy with ragged-red fibres)", + "clinvar_id": "9579", + "pmids": "2112427", + "note": "Usually heteroplasmic; the level differs between tissues.", + "pos": 8344, + "filter": "weak_evidence", + "af": "0.30" + }, + { + "hgvs": "m.11778G>A", + "gene": "MT-ND4", + "disease": "Leber hereditary optic neuropathy (LHON)", + "clinvar_id": "9708", + "pmids": "3201231", + "note": "Usually homoplasmic; not everyone who carries it loses vision.", + "pos": 11778, + "filter": "PASS", + "af": "0.99" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": 0.5, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + }, + { + "bkp5": 100, + "bkp3": 200, + "length": null, + "level_pct": 2.5 + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 7, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [ + { + "variant": "chrM:302 A>AC,ACC", + "af": 0.3 + }, + { + "variant": "chrM:16189 T>C", + "af": 0.5 + } + ], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3", + "gene": "X", + "trait": "x", + "genotype": "C/T", + "status": "low_gq", + "reading": "A low-quality call at this position (genotype quality 5, below 20): not read; check the reads.", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "low_gq": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "30.5", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TET2", + "mean_depth": "40", + "detection_limit_vaf": "0.08", + "status": "covered" + }, + { + "gene": "ASXL1", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + }, + { + "gene": "JAK2", + "mean_depth": "35", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TP53", + "mean_depth": "25", + "detection_limit_vaf": "0.1", + "status": "covered" + } + ], + "median_depth": 32.75, + "median_limit": 0.12901665405107993, + "not_covered": [ + "ASXL1" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "ok", + "source": "cpsr/S.cpsr.grch38.classification.tsv.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "html_report": true, + "classification_column": "CLASSIFICATION", + "classification": { + "Pathogenic": 1, + "Likely_Pathogenic": 1, + "VUS": 1, + "Benign": 1, + ".": 1 + }, + "plp": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "13:32338000 G>A", + "gene": "BRCA2", + "final": "Likely_Pathogenic", + "cpsr": "VUS", + "clinvar": null, + "authority": "CPSR", + "acmg_codes": null, + "dp": 40, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "5:112839000 A>T", + "gene": "APC", + "final": ".", + "cpsr": "Pathogenic", + "clinvar": null, + "authority": null, + "acmg_codes": null, + "dp": 30, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "disagreements": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "final_plp_own_benign": 1, + "own_class_column": "CPSR_CLASSIFICATION", + "low_confidence": 1, + "call_floor_dp": 8, + "cdna_flagged": 1 + } + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 5, + "by_impact": { + "HIGH": 3, + "MODERATE": 1, + ".": 1 + }, + "genes": 3, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "low_confidence": false, + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "low_confidence": false, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "1", + "significance": "Pathogenic", + "stars": 0, + "low_confidence": false, + "rule": "All hemi, het, homozygous P and LP", + "rule_note": null, + "severity": null + } + ], + "clinvar_checked": true, + "high_impact": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "consequence": "stop_gained", + "transcript": "MANE", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": 40, + "dp": 30, + "ad": "15,15", + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 2, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": true, + "structural_note": null, + "not_reported": [ + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "source": "ClinVar", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + }, + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "source": "ClinVar", + "reason": "one heterozygous variant; BTD is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + }, + "frequency_unknown": 1, + "weak_calls": 2 + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_slivar_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 2, + "frequency_unknown": 1, + "compound_het_variants": 4, + "compound_het_genes": 2, + "compound_het_pairs": 26, + "compound_het_phase": { + "trans": 23, + "cis": 1, + "unknown": 1 + }, + "compound_het_trans": [ + { + "gene": "G1", + "variants": "chr1:10 A>G + chr1:15 C>T", + "reads": "1 of 2" + }, + { + "gene": "G2", + "variants": "chr1:20 A>G + chr1:25 C>T", + "reads": "2 of 3" + }, + { + "gene": "G3", + "variants": "chr1:30 A>G + chr1:35 C>T", + "reads": "3 of 4" + }, + { + "gene": "G4", + "variants": "chr1:40 A>G + chr1:45 C>T", + "reads": "4 of 5" + }, + { + "gene": "G5", + "variants": "chr1:50 A>G + chr1:55 C>T", + "reads": "5 of 6" + }, + { + "gene": "G6", + "variants": "chr1:60 A>G + chr1:65 C>T", + "reads": "6 of 7" + }, + { + "gene": "G7", + "variants": "chr1:70 A>G + chr1:75 C>T", + "reads": "7 of 8" + }, + { + "gene": "G8", + "variants": "chr1:80 A>G + chr1:85 C>T", + "reads": "8 of 9" + }, + { + "gene": "G9", + "variants": "chr1:90 A>G + chr1:95 C>T", + "reads": "9 of 10" + }, + { + "gene": "G10", + "variants": "chr1:100 A>G + chr1:105 C>T", + "reads": "10 of 11" + }, + { + "gene": "G11", + "variants": "chr1:110 A>G + chr1:115 C>T", + "reads": "11 of 12" + }, + { + "gene": "G12", + "variants": "chr1:120 A>G + chr1:125 C>T", + "reads": "12 of 13" + }, + { + "gene": "G13", + "variants": "chr1:130 A>G + chr1:135 C>T", + "reads": "13 of 14" + }, + { + "gene": "G14", + "variants": "chr1:140 A>G + chr1:145 C>T", + "reads": "14 of 15" + }, + { + "gene": "G15", + "variants": "chr1:150 A>G + chr1:155 C>T", + "reads": "15 of 16" + }, + { + "gene": "G16", + "variants": "chr1:160 A>G + chr1:165 C>T", + "reads": "16 of 17" + }, + { + "gene": "G17", + "variants": "chr1:170 A>G + chr1:175 C>T", + "reads": "17 of 18" + }, + { + "gene": "G18", + "variants": "chr1:180 A>G + chr1:185 C>T", + "reads": "18 of 19" + }, + { + "gene": "G19", + "variants": "chr1:190 A>G + chr1:195 C>T", + "reads": "19 of 20" + }, + { + "gene": "G20", + "variants": "chr1:200 A>G + chr1:205 C>T", + "reads": "20 of 21" + } + ] + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": "*1/*2", + "Cyrius": "*2/*1" + }, + "agree": true, + "consensus": { + "result": "*1/*2", + "passed_to_pharmcat": true, + "reason": "pypgx and Cyrius agree", + "source": "pgx_consensus/S_pgx_consensus.tsv" + }, + "probable": null + }, + "not_run": [], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)" + ], + "recheck": { + "checked_utc": "2021-03-05 00:00:00 UTC", + "items": [ + { + "item": "clinvar", + "label": "ClinVar", + "class": "newer-release", + "ran_with": "2021-03-01", + "upstream": "2021-03-04", + "action": "rerun step 06" + }, + { + "item": "vep", + "label": "VEP", + "class": "no-newer-release", + "ran_with": null, + "upstream": null, + "action": null + } + ], + "affected_steps": [ + "06" + ], + "rerun": { + "command": "run-all.sh" + } + }, + "new_since": { + "previous_utc": "2021-01-01 00:00 UTC", + "sections": [ + "ancestry_fine", + "ancient", + "annotsv", + "archaic", + "bloodtype", + "callability", + "carrier", + "chip", + "clinical", + "cnvpytor", + "consumer_export", + "couple", + "coverage", + "cpsr", + "cyrius", + "delly", + "exomiser", + "expansions", + "haplogroup", + "hla", + "locityper", + "lpa", + "manta", + "mito", + "mosaic", + "mosaic_sex_loss", + "paralogs", + "parascopy", + "pharmcat", + "read_count", + "roh", + "sample_qc", + "sex_check", + "slivar", + "sv_consensus", + "telomere", + "traits", + "variants", + "viral", + "vntyper", + "y_haplogroup", + "y_str", + "y_tree", + "clinvar", + "cpic", + "pypgx", + "prs" + ] + } +} diff --git a/tests/parity/golden/summary-collect/all-nf/all-nf.json b/tests/parity/golden/summary-collect/all-nf/all-nf.json new file mode 100644 index 000000000..843b6f4eb --- /dev/null +++ b/tests/parity/golden/summary-collect/all-nf/all-nf.json @@ -0,0 +1,14035 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:15 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "female" + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "coverage/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": NaN + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "ok", + "source": "coverage/S_callability.bed.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "min_depth": 10, + "min_mapq": 20, + "min_share": 0.9, + "catalog_genes": 1572, + "read": 217, + "not_read": [ + { + "gene": "OR4F5", + "hgnc_id": "HGNC:14825", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR4F29", + "hgnc_id": "HGNC:31275", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F16", + "hgnc_id": "HGNC:15079", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "AGRN", + "hgnc_id": "HGNC:329", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ATAD3B", + "hgnc_id": "HGNC:24007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MMP23B", + "hgnc_id": "HGNC:7171", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CDK11B", + "hgnc_id": "HGNC:1729", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35E2B", + "hgnc_id": "HGNC:33941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDK11A", + "hgnc_id": "HGNC:1730", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TTC34", + "hgnc_id": "HGNC:34297", + "reasons": [ + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPL22", + "hgnc_id": "HGNC:10315", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MASP2", + "hgnc_id": "HGNC:6902", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPPA", + "hgnc_id": "HGNC:7939", + "reasons": [ + "CMRG" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KIAA2013", + "hgnc_id": "HGNC:28513", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF1", + "hgnc_id": "HGNC:28840", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF11", + "hgnc_id": "HGNC:14086", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL1", + "hgnc_id": "HGNC:29295", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRAMEF4", + "hgnc_id": "HGNC:31971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF10", + "hgnc_id": "HGNC:27997", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRAMEF7", + "hgnc_id": "HGNC:28415", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF6", + "hgnc_id": "HGNC:30583", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF27", + "hgnc_id": "HGNC:51234", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL3", + "hgnc_id": "HGNC:51235", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HNRNPCL2", + "hgnc_id": "HGNC:48813", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF26", + "hgnc_id": "HGNC:49178", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HNRNPCL4", + "hgnc_id": "HGNC:51333", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "PRAMEF9", + "hgnc_id": "HGNC:27996", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF13", + "hgnc_id": "HGNC:13262", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRAMEF18", + "hgnc_id": "HGNC:30693", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRAMEF8", + "hgnc_id": "HGNC:24074", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF33", + "hgnc_id": "HGNC:49193", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PRAMEF15", + "hgnc_id": "HGNC:26764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRAMEF14", + "hgnc_id": "HGNC:13576", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF19", + "hgnc_id": "HGNC:24908", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "PRAMEF17", + "hgnc_id": "HGNC:29485", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CELA2B", + "hgnc_id": "HGNC:29995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CLCNKA", + "hgnc_id": "HGNC:2026", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CLCNKB", + "hgnc_id": "HGNC:2027", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NBPF1", + "hgnc_id": "HGNC:26088", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CROCC", + "hgnc_id": "HGNC:21299", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AKR7L", + "hgnc_id": "HGNC:24056", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF3", + "hgnc_id": "HGNC:25076", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CELA3B", + "hgnc_id": "HGNC:15945", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CELA3A", + "hgnc_id": "HGNC:15944", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HMGCL", + "hgnc_id": "HGNC:5005", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNR2", + "hgnc_id": "HGNC:2160", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RSRP1", + "hgnc_id": "HGNC:25234", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGV", + "hgnc_id": "HGNC:26031", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "XKR8", + "hgnc_id": "HGNC:25508", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BMP8A", + "hgnc_id": "HGNC:21650", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BMP8B", + "hgnc_id": "HGNC:1075", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OXCT2", + "hgnc_id": "HGNC:18606", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CYP4Z1", + "hgnc_id": "HGNC:20583", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4A22", + "hgnc_id": "HGNC:20575", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GBP1", + "hgnc_id": "HGNC:4182", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SETSIP", + "hgnc_id": "HGNC:42937", + "reasons": [ + "segdup" + ], + "share": 0.4994, + "status": "not read" + }, + { + "gene": "AGL", + "hgnc_id": "HGNC:321", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "EXTL2", + "hgnc_id": "HGNC:3516", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AMY2A", + "hgnc_id": "HGNC:477", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF4", + "hgnc_id": "HGNC:26550", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NBPF6", + "hgnc_id": "HGNC:31988", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LRIG2", + "hgnc_id": "HGNC:20889", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "VANGL1", + "hgnc_id": "HGNC:15512", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IGSF3", + "hgnc_id": "HGNC:5950", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SEC22B", + "hgnc_id": "HGNC:10700", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF26", + "hgnc_id": "HGNC:49571", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2NLR", + "hgnc_id": "HGNC:53925", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4A", + "hgnc_id": "HGNC:24369", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM72B", + "hgnc_id": "HGNC:24805", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SRGAP2C", + "hgnc_id": "HGNC:30584", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3-7", + "hgnc_id": "HGNC:32060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPIAL4E", + "hgnc_id": "HGNC:33997", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF15", + "hgnc_id": "HGNC:28791", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4F", + "hgnc_id": "HGNC:33999", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "SRGAP2B", + "hgnc_id": "HGNC:35237", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM72D", + "hgnc_id": "HGNC:33593", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PPIAL4D", + "hgnc_id": "HGNC:33998", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "GPR89A", + "hgnc_id": "HGNC:31984", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBM8A", + "hgnc_id": "HGNC:9905", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "NBPF10", + "hgnc_id": "HGNC:31992", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLA", + "hgnc_id": "HGNC:31862", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPIAL4H", + "hgnc_id": "HGNC:53889", + "reasons": [ + "segdup" + ], + "share": 0.6658, + "status": "not read" + }, + { + "gene": "NBPF12", + "hgnc_id": "HGNC:24297", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF11", + "hgnc_id": "HGNC:31993", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PPIAL4G", + "hgnc_id": "HGNC:33996", + "reasons": [ + "segdup" + ], + "share": 0.1656, + "status": "not read" + }, + { + "gene": "NBPF14", + "hgnc_id": "HGNC:25232", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLB", + "hgnc_id": "HGNC:53923", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NUDT4B", + "hgnc_id": "HGNC:18012", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PDE4DIP", + "hgnc_id": "HGNC:15580", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NOTCH2NLC", + "hgnc_id": "HGNC:53924", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF19", + "hgnc_id": "HGNC:31999", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4C", + "hgnc_id": "HGNC:33995", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "H2BC18", + "hgnc_id": "HGNC:24700", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3C13", + "hgnc_id": "HGNC:25311", + "reasons": [ + "segdup" + ], + "share": 0.8316, + "status": "not read" + }, + { + "gene": "H3C14", + "hgnc_id": "HGNC:20503", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H2AC18", + "hgnc_id": "HGNC:4736", + "reasons": [ + "segdup" + ], + "share": 0.1651, + "status": "not read" + }, + { + "gene": "H2AC19", + "hgnc_id": "HGNC:29668", + "reasons": [ + "segdup" + ], + "share": 0.3328, + "status": "not read" + }, + { + "gene": "H3C15", + "hgnc_id": "HGNC:20505", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H4C15", + "hgnc_id": "HGNC:29607", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HRNR", + "hgnc_id": "HGNC:20846", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LCE3E", + "hgnc_id": "HGNC:29463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCE3D", + "hgnc_id": "HGNC:16615", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "S100A8", + "hgnc_id": "HGNC:10498", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "S100A7A", + "hgnc_id": "HGNC:21657", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "S100A7", + "hgnc_id": "HGNC:10497", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FLAD1", + "hgnc_id": "HGNC:24671", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MTX1", + "hgnc_id": "HGNC:7504", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN3", + "hgnc_id": "HGNC:19183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FCRL5", + "hgnc_id": "HGNC:18508", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA6", + "hgnc_id": "HGNC:5239", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FCGR2B", + "hgnc_id": "HGNC:3618", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD17B7", + "hgnc_id": "HGNC:5215", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD247", + "hgnc_id": "HGNC:1677", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "XCL2", + "hgnc_id": "HGNC:10646", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "PRG4", + "hgnc_id": "HGNC:9364", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PTPRC", + "hgnc_id": "HGNC:9666", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPP1R12B", + "hgnc_id": "HGNC:7619", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KISS1", + "hgnc_id": "HGNC:6341", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SRGAP2", + "hgnc_id": "HGNC:19751", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CR1", + "hgnc_id": "HGNC:2334", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CR1L", + "hgnc_id": "HGNC:2335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZC3H11B", + "hgnc_id": "HGNC:25659", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LBR", + "hgnc_id": "HGNC:6518", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "LEFTY2", + "hgnc_id": "HGNC:3122", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "OR2T33", + "hgnc_id": "HGNC:31255", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T7", + "hgnc_id": "HGNC:15019", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T2", + "hgnc_id": "HGNC:14725", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T3", + "hgnc_id": "HGNC:14727", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR2T5", + "hgnc_id": "HGNC:15017", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "OR2G6", + "hgnc_id": "HGNC:27019", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR2T34", + "hgnc_id": "HGNC:31256", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T11", + "hgnc_id": "HGNC:19574", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T35", + "hgnc_id": "HGNC:31257", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T27", + "hgnc_id": "HGNC:31252", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SNTG2", + "hgnc_id": "HGNC:13741", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TPO", + "hgnc_id": "HGNC:12015", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KLF11", + "hgnc_id": "HGNC:11811", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CCDC121", + "hgnc_id": "HGNC:25833", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "ABCG8", + "hgnc_id": "HGNC:13887", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PCBP1", + "hgnc_id": "HGNC:8647", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "NAT8", + "hgnc_id": "HGNC:18069", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "C2orf78", + "hgnc_id": "HGNC:34349", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CD8B", + "hgnc_id": "HGNC:1707", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RGPD1", + "hgnc_id": "HGNC:32414", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLGLB1", + "hgnc_id": "HGNC:9072", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLGLB2", + "hgnc_id": "HGNC:9073", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RGPD2", + "hgnc_id": "HGNC:32415", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPIA", + "hgnc_id": "HGNC:10297", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEKT4", + "hgnc_id": "HGNC:31012", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAHD2A", + "hgnc_id": "HGNC:24252", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TRIM43B", + "hgnc_id": "HGNC:37146", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM43", + "hgnc_id": "HGNC:19015", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANKRD36C", + "hgnc_id": "HGNC:32946", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPAT2", + "hgnc_id": "HGNC:27168", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD36", + "hgnc_id": "HGNC:24079", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANKRD36B", + "hgnc_id": "HGNC:29333", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "RGPD3", + "hgnc_id": "HGNC:32416", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD4", + "hgnc_id": "HGNC:32417", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RANBP2", + "hgnc_id": "HGNC:9848", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD5", + "hgnc_id": "HGNC:32418", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MALL", + "hgnc_id": "HGNC:6818", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MTLN", + "hgnc_id": "HGNC:27339", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LIMS4", + "hgnc_id": "HGNC:39941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD6", + "hgnc_id": "HGNC:32419", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANAPC1", + "hgnc_id": "HGNC:19988", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD8", + "hgnc_id": "HGNC:9849", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FOXD4L1", + "hgnc_id": "HGNC:18521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RABL2A", + "hgnc_id": "HGNC:9799", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RAB6C", + "hgnc_id": "HGNC:16525", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "POTEF", + "hgnc_id": "HGNC:33905", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CCDC74B", + "hgnc_id": "HGNC:25267", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SMPD4", + "hgnc_id": "HGNC:32949", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3E", + "hgnc_id": "HGNC:20765", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEI", + "hgnc_id": "HGNC:37093", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CFC1B", + "hgnc_id": "HGNC:33983", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CFC1", + "hgnc_id": "HGNC:18292", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "POTEJ", + "hgnc_id": "HGNC:37094", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PLEKHB2", + "hgnc_id": "HGNC:19236", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB6D", + "hgnc_id": "HGNC:30272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MZT2A", + "hgnc_id": "HGNC:33187", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TUBA3D", + "hgnc_id": "HGNC:24071", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CCDC74A", + "hgnc_id": "HGNC:25197", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "ANKRD30BL", + "hgnc_id": "HGNC:35167", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NOSTRIN", + "hgnc_id": "HGNC:20203", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IFT70B", + "hgnc_id": "HGNC:26425", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CASP10", + "hgnc_id": "HGNC:1500", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ALPP", + "hgnc_id": "HGNC:439", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ALPG", + "hgnc_id": "HGNC:441", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT1A6", + "hgnc_id": "HGNC:12538", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UGT1A5", + "hgnc_id": "HGNC:12537", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT1A3", + "hgnc_id": "HGNC:12535", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MLPH", + "hgnc_id": "HGNC:29643", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR6B2", + "hgnc_id": "HGNC:15041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AQP12B", + "hgnc_id": "HGNC:6096", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "AQP12A", + "hgnc_id": "HGNC:19941", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ANO7", + "hgnc_id": "HGNC:31677", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CHL1", + "hgnc_id": "HGNC:1939", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMC3", + "hgnc_id": "HGNC:23999", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANCD2", + "hgnc_id": "HGNC:3585", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DAZL", + "hgnc_id": "HGNC:2685", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF35", + "hgnc_id": "HGNC:13099", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LZTFL1", + "hgnc_id": "HGNC:6741", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MST1", + "hgnc_id": "HGNC:7380", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MST1R", + "hgnc_id": "HGNC:7381", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HYAL1", + "hgnc_id": "HGNC:5320", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FRG2C", + "hgnc_id": "HGNC:33626", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "ZNF717", + "hgnc_id": "HGNC:29448", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR5H6", + "hgnc_id": "HGNC:14767", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ROPN1", + "hgnc_id": "HGNC:17692", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ROPN1B", + "hgnc_id": "HGNC:31927", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RPN1", + "hgnc_id": "HGNC:10381", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ALG1L2", + "hgnc_id": "HGNC:37258", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BFSP2", + "hgnc_id": "HGNC:1041", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRR23B", + "hgnc_id": "HGNC:33764", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SOX2", + "hgnc_id": "HGNC:11195", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "EIF2B5", + "hgnc_id": "HGNC:3261", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KNG1", + "hgnc_id": "HGNC:6383", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TM4SF19", + "hgnc_id": "HGNC:25167", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF595", + "hgnc_id": "HGNC:27196", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDE6B", + "hgnc_id": "HGNC:8786", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAK", + "hgnc_id": "HGNC:4113", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FGFRL1", + "hgnc_id": "HGNC:3693", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RNF212", + "hgnc_id": "HGNC:27729", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UVSSA", + "hgnc_id": "HGNC:29304", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DOK7", + "hgnc_id": "HGNC:26594", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HMX1", + "hgnc_id": "HGNC:5017", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A26", + "hgnc_id": "HGNC:43746", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "USP17L10", + "hgnc_id": "HGNC:44438", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L11", + "hgnc_id": "HGNC:44439", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L12", + "hgnc_id": "HGNC:44440", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L13", + "hgnc_id": "HGNC:44441", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L17", + "hgnc_id": "HGNC:44445", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L18", + "hgnc_id": "HGNC:44446", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L19", + "hgnc_id": "HGNC:44447", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L20", + "hgnc_id": "HGNC:44448", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L21", + "hgnc_id": "HGNC:44449", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L22", + "hgnc_id": "HGNC:44450", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L24", + "hgnc_id": "HGNC:44453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L25", + "hgnc_id": "HGNC:44452", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L26", + "hgnc_id": "HGNC:44454", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L5", + "hgnc_id": "HGNC:37177", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L27", + "hgnc_id": "HGNC:44455", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L28", + "hgnc_id": "HGNC:44456", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L30", + "hgnc_id": "HGNC:44458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB131A", + "hgnc_id": "HGNC:18108", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRD5", + "hgnc_id": "HGNC:3026", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "UGT2B10", + "hgnc_id": "HGNC:12544", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B28", + "hgnc_id": "HGNC:13479", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UGT2A1", + "hgnc_id": "HGNC:12542", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UGT2A2", + "hgnc_id": "HGNC:28183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HTN3", + "hgnc_id": "HGNC:5284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HTN1", + "hgnc_id": "HGNC:5283", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "AFP", + "hgnc_id": "HGNC:317", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGY", + "hgnc_id": "HGNC:28213", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PYURF", + "hgnc_id": "HGNC:44317", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CISD2", + "hgnc_id": "HGNC:24212", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC9B1", + "hgnc_id": "HGNC:24244", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KLHL2", + "hgnc_id": "HGNC:6353", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GK3", + "hgnc_id": "HGNC:4292", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "HPF1", + "hgnc_id": "HGNC:26051", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDLIM3", + "hgnc_id": "HGNC:20767", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAT1", + "hgnc_id": "HGNC:3595", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG1", + "hgnc_id": "HGNC:3954", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FRG2", + "hgnc_id": "HGNC:19136", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DUX4", + "hgnc_id": "HGNC:50800", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SDHA", + "hgnc_id": "HGNC:10680", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDCD6", + "hgnc_id": "HGNC:8765", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZDHHC11B", + "hgnc_id": "HGNC:32962", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZDHHC11", + "hgnc_id": "HGNC:19158", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TERT", + "hgnc_id": "HGNC:11730", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A3", + "hgnc_id": "HGNC:11049", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAF11L2", + "hgnc_id": "HGNC:53845", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L3", + "hgnc_id": "HGNC:53846", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L4", + "hgnc_id": "HGNC:53847", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L5", + "hgnc_id": "HGNC:53848", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "TAF11L7", + "hgnc_id": "HGNC:53850", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAF11L8", + "hgnc_id": "HGNC:53851", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "TAF11L9", + "hgnc_id": "HGNC:53852", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L10", + "hgnc_id": "HGNC:53853", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L11", + "hgnc_id": "HGNC:53854", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L12", + "hgnc_id": "HGNC:53855", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "TAF11L14", + "hgnc_id": "HGNC:53857", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMB", + "hgnc_id": "HGNC:30465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MARVELD2", + "hgnc_id": "HGNC:26401", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERF1B", + "hgnc_id": "HGNC:10756", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NAIP", + "hgnc_id": "HGNC:7634", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SKIC3", + "hgnc_id": "HGNC:23639", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIX1", + "hgnc_id": "HGNC:18581", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SAR1B", + "hgnc_id": "HGNC:10535", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DND1", + "hgnc_id": "HGNC:23799", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDHA10", + "hgnc_id": "HGNC:8664", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PCDHB7", + "hgnc_id": "HGNC:8692", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PCDHB8", + "hgnc_id": "HGNC:8693", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PCDHB11", + "hgnc_id": "HGNC:8682", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PCDHB12", + "hgnc_id": "HGNC:8683", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPP1R2B", + "hgnc_id": "HGNC:16318", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPM1", + "hgnc_id": "HGNC:7910", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "THOC3", + "hgnc_id": "HGNC:19072", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SIMC1", + "hgnc_id": "HGNC:24779", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM153A", + "hgnc_id": "HGNC:29940", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FAM153CP", + "hgnc_id": "HGNC:33936", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FLT4", + "hgnc_id": "HGNC:3767", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR4F3", + "hgnc_id": "HGNC:8300", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "DUSP22", + "hgnc_id": "HGNC:16077", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TUBB2A", + "hgnc_id": "HGNC:12412", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "H2AC13", + "hgnc_id": "HGNC:4725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H2AC14", + "hgnc_id": "HGNC:4727", + "reasons": [ + "segdup" + ], + "share": 0.8319, + "status": "not read" + }, + { + "gene": "OR2J1", + "hgnc_id": "HGNC:8259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2J2", + "hgnc_id": "HGNC:8260", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HLA-G", + "hgnc_id": "HGNC:4964", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HLA-A", + "hgnc_id": "HGNC:4931", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "MICA", + "hgnc_id": "HGNC:7090", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MICB", + "hgnc_id": "HGNC:7091", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZBTB12", + "hgnc_id": "HGNC:19066", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PBX2", + "hgnc_id": "HGNC:8633", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HLA-DRB1", + "hgnc_id": "HGNC:4948", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GSTA2", + "hgnc_id": "HGNC:4627", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GSTA1", + "hgnc_id": "HGNC:4626", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTA5", + "hgnc_id": "HGNC:19662", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRIM2", + "hgnc_id": "HGNC:9370", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FKBP1C", + "hgnc_id": "HGNC:21376", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SLC17A5", + "hgnc_id": "HGNC:10933", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEC63", + "hgnc_id": "HGNC:21082", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR2A4", + "hgnc_id": "HGNC:14729", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE9", + "hgnc_id": "HGNC:37275", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "PCMT1", + "hgnc_id": "HGNC:8728", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RAET1G", + "hgnc_id": "HGNC:16795", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ULBP1", + "hgnc_id": "HGNC:14893", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RAET1L", + "hgnc_id": "HGNC:16798", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ULBP3", + "hgnc_id": "HGNC:14895", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC22A1", + "hgnc_id": "HGNC:10963", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMOC2", + "hgnc_id": "HGNC:20323", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RBAK-RBAKDN", + "hgnc_id": "HGNC:42971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBAK", + "hgnc_id": "HGNC:17680", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC29A4", + "hgnc_id": "HGNC:23097", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FSCN1", + "hgnc_id": "HGNC:11148", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM", + "hgnc_id": "HGNC:8105", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCZ1", + "hgnc_id": "HGNC:21691", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RSPH10B2", + "hgnc_id": "HGNC:34385", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CCZ1B", + "hgnc_id": "HGNC:21717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOD1", + "hgnc_id": "HGNC:16390", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FKBP9", + "hgnc_id": "HGNC:3725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DPY19L1", + "hgnc_id": "HGNC:22205", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SEPTIN7", + "hgnc_id": "HGNC:1717", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE1", + "hgnc_id": "HGNC:16408", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIA", + "hgnc_id": "HGNC:9253", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDC14C", + "hgnc_id": "HGNC:22427", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSPH", + "hgnc_id": "HGNC:9577", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCT6A", + "hgnc_id": "HGNC:1620", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF716", + "hgnc_id": "HGNC:32458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF727", + "hgnc_id": "HGNC:22785", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF735", + "hgnc_id": "HGNC:32466", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF736", + "hgnc_id": "HGNC:32467", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GUSB", + "hgnc_id": "HGNC:4696", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RABGEF1", + "hgnc_id": "HGNC:17676", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TYW1", + "hgnc_id": "HGNC:25598", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE21", + "hgnc_id": "HGNC:51517", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYW1B", + "hgnc_id": "HGNC:33908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POM121", + "hgnc_id": "HGNC:19702", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM74", + "hgnc_id": "HGNC:17453", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPDYE8", + "hgnc_id": "HGNC:33771", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE9", + "hgnc_id": "HGNC:45034", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE10", + "hgnc_id": "HGNC:51506", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NSUN5", + "hgnc_id": "HGNC:16385", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM50", + "hgnc_id": "HGNC:19017", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLIP2", + "hgnc_id": "HGNC:2586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GTF2IRD2", + "hgnc_id": "HGNC:30775", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE12", + "hgnc_id": "HGNC:51508", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CASTOR2", + "hgnc_id": "HGNC:37073", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GTF2IRD2B", + "hgnc_id": "HGNC:33125", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPDYE13", + "hgnc_id": "HGNC:51509", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE14", + "hgnc_id": "HGNC:51510", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM73", + "hgnc_id": "HGNC:18162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POM121C", + "hgnc_id": "HGNC:34005", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPDYE5", + "hgnc_id": "HGNC:35464", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DTX2", + "hgnc_id": "HGNC:15973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UPK3B", + "hgnc_id": "HGNC:21444", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE16", + "hgnc_id": "HGNC:51512", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE17", + "hgnc_id": "HGNC:51513", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE18", + "hgnc_id": "HGNC:51514", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "STEAP1", + "hgnc_id": "HGNC:11378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ASNS", + "hgnc_id": "HGNC:753", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM2", + "hgnc_id": "HGNC:34396", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP3A7-CYP3A51P", + "hgnc_id": "HGNC:51504", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP3A4", + "hgnc_id": "HGNC:2637", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "STAG3", + "hgnc_id": "HGNC:11356", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PVRIG", + "hgnc_id": "HGNC:32190", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SPDYE3", + "hgnc_id": "HGNC:35462", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SAP25", + "hgnc_id": "HGNC:41908", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZAN", + "hgnc_id": "HGNC:12857", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC12", + "hgnc_id": "HGNC:7510", + "reasons": [ + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE6", + "hgnc_id": "HGNC:35465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "POLR2J", + "hgnc_id": "HGNC:9197", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RASA4B", + "hgnc_id": "HGNC:35202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J3", + "hgnc_id": "HGNC:33853", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UPK3BL2", + "hgnc_id": "HGNC:53444", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "RASA4", + "hgnc_id": "HGNC:23181", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UPK3BL1", + "hgnc_id": "HGNC:37278", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "POLR2J3-UPK3BL2", + "hgnc_id": "HGNC:58364", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SPDYE2B", + "hgnc_id": "HGNC:48334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J2", + "hgnc_id": "HGNC:23208", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAMB1", + "hgnc_id": "HGNC:6486", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2B", + "hgnc_id": "HGNC:18272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KLF14", + "hgnc_id": "HGNC:23025", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AKR1B10", + "hgnc_id": "HGNC:382", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1B15", + "hgnc_id": "HGNC:37281", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BRAF", + "hgnc_id": "HGNC:1097", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRBV9", + "hgnc_id": "HGNC:12246", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRSS2", + "hgnc_id": "HGNC:9483", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAGE15", + "hgnc_id": "HGNC:37295", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TCAF2", + "hgnc_id": "HGNC:26878", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TCAF2C", + "hgnc_id": "HGNC:53641", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CTAGE6", + "hgnc_id": "HGNC:28644", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "OR2F2", + "hgnc_id": "HGNC:8247", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CTAGE4", + "hgnc_id": "HGNC:24772", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2A42", + "hgnc_id": "HGNC:31230", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2A7", + "hgnc_id": "HGNC:8234", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE8", + "hgnc_id": "HGNC:37294", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "OR2A1", + "hgnc_id": "HGNC:8229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARHGEF5", + "hgnc_id": "HGNC:13209", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACTR3C", + "hgnc_id": "HGNC:37282", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KMT2C", + "hgnc_id": "HGNC:13726", + "reasons": [ + "CMRG", + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ACTR3B", + "hgnc_id": "HGNC:17256", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F21", + "hgnc_id": "HGNC:19583", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "FBXO25", + "hgnc_id": "HGNC:13596", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARHGEF10", + "hgnc_id": "HGNC:14103", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A15", + "hgnc_id": "HGNC:32263", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A3", + "hgnc_id": "HGNC:32251", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A13", + "hgnc_id": "HGNC:32261", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A5", + "hgnc_id": "HGNC:32253", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM90A20", + "hgnc_id": "HGNC:32268", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L1", + "hgnc_id": "HGNC:37182", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L4", + "hgnc_id": "HGNC:37176", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNF705G", + "hgnc_id": "HGNC:37134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB108C", + "hgnc_id": "HGNC:30847", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB4B", + "hgnc_id": "HGNC:30193", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "DEFB103B", + "hgnc_id": "HGNC:31702", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "DEFB104B", + "hgnc_id": "HGNC:26165", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB106B", + "hgnc_id": "HGNC:28879", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "DEFB105B", + "hgnc_id": "HGNC:29930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "DEFB107B", + "hgnc_id": "HGNC:31918", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRR23D1", + "hgnc_id": "HGNC:49420", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A7", + "hgnc_id": "HGNC:32255", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FAM90A23", + "hgnc_id": "HGNC:32271", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A14", + "hgnc_id": "HGNC:32262", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A18", + "hgnc_id": "HGNC:32266", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A16", + "hgnc_id": "HGNC:32264", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A8", + "hgnc_id": "HGNC:32256", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A17", + "hgnc_id": "HGNC:32265", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A9", + "hgnc_id": "HGNC:32257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A10", + "hgnc_id": "HGNC:32258", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PRR23D2", + "hgnc_id": "HGNC:49396", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB107A", + "hgnc_id": "HGNC:18086", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB105A", + "hgnc_id": "HGNC:18087", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DEFB106A", + "hgnc_id": "HGNC:18088", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPAG11A", + "hgnc_id": "HGNC:33342", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB103A", + "hgnc_id": "HGNC:15967", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DEFB4A", + "hgnc_id": "HGNC:2767", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNF705B", + "hgnc_id": "HGNC:32284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "USP17L8", + "hgnc_id": "HGNC:37181", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L3", + "hgnc_id": "HGNC:37175", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "FAM90A11", + "hgnc_id": "HGNC:32259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A24", + "hgnc_id": "HGNC:32272", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A12", + "hgnc_id": "HGNC:32260", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLC35G5", + "hgnc_id": "HGNC:15546", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "DEFB130B", + "hgnc_id": "HGNC:39814", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF705D", + "hgnc_id": "HGNC:33202", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L2", + "hgnc_id": "HGNC:34434", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB109D", + "hgnc_id": "HGNC:30838", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM86B1", + "hgnc_id": "HGNC:28268", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB130A", + "hgnc_id": "HGNC:18107", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM86B2", + "hgnc_id": "HGNC:32222", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RBPMS", + "hgnc_id": "HGNC:19097", + "reasons": [ + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IMPA1", + "hgnc_id": "HGNC:6050", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC10A5", + "hgnc_id": "HGNC:22981", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH17", + "hgnc_id": "HGNC:1756", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP11B1", + "hgnc_id": "HGNC:2591", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP11B2", + "hgnc_id": "HGNC:2592", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GPIHBP1", + "hgnc_id": "HGNC:24945", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "WASHC1", + "hgnc_id": "HGNC:24361", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOXD4", + "hgnc_id": "HGNC:3805", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNG1A", + "hgnc_id": "HGNC:17134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KCNV2", + "hgnc_id": "HGNC:19698", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RLN2", + "hgnc_id": "HGNC:10027", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RLN1", + "hgnc_id": "HGNC:10026", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "IFNA21", + "hgnc_id": "HGNC:5424", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA4", + "hgnc_id": "HGNC:5425", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "IFNA7", + "hgnc_id": "HGNC:5428", + "reasons": [ + "segdup" + ], + "share": 0.3324, + "status": "not read" + }, + { + "gene": "IFNA10", + "hgnc_id": "HGNC:5418", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "IFNA16", + "hgnc_id": "HGNC:5421", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "IFNA17", + "hgnc_id": "HGNC:5422", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "IFNA13", + "hgnc_id": "HGNC:5419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA1", + "hgnc_id": "HGNC:5417", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "ANKRD18B", + "hgnc_id": "HGNC:23644", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRSS3", + "hgnc_id": "HGNC:9486", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALT", + "hgnc_id": "HGNC:4135", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31F1", + "hgnc_id": "HGNC:41911", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CNTNAP3", + "hgnc_id": "HGNC:13834", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31A1", + "hgnc_id": "HGNC:23394", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L6", + "hgnc_id": "HGNC:31986", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNG1F", + "hgnc_id": "HGNC:31978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNTNAP3B", + "hgnc_id": "HGNC:32035", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31A6", + "hgnc_id": "HGNC:32006", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPATA31A7", + "hgnc_id": "HGNC:32007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CNTNAP3C", + "hgnc_id": "HGNC:53878", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L5", + "hgnc_id": "HGNC:18522", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNG1E", + "hgnc_id": "HGNC:24584", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXD4L4", + "hgnc_id": "HGNC:23762", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF658", + "hgnc_id": "HGNC:25226", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD20A1", + "hgnc_id": "HGNC:23665", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNG1C", + "hgnc_id": "HGNC:18519", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L3", + "hgnc_id": "HGNC:18523", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PGM5", + "hgnc_id": "HGNC:8908", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FXN", + "hgnc_id": "HGNC:3951", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TJP2", + "hgnc_id": "HGNC:11828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPATA31D4", + "hgnc_id": "HGNC:38601", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31D3", + "hgnc_id": "HGNC:38603", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPATA31D1", + "hgnc_id": "HGNC:37283", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "KIF27", + "hgnc_id": "HGNC:18632", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPATA31C1", + "hgnc_id": "HGNC:27846", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPATA31C2", + "hgnc_id": "HGNC:24508", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC71A2", + "hgnc_id": "HGNC:23376", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2G", + "hgnc_id": "HGNC:23449", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CTSV", + "hgnc_id": "HGNC:2538", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR13C5", + "hgnc_id": "HGNC:15100", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR13C2", + "hgnc_id": "HGNC:14701", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR13C9", + "hgnc_id": "HGNC:15104", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "POLE3", + "hgnc_id": "HGNC:13546", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ORM1", + "hgnc_id": "HGNC:8498", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ORM2", + "hgnc_id": "HGNC:8499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SLC27A4", + "hgnc_id": "HGNC:10998", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PKN3", + "hgnc_id": "HGNC:17999", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ABO", + "hgnc_id": "HGNC:79", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCN1", + "hgnc_id": "HGNC:6525", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OBP2A", + "hgnc_id": "HGNC:23380", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SOHLH1", + "hgnc_id": "HGNC:27845", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "INPP5E", + "hgnc_id": "HGNC:21474", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AJM1", + "hgnc_id": "HGNC:37284", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRIN1", + "hgnc_id": "HGNC:4584", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8", + "hgnc_id": "HGNC:20773", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "AKR1C1", + "hgnc_id": "HGNC:384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1C2", + "hgnc_id": "HGNC:385", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DCLRE1C", + "hgnc_id": "HGNC:17642", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRC1", + "hgnc_id": "HGNC:7228", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD30A", + "hgnc_id": "HGNC:17234", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF33A", + "hgnc_id": "HGNC:13096", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF37A", + "hgnc_id": "HGNC:13102", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLD5P1", + "hgnc_id": "HGNC:55072", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF33B", + "hgnc_id": "HGNC:13097", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BMS1", + "hgnc_id": "HGNC:23505", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AGAP4", + "hgnc_id": "HGNC:23459", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23", + "hgnc_id": "HGNC:17312", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ANTXRL", + "hgnc_id": "HGNC:27277", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANXA8L1", + "hgnc_id": "HGNC:23334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPY4R", + "hgnc_id": "HGNC:9329", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPRIN2", + "hgnc_id": "HGNC:23730", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PTPN20", + "hgnc_id": "HGNC:23423", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANXA8", + "hgnc_id": "HGNC:546", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM25G", + "hgnc_id": "HGNC:23590", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP9", + "hgnc_id": "HGNC:23463", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SYT15B", + "hgnc_id": "HGNC:51487", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPY4R2", + "hgnc_id": "HGNC:52383", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PARG", + "hgnc_id": "HGNC:8605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23B", + "hgnc_id": "HGNC:23581", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TIMM23B-AGAP6", + "hgnc_id": "HGNC:45009", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP6", + "hgnc_id": "HGNC:23466", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASHC2A", + "hgnc_id": "HGNC:23416", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ASAH2B", + "hgnc_id": "HGNC:23456", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EIF5AL1", + "hgnc_id": "HGNC:17419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2B", + "hgnc_id": "HGNC:23445", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NUTM2E", + "hgnc_id": "HGNC:23448", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMEM254", + "hgnc_id": "HGNC:25804", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PLAC9", + "hgnc_id": "HGNC:19255", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ADIRF", + "hgnc_id": "HGNC:24043", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NUTM2A", + "hgnc_id": "HGNC:23438", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2D", + "hgnc_id": "HGNC:23447", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PAPSS2", + "hgnc_id": "HGNC:8604", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PTEN", + "hgnc_id": "HGNC:9588", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIPN", + "hgnc_id": "HGNC:23452", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "BTRC", + "hgnc_id": "HGNC:1144", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SFR1", + "hgnc_id": "HGNC:29574", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DHX32", + "hgnc_id": "HGNC:16717", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANK1", + "hgnc_id": "HGNC:23527", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ECHS1", + "hgnc_id": "HGNC:3151", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPRN", + "hgnc_id": "HGNC:16871", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG2B", + "hgnc_id": "HGNC:33518", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SCGB1C1", + "hgnc_id": "HGNC:18394", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFITM3", + "hgnc_id": "HGNC:5414", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DRD4", + "hgnc_id": "HGNC:3025", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEAF1", + "hgnc_id": "HGNC:14677", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP5-2", + "hgnc_id": "HGNC:23597", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP5-4", + "hgnc_id": "HGNC:23599", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MRPL23", + "hgnc_id": "HGNC:10322", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H19", + "hgnc_id": "HGNC:4713", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSU72L5", + "hgnc_id": "HGNC:43624", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "SSU72L2", + "hgnc_id": "HGNC:43621", + "reasons": [ + "segdup" + ], + "share": 0.666, + "status": "not read" + }, + { + "gene": "SSU72L4", + "hgnc_id": "HGNC:43623", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "SSU72L1", + "hgnc_id": "HGNC:43626", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR51A4", + "hgnc_id": "HGNC:14795", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR51A2", + "hgnc_id": "HGNC:14764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HBG1", + "hgnc_id": "HGNC:4831", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR10A5", + "hgnc_id": "HGNC:15131", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRGPRX4", + "hgnc_id": "HGNC:17617", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MDK", + "hgnc_id": "HGNC:6972", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPI1", + "hgnc_id": "HGNC:11241", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C3", + "hgnc_id": "HGNC:14697", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4C5", + "hgnc_id": "HGNC:14702", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4A47", + "hgnc_id": "HGNC:31266", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM51G", + "hgnc_id": "HGNC:43972", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64C", + "hgnc_id": "HGNC:37148", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOLH1", + "hgnc_id": "HGNC:3788", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C13", + "hgnc_id": "HGNC:15169", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4C12", + "hgnc_id": "HGNC:15168", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4C45", + "hgnc_id": "HGNC:31270", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4C46", + "hgnc_id": "HGNC:31271", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4A5", + "hgnc_id": "HGNC:15162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM48", + "hgnc_id": "HGNC:19021", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR4A15", + "hgnc_id": "HGNC:15152", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM51", + "hgnc_id": "HGNC:19023", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR5F1", + "hgnc_id": "HGNC:8343", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR8H2", + "hgnc_id": "HGNC:15308", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR5M8", + "hgnc_id": "HGNC:14846", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR9G1", + "hgnc_id": "HGNC:15319", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR1S2", + "hgnc_id": "HGNC:15141", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1S1", + "hgnc_id": "HGNC:8227", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GLYATL1B", + "hgnc_id": "HGNC:37865", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PGA3", + "hgnc_id": "HGNC:8885", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "B3GAT3", + "hgnc_id": "HGNC:923", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ESRRA", + "hgnc_id": "HGNC:3471", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC22A12", + "hgnc_id": "HGNC:17989", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RBM4B", + "hgnc_id": "HGNC:28842", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TPCN2", + "hgnc_id": "HGNC:20820", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FGF3", + "hgnc_id": "HGNC:3681", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KRTAP5-8", + "hgnc_id": "HGNC:23603", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP5-10", + "hgnc_id": "HGNC:23605", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "KRTAP5-11", + "hgnc_id": "HGNC:23606", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "XNDC1N-ZNF705EP-ALG1L9P", + "hgnc_id": "HGNC:55871", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DEFB108B", + "hgnc_id": "HGNC:29966", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "XNDC1N", + "hgnc_id": "HGNC:54661", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "INTS4", + "hgnc_id": "HGNC:25048", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOX4", + "hgnc_id": "HGNC:7891", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TRIM77", + "hgnc_id": "HGNC:34228", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "TRIM49", + "hgnc_id": "HGNC:13431", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM64B", + "hgnc_id": "HGNC:37147", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM49D1", + "hgnc_id": "HGNC:43973", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64", + "hgnc_id": "HGNC:14663", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM49C", + "hgnc_id": "HGNC:38877", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UBTFL1", + "hgnc_id": "HGNC:14533", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CARD16", + "hgnc_id": "HGNC:33701", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA8", + "hgnc_id": "HGNC:5241", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR8B2", + "hgnc_id": "HGNC:8471", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TMEM45B", + "hgnc_id": "HGNC:25194", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CD4", + "hgnc_id": "HGNC:1678", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PEX5", + "hgnc_id": "HGNC:9719", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC1", + "hgnc_id": "HGNC:604", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GDF3", + "hgnc_id": "HGNC:4218", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NANOG", + "hgnc_id": "HGNC:20857", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A1", + "hgnc_id": "HGNC:25526", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRH1", + "hgnc_id": "HGNC:9366", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRR4", + "hgnc_id": "HGNC:18020", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRH2", + "hgnc_id": "HGNC:9367", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAS2R14", + "hgnc_id": "HGNC:14920", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TAS2R31", + "hgnc_id": "HGNC:19113", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAS2R46", + "hgnc_id": "HGNC:18877", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAS2R43", + "hgnc_id": "HGNC:18875", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "TAS2R30", + "hgnc_id": "HGNC:19112", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "PRB4", + "hgnc_id": "HGNC:9340", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRB1", + "hgnc_id": "HGNC:9337", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "DDX11", + "hgnc_id": "HGNC:2736", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H3-5", + "hgnc_id": "HGNC:33164", + "reasons": [ + "segdup" + ], + "share": 0.1661, + "status": "not read" + }, + { + "gene": "ALG10", + "hgnc_id": "HGNC:23162", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ALG10B", + "hgnc_id": "HGNC:31088", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBA1B", + "hgnc_id": "HGNC:18809", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "COX14", + "hgnc_id": "HGNC:28216", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NACA", + "hgnc_id": "HGNC:7629", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DPY19L2", + "hgnc_id": "HGNC:19414", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNF1A", + "hgnc_id": "HGNC:11621", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HPD", + "hgnc_id": "HGNC:5147", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCAR2", + "hgnc_id": "HGNC:24827", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HCAR3", + "hgnc_id": "HGNC:16824", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "P2RX2", + "hgnc_id": "HGNC:15459", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PGAM5", + "hgnc_id": "HGNC:28763", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA3", + "hgnc_id": "HGNC:4426", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "C1QTNF9B", + "hgnc_id": "HGNC:34072", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C1QTNF9", + "hgnc_id": "HGNC:28732", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PARP4", + "hgnc_id": "HGNC:271", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PABPC3", + "hgnc_id": "HGNC:8556", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "MRPS31", + "hgnc_id": "HGNC:16632", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNRNPA1L2", + "hgnc_id": "HGNC:27067", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRR20C", + "hgnc_id": "HGNC:37221", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRR20B", + "hgnc_id": "HGNC:37220", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "PRR20D", + "hgnc_id": "HGNC:37222", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRR20E", + "hgnc_id": "HGNC:37223", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "F7", + "hgnc_id": "HGNC:3544", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PROZ", + "hgnc_id": "HGNC:9460", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEM", + "hgnc_id": "HGNC:37096", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEG", + "hgnc_id": "HGNC:33896", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR11H2", + "hgnc_id": "HGNC:14716", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4N2", + "hgnc_id": "HGNC:14742", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR4Q3", + "hgnc_id": "HGNC:15426", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR4M1", + "hgnc_id": "HGNC:14735", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RNASE3", + "hgnc_id": "HGNC:10046", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RNASE2", + "hgnc_id": "HGNC:10045", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10G2", + "hgnc_id": "HGNC:8170", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "DHRS4", + "hgnc_id": "HGNC:16985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHRS4L2", + "hgnc_id": "HGNC:19731", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARF6", + "hgnc_id": "HGNC:659", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACOT2", + "hgnc_id": "HGNC:18431", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NDUFB1", + "hgnc_id": "HGNC:7695", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "JAG2", + "hgnc_id": "HGNC:6189", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BTBD6", + "hgnc_id": "HGNC:19897", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "IGHV3-21", + "hgnc_id": "HGNC:5586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA6L6", + "hgnc_id": "HGNC:37225", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEB3", + "hgnc_id": "HGNC:51240", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LINC02203", + "hgnc_id": "HGNC:53069", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4M2B", + "hgnc_id": "HGNC:55109", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4N4C", + "hgnc_id": "HGNC:55110", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "POTEB", + "hgnc_id": "HGNC:33734", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4M2-OT1", + "hgnc_id": "HGNC:56199", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4N4", + "hgnc_id": "HGNC:15375", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L22", + "hgnc_id": "HGNC:50289", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6L1", + "hgnc_id": "HGNC:37444", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L26", + "hgnc_id": "HGNC:56306", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8S", + "hgnc_id": "HGNC:44409", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6L2", + "hgnc_id": "HGNC:26695", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HERC2", + "hgnc_id": "HGNC:4868", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L24", + "hgnc_id": "HGNC:55710", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA8F", + "hgnc_id": "HGNC:32378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8G", + "hgnc_id": "HGNC:25328", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6L25", + "hgnc_id": "HGNC:55711", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8M", + "hgnc_id": "HGNC:44404", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8J", + "hgnc_id": "HGNC:38650", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8T", + "hgnc_id": "HGNC:44410", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8R", + "hgnc_id": "HGNC:44407", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8Q", + "hgnc_id": "HGNC:44408", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8H", + "hgnc_id": "HGNC:37443", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8O", + "hgnc_id": "HGNC:44406", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8N", + "hgnc_id": "HGNC:44405", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8A", + "hgnc_id": "HGNC:31972", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8B", + "hgnc_id": "HGNC:31973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NANOGP8", + "hgnc_id": "HGNC:23106", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "NDUFAF1", + "hgnc_id": "HGNC:18828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CKMT1B", + "hgnc_id": "HGNC:1995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CATSPER2", + "hgnc_id": "HGNC:18810", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CKMT1A", + "hgnc_id": "HGNC:31736", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TERB2", + "hgnc_id": "HGNC:28520", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LDHAL6B", + "hgnc_id": "HGNC:21481", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6B", + "hgnc_id": "HGNC:32205", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6A", + "hgnc_id": "HGNC:13567", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ISLR2", + "hgnc_id": "HGNC:29286", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6C", + "hgnc_id": "HGNC:32206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6D", + "hgnc_id": "HGNC:32204", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ADAMTS7", + "hgnc_id": "HGNC:223", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EFL1", + "hgnc_id": "HGNC:25789", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SAXO2", + "hgnc_id": "HGNC:33727", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L10", + "hgnc_id": "HGNC:37228", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GOLGA6L9", + "hgnc_id": "HGNC:37229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "WHAMM", + "hgnc_id": "HGNC:30493", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4F4", + "hgnc_id": "HGNC:8301", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MPG", + "hgnc_id": "HGNC:7211", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HBM", + "hgnc_id": "HGNC:4826", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AXIN1", + "hgnc_id": "HGNC:903", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSTR5", + "hgnc_id": "HGNC:11334", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TPSD1", + "hgnc_id": "HGNC:14118", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GNPTG", + "hgnc_id": "HGNC:23026", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UNKL", + "hgnc_id": "HGNC:14184", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PKD1", + "hgnc_id": "HGNC:9008", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PDPK1", + "hgnc_id": "HGNC:8816", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "EEF2KMT", + "hgnc_id": "HGNC:32221", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NPIPA3", + "hgnc_id": "HGNC:41978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPA2", + "hgnc_id": "HGNC:41979", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NOMO1", + "hgnc_id": "HGNC:30060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDXDC1", + "hgnc_id": "HGNC:28995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RRN3", + "hgnc_id": "HGNC:30346", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPA5", + "hgnc_id": "HGNC:41980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MPV17L", + "hgnc_id": "HGNC:26827", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NOMO3", + "hgnc_id": "HGNC:25242", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPA6", + "hgnc_id": "HGNC:27893", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPA8", + "hgnc_id": "HGNC:41983", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPA9", + "hgnc_id": "HGNC:41984", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NOMO2", + "hgnc_id": "HGNC:22652", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SMG1", + "hgnc_id": "HGNC:30045", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IQCK", + "hgnc_id": "HGNC:28556", + "reasons": [ + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACSM5", + "hgnc_id": "HGNC:26060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACSM2B", + "hgnc_id": "HGNC:30931", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPB3", + "hgnc_id": "HGNC:28989", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB4", + "hgnc_id": "HGNC:41985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB5", + "hgnc_id": "HGNC:37233", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB6", + "hgnc_id": "HGNC:37454", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB7", + "hgnc_id": "HGNC:33832", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A2", + "hgnc_id": "HGNC:11454", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB8", + "hgnc_id": "HGNC:37490", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EIF3C", + "hgnc_id": "HGNC:3279", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB9", + "hgnc_id": "HGNC:41987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB11", + "hgnc_id": "HGNC:37453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BOLA2-SMG1P6", + "hgnc_id": "HGNC:53563", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BOLA2", + "hgnc_id": "HGNC:29488", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLX1B", + "hgnc_id": "HGNC:28748", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SULT1A4", + "hgnc_id": "HGNC:30004", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NPIPB12", + "hgnc_id": "HGNC:37491", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLX1A", + "hgnc_id": "HGNC:20922", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A3", + "hgnc_id": "HGNC:11455", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB13", + "hgnc_id": "HGNC:41989", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PHKG2", + "hgnc_id": "HGNC:8931", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "VKORC1", + "hgnc_id": "HGNC:23663", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRABD5", + "hgnc_id": "HGNC:26987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TP53TG3", + "hgnc_id": "HGNC:30759", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TP53TG3C", + "hgnc_id": "HGNC:42962", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "TP53TG3E", + "hgnc_id": "HGNC:51816", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TP53TG3B", + "hgnc_id": "HGNC:37202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TP53TG3F", + "hgnc_id": "HGNC:51817", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "CCNYL1B", + "hgnc_id": "HGNC:56313", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CES1", + "hgnc_id": "HGNC:1863", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MT1M", + "hgnc_id": "HGNC:14296", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MATCAP1", + "hgnc_id": "HGNC:34408", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD11B2", + "hgnc_id": "HGNC:5209", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLEC18A", + "hgnc_id": "HGNC:30388", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PDPR", + "hgnc_id": "HGNC:30264", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EXOSC6", + "hgnc_id": "HGNC:19055", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HPR", + "hgnc_id": "HGNC:5156", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB15", + "hgnc_id": "HGNC:34409", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CLEC18B", + "hgnc_id": "HGNC:33849", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CA5A", + "hgnc_id": "HGNC:1377", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF469", + "hgnc_id": "HGNC:23216", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH15", + "hgnc_id": "HGNC:1754", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANKRD11", + "hgnc_id": "HGNC:21316", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MC1R", + "hgnc_id": "HGNC:6929", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PRDM7", + "hgnc_id": "HGNC:9351", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ABR", + "hgnc_id": "HGNC:81", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SERPINF2", + "hgnc_id": "HGNC:9075", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SRR", + "hgnc_id": "HGNC:14398", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1D5", + "hgnc_id": "HGNC:8186", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "P2RX5", + "hgnc_id": "HGNC:8536", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ALOX15", + "hgnc_id": "HGNC:433", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ENO3", + "hgnc_id": "HGNC:3354", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP6", + "hgnc_id": "HGNC:12629", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35G6", + "hgnc_id": "HGNC:31351", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "ALOXE3", + "hgnc_id": "HGNC:13743", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HES7", + "hgnc_id": "HGNC:15977", + "reasons": [ + "CMRG" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CDRT15", + "hgnc_id": "HGNC:14395", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "FBXW10B", + "hgnc_id": "HGNC:14379", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM16", + "hgnc_id": "HGNC:17241", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF286A-TBC1D26", + "hgnc_id": "HGNC:55384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF286A", + "hgnc_id": "HGNC:13501", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D26", + "hgnc_id": "HGNC:28745", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UBB", + "hgnc_id": "HGNC:12463", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "ATPAF2", + "hgnc_id": "HGNC:18802", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EVPLL", + "hgnc_id": "HGNC:35236", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS9C", + "hgnc_id": "HGNC:33874", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D28", + "hgnc_id": "HGNC:26858", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXO3B", + "hgnc_id": "HGNC:3822", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FBXW10", + "hgnc_id": "HGNC:1211", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRAP", + "hgnc_id": "HGNC:4562", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GRAPL", + "hgnc_id": "HGNC:37240", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LGALS9B", + "hgnc_id": "HGNC:24842", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDRT15L2", + "hgnc_id": "HGNC:34075", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MEIS3P2", + "hgnc_id": "HGNC:17638", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAP2K3", + "hgnc_id": "HGNC:6843", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KCNJ18", + "hgnc_id": "HGNC:39080", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS9", + "hgnc_id": "HGNC:6570", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXN1", + "hgnc_id": "HGNC:12765", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SUZ12", + "hgnc_id": "HGNC:17101", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37B", + "hgnc_id": "HGNC:29070", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SLC35G3", + "hgnc_id": "HGNC:26848", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CCL4", + "hgnc_id": "HGNC:10630", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3B", + "hgnc_id": "HGNC:27011", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCL4L2", + "hgnc_id": "HGNC:24066", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "TBC1D3I", + "hgnc_id": "HGNC:32709", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TBC1D3G", + "hgnc_id": "HGNC:29860", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TBC1D3F", + "hgnc_id": "HGNC:18257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3K", + "hgnc_id": "HGNC:51245", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TBC1D3L", + "hgnc_id": "HGNC:51246", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D3D", + "hgnc_id": "HGNC:28944", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D3C", + "hgnc_id": "HGNC:24889", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TBC1D3E", + "hgnc_id": "HGNC:27071", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MRPL45", + "hgnc_id": "HGNC:16651", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP1-1", + "hgnc_id": "HGNC:16772", + "reasons": [ + "CMRG" + ], + "share": 0.1659, + "status": "not read" + }, + { + "gene": "KRTAP4-7", + "hgnc_id": "HGNC:18898", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP4-8", + "hgnc_id": "HGNC:17230", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP4-16", + "hgnc_id": "HGNC:18921", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRTAP4-9", + "hgnc_id": "HGNC:18910", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP4-12", + "hgnc_id": "HGNC:16776", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP9-7", + "hgnc_id": "HGNC:18915", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "KRT14", + "hgnc_id": "HGNC:6416", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRT16", + "hgnc_id": "HGNC:6423", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRT17", + "hgnc_id": "HGNC:6427", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HSD17B1", + "hgnc_id": "HGNC:5210", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "G6PC3", + "hgnc_id": "HGNC:24861", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ARHGAP27", + "hgnc_id": "HGNC:31813", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLEKHM1", + "hgnc_id": "HGNC:29017", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ARL17B", + "hgnc_id": "HGNC:32387", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37A", + "hgnc_id": "HGNC:29069", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARL17A", + "hgnc_id": "HGNC:24096", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NSF", + "hgnc_id": "HGNC:8016", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPEPPS", + "hgnc_id": "HGNC:7900", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HOXB8", + "hgnc_id": "HGNC:5119", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GIP", + "hgnc_id": "HGNC:4270", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHX40", + "hgnc_id": "HGNC:18018", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPS6KB1", + "hgnc_id": "HGNC:10436", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2A", + "hgnc_id": "HGNC:25755", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TLK2", + "hgnc_id": "HGNC:11842", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CSH2", + "hgnc_id": "HGNC:2441", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "GH2", + "hgnc_id": "HGNC:4262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CSH1", + "hgnc_id": "HGNC:2440", + "reasons": [ + "segdup" + ], + "share": 0.6663, + "status": "not read" + }, + { + "gene": "CSHL1", + "hgnc_id": "HGNC:2442", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "LRRC37A3", + "hgnc_id": "HGNC:32427", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C17orf58", + "hgnc_id": "HGNC:27568", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "AMZ2", + "hgnc_id": "HGNC:28041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMC6", + "hgnc_id": "HGNC:18021", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CANT1", + "hgnc_id": "HGNC:19721", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RNF213", + "hgnc_id": "HGNC:14539", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ARL16", + "hgnc_id": "HGNC:27902", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8B", + "hgnc_id": "HGNC:24983", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYMS", + "hgnc_id": "HGNC:12441", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC35G4", + "hgnc_id": "HGNC:31043", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "POTEC", + "hgnc_id": "HGNC:33894", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ANKRD30B", + "hgnc_id": "HGNC:24165", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC25A52", + "hgnc_id": "HGNC:23324", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KATNAL2", + "hgnc_id": "HGNC:25387", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SERPINB4", + "hgnc_id": "HGNC:10570", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERPINB3", + "hgnc_id": "HGNC:10569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALR1", + "hgnc_id": "HGNC:4132", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CTDP1", + "hgnc_id": "HGNC:2498", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "BSG", + "hgnc_id": "HGNC:1116", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN2", + "hgnc_id": "HGNC:4846", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KISS1R", + "hgnc_id": "HGNC:4510", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "STK11", + "hgnc_id": "HGNC:11389", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TCF3", + "hgnc_id": "HGNC:11633", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ABHD17A", + "hgnc_id": "HGNC:28756", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "GIPC3", + "hgnc_id": "HGNC:18183", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBXA2R", + "hgnc_id": "HGNC:11608", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CREB3L3", + "hgnc_id": "HGNC:18855", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RFX2", + "hgnc_id": "HGNC:9983", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "C3", + "hgnc_id": "HGNC:1318", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MBD3L2B", + "hgnc_id": "HGNC:53435", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MBD3L4", + "hgnc_id": "HGNC:37206", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBD3L2", + "hgnc_id": "HGNC:18532", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "STXBP2", + "hgnc_id": "HGNC:11445", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TYK2", + "hgnc_id": "HGNC:12440", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF799", + "hgnc_id": "HGNC:28071", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF443", + "hgnc_id": "HGNC:20878", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP4F3", + "hgnc_id": "HGNC:2646", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4F12", + "hgnc_id": "HGNC:18857", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10H2", + "hgnc_id": "HGNC:8173", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CALR3", + "hgnc_id": "HGNC:20407", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MYO9B", + "hgnc_id": "HGNC:7609", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "INSL3", + "hgnc_id": "HGNC:6086", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF738", + "hgnc_id": "HGNC:32469", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF100", + "hgnc_id": "HGNC:12880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF208", + "hgnc_id": "HGNC:12999", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF676", + "hgnc_id": "HGNC:20429", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF729", + "hgnc_id": "HGNC:32464", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF492", + "hgnc_id": "HGNC:23707", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF728", + "hgnc_id": "HGNC:32463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHPN2", + "hgnc_id": "HGNC:19974", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GPI", + "hgnc_id": "HGNC:4458", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF302", + "hgnc_id": "HGNC:13848", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF181", + "hgnc_id": "HGNC:12971", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FFAR3", + "hgnc_id": "HGNC:4499", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "COX6B1", + "hgnc_id": "HGNC:2280", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS7", + "hgnc_id": "HGNC:6568", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS7B", + "hgnc_id": "HGNC:34447", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "IFNL3", + "hgnc_id": "HGNC:18365", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IFNL4", + "hgnc_id": "HGNC:44480", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "IFNL2", + "hgnc_id": "HGNC:18364", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CYP2A7", + "hgnc_id": "HGNC:2611", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CYP2G1P", + "hgnc_id": "HGNC:2633", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A13", + "hgnc_id": "HGNC:2608", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CYP2F1", + "hgnc_id": "HGNC:2632", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG8", + "hgnc_id": "HGNC:9525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG1", + "hgnc_id": "HGNC:9514", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PSG6", + "hgnc_id": "HGNC:9523", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PSG7", + "hgnc_id": "HGNC:9524", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSG11", + "hgnc_id": "HGNC:9516", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PSG2", + "hgnc_id": "HGNC:9519", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG4", + "hgnc_id": "HGNC:9521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG9", + "hgnc_id": "HGNC:9526", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ETHE1", + "hgnc_id": "HGNC:23287", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF285", + "hgnc_id": "HGNC:13079", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "APOC1", + "hgnc_id": "HGNC:607", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOC4", + "hgnc_id": "HGNC:611", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BLOC1S3", + "hgnc_id": "HGNC:20914", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DMPK", + "hgnc_id": "HGNC:2933", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FUT1", + "hgnc_id": "HGNC:4012", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BAX", + "hgnc_id": "HGNC:959", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LHB", + "hgnc_id": "HGNC:6584", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "CGB3", + "hgnc_id": "HGNC:1886", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CGB2", + "hgnc_id": "HGNC:16722", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "CGB1", + "hgnc_id": "HGNC:16721", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "CGB5", + "hgnc_id": "HGNC:16452", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CGB8", + "hgnc_id": "HGNC:16453", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CGB7", + "hgnc_id": "HGNC:16451", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PNKP", + "hgnc_id": "HGNC:9154", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SIGLEC16", + "hgnc_id": "HGNC:24851", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KLK4", + "hgnc_id": "HGNC:6365", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ETFB", + "hgnc_id": "HGNC:3482", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF808", + "hgnc_id": "HGNC:33230", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF600", + "hgnc_id": "HGNC:30951", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF468", + "hgnc_id": "HGNC:33105", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VN1R4", + "hgnc_id": "HGNC:19871", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "FAM90A27P", + "hgnc_id": "HGNC:43617", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF765", + "hgnc_id": "HGNC:25092", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF761", + "hgnc_id": "HGNC:23179", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF813", + "hgnc_id": "HGNC:33257", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRKCG", + "hgnc_id": "HGNC:9402", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBOAT7", + "hgnc_id": "HGNC:15505", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LILRA6", + "hgnc_id": "HGNC:15495", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "LILRB2", + "hgnc_id": "HGNC:6606", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "LAIR1", + "hgnc_id": "HGNC:6477", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAIR2", + "hgnc_id": "HGNC:6478", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LILRB1", + "hgnc_id": "HGNC:6605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KIR3DL3", + "hgnc_id": "HGNC:16312", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KIR2DL3", + "hgnc_id": "HGNC:6331", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KIR2DL1", + "hgnc_id": "HGNC:6329", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KIR2DL4", + "hgnc_id": "HGNC:6332", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KIR3DL1", + "hgnc_id": "HGNC:6338", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KIR3DL2", + "hgnc_id": "HGNC:6339", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NLRP7", + "hgnc_id": "HGNC:22947", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NLRP2", + "hgnc_id": "HGNC:22948", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GP6", + "hgnc_id": "HGNC:14388", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TNNT1", + "hgnc_id": "HGNC:11948", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TNNI3", + "hgnc_id": "HGNC:11947", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL4A", + "hgnc_id": "HGNC:16449", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RFPL4AL1", + "hgnc_id": "HGNC:45147", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZNF419", + "hgnc_id": "HGNC:20648", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF773", + "hgnc_id": "HGNC:30487", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF587B", + "hgnc_id": "HGNC:37142", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF814", + "hgnc_id": "HGNC:33258", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF417", + "hgnc_id": "HGNC:20646", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC27A5", + "hgnc_id": "HGNC:10999", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SIRPB1", + "hgnc_id": "HGNC:15928", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CST4", + "hgnc_id": "HGNC:2476", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CST1", + "hgnc_id": "HGNC:2473", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CST2", + "hgnc_id": "HGNC:2474", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PLAGL2", + "hgnc_id": "HGNC:9047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEMG1", + "hgnc_id": "HGNC:10742", + "reasons": [ + "CMRG" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PLTP", + "hgnc_id": "HGNC:9093", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRNA4", + "hgnc_id": "HGNC:1958", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EEF1A2", + "hgnc_id": "HGNC:3192", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PTK6", + "hgnc_id": "HGNC:9617", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MYT1", + "hgnc_id": "HGNC:7622", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCMTD2", + "hgnc_id": "HGNC:15882", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TPTE", + "hgnc_id": "HGNC:12023", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POTED", + "hgnc_id": "HGNC:23822", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP21-1", + "hgnc_id": "HGNC:18945", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SMIM11", + "hgnc_id": "HGNC:1293", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMIM34", + "hgnc_id": "HGNC:39601", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KCNE1", + "hgnc_id": "HGNC:6240", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CBR3", + "hgnc_id": "HGNC:1549", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NDUFV3", + "hgnc_id": "HGNC:7719", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CBS", + "hgnc_id": "HGNC:1550", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "U2AF1", + "hgnc_id": "HGNC:12453", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SIK1", + "hgnc_id": "HGNC:11142", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSF2BP", + "hgnc_id": "HGNC:5226", + "reasons": [ + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BC12L", + "hgnc_id": "HGNC:4762", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3319, + "status": "not read" + }, + { + "gene": "TRAPPC10", + "hgnc_id": "HGNC:11868", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PWP2", + "hgnc_id": "HGNC:9711", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GATD3", + "hgnc_id": "HGNC:1273", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DNMT3L", + "hgnc_id": "HGNC:2980", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-1", + "hgnc_id": "HGNC:22966", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "KRTAP10-2", + "hgnc_id": "HGNC:22967", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRTAP10-4", + "hgnc_id": "HGNC:20521", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "KRTAP10-5", + "hgnc_id": "HGNC:22969", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "KRTAP10-6", + "hgnc_id": "HGNC:20523", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "KRTAP10-8", + "hgnc_id": "HGNC:20525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-9", + "hgnc_id": "HGNC:22971", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KRTAP10-10", + "hgnc_id": "HGNC:22972", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP12-2", + "hgnc_id": "HGNC:20530", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "KRTAP12-1", + "hgnc_id": "HGNC:20529", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "COL18A1", + "hgnc_id": "HGNC:2195", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR11H1", + "hgnc_id": "HGNC:15404", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEH", + "hgnc_id": "HGNC:133", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCT8L2", + "hgnc_id": "HGNC:15553", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "USP18", + "hgnc_id": "HGNC:12616", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GGTLC3", + "hgnc_id": "HGNC:33426", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TMEM191B", + "hgnc_id": "HGNC:33600", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM246B", + "hgnc_id": "HGNC:54843", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DGCR6", + "hgnc_id": "HGNC:2846", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRODH", + "hgnc_id": "HGNC:9453", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM246C", + "hgnc_id": "HGNC:54842", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "CCDC188", + "hgnc_id": "HGNC:51899", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RTN4R", + "hgnc_id": "HGNC:18601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3FP", + "hgnc_id": "HGNC:24067", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LRRC74B", + "hgnc_id": "HGNC:34301", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM246A", + "hgnc_id": "HGNC:54844", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RIMBP3B", + "hgnc_id": "HGNC:33891", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HIC2", + "hgnc_id": "HGNC:18595", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TMEM191C", + "hgnc_id": "HGNC:33601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GGTLC2", + "hgnc_id": "HGNC:18596", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IGLL1", + "hgnc_id": "HGNC:5870", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRICH1", + "hgnc_id": "HGNC:28031", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTT2B", + "hgnc_id": "HGNC:33437", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DDTL", + "hgnc_id": "HGNC:33446", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GSTT2", + "hgnc_id": "HGNC:4642", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL1", + "hgnc_id": "HGNC:9977", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C22orf42", + "hgnc_id": "HGNC:27160", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RFPL3", + "hgnc_id": "HGNC:9980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3D", + "hgnc_id": "HGNC:17354", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOBEC3H", + "hgnc_id": "HGNC:24100", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ATF4", + "hgnc_id": "HGNC:786", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NDUFA6", + "hgnc_id": "HGNC:7690", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2D7", + "hgnc_id": "HGNC:2624", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RRP7A", + "hgnc_id": "HGNC:24286", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYB5R3", + "hgnc_id": "HGNC:2873", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "A4GALT", + "hgnc_id": "HGNC:18149", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TTLL1", + "hgnc_id": "HGNC:1312", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MLC1", + "hgnc_id": "HGNC:17082", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBGCP6", + "hgnc_id": "HGNC:18127", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACR", + "hgnc_id": "HGNC:126", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "XG", + "hgnc_id": "HGNC:12806", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GYG2", + "hgnc_id": "HGNC:4700", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ARSD", + "hgnc_id": "HGNC:717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MXRA5", + "hgnc_id": "HGNC:7539", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRKX", + "hgnc_id": "HGNC:9441", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "VCX3A", + "hgnc_id": "HGNC:18159", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "VCX2", + "hgnc_id": "HGNC:18158", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VCX3B", + "hgnc_id": "HGNC:31838", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM9A", + "hgnc_id": "HGNC:18403", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM9C", + "hgnc_id": "HGNC:18405", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SUPT20HL2", + "hgnc_id": "HGNC:31797", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SUPT20HL1", + "hgnc_id": "HGNC:30773", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "MAGEB6", + "hgnc_id": "HGNC:23796", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DCAF8L1", + "hgnc_id": "HGNC:31810", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM47B", + "hgnc_id": "HGNC:26659", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "H2AL1Q", + "hgnc_id": "HGNC:53959", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "FAM47C", + "hgnc_id": "HGNC:25301", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "H2AL3", + "hgnc_id": "HGNC:53960", + "reasons": [ + "segdup" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "SPACA5B", + "hgnc_id": "HGNC:19142", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SSX5", + "hgnc_id": "HGNC:11339", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SSX1", + "hgnc_id": "HGNC:11335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSX3", + "hgnc_id": "HGNC:11337", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SSX4", + "hgnc_id": "HGNC:11338", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX4B", + "hgnc_id": "HGNC:16880", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12J", + "hgnc_id": "HGNC:17778", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE13", + "hgnc_id": "HGNC:29081", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE2E", + "hgnc_id": "HGNC:31960", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE12B", + "hgnc_id": "HGNC:26779", + "reasons": [ + "segdup" + ], + "share": 0.4957, + "status": "not read" + }, + { + "gene": "GAGE12C", + "hgnc_id": "HGNC:28402", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GAGE12D", + "hgnc_id": "HGNC:31904", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12F", + "hgnc_id": "HGNC:31906", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE12G", + "hgnc_id": "HGNC:31907", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE12H", + "hgnc_id": "HGNC:31908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE2A", + "hgnc_id": "HGNC:4099", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GAGE1", + "hgnc_id": "HGNC:4098", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CENPVL3", + "hgnc_id": "HGNC:43880", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CENPVL1", + "hgnc_id": "HGNC:31851", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MAGED4B", + "hgnc_id": "HGNC:22880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MAGED4", + "hgnc_id": "HGNC:23793", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE1A", + "hgnc_id": "HGNC:4111", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "XAGE1B", + "hgnc_id": "HGNC:25400", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX7", + "hgnc_id": "HGNC:19653", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSX2B", + "hgnc_id": "HGNC:22263", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN5", + "hgnc_id": "HGNC:33178", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "XAGE5", + "hgnc_id": "HGNC:30930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE3", + "hgnc_id": "HGNC:14618", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM156B", + "hgnc_id": "HGNC:31962", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM156A", + "hgnc_id": "HGNC:30114", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPIN2B", + "hgnc_id": "HGNC:33147", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPIN2A", + "hgnc_id": "HGNC:20694", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZXDA", + "hgnc_id": "HGNC:13198", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CXorf49", + "hgnc_id": "HGNC:30891", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CXorf49B", + "hgnc_id": "HGNC:34229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "DMRTC1B", + "hgnc_id": "HGNC:31686", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM236D", + "hgnc_id": "HGNC:52642", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DMRTC1", + "hgnc_id": "HGNC:13910", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM236C", + "hgnc_id": "HGNC:52641", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM236A", + "hgnc_id": "HGNC:44268", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PABPC1L2B", + "hgnc_id": "HGNC:31852", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PABPC1L2A", + "hgnc_id": "HGNC:27989", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TGIF2LX", + "hgnc_id": "HGNC:18570", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDH11X", + "hgnc_id": "HGNC:8656", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TCEAL6", + "hgnc_id": "HGNC:24553", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "TCP11X1", + "hgnc_id": "HGNC:48369", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NXF2", + "hgnc_id": "HGNC:8072", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NXF2B", + "hgnc_id": "HGNC:23984", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB40AL", + "hgnc_id": "HGNC:25410", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TMSB15B", + "hgnc_id": "HGNC:28612", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BW1", + "hgnc_id": "HGNC:27252", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "H2BW2", + "hgnc_id": "HGNC:27867", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "TMSB15C", + "hgnc_id": "HGNC:55173", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PSMD10", + "hgnc_id": "HGNC:9555", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CT47C1", + "hgnc_id": "HGNC:53820", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHOXF2B", + "hgnc_id": "HGNC:33519", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RHOXF2", + "hgnc_id": "HGNC:30011", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT47B1", + "hgnc_id": "HGNC:33293", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A12", + "hgnc_id": "HGNC:33292", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT47A11", + "hgnc_id": "HGNC:27397", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CT47A9", + "hgnc_id": "HGNC:33290", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A8", + "hgnc_id": "HGNC:33289", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "CT47A7", + "hgnc_id": "HGNC:33288", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT47A6", + "hgnc_id": "HGNC:33287", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A5", + "hgnc_id": "HGNC:33286", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CT47A4", + "hgnc_id": "HGNC:33285", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT47A2", + "hgnc_id": "HGNC:33283", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A1", + "hgnc_id": "HGNC:33282", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "RTL8B", + "hgnc_id": "HGNC:33156", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "RTL8C", + "hgnc_id": "HGNC:2569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RTL8A", + "hgnc_id": "HGNC:24514", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "SMIM10L2B", + "hgnc_id": "HGNC:34500", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT55", + "hgnc_id": "HGNC:26047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ETDA", + "hgnc_id": "HGNC:53449", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "SMIM10L2A", + "hgnc_id": "HGNC:34499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT45A1", + "hgnc_id": "HGNC:33267", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A3", + "hgnc_id": "HGNC:33268", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT45A5", + "hgnc_id": "HGNC:33270", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT45A2", + "hgnc_id": "HGNC:28400", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT45A7", + "hgnc_id": "HGNC:51260", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CT45A8", + "hgnc_id": "HGNC:51261", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT45A9", + "hgnc_id": "HGNC:51262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A10", + "hgnc_id": "HGNC:51263", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SAGE1", + "hgnc_id": "HGNC:30369", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPANXC", + "hgnc_id": "HGNC:14331", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXA1", + "hgnc_id": "HGNC:11218", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "SPANXA2", + "hgnc_id": "HGNC:14328", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "SPANXD", + "hgnc_id": "HGNC:14332", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "MAGEC1", + "hgnc_id": "HGNC:6812", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPANXN3", + "hgnc_id": "HGNC:33176", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UBE2NL", + "hgnc_id": "HGNC:31710", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN1", + "hgnc_id": "HGNC:33174", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CXorf51B", + "hgnc_id": "HGNC:42787", + "reasons": [ + "segdup" + ], + "share": 0.3323, + "status": "not read" + }, + { + "gene": "CXorf51A", + "hgnc_id": "HGNC:30533", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "EOLA1", + "hgnc_id": "HGNC:28089", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HSFX3", + "hgnc_id": "HGNC:52395", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HSFX2", + "hgnc_id": "HGNC:32701", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSFX1", + "hgnc_id": "HGNC:29603", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MAGEA9", + "hgnc_id": "HGNC:6807", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "EOLA2", + "hgnc_id": "HGNC:17402", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSFX4", + "hgnc_id": "HGNC:52398", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAGEA3", + "hgnc_id": "HGNC:6801", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MAGEA2B", + "hgnc_id": "HGNC:19340", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CSAG1", + "hgnc_id": "HGNC:24294", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "MAGEA12", + "hgnc_id": "HGNC:6799", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "MAGEA2", + "hgnc_id": "HGNC:6800", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CSAG3", + "hgnc_id": "HGNC:26237", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MAGEA6", + "hgnc_id": "HGNC:6804", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PWWP4", + "hgnc_id": "HGNC:55197", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A8", + "hgnc_id": "HGNC:11055", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BCAP31", + "hgnc_id": "HGNC:16695", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OPN1LW", + "hgnc_id": "HGNC:9936", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OPN1MW", + "hgnc_id": "HGNC:4206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OPN1MW2", + "hgnc_id": "HGNC:26952", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEX28", + "hgnc_id": "HGNC:2563", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAG1A", + "hgnc_id": "HGNC:24198", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "CTAG1B", + "hgnc_id": "HGNC:2491", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAG2", + "hgnc_id": "HGNC:2492", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H2AB1", + "hgnc_id": "HGNC:22516", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "F8A1", + "hgnc_id": "HGNC:3547", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "F8A2", + "hgnc_id": "HGNC:31849", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "F8A3", + "hgnc_id": "HGNC:31850", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "H2AB3", + "hgnc_id": "HGNC:14455", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IL9R", + "hgnc_id": "HGNC:6030", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASH6P", + "hgnc_id": "HGNC:31685", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "TGIF2LY", + "hgnc_id": "HGNC:18569", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "PCDH11Y", + "hgnc_id": "HGNC:15813", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY2", + "hgnc_id": "HGNC:23924", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY4", + "hgnc_id": "HGNC:37287", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY8", + "hgnc_id": "HGNC:37471", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY3", + "hgnc_id": "HGNC:33876", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY1", + "hgnc_id": "HGNC:12381", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY9", + "hgnc_id": "HGNC:37472", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY10", + "hgnc_id": "HGNC:37473", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY", + "hgnc_id": "HGNC:12668", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY1B", + "hgnc_id": "HGNC:31751", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2B", + "hgnc_id": "HGNC:23921", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2A", + "hgnc_id": "HGNC:1810", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY1", + "hgnc_id": "HGNC:18568", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY2", + "hgnc_id": "HGNC:23950", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1B", + "hgnc_id": "HGNC:23914", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1A1", + "hgnc_id": "HGNC:9912", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1D", + "hgnc_id": "HGNC:23915", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1E", + "hgnc_id": "HGNC:23916", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1F", + "hgnc_id": "HGNC:23974", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1J", + "hgnc_id": "HGNC:23917", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2", + "hgnc_id": "HGNC:13508", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ1", + "hgnc_id": "HGNC:2682", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ2", + "hgnc_id": "HGNC:15964", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1B", + "hgnc_id": "HGNC:23920", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2B", + "hgnc_id": "HGNC:25449", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ3", + "hgnc_id": "HGNC:15965", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ4", + "hgnc_id": "HGNC:15966", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2C", + "hgnc_id": "HGNC:18225", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1", + "hgnc_id": "HGNC:1809", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "C4B_2", + "hgnc_id": "HGNC:42398", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTT1", + "hgnc_id": "HGNC:4641", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTTP2", + "hgnc_id": "HGNC:33606", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GTF2H2C_2", + "hgnc_id": "HGNC:35418", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB3", + "hgnc_id": "HGNC:4951", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB4", + "hgnc_id": "HGNC:4952", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL2", + "hgnc_id": "HGNC:6330", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5A", + "hgnc_id": "HGNC:16345", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5B", + "hgnc_id": "HGNC:16346", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS1", + "hgnc_id": "HGNC:6333", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS2", + "hgnc_id": "HGNC:6334", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS3", + "hgnc_id": "HGNC:6335", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS5", + "hgnc_id": "HGNC:6337", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR3DS1", + "hgnc_id": "HGNC:6340", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "LILRA3", + "hgnc_id": "HGNC:6604", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U8", + "hgnc_id": "HGNC:27538", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U9", + "hgnc_id": "HGNC:29166", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR9G9", + "hgnc_id": "HGNC:31940", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TAS2R45", + "hgnc_id": "HGNC:18876", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + } + ], + "not_read_owned": [ + { + "gene": "RHCE", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "AMY1A", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AMY1B", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AMY1C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GSTM1", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FCGR1A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GBA1", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FCGR2A", + "owner": "parascopy", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FCGR2C", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "FCGR3B", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR3", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR1", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PROS1", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B17", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT2B15", + "owner": "pypgx", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPE", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPB", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GTF2H2C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMN2", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMN1", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2H2", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSPA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "C4A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "C4B", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP21A2", + "owner": "cn_loci:CYP21A2", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LPA", + "owner": "cn_loci:LPA_KIV2", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PMS2", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2I", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DEFA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CEL", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC5B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KLRC2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KRT6B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRT6C", + "owner": "parascopy", + "share": 0.1665, + "status": "not read" + }, + { + "gene": "KRT6A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRFAM7A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "STRC", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SORD", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HBA2", + "owner": "cn_loci:HBA", + "share": 0.5, + "status": "not read" + }, + { + "gene": "HBA1", + "owner": "cn_loci:HBA", + "share": 0.6659, + "status": "not read" + }, + { + "gene": "TPSB2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TPSAB1", + "owner": "parascopy", + "share": 0.3331, + "status": "not read" + }, + { + "gene": "OTOA", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SULT1A1", + "owner": "pypgx", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HYDIN", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HP", + "owner": "cn_loci:HP", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CCL3L3", + "owner": "cn_loci:CCL3L1", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A6", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP2B6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD177", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "APOBEC3B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP2D6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + } + ], + "not_on_reference": [ + "TGIF2LY", + "PCDH11Y", + "TSPY2", + "TSPY4", + "TSPY8", + "TSPY3", + "TSPY1", + "TSPY9", + "TSPY10", + "VCY", + "VCY1B", + "CDY2B", + "CDY2A", + "HSFY1", + "HSFY2", + "RBMY1B", + "RBMY1A1", + "RBMY1D", + "RBMY1E", + "RBMY1F", + "RBMY1J", + "BPY2", + "DAZ1", + "DAZ2", + "CDY1B", + "BPY2B", + "DAZ3", + "DAZ4", + "BPY2C", + "CDY1", + "C4B_2", + "GSTT1", + "GSTTP2", + "GTF2H2C_2", + "HLA-DRB3", + "HLA-DRB4", + "KIR2DL2", + "KIR2DL5A", + "KIR2DL5B", + "KIR2DS1", + "KIR2DS2", + "KIR2DS3", + "KIR2DS5", + "KIR3DS1", + "LILRA3", + "OR8U8", + "OR8U9", + "OR9G9", + "TAS2R45" + ] + } + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "ok", + "source": "indexcov/A-indexcov.ped", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "inferred_sex": "female", + "cn_chrX": "2.1", + "cn_chrY": "0.02", + "karyotype": null + } + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 8, + "by_stars": { + "4": 0, + "3": 2, + "2": 3, + "1": 1, + "0": 2 + }, + "hits": [ + { + "stars": 4, + "chrom": "chr7", + "pos": 117559590, + "ref": "ATCT", + "alt": "A", + "genotype": "het", + "gene": "CFTR", + "significance": "Pathogenic", + "review_status": "practice guideline", + "combined_allele": ".", + "dp": 30, + "gq": 15, + "low_confidence": true, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 3, + "chrom": "chr17", + "pos": 43057100, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "BRCA1,NBR2", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": "part of a combined allele: chr17:43057100 C>T, chr17:43057102 A>G; unphased, so the parts may sit on different haplotypes", + "dp": 35, + "gq": 60, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 3, + "chrom": "chr6", + "pos": 26092913, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HFE", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": ".", + "dp": 40, + "gq": 99, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 2, + "chrom": "chr1", + "pos": 45331556, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "MUTYH", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 30, + "gq": 50, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 2, + "chrom": "chr16", + "pos": 176700, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HBA1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 2, + "chrom": "chr5", + "pos": 70951946, + "ref": "C", + "alt": "T", + "genotype": "hom", + "gene": "SMN1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 1, + "chrom": "chr13", + "pos": 20189473, + "ref": "C", + "alt": "T", + "genotype": "one copy, other copy not called ref (1/.)", + "gene": "GJB2", + "significance": "Likely pathogenic", + "review_status": "criteria provided, single submitter", + "combined_allele": "combined allele of chr13:20189473 C>T + chr13:20189475 G>A", + "dp": 5, + "gq": 40, + "low_confidence": true, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 1, + "chrom": "chr3", + "pos": 15645186, + "ref": "G", + "alt": "C", + "genotype": "het", + "gene": "BTD", + "significance": "Pathogenic/Likely pathogenic", + "review_status": "criteria provided, conflicting classifications", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 0, + "chrom": "chr2", + "pos": 100, + "ref": "A", + "alt": "G", + "genotype": "het", + "gene": ".", + "significance": ".", + "review_status": ".", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + }, + { + "stars": 0, + "chrom": "chrX", + "pos": 101398000, + "ref": "C", + "alt": "T", + "genotype": "1", + "gene": "GLA", + "significance": "Pathogenic", + "review_status": "no assertion criteria provided", + "combined_allele": "possibly one allele with chrX:101398004 G>A: check ClinVar for a combined record", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false + } + ], + "low_confidence": 2, + "call_floor": { + "GQ": 20, + "DP": 8 + }, + "frequency": { + "source": null, + "note": "no VEP output (step 13) in this folder", + "common_af": 0.01 + } + } + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_on_list": 3292, + "list": "assets/carrier_genes.tsv", + "carriers": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "MUTYH-related polyposis" + }, + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Symptomatic form of HFE-related hemochromatosis" + }, + { + "gene": "CFTR", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 4, + "inheritance": "AR", + "condition": "Cystic fibrosis" + }, + { + "gene": "GJB2", + "variant": "chr13:20189473 C>T", + "genotype": "one copy, other copy not called ref (1/.)", + "significance": "Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Nonsyndromic hearing loss, recessive (DFNB1A)" + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Fanconi anemia" + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 0, + "inheritance": "XL", + "condition": "Fabry disease" + }, + { + "gene": "HBA1", + "variant": "chr16:176700 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Alpha-thalassemia" + }, + { + "gene": "SMN1", + "variant": "chr5:70951946 C>T", + "genotype": "hom", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Spinal muscular atrophy" + } + ], + "set_aside": [ + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "significance": "Pathogenic/Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Biotinidase deficiency", + "kind": "hypomorph", + "name": "p.Asp444His (p.Asp424His on NM_001370658.1), partial biotinidase deficiency" + } + ], + "two_hits": { + "genes": [ + { + "gene": "CFTR", + "verdict": "possibly biallelic", + "why": "a heterozygous call inside a called deletion: one of the two calls may be wrong; check the reads", + "deletion": "CFTR_del deletion, 1 of 2 copies (read depth, step 35)", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "allele": null, + "condition": "Cystic fibrosis", + "inheritance": "AR" + } + ], + "deletions_read": [ + "AnnotSV class 4 and 5 deletions (step 05)", + "step 35's interval table (read depth)" + ], + "deletions_not_read": [ + "step 35's HBA1/HBA2 count: not assessed: S_hba_copy_number.tsv has no status row", + "Parascopy's SMN1 count: not resolved" + ] + } + } + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "ok", + "source": "vcf/B.report.json", + "file_date": "2021-03-04 05:07 UTC", + "note": null, + "data": { + "version": "", + "html_report": false + } + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "stale", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 27 was not run in the run of 2020-01-01 00:00 UTC", + "data": { + "genes": [ + { + "gene": "CYP2D6", + "diplotype": "*4/*4", + "phenotype": "Poor", + "status": "not called" + } + ], + "non_normal": 0, + "ambiguous": 0, + "not_called": 1, + "unclassified": 0, + "parse_failed": false, + "warnings": [] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "ok", + "source": "pypgx/S_pypgx_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_total": 6, + "genes_called": 3, + "cyp2d6": "*1/*2", + "cnv_call": { + "CYP2D6": "Normal", + "CYP2A6": "not called: pypgx failed", + "CYP2B6": "not called: the CYP2D6 depth check failed", + "SLCO1B1": "not called: pypgx gave no copy-number call" + }, + "not_callable": [ + "CYP2E1" + ], + "low_gq": [ + { + "gene": "UGT1A1", + "diplotype": "Indeterminate", + "min_gq": 12, + "weakest": "chr2:1:A:G" + }, + { + "gene": "SLCO1B1", + "diplotype": "*1/*5", + "min_gq": null, + "weakest": "" + } + ], + "cyp2d6_evidence": { + "phenotype": "Normal Metabolizer", + "defining_filter": "PASS", + "quality": "ok" + } + } + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/T_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "*1/*2", + "filter": "LowQ" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": null, + "text": "no row covers exon 7", + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": "." + }, + "smn2": { + "cn": null, + "text": "no row covers exon 7" + }, + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": ".", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "SMN1 g.27134T>G: not assessed, step 35 wrote no S_smn1_marker.tsv (a run from before this reader, or the check did not run)." + }, + "rows": 1, + "hba": { + "status": "not assessed", + "message": "not assessed: S_hba_copy_number.tsv has no status row" + }, + "cn_intervals": { + "rows": 3, + "by_status": { + "called": 3 + }, + "changes": [ + { + "locus": "CFTR_del", + "name": "CFTR region", + "chrom": "chr7", + "start": "117500000", + "end": "117600000", + "copies": "1", + "expected": "2", + "change": "deletion" + }, + { + "locus": "DUP1", + "name": "x", + "chrom": "chr1", + "start": "1", + "end": "100", + "copies": "3", + "expected": "2", + "change": "duplication" + }, + { + "locus": "GJB6", + "name": "x", + "chrom": "chr13", + "start": "x", + "end": "2", + "copies": "1", + "expected": "2", + "change": "deletion" + } + ] + }, + "cyp21a2": null, + "dosage": null, + "reading": "SMN1 and SMN2 not assessed: no model: no carrier reading", + "loci": [] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "x", + "status": "called", + "detail": "." + } + ], + "other_regions": 0, + "other_regions_called": 0, + "variants": {}, + "smn1": { + "paraphase": null, + "parascopy": { + "cn": null, + "text": "no row covers SMN1 exon 7" + }, + "agree": null + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*03:01", + "DQA1*01:01" + ], + "quality": [ + "0", + "30" + ], + "low_confidence": true, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*04:01", + "DRB1*07:01" + ], + "quality": [ + "40", + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 low confidence", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A not typed" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "present", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [], + "reason": "OptiType did not run: no class I reads" + }, + "kir": { + "typed": [], + "not_called": [], + "reason": "too few KIR reads" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "stale", + "source": "prs/S_prs_summary.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 25 was failed (exit 2) in the run of 2020-01-01 00:00 UTC", + "data": { + "scores": [ + { + "condition": "X", + "pgs_id": "PGS1", + "score": "", + "matched": "", + "total": "", + "chrx": "", + "ambiguous": "", + "applies_to": "", + "sex_check": "", + "percentile": "", + "group": "", + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": false, + "ancestry": { + "population": "", + "panel": "", + "low_confidence": "" + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/x/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 0, + "pass": 0 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "stale", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 18 was skipped (opt-in) in the run of 2020-01-01 00:00 UTC", + "data": { + "total": 2, + "deletions": 2, + "duplications": 0, + "significant": 2, + "significant_filtered": 1, + "filtered_chry": 1, + "dropped": { + "q0": 0, + "gap": 0, + "band": 0, + "chry_female": 1 + }, + "declared_sex": "female" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "stale", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 22 was failed (exit 1) in the run of 2020-01-01 00:00 UTC", + "data": { + "consensus": 1 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 0, + "rows": [], + "downgraded": [], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 0, + "exon_edge": 0, + "exon_edge_checked": true, + "common": 0 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 3, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "CFTR_T", + "repeat_count": "5/5", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/5-5", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "12/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "12-12/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "UGT1A1", + "repeat_count": "8/10", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "8-8/10-10", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": false, + "stranger_outdated": true, + "unsized": [], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG12-5T/TG12-5T", + "pairings": [ + "TG12-5T/TG12-5T" + ], + "tg": [ + 12, + 12 + ], + "poly_t": [ + 5, + 5 + ], + "five_t": true, + "phase": "not_read", + "r117h": ".", + "reading": "5T: 2 copys with the 5T poly-T tract, TG12 on the 5T copies. 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. R117H was not read (no step 42 traits row for rs78655421 was called), so whether it sits on the 5T copy is not known." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*28/TA9", + "ta_units": [ + 8, + 10 + ], + "pharmcat": ".", + "agreement": "not_compared", + "reading": "*28/TA9 (TA7/TA9, 8/10 TA units as ExpansionHunter counts them): a TA length outside TA5 to TA8 is not a named allele here, so its function is not read and it is not compared with PharmCAT (step 7)." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ] + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/x/deep/T_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "334", + "intratelomeric_reads": 61873, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "roh/S_roh.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": null, + "data": { + "segments": 1, + "total_mb": 7.0, + "largest_mb": 7.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + } + ], + "masked_mb": 0.0, + "sex_chromosome_segments": 0, + "long_mb": 7.0, + "f_roh": 0.0026, + "relationship": "no sign of related parents", + "upd_flag": null + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "stale", + "source": "mito/S_haplogroup.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 12 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "haplogroup": "U5a", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/U5a" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/U5a/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "insufficient markers", + "valid_markers": ".", + "qc_score": ".", + "explore": [] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "no branch has a derived call", + "level": ".", + "tree": ".", + "path_derived": ".", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": ".", + "min_depth": ".", + "explore": [], + "path": [] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 10, + "called": 10, + "no_call": 0, + "from_reference_blocks": 0, + "template_sites": ".", + "file_23andme": ".", + "file_ancestrydna": ".", + "no_call_reasons": {} + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "too little depth", + "kiv2_copies": ".", + "ci95_low": ".", + "ci95_high": ".", + "class": ".", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": ".", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "LPA KIV-2 4925G>A: not assessed, step 46 wrote no S_lpa_kiv2_4925.tsv (a run from before this reader, or the check did not run)." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": ".", + "status": "not_covered", + "risk_alleles": null + } + ], + "risk_alleles": 0, + "flag": ".", + "flag_text": "Not determined: neither the KIV-2 count nor the two risk variants could be read. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "confidence": ".", + "variant": ".", + "flag": ".", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "." + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "stale", + "source": "locityper/S_locityper.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 53 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*04:01/DRB1*07:01" + }, + { + "locus": "HLA-DQB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DQB1*02:01" + }, + { + "locus": "CYP2D6", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "pypgx (step 32): *1/*2; Cyrius (step 21): *1/*2 (not usable: Filter LowQ)" + } + ], + "called": 3, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + } + ], + "deletions_note": "no deletion table (mito/_chrM_deletions.tsv): step 20 ran before it looked for large deletions, eKLIPse wrote no table (see the step's log), or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "copies_note": "no copy-number table (mito/_mtdna_copies.tsv): step 20 ran before it counted mtDNA copies, or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "pass": 1, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "note": "mito/S_chrM_annotated.tsv is older than the chrM calls: rerun step 20" + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [], + "called": 0, + "not_covered": 1, + "other_allele": 0, + "low_gq": 0, + "rules": null + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "not assessed (no human depth)", + "human_depth": ".", + "found": [], + "genomes": 1 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [], + "genes": [] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [], + "other": 0, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + } + ], + "median_depth": null, + "median_limit": null, + "not_covered": [ + "DNMT3A" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 0, + "by_element": {}, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "F", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "seen" + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "message": "too shallow", + "depth": ".", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": ".", + "calls_mocha": ".", + "calls_reported": ".", + "phasing": ".", + "calls": [], + "not_reported": 0 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "ok", + "source": "cpsr/T.cpsr.grch37.classification.tsv.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "html_report": true, + "classification_column": "FINAL_CLASSIFICATION", + "classification": { + "Pathogenic": 1, + ".": 1 + }, + "plp": [ + { + "variant": ".", + "gene": "A", + "final": "Pathogenic", + "cpsr": "Benign", + "clinvar": null, + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false + } + ], + "disagreements": [], + "final_plp_own_benign": 1, + "own_class_column": "CPSR_CLASSIFICATION", + "low_confidence": 0, + "call_floor_dp": 8 + } + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 6, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "low_confidence": false, + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "low_confidence": false, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "1", + "significance": "Pathogenic", + "stars": 0, + "low_confidence": false, + "rule": "All hemi, het, homozygous P and LP", + "rule_note": null, + "severity": null + } + ], + "clinvar_checked": true, + "high_impact": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "consequence": "stop_gained", + "transcript": "MANE", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": 40, + "dp": 30, + "ad": "15,15", + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 2, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": true, + "structural_note": null, + "not_reported": [ + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "source": "ClinVar", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + }, + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "source": "ClinVar", + "reason": "one heterozygous variant; BTD is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + } + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_prioritized.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 2, + "compound_het_pairs": 26, + "compound_het_phase": { + "trans": 23, + "cis": 1, + "unknown": 1 + }, + "compound_het_trans": [ + { + "gene": "G1", + "variants": "chr1:10 A>G + chr1:15 C>T", + "reads": "1 of 2" + }, + { + "gene": "G2", + "variants": "chr1:20 A>G + chr1:25 C>T", + "reads": "2 of 3" + }, + { + "gene": "G3", + "variants": "chr1:30 A>G + chr1:35 C>T", + "reads": "3 of 4" + }, + { + "gene": "G4", + "variants": "chr1:40 A>G + chr1:45 C>T", + "reads": "4 of 5" + }, + { + "gene": "G5", + "variants": "chr1:50 A>G + chr1:55 C>T", + "reads": "5 of 6" + }, + { + "gene": "G6", + "variants": "chr1:60 A>G + chr1:65 C>T", + "reads": "6 of 7" + }, + { + "gene": "G7", + "variants": "chr1:70 A>G + chr1:75 C>T", + "reads": "7 of 8" + }, + { + "gene": "G8", + "variants": "chr1:80 A>G + chr1:85 C>T", + "reads": "8 of 9" + }, + { + "gene": "G9", + "variants": "chr1:90 A>G + chr1:95 C>T", + "reads": "9 of 10" + }, + { + "gene": "G10", + "variants": "chr1:100 A>G + chr1:105 C>T", + "reads": "10 of 11" + }, + { + "gene": "G11", + "variants": "chr1:110 A>G + chr1:115 C>T", + "reads": "11 of 12" + }, + { + "gene": "G12", + "variants": "chr1:120 A>G + chr1:125 C>T", + "reads": "12 of 13" + }, + { + "gene": "G13", + "variants": "chr1:130 A>G + chr1:135 C>T", + "reads": "13 of 14" + }, + { + "gene": "G14", + "variants": "chr1:140 A>G + chr1:145 C>T", + "reads": "14 of 15" + }, + { + "gene": "G15", + "variants": "chr1:150 A>G + chr1:155 C>T", + "reads": "15 of 16" + }, + { + "gene": "G16", + "variants": "chr1:160 A>G + chr1:165 C>T", + "reads": "16 of 17" + }, + { + "gene": "G17", + "variants": "chr1:170 A>G + chr1:175 C>T", + "reads": "17 of 18" + }, + { + "gene": "G18", + "variants": "chr1:180 A>G + chr1:185 C>T", + "reads": "18 of 19" + }, + { + "gene": "G19", + "variants": "chr1:190 A>G + chr1:195 C>T", + "reads": "19 of 20" + }, + { + "gene": "G20", + "variants": "chr1:200 A>G + chr1:205 C>T", + "reads": "20 of 21" + } + ] + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": "not called", + "pypgx": "*1/*2", + "Cyrius": "*1/*2 (not usable: Filter LowQ)" + }, + "agree": null, + "consensus": null, + "probable": { + "diplotype": "*1/*2", + "phenotype": "Normal Metabolizer", + "label": "probable (one caller: pypgx)" + } + }, + "not_run": [ + "Mosaic loss of chromosome Y or X", + "Variant calling", + "Fine-grained ancestry (ADMIXTURE, RFMix)", + "Structural variants (Delly)", + "Y-STR markers (HipSTR)", + "Blood groups (predicted)", + "Archaic ancestry (IBDmix)", + "Ancient populations (AADR PCA)" + ], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)", + "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums", + "HBA1/HBA2 copy number (alpha-thalassaemia deletions): neither step 35's read-depth count (short reads) nor step 35b (Paraphase, PacBio HiFi) made a call", + "MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; no standard caller sees them", + "Autosomal mosaic chromosomal alterations (large gains, losses and copy-neutral LOH in a share of the blood cells): step 51 (MoChA) did not assess this sample" + ] +} diff --git a/tests/parity/golden/summary-collect/all/all.json b/tests/parity/golden/summary-collect/all/all.json new file mode 100644 index 000000000..47d509ef4 --- /dev/null +++ b/tests/parity/golden/summary-collect/all/all.json @@ -0,0 +1,15476 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:15 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "male" + }, + "databases": { + "clinvar_release": "2026-10-01", + "cpsr_clinvar_release": "20250901", + "hla_database": "IPD-IMGT/HLA 3.60\textra" + }, + "manifest": [ + [ + "run", + "declared_sex", + "male" + ], + [ + "data", + "clinvar_file_date", + "2026-10-01" + ], + [ + "data", + "cpsr_clinvar_release", + "20250901" + ], + [ + "data", + "hla_database", + "IPD-IMGT/HLA 3.60\textra" + ] + ], + "sections": { + "coverage": { + "title": "Coverage (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S.mosdepth.summary.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "mean_depth": 31.25 + } + }, + "callability": { + "title": "Gene callability (mosdepth)", + "step": "16b", + "state": "ok", + "source": "mosdepth/S_callability.bed.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "min_depth": 10, + "min_mapq": 20, + "min_share": 0.9, + "catalog_genes": 1572, + "read": 217, + "not_read": [ + { + "gene": "OR4F5", + "hgnc_id": "HGNC:14825", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR4F29", + "hgnc_id": "HGNC:31275", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F16", + "hgnc_id": "HGNC:15079", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "AGRN", + "hgnc_id": "HGNC:329", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ATAD3B", + "hgnc_id": "HGNC:24007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MMP23B", + "hgnc_id": "HGNC:7171", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CDK11B", + "hgnc_id": "HGNC:1729", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35E2B", + "hgnc_id": "HGNC:33941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDK11A", + "hgnc_id": "HGNC:1730", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TTC34", + "hgnc_id": "HGNC:34297", + "reasons": [ + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPL22", + "hgnc_id": "HGNC:10315", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MASP2", + "hgnc_id": "HGNC:6902", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPPA", + "hgnc_id": "HGNC:7939", + "reasons": [ + "CMRG" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KIAA2013", + "hgnc_id": "HGNC:28513", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF1", + "hgnc_id": "HGNC:28840", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF11", + "hgnc_id": "HGNC:14086", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL1", + "hgnc_id": "HGNC:29295", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRAMEF4", + "hgnc_id": "HGNC:31971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF10", + "hgnc_id": "HGNC:27997", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRAMEF7", + "hgnc_id": "HGNC:28415", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRAMEF6", + "hgnc_id": "HGNC:30583", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF27", + "hgnc_id": "HGNC:51234", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HNRNPCL3", + "hgnc_id": "HGNC:51235", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HNRNPCL2", + "hgnc_id": "HGNC:48813", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF26", + "hgnc_id": "HGNC:49178", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HNRNPCL4", + "hgnc_id": "HGNC:51333", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "PRAMEF9", + "hgnc_id": "HGNC:27996", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRAMEF13", + "hgnc_id": "HGNC:13262", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRAMEF18", + "hgnc_id": "HGNC:30693", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRAMEF8", + "hgnc_id": "HGNC:24074", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRAMEF33", + "hgnc_id": "HGNC:49193", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PRAMEF15", + "hgnc_id": "HGNC:26764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRAMEF14", + "hgnc_id": "HGNC:13576", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRAMEF19", + "hgnc_id": "HGNC:24908", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "PRAMEF17", + "hgnc_id": "HGNC:29485", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CELA2B", + "hgnc_id": "HGNC:29995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CLCNKA", + "hgnc_id": "HGNC:2026", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CLCNKB", + "hgnc_id": "HGNC:2027", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NBPF1", + "hgnc_id": "HGNC:26088", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CROCC", + "hgnc_id": "HGNC:21299", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AKR7L", + "hgnc_id": "HGNC:24056", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF3", + "hgnc_id": "HGNC:25076", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CELA3B", + "hgnc_id": "HGNC:15945", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CELA3A", + "hgnc_id": "HGNC:15944", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HMGCL", + "hgnc_id": "HGNC:5005", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNR2", + "hgnc_id": "HGNC:2160", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RSRP1", + "hgnc_id": "HGNC:25234", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGV", + "hgnc_id": "HGNC:26031", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "XKR8", + "hgnc_id": "HGNC:25508", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BMP8A", + "hgnc_id": "HGNC:21650", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BMP8B", + "hgnc_id": "HGNC:1075", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OXCT2", + "hgnc_id": "HGNC:18606", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CYP4Z1", + "hgnc_id": "HGNC:20583", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4A22", + "hgnc_id": "HGNC:20575", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GBP1", + "hgnc_id": "HGNC:4182", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SETSIP", + "hgnc_id": "HGNC:42937", + "reasons": [ + "segdup" + ], + "share": 0.4994, + "status": "not read" + }, + { + "gene": "AGL", + "hgnc_id": "HGNC:321", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "EXTL2", + "hgnc_id": "HGNC:3516", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AMY2A", + "hgnc_id": "HGNC:477", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF4", + "hgnc_id": "HGNC:26550", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NBPF6", + "hgnc_id": "HGNC:31988", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LRIG2", + "hgnc_id": "HGNC:20889", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "VANGL1", + "hgnc_id": "HGNC:15512", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IGSF3", + "hgnc_id": "HGNC:5950", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SEC22B", + "hgnc_id": "HGNC:10700", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF26", + "hgnc_id": "HGNC:49571", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2NLR", + "hgnc_id": "HGNC:53925", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4A", + "hgnc_id": "HGNC:24369", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM72B", + "hgnc_id": "HGNC:24805", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SRGAP2C", + "hgnc_id": "HGNC:30584", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3-7", + "hgnc_id": "HGNC:32060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPIAL4E", + "hgnc_id": "HGNC:33997", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF15", + "hgnc_id": "HGNC:28791", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4F", + "hgnc_id": "HGNC:33999", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "SRGAP2B", + "hgnc_id": "HGNC:35237", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM72D", + "hgnc_id": "HGNC:33593", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PPIAL4D", + "hgnc_id": "HGNC:33998", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "GPR89A", + "hgnc_id": "HGNC:31984", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBM8A", + "hgnc_id": "HGNC:9905", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "NBPF10", + "hgnc_id": "HGNC:31992", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLA", + "hgnc_id": "HGNC:31862", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPIAL4H", + "hgnc_id": "HGNC:53889", + "reasons": [ + "segdup" + ], + "share": 0.6658, + "status": "not read" + }, + { + "gene": "NBPF12", + "hgnc_id": "HGNC:24297", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NBPF11", + "hgnc_id": "HGNC:31993", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PPIAL4G", + "hgnc_id": "HGNC:33996", + "reasons": [ + "segdup" + ], + "share": 0.1656, + "status": "not read" + }, + { + "gene": "NBPF14", + "hgnc_id": "HGNC:25232", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOTCH2NLB", + "hgnc_id": "HGNC:53923", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NUDT4B", + "hgnc_id": "HGNC:18012", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PDE4DIP", + "hgnc_id": "HGNC:15580", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NOTCH2NLC", + "hgnc_id": "HGNC:53924", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NBPF19", + "hgnc_id": "HGNC:31999", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIAL4C", + "hgnc_id": "HGNC:33995", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "H2BC18", + "hgnc_id": "HGNC:24700", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H3C13", + "hgnc_id": "HGNC:25311", + "reasons": [ + "segdup" + ], + "share": 0.8316, + "status": "not read" + }, + { + "gene": "H3C14", + "hgnc_id": "HGNC:20503", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H2AC18", + "hgnc_id": "HGNC:4736", + "reasons": [ + "segdup" + ], + "share": 0.1651, + "status": "not read" + }, + { + "gene": "H2AC19", + "hgnc_id": "HGNC:29668", + "reasons": [ + "segdup" + ], + "share": 0.3328, + "status": "not read" + }, + { + "gene": "H3C15", + "hgnc_id": "HGNC:20505", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H4C15", + "hgnc_id": "HGNC:29607", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HRNR", + "hgnc_id": "HGNC:20846", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LCE3E", + "hgnc_id": "HGNC:29463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCE3D", + "hgnc_id": "HGNC:16615", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "S100A8", + "hgnc_id": "HGNC:10498", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "S100A7A", + "hgnc_id": "HGNC:21657", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "S100A7", + "hgnc_id": "HGNC:10497", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FLAD1", + "hgnc_id": "HGNC:24671", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MTX1", + "hgnc_id": "HGNC:7504", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN3", + "hgnc_id": "HGNC:19183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FCRL5", + "hgnc_id": "HGNC:18508", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA6", + "hgnc_id": "HGNC:5239", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FCGR2B", + "hgnc_id": "HGNC:3618", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD17B7", + "hgnc_id": "HGNC:5215", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD247", + "hgnc_id": "HGNC:1677", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "XCL2", + "hgnc_id": "HGNC:10646", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "PRG4", + "hgnc_id": "HGNC:9364", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PTPRC", + "hgnc_id": "HGNC:9666", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PPP1R12B", + "hgnc_id": "HGNC:7619", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KISS1", + "hgnc_id": "HGNC:6341", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SRGAP2", + "hgnc_id": "HGNC:19751", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CR1", + "hgnc_id": "HGNC:2334", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CR1L", + "hgnc_id": "HGNC:2335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZC3H11B", + "hgnc_id": "HGNC:25659", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LBR", + "hgnc_id": "HGNC:6518", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "LEFTY2", + "hgnc_id": "HGNC:3122", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "OR2T33", + "hgnc_id": "HGNC:31255", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T7", + "hgnc_id": "HGNC:15019", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T2", + "hgnc_id": "HGNC:14725", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T3", + "hgnc_id": "HGNC:14727", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR2T5", + "hgnc_id": "HGNC:15017", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "OR2G6", + "hgnc_id": "HGNC:27019", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR2T34", + "hgnc_id": "HGNC:31256", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2T11", + "hgnc_id": "HGNC:19574", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2T35", + "hgnc_id": "HGNC:31257", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR2T27", + "hgnc_id": "HGNC:31252", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SNTG2", + "hgnc_id": "HGNC:13741", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TPO", + "hgnc_id": "HGNC:12015", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KLF11", + "hgnc_id": "HGNC:11811", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CCDC121", + "hgnc_id": "HGNC:25833", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "ABCG8", + "hgnc_id": "HGNC:13887", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PCBP1", + "hgnc_id": "HGNC:8647", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "NAT8", + "hgnc_id": "HGNC:18069", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "C2orf78", + "hgnc_id": "HGNC:34349", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CD8B", + "hgnc_id": "HGNC:1707", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RGPD1", + "hgnc_id": "HGNC:32414", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLGLB1", + "hgnc_id": "HGNC:9072", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLGLB2", + "hgnc_id": "HGNC:9073", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RGPD2", + "hgnc_id": "HGNC:32415", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPIA", + "hgnc_id": "HGNC:10297", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEKT4", + "hgnc_id": "HGNC:31012", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAHD2A", + "hgnc_id": "HGNC:24252", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TRIM43B", + "hgnc_id": "HGNC:37146", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM43", + "hgnc_id": "HGNC:19015", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANKRD36C", + "hgnc_id": "HGNC:32946", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPAT2", + "hgnc_id": "HGNC:27168", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD36", + "hgnc_id": "HGNC:24079", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANKRD36B", + "hgnc_id": "HGNC:29333", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "RGPD3", + "hgnc_id": "HGNC:32416", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD4", + "hgnc_id": "HGNC:32417", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RANBP2", + "hgnc_id": "HGNC:9848", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD5", + "hgnc_id": "HGNC:32418", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MALL", + "hgnc_id": "HGNC:6818", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MTLN", + "hgnc_id": "HGNC:27339", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LIMS4", + "hgnc_id": "HGNC:39941", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RGPD6", + "hgnc_id": "HGNC:32419", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ANAPC1", + "hgnc_id": "HGNC:19988", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RGPD8", + "hgnc_id": "HGNC:9849", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FOXD4L1", + "hgnc_id": "HGNC:18521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RABL2A", + "hgnc_id": "HGNC:9799", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RAB6C", + "hgnc_id": "HGNC:16525", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "POTEF", + "hgnc_id": "HGNC:33905", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CCDC74B", + "hgnc_id": "HGNC:25267", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SMPD4", + "hgnc_id": "HGNC:32949", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3E", + "hgnc_id": "HGNC:20765", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEI", + "hgnc_id": "HGNC:37093", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CFC1B", + "hgnc_id": "HGNC:33983", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CFC1", + "hgnc_id": "HGNC:18292", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "POTEJ", + "hgnc_id": "HGNC:37094", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PLEKHB2", + "hgnc_id": "HGNC:19236", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB6D", + "hgnc_id": "HGNC:30272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MZT2A", + "hgnc_id": "HGNC:33187", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TUBA3D", + "hgnc_id": "HGNC:24071", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CCDC74A", + "hgnc_id": "HGNC:25197", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "ANKRD30BL", + "hgnc_id": "HGNC:35167", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NOSTRIN", + "hgnc_id": "HGNC:20203", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IFT70B", + "hgnc_id": "HGNC:26425", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CASP10", + "hgnc_id": "HGNC:1500", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ALPP", + "hgnc_id": "HGNC:439", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ALPG", + "hgnc_id": "HGNC:441", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT1A6", + "hgnc_id": "HGNC:12538", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UGT1A5", + "hgnc_id": "HGNC:12537", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT1A3", + "hgnc_id": "HGNC:12535", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MLPH", + "hgnc_id": "HGNC:29643", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR6B2", + "hgnc_id": "HGNC:15041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AQP12B", + "hgnc_id": "HGNC:6096", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "AQP12A", + "hgnc_id": "HGNC:19941", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ANO7", + "hgnc_id": "HGNC:31677", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CHL1", + "hgnc_id": "HGNC:1939", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMC3", + "hgnc_id": "HGNC:23999", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANCD2", + "hgnc_id": "HGNC:3585", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DAZL", + "hgnc_id": "HGNC:2685", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF35", + "hgnc_id": "HGNC:13099", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LZTFL1", + "hgnc_id": "HGNC:6741", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MST1", + "hgnc_id": "HGNC:7380", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MST1R", + "hgnc_id": "HGNC:7381", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HYAL1", + "hgnc_id": "HGNC:5320", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FRG2C", + "hgnc_id": "HGNC:33626", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "ZNF717", + "hgnc_id": "HGNC:29448", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR5H6", + "hgnc_id": "HGNC:14767", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ROPN1", + "hgnc_id": "HGNC:17692", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ROPN1B", + "hgnc_id": "HGNC:31927", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RPN1", + "hgnc_id": "HGNC:10381", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ALG1L2", + "hgnc_id": "HGNC:37258", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BFSP2", + "hgnc_id": "HGNC:1041", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRR23B", + "hgnc_id": "HGNC:33764", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SOX2", + "hgnc_id": "HGNC:11195", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "EIF2B5", + "hgnc_id": "HGNC:3261", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KNG1", + "hgnc_id": "HGNC:6383", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TM4SF19", + "hgnc_id": "HGNC:25167", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF595", + "hgnc_id": "HGNC:27196", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDE6B", + "hgnc_id": "HGNC:8786", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAK", + "hgnc_id": "HGNC:4113", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FGFRL1", + "hgnc_id": "HGNC:3693", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RNF212", + "hgnc_id": "HGNC:27729", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UVSSA", + "hgnc_id": "HGNC:29304", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DOK7", + "hgnc_id": "HGNC:26594", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HMX1", + "hgnc_id": "HGNC:5017", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A26", + "hgnc_id": "HGNC:43746", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "USP17L10", + "hgnc_id": "HGNC:44438", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L11", + "hgnc_id": "HGNC:44439", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L12", + "hgnc_id": "HGNC:44440", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L13", + "hgnc_id": "HGNC:44441", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L17", + "hgnc_id": "HGNC:44445", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L18", + "hgnc_id": "HGNC:44446", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L19", + "hgnc_id": "HGNC:44447", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L20", + "hgnc_id": "HGNC:44448", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L21", + "hgnc_id": "HGNC:44449", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L22", + "hgnc_id": "HGNC:44450", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L24", + "hgnc_id": "HGNC:44453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L25", + "hgnc_id": "HGNC:44452", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "USP17L26", + "hgnc_id": "HGNC:44454", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "USP17L5", + "hgnc_id": "HGNC:37177", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "USP17L27", + "hgnc_id": "HGNC:44455", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L28", + "hgnc_id": "HGNC:44456", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "USP17L30", + "hgnc_id": "HGNC:44458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB131A", + "hgnc_id": "HGNC:18108", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRD5", + "hgnc_id": "HGNC:3026", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "UGT2B10", + "hgnc_id": "HGNC:12544", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B28", + "hgnc_id": "HGNC:13479", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UGT2A1", + "hgnc_id": "HGNC:12542", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UGT2A2", + "hgnc_id": "HGNC:28183", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HTN3", + "hgnc_id": "HGNC:5284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HTN1", + "hgnc_id": "HGNC:5283", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "AFP", + "hgnc_id": "HGNC:317", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PIGY", + "hgnc_id": "HGNC:28213", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PYURF", + "hgnc_id": "HGNC:44317", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CISD2", + "hgnc_id": "HGNC:24212", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC9B1", + "hgnc_id": "HGNC:24244", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KLHL2", + "hgnc_id": "HGNC:6353", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GK3", + "hgnc_id": "HGNC:4292", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "HPF1", + "hgnc_id": "HGNC:26051", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDLIM3", + "hgnc_id": "HGNC:20767", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAT1", + "hgnc_id": "HGNC:3595", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG1", + "hgnc_id": "HGNC:3954", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FRG2", + "hgnc_id": "HGNC:19136", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DUX4", + "hgnc_id": "HGNC:50800", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SDHA", + "hgnc_id": "HGNC:10680", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PDCD6", + "hgnc_id": "HGNC:8765", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZDHHC11B", + "hgnc_id": "HGNC:32962", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZDHHC11", + "hgnc_id": "HGNC:19158", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TERT", + "hgnc_id": "HGNC:11730", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A3", + "hgnc_id": "HGNC:11049", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAF11L2", + "hgnc_id": "HGNC:53845", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L3", + "hgnc_id": "HGNC:53846", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L4", + "hgnc_id": "HGNC:53847", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L5", + "hgnc_id": "HGNC:53848", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "TAF11L7", + "hgnc_id": "HGNC:53850", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAF11L8", + "hgnc_id": "HGNC:53851", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "TAF11L9", + "hgnc_id": "HGNC:53852", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TAF11L10", + "hgnc_id": "HGNC:53853", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "TAF11L11", + "hgnc_id": "HGNC:53854", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAF11L12", + "hgnc_id": "HGNC:53855", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "TAF11L14", + "hgnc_id": "HGNC:53857", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EMB", + "hgnc_id": "HGNC:30465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MARVELD2", + "hgnc_id": "HGNC:26401", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERF1B", + "hgnc_id": "HGNC:10756", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NAIP", + "hgnc_id": "HGNC:7634", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SKIC3", + "hgnc_id": "HGNC:23639", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIX1", + "hgnc_id": "HGNC:18581", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SAR1B", + "hgnc_id": "HGNC:10535", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DND1", + "hgnc_id": "HGNC:23799", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDHA10", + "hgnc_id": "HGNC:8664", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PCDHB7", + "hgnc_id": "HGNC:8692", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PCDHB8", + "hgnc_id": "HGNC:8693", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PCDHB11", + "hgnc_id": "HGNC:8682", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PCDHB12", + "hgnc_id": "HGNC:8683", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PPP1R2B", + "hgnc_id": "HGNC:16318", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPM1", + "hgnc_id": "HGNC:7910", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "THOC3", + "hgnc_id": "HGNC:19072", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SIMC1", + "hgnc_id": "HGNC:24779", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FAM153A", + "hgnc_id": "HGNC:29940", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FAM153CP", + "hgnc_id": "HGNC:33936", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FLT4", + "hgnc_id": "HGNC:3767", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR4F3", + "hgnc_id": "HGNC:8300", + "reasons": [ + "segdup" + ], + "share": 0.1658, + "status": "not read" + }, + { + "gene": "DUSP22", + "hgnc_id": "HGNC:16077", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TUBB2A", + "hgnc_id": "HGNC:12412", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "H2AC13", + "hgnc_id": "HGNC:4725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "H2AC14", + "hgnc_id": "HGNC:4727", + "reasons": [ + "segdup" + ], + "share": 0.8319, + "status": "not read" + }, + { + "gene": "OR2J1", + "hgnc_id": "HGNC:8259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2J2", + "hgnc_id": "HGNC:8260", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HLA-G", + "hgnc_id": "HGNC:4964", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HLA-A", + "hgnc_id": "HGNC:4931", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "MICA", + "hgnc_id": "HGNC:7090", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MICB", + "hgnc_id": "HGNC:7091", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZBTB12", + "hgnc_id": "HGNC:19066", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PBX2", + "hgnc_id": "HGNC:8633", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HLA-DRB1", + "hgnc_id": "HGNC:4948", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GSTA2", + "hgnc_id": "HGNC:4627", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GSTA1", + "hgnc_id": "HGNC:4626", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTA5", + "hgnc_id": "HGNC:19662", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "PRIM2", + "hgnc_id": "HGNC:9370", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FKBP1C", + "hgnc_id": "HGNC:21376", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SLC17A5", + "hgnc_id": "HGNC:10933", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEC63", + "hgnc_id": "HGNC:21082", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR2A4", + "hgnc_id": "HGNC:14729", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE9", + "hgnc_id": "HGNC:37275", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "PCMT1", + "hgnc_id": "HGNC:8728", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RAET1G", + "hgnc_id": "HGNC:16795", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ULBP1", + "hgnc_id": "HGNC:14893", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RAET1L", + "hgnc_id": "HGNC:16798", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ULBP3", + "hgnc_id": "HGNC:14895", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC22A1", + "hgnc_id": "HGNC:10963", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMOC2", + "hgnc_id": "HGNC:20323", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RBAK-RBAKDN", + "hgnc_id": "HGNC:42971", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RBAK", + "hgnc_id": "HGNC:17680", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC29A4", + "hgnc_id": "HGNC:23097", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FSCN1", + "hgnc_id": "HGNC:11148", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM", + "hgnc_id": "HGNC:8105", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCZ1", + "hgnc_id": "HGNC:21691", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RSPH10B2", + "hgnc_id": "HGNC:34385", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CCZ1B", + "hgnc_id": "HGNC:21717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NOD1", + "hgnc_id": "HGNC:16390", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FKBP9", + "hgnc_id": "HGNC:3725", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DPY19L1", + "hgnc_id": "HGNC:22205", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SEPTIN7", + "hgnc_id": "HGNC:1717", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE1", + "hgnc_id": "HGNC:16408", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PPIA", + "hgnc_id": "HGNC:9253", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDC14C", + "hgnc_id": "HGNC:22427", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSPH", + "hgnc_id": "HGNC:9577", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CCT6A", + "hgnc_id": "HGNC:1620", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF716", + "hgnc_id": "HGNC:32458", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF727", + "hgnc_id": "HGNC:22785", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF735", + "hgnc_id": "HGNC:32466", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF736", + "hgnc_id": "HGNC:32467", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GUSB", + "hgnc_id": "HGNC:4696", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RABGEF1", + "hgnc_id": "HGNC:17676", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TYW1", + "hgnc_id": "HGNC:25598", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE21", + "hgnc_id": "HGNC:51517", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYW1B", + "hgnc_id": "HGNC:33908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POM121", + "hgnc_id": "HGNC:19702", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM74", + "hgnc_id": "HGNC:17453", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPDYE8", + "hgnc_id": "HGNC:33771", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE9", + "hgnc_id": "HGNC:45034", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE10", + "hgnc_id": "HGNC:51506", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NSUN5", + "hgnc_id": "HGNC:16385", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM50", + "hgnc_id": "HGNC:19017", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLIP2", + "hgnc_id": "HGNC:2586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GTF2IRD2", + "hgnc_id": "HGNC:30775", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE12", + "hgnc_id": "HGNC:51508", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CASTOR2", + "hgnc_id": "HGNC:37073", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GTF2IRD2B", + "hgnc_id": "HGNC:33125", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPDYE13", + "hgnc_id": "HGNC:51509", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE14", + "hgnc_id": "HGNC:51510", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM73", + "hgnc_id": "HGNC:18162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POM121C", + "hgnc_id": "HGNC:34005", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPDYE5", + "hgnc_id": "HGNC:35464", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DTX2", + "hgnc_id": "HGNC:15973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "UPK3B", + "hgnc_id": "HGNC:21444", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPDYE16", + "hgnc_id": "HGNC:51512", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPDYE17", + "hgnc_id": "HGNC:51513", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE18", + "hgnc_id": "HGNC:51514", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "STEAP1", + "hgnc_id": "HGNC:11378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ASNS", + "hgnc_id": "HGNC:753", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OCM2", + "hgnc_id": "HGNC:34396", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP3A7-CYP3A51P", + "hgnc_id": "HGNC:51504", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP3A4", + "hgnc_id": "HGNC:2637", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "STAG3", + "hgnc_id": "HGNC:11356", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PVRIG", + "hgnc_id": "HGNC:32190", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SPDYE3", + "hgnc_id": "HGNC:35462", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SAP25", + "hgnc_id": "HGNC:41908", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZAN", + "hgnc_id": "HGNC:12857", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC12", + "hgnc_id": "HGNC:7510", + "reasons": [ + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPDYE6", + "hgnc_id": "HGNC:35465", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "POLR2J", + "hgnc_id": "HGNC:9197", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RASA4B", + "hgnc_id": "HGNC:35202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J3", + "hgnc_id": "HGNC:33853", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "UPK3BL2", + "hgnc_id": "HGNC:53444", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "RASA4", + "hgnc_id": "HGNC:23181", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "UPK3BL1", + "hgnc_id": "HGNC:37278", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "POLR2J3-UPK3BL2", + "hgnc_id": "HGNC:58364", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SPDYE2B", + "hgnc_id": "HGNC:48334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "POLR2J2", + "hgnc_id": "HGNC:23208", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAMB1", + "hgnc_id": "HGNC:6486", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2B", + "hgnc_id": "HGNC:18272", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KLF14", + "hgnc_id": "HGNC:23025", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AKR1B10", + "hgnc_id": "HGNC:382", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1B15", + "hgnc_id": "HGNC:37281", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BRAF", + "hgnc_id": "HGNC:1097", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRBV9", + "hgnc_id": "HGNC:12246", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRSS2", + "hgnc_id": "HGNC:9483", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAGE15", + "hgnc_id": "HGNC:37295", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TCAF2", + "hgnc_id": "HGNC:26878", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TCAF2C", + "hgnc_id": "HGNC:53641", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CTAGE6", + "hgnc_id": "HGNC:28644", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "OR2F2", + "hgnc_id": "HGNC:8247", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CTAGE4", + "hgnc_id": "HGNC:24772", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR2A42", + "hgnc_id": "HGNC:31230", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR2A7", + "hgnc_id": "HGNC:8234", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAGE8", + "hgnc_id": "HGNC:37294", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "OR2A1", + "hgnc_id": "HGNC:8229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARHGEF5", + "hgnc_id": "HGNC:13209", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACTR3C", + "hgnc_id": "HGNC:37282", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KMT2C", + "hgnc_id": "HGNC:13726", + "reasons": [ + "CMRG", + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ACTR3B", + "hgnc_id": "HGNC:17256", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4F21", + "hgnc_id": "HGNC:19583", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "FBXO25", + "hgnc_id": "HGNC:13596", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARHGEF10", + "hgnc_id": "HGNC:14103", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A15", + "hgnc_id": "HGNC:32263", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A3", + "hgnc_id": "HGNC:32251", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A13", + "hgnc_id": "HGNC:32261", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A5", + "hgnc_id": "HGNC:32253", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM90A20", + "hgnc_id": "HGNC:32268", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L1", + "hgnc_id": "HGNC:37182", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP17L4", + "hgnc_id": "HGNC:37176", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNF705G", + "hgnc_id": "HGNC:37134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB108C", + "hgnc_id": "HGNC:30847", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB4B", + "hgnc_id": "HGNC:30193", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "DEFB103B", + "hgnc_id": "HGNC:31702", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "DEFB104B", + "hgnc_id": "HGNC:26165", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB106B", + "hgnc_id": "HGNC:28879", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "DEFB105B", + "hgnc_id": "HGNC:29930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "DEFB107B", + "hgnc_id": "HGNC:31918", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRR23D1", + "hgnc_id": "HGNC:49420", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A7", + "hgnc_id": "HGNC:32255", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "FAM90A23", + "hgnc_id": "HGNC:32271", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A14", + "hgnc_id": "HGNC:32262", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A18", + "hgnc_id": "HGNC:32266", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "FAM90A16", + "hgnc_id": "HGNC:32264", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "FAM90A8", + "hgnc_id": "HGNC:32256", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "FAM90A17", + "hgnc_id": "HGNC:32265", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A9", + "hgnc_id": "HGNC:32257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A10", + "hgnc_id": "HGNC:32258", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PRR23D2", + "hgnc_id": "HGNC:49396", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB107A", + "hgnc_id": "HGNC:18086", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "DEFB105A", + "hgnc_id": "HGNC:18087", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "DEFB106A", + "hgnc_id": "HGNC:18088", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPAG11A", + "hgnc_id": "HGNC:33342", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB103A", + "hgnc_id": "HGNC:15967", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DEFB4A", + "hgnc_id": "HGNC:2767", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNF705B", + "hgnc_id": "HGNC:32284", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "USP17L8", + "hgnc_id": "HGNC:37181", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "USP17L3", + "hgnc_id": "HGNC:37175", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "FAM90A11", + "hgnc_id": "HGNC:32259", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FAM90A24", + "hgnc_id": "HGNC:32272", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "FAM90A12", + "hgnc_id": "HGNC:32260", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLC35G5", + "hgnc_id": "HGNC:15546", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "DEFB130B", + "hgnc_id": "HGNC:39814", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF705D", + "hgnc_id": "HGNC:33202", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "USP17L2", + "hgnc_id": "HGNC:34434", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DEFB109D", + "hgnc_id": "HGNC:30838", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM86B1", + "hgnc_id": "HGNC:28268", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEFB130A", + "hgnc_id": "HGNC:18107", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM86B2", + "hgnc_id": "HGNC:32222", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RBPMS", + "hgnc_id": "HGNC:19097", + "reasons": [ + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "IMPA1", + "hgnc_id": "HGNC:6050", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC10A5", + "hgnc_id": "HGNC:22981", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH17", + "hgnc_id": "HGNC:1756", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP11B1", + "hgnc_id": "HGNC:2591", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP11B2", + "hgnc_id": "HGNC:2592", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GPIHBP1", + "hgnc_id": "HGNC:24945", + "reasons": [ + "CMRG" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "WASHC1", + "hgnc_id": "HGNC:24361", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOXD4", + "hgnc_id": "HGNC:3805", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "ZNG1A", + "hgnc_id": "HGNC:17134", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KCNV2", + "hgnc_id": "HGNC:19698", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RLN2", + "hgnc_id": "HGNC:10027", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RLN1", + "hgnc_id": "HGNC:10026", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "IFNA21", + "hgnc_id": "HGNC:5424", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA4", + "hgnc_id": "HGNC:5425", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "IFNA7", + "hgnc_id": "HGNC:5428", + "reasons": [ + "segdup" + ], + "share": 0.3324, + "status": "not read" + }, + { + "gene": "IFNA10", + "hgnc_id": "HGNC:5418", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "IFNA16", + "hgnc_id": "HGNC:5421", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "IFNA17", + "hgnc_id": "HGNC:5422", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "IFNA13", + "hgnc_id": "HGNC:5419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFNA1", + "hgnc_id": "HGNC:5417", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "ANKRD18B", + "hgnc_id": "HGNC:23644", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRSS3", + "hgnc_id": "HGNC:9486", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALT", + "hgnc_id": "HGNC:4135", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31F1", + "hgnc_id": "HGNC:41911", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CNTNAP3", + "hgnc_id": "HGNC:13834", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31A1", + "hgnc_id": "HGNC:23394", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L6", + "hgnc_id": "HGNC:31986", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNG1F", + "hgnc_id": "HGNC:31978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CNTNAP3B", + "hgnc_id": "HGNC:32035", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPATA31A6", + "hgnc_id": "HGNC:32006", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SPATA31A7", + "hgnc_id": "HGNC:32007", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CNTNAP3C", + "hgnc_id": "HGNC:53878", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L5", + "hgnc_id": "HGNC:18522", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ZNG1E", + "hgnc_id": "HGNC:24584", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXD4L4", + "hgnc_id": "HGNC:23762", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF658", + "hgnc_id": "HGNC:25226", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD20A1", + "hgnc_id": "HGNC:23665", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNG1C", + "hgnc_id": "HGNC:18519", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXD4L3", + "hgnc_id": "HGNC:18523", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PGM5", + "hgnc_id": "HGNC:8908", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FXN", + "hgnc_id": "HGNC:3951", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TJP2", + "hgnc_id": "HGNC:11828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPATA31D4", + "hgnc_id": "HGNC:38601", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPATA31D3", + "hgnc_id": "HGNC:38603", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SPATA31D1", + "hgnc_id": "HGNC:37283", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "KIF27", + "hgnc_id": "HGNC:18632", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPATA31C1", + "hgnc_id": "HGNC:27846", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SPATA31C2", + "hgnc_id": "HGNC:24508", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC71A2", + "hgnc_id": "HGNC:23376", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2G", + "hgnc_id": "HGNC:23449", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CTSV", + "hgnc_id": "HGNC:2538", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR13C5", + "hgnc_id": "HGNC:15100", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR13C2", + "hgnc_id": "HGNC:14701", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR13C9", + "hgnc_id": "HGNC:15104", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "POLE3", + "hgnc_id": "HGNC:13546", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ORM1", + "hgnc_id": "HGNC:8498", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ORM2", + "hgnc_id": "HGNC:8499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SLC27A4", + "hgnc_id": "HGNC:10998", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PKN3", + "hgnc_id": "HGNC:17999", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ABO", + "hgnc_id": "HGNC:79", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LCN1", + "hgnc_id": "HGNC:6525", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OBP2A", + "hgnc_id": "HGNC:23380", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "SOHLH1", + "hgnc_id": "HGNC:27845", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "INPP5E", + "hgnc_id": "HGNC:21474", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "AJM1", + "hgnc_id": "HGNC:37284", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRIN1", + "hgnc_id": "HGNC:4584", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8", + "hgnc_id": "HGNC:20773", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "AKR1C1", + "hgnc_id": "HGNC:384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AKR1C2", + "hgnc_id": "HGNC:385", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DCLRE1C", + "hgnc_id": "HGNC:17642", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRC1", + "hgnc_id": "HGNC:7228", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ANKRD30A", + "hgnc_id": "HGNC:17234", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF33A", + "hgnc_id": "HGNC:13096", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF37A", + "hgnc_id": "HGNC:13102", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PLD5P1", + "hgnc_id": "HGNC:55072", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF33B", + "hgnc_id": "HGNC:13097", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "BMS1", + "hgnc_id": "HGNC:23505", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "AGAP4", + "hgnc_id": "HGNC:23459", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23", + "hgnc_id": "HGNC:17312", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ANTXRL", + "hgnc_id": "HGNC:27277", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANXA8L1", + "hgnc_id": "HGNC:23334", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPY4R", + "hgnc_id": "HGNC:9329", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GPRIN2", + "hgnc_id": "HGNC:23730", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PTPN20", + "hgnc_id": "HGNC:23423", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ANXA8", + "hgnc_id": "HGNC:546", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM25G", + "hgnc_id": "HGNC:23590", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP9", + "hgnc_id": "HGNC:23463", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SYT15B", + "hgnc_id": "HGNC:51487", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPY4R2", + "hgnc_id": "HGNC:52383", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PARG", + "hgnc_id": "HGNC:8605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TIMM23B", + "hgnc_id": "HGNC:23581", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TIMM23B-AGAP6", + "hgnc_id": "HGNC:45009", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AGAP6", + "hgnc_id": "HGNC:23466", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASHC2A", + "hgnc_id": "HGNC:23416", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ASAH2B", + "hgnc_id": "HGNC:23456", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EIF5AL1", + "hgnc_id": "HGNC:17419", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2B", + "hgnc_id": "HGNC:23445", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NUTM2E", + "hgnc_id": "HGNC:23448", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMEM254", + "hgnc_id": "HGNC:25804", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PLAC9", + "hgnc_id": "HGNC:19255", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ADIRF", + "hgnc_id": "HGNC:24043", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NUTM2A", + "hgnc_id": "HGNC:23438", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NUTM2D", + "hgnc_id": "HGNC:23447", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PAPSS2", + "hgnc_id": "HGNC:8604", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PTEN", + "hgnc_id": "HGNC:9588", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LIPN", + "hgnc_id": "HGNC:23452", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "BTRC", + "hgnc_id": "HGNC:1144", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SFR1", + "hgnc_id": "HGNC:29574", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DHX32", + "hgnc_id": "HGNC:16717", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FANK1", + "hgnc_id": "HGNC:23527", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ECHS1", + "hgnc_id": "HGNC:3151", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SPRN", + "hgnc_id": "HGNC:16871", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FRG2B", + "hgnc_id": "HGNC:33518", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "SCGB1C1", + "hgnc_id": "HGNC:18394", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IFITM3", + "hgnc_id": "HGNC:5414", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DRD4", + "hgnc_id": "HGNC:3025", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "DEAF1", + "hgnc_id": "HGNC:14677", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP5-2", + "hgnc_id": "HGNC:23597", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP5-4", + "hgnc_id": "HGNC:23599", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MRPL23", + "hgnc_id": "HGNC:10322", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H19", + "hgnc_id": "HGNC:4713", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSU72L5", + "hgnc_id": "HGNC:43624", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "SSU72L2", + "hgnc_id": "HGNC:43621", + "reasons": [ + "segdup" + ], + "share": 0.666, + "status": "not read" + }, + { + "gene": "SSU72L4", + "hgnc_id": "HGNC:43623", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "SSU72L1", + "hgnc_id": "HGNC:43626", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR51A4", + "hgnc_id": "HGNC:14795", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "OR51A2", + "hgnc_id": "HGNC:14764", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HBG1", + "hgnc_id": "HGNC:4831", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR10A5", + "hgnc_id": "HGNC:15131", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MRGPRX4", + "hgnc_id": "HGNC:17617", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MDK", + "hgnc_id": "HGNC:6972", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPI1", + "hgnc_id": "HGNC:11241", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C3", + "hgnc_id": "HGNC:14697", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4C5", + "hgnc_id": "HGNC:14702", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4A47", + "hgnc_id": "HGNC:31266", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM51G", + "hgnc_id": "HGNC:43972", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64C", + "hgnc_id": "HGNC:37148", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "FOLH1", + "hgnc_id": "HGNC:3788", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4C13", + "hgnc_id": "HGNC:15169", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4C12", + "hgnc_id": "HGNC:15168", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "OR4C45", + "hgnc_id": "HGNC:31270", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4C46", + "hgnc_id": "HGNC:31271", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4A5", + "hgnc_id": "HGNC:15162", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM48", + "hgnc_id": "HGNC:19021", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR4A15", + "hgnc_id": "HGNC:15152", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TRIM51", + "hgnc_id": "HGNC:19023", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR5F1", + "hgnc_id": "HGNC:8343", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR8H2", + "hgnc_id": "HGNC:15308", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR5M8", + "hgnc_id": "HGNC:14846", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "OR9G1", + "hgnc_id": "HGNC:15319", + "reasons": [ + "collapsed" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "OR1S2", + "hgnc_id": "HGNC:15141", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1S1", + "hgnc_id": "HGNC:8227", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GLYATL1B", + "hgnc_id": "HGNC:37865", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PGA3", + "hgnc_id": "HGNC:8885", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "B3GAT3", + "hgnc_id": "HGNC:923", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ESRRA", + "hgnc_id": "HGNC:3471", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC22A12", + "hgnc_id": "HGNC:17989", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RBM4B", + "hgnc_id": "HGNC:28842", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TPCN2", + "hgnc_id": "HGNC:20820", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FGF3", + "hgnc_id": "HGNC:3681", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KRTAP5-8", + "hgnc_id": "HGNC:23603", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP5-10", + "hgnc_id": "HGNC:23605", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "KRTAP5-11", + "hgnc_id": "HGNC:23606", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "XNDC1N-ZNF705EP-ALG1L9P", + "hgnc_id": "HGNC:55871", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DEFB108B", + "hgnc_id": "HGNC:29966", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "XNDC1N", + "hgnc_id": "HGNC:54661", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "INTS4", + "hgnc_id": "HGNC:25048", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOX4", + "hgnc_id": "HGNC:7891", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TRIM77", + "hgnc_id": "HGNC:34228", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "TRIM49", + "hgnc_id": "HGNC:13431", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TRIM64B", + "hgnc_id": "HGNC:37147", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TRIM49D1", + "hgnc_id": "HGNC:43973", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TRIM64", + "hgnc_id": "HGNC:14663", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM49C", + "hgnc_id": "HGNC:38877", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "UBTFL1", + "hgnc_id": "HGNC:14533", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CARD16", + "hgnc_id": "HGNC:33701", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSPA8", + "hgnc_id": "HGNC:5241", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR8B2", + "hgnc_id": "HGNC:8471", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TMEM45B", + "hgnc_id": "HGNC:25194", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CD4", + "hgnc_id": "HGNC:1678", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PEX5", + "hgnc_id": "HGNC:9719", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC1", + "hgnc_id": "HGNC:604", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GDF3", + "hgnc_id": "HGNC:4218", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NANOG", + "hgnc_id": "HGNC:20857", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM90A1", + "hgnc_id": "HGNC:25526", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRH1", + "hgnc_id": "HGNC:9366", + "reasons": [ + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PRR4", + "hgnc_id": "HGNC:18020", + "reasons": [ + "collapsed" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRH2", + "hgnc_id": "HGNC:9367", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TAS2R14", + "hgnc_id": "HGNC:14920", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TAS2R31", + "hgnc_id": "HGNC:19113", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TAS2R46", + "hgnc_id": "HGNC:18877", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TAS2R43", + "hgnc_id": "HGNC:18875", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "TAS2R30", + "hgnc_id": "HGNC:19112", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "PRB4", + "hgnc_id": "HGNC:9340", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PRB1", + "hgnc_id": "HGNC:9337", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "DDX11", + "hgnc_id": "HGNC:2736", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "H3-5", + "hgnc_id": "HGNC:33164", + "reasons": [ + "segdup" + ], + "share": 0.1661, + "status": "not read" + }, + { + "gene": "ALG10", + "hgnc_id": "HGNC:23162", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ALG10B", + "hgnc_id": "HGNC:31088", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBA1B", + "hgnc_id": "HGNC:18809", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "COX14", + "hgnc_id": "HGNC:28216", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NACA", + "hgnc_id": "HGNC:7629", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DPY19L2", + "hgnc_id": "HGNC:19414", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNF1A", + "hgnc_id": "HGNC:11621", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HPD", + "hgnc_id": "HGNC:5147", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCAR2", + "hgnc_id": "HGNC:24827", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "HCAR3", + "hgnc_id": "HGNC:16824", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "P2RX2", + "hgnc_id": "HGNC:15459", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PGAM5", + "hgnc_id": "HGNC:28763", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA3", + "hgnc_id": "HGNC:4426", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "C1QTNF9B", + "hgnc_id": "HGNC:34072", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C1QTNF9", + "hgnc_id": "HGNC:28732", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PARP4", + "hgnc_id": "HGNC:271", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PABPC3", + "hgnc_id": "HGNC:8556", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "MRPS31", + "hgnc_id": "HGNC:16632", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HNRNPA1L2", + "hgnc_id": "HGNC:27067", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PRR20C", + "hgnc_id": "HGNC:37221", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PRR20B", + "hgnc_id": "HGNC:37220", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "PRR20D", + "hgnc_id": "HGNC:37222", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "PRR20E", + "hgnc_id": "HGNC:37223", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "F7", + "hgnc_id": "HGNC:3544", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PROZ", + "hgnc_id": "HGNC:9460", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEM", + "hgnc_id": "HGNC:37096", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEG", + "hgnc_id": "HGNC:33896", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR11H2", + "hgnc_id": "HGNC:14716", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR4N2", + "hgnc_id": "HGNC:14742", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OR4Q3", + "hgnc_id": "HGNC:15426", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OR4M1", + "hgnc_id": "HGNC:14735", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RNASE3", + "hgnc_id": "HGNC:10046", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RNASE2", + "hgnc_id": "HGNC:10045", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10G2", + "hgnc_id": "HGNC:8170", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "DHRS4", + "hgnc_id": "HGNC:16985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHRS4L2", + "hgnc_id": "HGNC:19731", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ARF6", + "hgnc_id": "HGNC:659", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACOT2", + "hgnc_id": "HGNC:18431", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NDUFB1", + "hgnc_id": "HGNC:7695", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "JAG2", + "hgnc_id": "HGNC:6189", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "BTBD6", + "hgnc_id": "HGNC:19897", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "IGHV3-21", + "hgnc_id": "HGNC:5586", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA6L6", + "hgnc_id": "HGNC:37225", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "POTEB3", + "hgnc_id": "HGNC:51240", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LINC02203", + "hgnc_id": "HGNC:53069", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR4M2B", + "hgnc_id": "HGNC:55109", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "OR4N4C", + "hgnc_id": "HGNC:55110", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "POTEB", + "hgnc_id": "HGNC:33734", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "OR4M2-OT1", + "hgnc_id": "HGNC:56199", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4N4", + "hgnc_id": "HGNC:15375", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L22", + "hgnc_id": "HGNC:50289", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6L1", + "hgnc_id": "HGNC:37444", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L26", + "hgnc_id": "HGNC:56306", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8S", + "hgnc_id": "HGNC:44409", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6L2", + "hgnc_id": "HGNC:26695", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HERC2", + "hgnc_id": "HGNC:4868", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6L24", + "hgnc_id": "HGNC:55710", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA8F", + "hgnc_id": "HGNC:32378", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8G", + "hgnc_id": "HGNC:25328", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6L25", + "hgnc_id": "HGNC:55711", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8M", + "hgnc_id": "HGNC:44404", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8J", + "hgnc_id": "HGNC:38650", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8T", + "hgnc_id": "HGNC:44410", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8R", + "hgnc_id": "HGNC:44407", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA8Q", + "hgnc_id": "HGNC:44408", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GOLGA8H", + "hgnc_id": "HGNC:37443", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GOLGA8O", + "hgnc_id": "HGNC:44406", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA8N", + "hgnc_id": "HGNC:44405", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GOLGA8A", + "hgnc_id": "HGNC:31972", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA8B", + "hgnc_id": "HGNC:31973", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NANOGP8", + "hgnc_id": "HGNC:23106", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "NDUFAF1", + "hgnc_id": "HGNC:18828", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CKMT1B", + "hgnc_id": "HGNC:1995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CATSPER2", + "hgnc_id": "HGNC:18810", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CKMT1A", + "hgnc_id": "HGNC:31736", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TERB2", + "hgnc_id": "HGNC:28520", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LDHAL6B", + "hgnc_id": "HGNC:21481", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GOLGA6B", + "hgnc_id": "HGNC:32205", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GOLGA6A", + "hgnc_id": "HGNC:13567", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ISLR2", + "hgnc_id": "HGNC:29286", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GOLGA6C", + "hgnc_id": "HGNC:32206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GOLGA6D", + "hgnc_id": "HGNC:32204", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ADAMTS7", + "hgnc_id": "HGNC:223", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EFL1", + "hgnc_id": "HGNC:25789", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SAXO2", + "hgnc_id": "HGNC:33727", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GOLGA6L10", + "hgnc_id": "HGNC:37228", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GOLGA6L9", + "hgnc_id": "HGNC:37229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "WHAMM", + "hgnc_id": "HGNC:30493", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR4F4", + "hgnc_id": "HGNC:8301", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MPG", + "hgnc_id": "HGNC:7211", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HBM", + "hgnc_id": "HGNC:4826", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AXIN1", + "hgnc_id": "HGNC:903", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSTR5", + "hgnc_id": "HGNC:11334", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TPSD1", + "hgnc_id": "HGNC:14118", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GNPTG", + "hgnc_id": "HGNC:23026", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UNKL", + "hgnc_id": "HGNC:14184", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PKD1", + "hgnc_id": "HGNC:9008", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PDPK1", + "hgnc_id": "HGNC:8816", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "EEF2KMT", + "hgnc_id": "HGNC:32221", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NPIPA3", + "hgnc_id": "HGNC:41978", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPA2", + "hgnc_id": "HGNC:41979", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NOMO1", + "hgnc_id": "HGNC:30060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PDXDC1", + "hgnc_id": "HGNC:28995", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RRN3", + "hgnc_id": "HGNC:30346", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPA5", + "hgnc_id": "HGNC:41980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MPV17L", + "hgnc_id": "HGNC:26827", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NOMO3", + "hgnc_id": "HGNC:25242", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPA6", + "hgnc_id": "HGNC:27893", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPA8", + "hgnc_id": "HGNC:41983", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPA9", + "hgnc_id": "HGNC:41984", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NOMO2", + "hgnc_id": "HGNC:22652", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SMG1", + "hgnc_id": "HGNC:30045", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IQCK", + "hgnc_id": "HGNC:28556", + "reasons": [ + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACSM5", + "hgnc_id": "HGNC:26060", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ACSM2B", + "hgnc_id": "HGNC:30931", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPIPB3", + "hgnc_id": "HGNC:28989", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB4", + "hgnc_id": "HGNC:41985", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB5", + "hgnc_id": "HGNC:37233", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB6", + "hgnc_id": "HGNC:37454", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB7", + "hgnc_id": "HGNC:33832", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A2", + "hgnc_id": "HGNC:11454", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB8", + "hgnc_id": "HGNC:37490", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EIF3C", + "hgnc_id": "HGNC:3279", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NPIPB9", + "hgnc_id": "HGNC:41987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NPIPB11", + "hgnc_id": "HGNC:37453", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BOLA2-SMG1P6", + "hgnc_id": "HGNC:53563", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BOLA2", + "hgnc_id": "HGNC:29488", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "SLX1B", + "hgnc_id": "HGNC:28748", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "SULT1A4", + "hgnc_id": "HGNC:30004", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "NPIPB12", + "hgnc_id": "HGNC:37491", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLX1A", + "hgnc_id": "HGNC:20922", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SULT1A3", + "hgnc_id": "HGNC:11455", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NPIPB13", + "hgnc_id": "HGNC:41989", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PHKG2", + "hgnc_id": "HGNC:8931", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "VKORC1", + "hgnc_id": "HGNC:23663", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRABD5", + "hgnc_id": "HGNC:26987", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TP53TG3", + "hgnc_id": "HGNC:30759", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TP53TG3C", + "hgnc_id": "HGNC:42962", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "TP53TG3E", + "hgnc_id": "HGNC:51816", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TP53TG3B", + "hgnc_id": "HGNC:37202", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TP53TG3F", + "hgnc_id": "HGNC:51817", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "CCNYL1B", + "hgnc_id": "HGNC:56313", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CES1", + "hgnc_id": "HGNC:1863", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MT1M", + "hgnc_id": "HGNC:14296", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MATCAP1", + "hgnc_id": "HGNC:34408", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSD11B2", + "hgnc_id": "HGNC:5209", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CLEC18A", + "hgnc_id": "HGNC:30388", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PDPR", + "hgnc_id": "HGNC:30264", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "EXOSC6", + "hgnc_id": "HGNC:19055", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HPR", + "hgnc_id": "HGNC:5156", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NPIPB15", + "hgnc_id": "HGNC:34409", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CLEC18B", + "hgnc_id": "HGNC:33849", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CA5A", + "hgnc_id": "HGNC:1377", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF469", + "hgnc_id": "HGNC:23216", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CDH15", + "hgnc_id": "HGNC:1754", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ANKRD11", + "hgnc_id": "HGNC:21316", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MC1R", + "hgnc_id": "HGNC:6929", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PRDM7", + "hgnc_id": "HGNC:9351", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ABR", + "hgnc_id": "HGNC:81", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SERPINF2", + "hgnc_id": "HGNC:9075", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SRR", + "hgnc_id": "HGNC:14398", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OR1D5", + "hgnc_id": "HGNC:8186", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "P2RX5", + "hgnc_id": "HGNC:8536", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ALOX15", + "hgnc_id": "HGNC:433", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ENO3", + "hgnc_id": "HGNC:3354", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "USP6", + "hgnc_id": "HGNC:12629", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SLC35G6", + "hgnc_id": "HGNC:31351", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "ALOXE3", + "hgnc_id": "HGNC:13743", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HES7", + "hgnc_id": "HGNC:15977", + "reasons": [ + "CMRG" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CDRT15", + "hgnc_id": "HGNC:14395", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "FBXW10B", + "hgnc_id": "HGNC:14379", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TRIM16", + "hgnc_id": "HGNC:17241", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF286A-TBC1D26", + "hgnc_id": "HGNC:55384", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF286A", + "hgnc_id": "HGNC:13501", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D26", + "hgnc_id": "HGNC:28745", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "UBB", + "hgnc_id": "HGNC:12463", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "ATPAF2", + "hgnc_id": "HGNC:18802", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "EVPLL", + "hgnc_id": "HGNC:35236", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS9C", + "hgnc_id": "HGNC:33874", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D28", + "hgnc_id": "HGNC:26858", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "FOXO3B", + "hgnc_id": "HGNC:3822", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FBXW10", + "hgnc_id": "HGNC:1211", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GRAP", + "hgnc_id": "HGNC:4562", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GRAPL", + "hgnc_id": "HGNC:37240", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LGALS9B", + "hgnc_id": "HGNC:24842", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CDRT15L2", + "hgnc_id": "HGNC:34075", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MEIS3P2", + "hgnc_id": "HGNC:17638", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAP2K3", + "hgnc_id": "HGNC:6843", + "reasons": [ + "collapsed" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KCNJ18", + "hgnc_id": "HGNC:39080", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS9", + "hgnc_id": "HGNC:6570", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FOXN1", + "hgnc_id": "HGNC:12765", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SUZ12", + "hgnc_id": "HGNC:17101", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37B", + "hgnc_id": "HGNC:29070", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SLC35G3", + "hgnc_id": "HGNC:26848", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "CCL4", + "hgnc_id": "HGNC:10630", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3B", + "hgnc_id": "HGNC:27011", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCL4L2", + "hgnc_id": "HGNC:24066", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "TBC1D3I", + "hgnc_id": "HGNC:32709", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TBC1D3G", + "hgnc_id": "HGNC:29860", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TBC1D3F", + "hgnc_id": "HGNC:18257", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBC1D3K", + "hgnc_id": "HGNC:51245", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TBC1D3L", + "hgnc_id": "HGNC:51246", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TBC1D3D", + "hgnc_id": "HGNC:28944", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TBC1D3C", + "hgnc_id": "HGNC:24889", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TBC1D3E", + "hgnc_id": "HGNC:27071", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MRPL45", + "hgnc_id": "HGNC:16651", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP1-1", + "hgnc_id": "HGNC:16772", + "reasons": [ + "CMRG" + ], + "share": 0.1659, + "status": "not read" + }, + { + "gene": "KRTAP4-7", + "hgnc_id": "HGNC:18898", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP4-8", + "hgnc_id": "HGNC:17230", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP4-16", + "hgnc_id": "HGNC:18921", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KRTAP4-9", + "hgnc_id": "HGNC:18910", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "KRTAP4-12", + "hgnc_id": "HGNC:16776", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP9-7", + "hgnc_id": "HGNC:18915", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "KRT14", + "hgnc_id": "HGNC:6416", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRT16", + "hgnc_id": "HGNC:6423", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRT17", + "hgnc_id": "HGNC:6427", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "HSD17B1", + "hgnc_id": "HGNC:5210", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "G6PC3", + "hgnc_id": "HGNC:24861", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ARHGAP27", + "hgnc_id": "HGNC:31813", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "PLEKHM1", + "hgnc_id": "HGNC:29017", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ARL17B", + "hgnc_id": "HGNC:32387", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "LRRC37A", + "hgnc_id": "HGNC:29069", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ARL17A", + "hgnc_id": "HGNC:24096", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "NSF", + "hgnc_id": "HGNC:8016", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NPEPPS", + "hgnc_id": "HGNC:7900", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HOXB8", + "hgnc_id": "HGNC:5119", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GIP", + "hgnc_id": "HGNC:4270", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DHX40", + "hgnc_id": "HGNC:18018", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "RPS6KB1", + "hgnc_id": "HGNC:10436", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "METTL2A", + "hgnc_id": "HGNC:25755", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TLK2", + "hgnc_id": "HGNC:11842", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CSH2", + "hgnc_id": "HGNC:2441", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "GH2", + "hgnc_id": "HGNC:4262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CSH1", + "hgnc_id": "HGNC:2440", + "reasons": [ + "segdup" + ], + "share": 0.6663, + "status": "not read" + }, + { + "gene": "CSHL1", + "hgnc_id": "HGNC:2442", + "reasons": [ + "segdup" + ], + "share": 0.8329, + "status": "not read" + }, + { + "gene": "LRRC37A3", + "hgnc_id": "HGNC:32427", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C17orf58", + "hgnc_id": "HGNC:27568", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "AMZ2", + "hgnc_id": "HGNC:28041", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TMC6", + "hgnc_id": "HGNC:18021", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CANT1", + "hgnc_id": "HGNC:19721", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RNF213", + "hgnc_id": "HGNC:14539", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ARL16", + "hgnc_id": "HGNC:27902", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TUBB8B", + "hgnc_id": "HGNC:24983", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TYMS", + "hgnc_id": "HGNC:12441", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SLC35G4", + "hgnc_id": "HGNC:31043", + "reasons": [ + "segdup" + ], + "share": 0.4995, + "status": "not read" + }, + { + "gene": "POTEC", + "hgnc_id": "HGNC:33894", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ANKRD30B", + "hgnc_id": "HGNC:24165", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SLC25A52", + "hgnc_id": "HGNC:23324", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KATNAL2", + "hgnc_id": "HGNC:25387", + "reasons": [ + "collapsed" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SERPINB4", + "hgnc_id": "HGNC:10570", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SERPINB3", + "hgnc_id": "HGNC:10569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GALR1", + "hgnc_id": "HGNC:4132", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CTDP1", + "hgnc_id": "HGNC:2498", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "BSG", + "hgnc_id": "HGNC:1116", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HCN2", + "hgnc_id": "HGNC:4846", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KISS1R", + "hgnc_id": "HGNC:4510", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "STK11", + "hgnc_id": "HGNC:11389", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TCF3", + "hgnc_id": "HGNC:11633", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ABHD17A", + "hgnc_id": "HGNC:28756", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "GIPC3", + "hgnc_id": "HGNC:18183", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TBXA2R", + "hgnc_id": "HGNC:11608", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CREB3L3", + "hgnc_id": "HGNC:18855", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RFX2", + "hgnc_id": "HGNC:9983", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "C3", + "hgnc_id": "HGNC:1318", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "MBD3L2B", + "hgnc_id": "HGNC:53435", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "MBD3L4", + "hgnc_id": "HGNC:37206", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBD3L2", + "hgnc_id": "HGNC:18532", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "STXBP2", + "hgnc_id": "HGNC:11445", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "TYK2", + "hgnc_id": "HGNC:12440", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF799", + "hgnc_id": "HGNC:28071", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF443", + "hgnc_id": "HGNC:20878", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYP4F3", + "hgnc_id": "HGNC:2646", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP4F12", + "hgnc_id": "HGNC:18857", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "OR10H2", + "hgnc_id": "HGNC:8173", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CALR3", + "hgnc_id": "HGNC:20407", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "MYO9B", + "hgnc_id": "HGNC:7609", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "INSL3", + "hgnc_id": "HGNC:6086", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF738", + "hgnc_id": "HGNC:32469", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ZNF100", + "hgnc_id": "HGNC:12880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ZNF208", + "hgnc_id": "HGNC:12999", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF676", + "hgnc_id": "HGNC:20429", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF729", + "hgnc_id": "HGNC:32464", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF492", + "hgnc_id": "HGNC:23707", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF728", + "hgnc_id": "HGNC:32463", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHPN2", + "hgnc_id": "HGNC:19974", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "GPI", + "hgnc_id": "HGNC:4458", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF302", + "hgnc_id": "HGNC:13848", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF181", + "hgnc_id": "HGNC:12971", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FFAR3", + "hgnc_id": "HGNC:4499", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "COX6B1", + "hgnc_id": "HGNC:2280", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LGALS7", + "hgnc_id": "HGNC:6568", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "LGALS7B", + "hgnc_id": "HGNC:34447", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "IFNL3", + "hgnc_id": "HGNC:18365", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "IFNL4", + "hgnc_id": "HGNC:44480", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "IFNL2", + "hgnc_id": "HGNC:18364", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CYP2A7", + "hgnc_id": "HGNC:2611", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CYP2G1P", + "hgnc_id": "HGNC:2633", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A13", + "hgnc_id": "HGNC:2608", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CYP2F1", + "hgnc_id": "HGNC:2632", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG8", + "hgnc_id": "HGNC:9525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG1", + "hgnc_id": "HGNC:9514", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PSG6", + "hgnc_id": "HGNC:9523", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PSG7", + "hgnc_id": "HGNC:9524", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PSG11", + "hgnc_id": "HGNC:9516", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "PSG2", + "hgnc_id": "HGNC:9519", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PSG4", + "hgnc_id": "HGNC:9521", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PSG9", + "hgnc_id": "HGNC:9526", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ETHE1", + "hgnc_id": "HGNC:23287", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF285", + "hgnc_id": "HGNC:13079", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "APOC1", + "hgnc_id": "HGNC:607", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOC4", + "hgnc_id": "HGNC:611", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "BLOC1S3", + "hgnc_id": "HGNC:20914", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "DMPK", + "hgnc_id": "HGNC:2933", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FUT1", + "hgnc_id": "HGNC:4012", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "BAX", + "hgnc_id": "HGNC:959", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LHB", + "hgnc_id": "HGNC:6584", + "reasons": [ + "segdup" + ], + "share": 0.8327, + "status": "not read" + }, + { + "gene": "CGB3", + "hgnc_id": "HGNC:1886", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CGB2", + "hgnc_id": "HGNC:16722", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "CGB1", + "hgnc_id": "HGNC:16721", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "CGB5", + "hgnc_id": "HGNC:16452", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CGB8", + "hgnc_id": "HGNC:16453", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CGB7", + "hgnc_id": "HGNC:16451", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PNKP", + "hgnc_id": "HGNC:9154", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SIGLEC16", + "hgnc_id": "HGNC:24851", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KLK4", + "hgnc_id": "HGNC:6365", + "reasons": [ + "CMRG" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "ETFB", + "hgnc_id": "HGNC:3482", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF808", + "hgnc_id": "HGNC:33230", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ZNF600", + "hgnc_id": "HGNC:30951", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF468", + "hgnc_id": "HGNC:33105", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VN1R4", + "hgnc_id": "HGNC:19871", + "reasons": [ + "segdup" + ], + "share": 0.166, + "status": "not read" + }, + { + "gene": "FAM90A27P", + "hgnc_id": "HGNC:43617", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF765", + "hgnc_id": "HGNC:25092", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF761", + "hgnc_id": "HGNC:23179", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF813", + "hgnc_id": "HGNC:33257", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PRKCG", + "hgnc_id": "HGNC:9402", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MBOAT7", + "hgnc_id": "HGNC:15505", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "LILRA6", + "hgnc_id": "HGNC:15495", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "LILRB2", + "hgnc_id": "HGNC:6606", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "LAIR1", + "hgnc_id": "HGNC:6477", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LAIR2", + "hgnc_id": "HGNC:6478", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "LILRB1", + "hgnc_id": "HGNC:6605", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KIR3DL3", + "hgnc_id": "HGNC:16312", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KIR2DL3", + "hgnc_id": "HGNC:6331", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KIR2DL1", + "hgnc_id": "HGNC:6329", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KIR2DL4", + "hgnc_id": "HGNC:6332", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KIR3DL1", + "hgnc_id": "HGNC:6338", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "KIR3DL2", + "hgnc_id": "HGNC:6339", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "NLRP7", + "hgnc_id": "HGNC:22947", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "NLRP2", + "hgnc_id": "HGNC:22948", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GP6", + "hgnc_id": "HGNC:14388", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TNNT1", + "hgnc_id": "HGNC:11948", + "reasons": [ + "CMRG" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TNNI3", + "hgnc_id": "HGNC:11947", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL4A", + "hgnc_id": "HGNC:16449", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RFPL4AL1", + "hgnc_id": "HGNC:45147", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZNF419", + "hgnc_id": "HGNC:20648", + "reasons": [ + "CMRG", + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "ZNF773", + "hgnc_id": "HGNC:30487", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "ZNF587B", + "hgnc_id": "HGNC:37142", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "ZNF814", + "hgnc_id": "HGNC:33258", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "ZNF417", + "hgnc_id": "HGNC:20646", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC27A5", + "hgnc_id": "HGNC:10999", + "reasons": [ + "CMRG" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SIRPB1", + "hgnc_id": "HGNC:15928", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CST4", + "hgnc_id": "HGNC:2476", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CST1", + "hgnc_id": "HGNC:2473", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "CST2", + "hgnc_id": "HGNC:2474", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "PLAGL2", + "hgnc_id": "HGNC:9047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SEMG1", + "hgnc_id": "HGNC:10742", + "reasons": [ + "CMRG" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "PLTP", + "hgnc_id": "HGNC:9093", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRNA4", + "hgnc_id": "HGNC:1958", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "EEF1A2", + "hgnc_id": "HGNC:3192", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PTK6", + "hgnc_id": "HGNC:9617", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MYT1", + "hgnc_id": "HGNC:7622", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCMTD2", + "hgnc_id": "HGNC:15882", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TPTE", + "hgnc_id": "HGNC:12023", + "reasons": [ + "segdup", + "collapsed" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "POTED", + "hgnc_id": "HGNC:23822", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "KRTAP21-1", + "hgnc_id": "HGNC:18945", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SMIM11", + "hgnc_id": "HGNC:1293", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMIM34", + "hgnc_id": "HGNC:39601", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KCNE1", + "hgnc_id": "HGNC:6240", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CBR3", + "hgnc_id": "HGNC:1549", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "NDUFV3", + "hgnc_id": "HGNC:7719", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CBS", + "hgnc_id": "HGNC:1550", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "U2AF1", + "hgnc_id": "HGNC:12453", + "reasons": [ + "CMRG", + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SIK1", + "hgnc_id": "HGNC:11142", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSF2BP", + "hgnc_id": "HGNC:5226", + "reasons": [ + "false_duplication" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BC12L", + "hgnc_id": "HGNC:4762", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3319, + "status": "not read" + }, + { + "gene": "TRAPPC10", + "hgnc_id": "HGNC:11868", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PWP2", + "hgnc_id": "HGNC:9711", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GATD3", + "hgnc_id": "HGNC:1273", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DNMT3L", + "hgnc_id": "HGNC:2980", + "reasons": [ + "CMRG", + "false_duplication" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-1", + "hgnc_id": "HGNC:22966", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "KRTAP10-2", + "hgnc_id": "HGNC:22967", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "KRTAP10-4", + "hgnc_id": "HGNC:20521", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "KRTAP10-5", + "hgnc_id": "HGNC:22969", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "KRTAP10-6", + "hgnc_id": "HGNC:20523", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "KRTAP10-8", + "hgnc_id": "HGNC:20525", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRTAP10-9", + "hgnc_id": "HGNC:22971", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "KRTAP10-10", + "hgnc_id": "HGNC:22972", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "KRTAP12-2", + "hgnc_id": "HGNC:20530", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "KRTAP12-1", + "hgnc_id": "HGNC:20529", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "COL18A1", + "hgnc_id": "HGNC:2195", + "reasons": [ + "CMRG" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "OR11H1", + "hgnc_id": "HGNC:15404", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "POTEH", + "hgnc_id": "HGNC:133", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CCT8L2", + "hgnc_id": "HGNC:15553", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "USP18", + "hgnc_id": "HGNC:12616", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "GGTLC3", + "hgnc_id": "HGNC:33426", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "TMEM191B", + "hgnc_id": "HGNC:33600", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM246B", + "hgnc_id": "HGNC:54843", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DGCR6", + "hgnc_id": "HGNC:2846", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "PRODH", + "hgnc_id": "HGNC:9453", + "reasons": [ + "segdup", + "false_duplication" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM246C", + "hgnc_id": "HGNC:54842", + "reasons": [ + "segdup" + ], + "share": 0.4997, + "status": "not read" + }, + { + "gene": "CCDC188", + "hgnc_id": "HGNC:51899", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RTN4R", + "hgnc_id": "HGNC:18601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TUBA3FP", + "hgnc_id": "HGNC:24067", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "LRRC74B", + "hgnc_id": "HGNC:34301", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM246A", + "hgnc_id": "HGNC:54844", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "RIMBP3B", + "hgnc_id": "HGNC:33891", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HIC2", + "hgnc_id": "HGNC:18595", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "TMEM191C", + "hgnc_id": "HGNC:33601", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GGTLC2", + "hgnc_id": "HGNC:18596", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "IGLL1", + "hgnc_id": "HGNC:5870", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "DRICH1", + "hgnc_id": "HGNC:28031", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GSTT2B", + "hgnc_id": "HGNC:33437", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "DDTL", + "hgnc_id": "HGNC:33446", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "GSTT2", + "hgnc_id": "HGNC:4642", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RFPL1", + "hgnc_id": "HGNC:9977", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "C22orf42", + "hgnc_id": "HGNC:27160", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RFPL3", + "hgnc_id": "HGNC:9980", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3D", + "hgnc_id": "HGNC:17354", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "APOBEC3H", + "hgnc_id": "HGNC:24100", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ATF4", + "hgnc_id": "HGNC:786", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "NDUFA6", + "hgnc_id": "HGNC:7690", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2D7", + "hgnc_id": "HGNC:2624", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "RRP7A", + "hgnc_id": "HGNC:24286", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CYB5R3", + "hgnc_id": "HGNC:2873", + "reasons": [ + "CMRG" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "A4GALT", + "hgnc_id": "HGNC:18149", + "reasons": [ + "CMRG" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TTLL1", + "hgnc_id": "HGNC:1312", + "reasons": [ + "CMRG" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MLC1", + "hgnc_id": "HGNC:17082", + "reasons": [ + "CMRG" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "TUBGCP6", + "hgnc_id": "HGNC:18127", + "reasons": [ + "CMRG" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "ACR", + "hgnc_id": "HGNC:126", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "XG", + "hgnc_id": "HGNC:12806", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GYG2", + "hgnc_id": "HGNC:4700", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "ARSD", + "hgnc_id": "HGNC:717", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "MXRA5", + "hgnc_id": "HGNC:7539", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PRKX", + "hgnc_id": "HGNC:9441", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "VCX3A", + "hgnc_id": "HGNC:18159", + "reasons": [ + "segdup" + ], + "share": 0.8328, + "status": "not read" + }, + { + "gene": "VCX2", + "hgnc_id": "HGNC:18158", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "VCX3B", + "hgnc_id": "HGNC:31838", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM9A", + "hgnc_id": "HGNC:18403", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "FAM9C", + "hgnc_id": "HGNC:18405", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SUPT20HL2", + "hgnc_id": "HGNC:31797", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SUPT20HL1", + "hgnc_id": "HGNC:30773", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "MAGEB6", + "hgnc_id": "HGNC:23796", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DCAF8L1", + "hgnc_id": "HGNC:31810", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "FAM47B", + "hgnc_id": "HGNC:26659", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "H2AL1Q", + "hgnc_id": "HGNC:53959", + "reasons": [ + "segdup" + ], + "share": 0.4993, + "status": "not read" + }, + { + "gene": "FAM47C", + "hgnc_id": "HGNC:25301", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "H2AL3", + "hgnc_id": "HGNC:53960", + "reasons": [ + "segdup" + ], + "share": 0.8325, + "status": "not read" + }, + { + "gene": "SPACA5B", + "hgnc_id": "HGNC:19142", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SSX5", + "hgnc_id": "HGNC:11339", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "SSX1", + "hgnc_id": "HGNC:11335", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SSX3", + "hgnc_id": "HGNC:11337", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "SSX4", + "hgnc_id": "HGNC:11338", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX4B", + "hgnc_id": "HGNC:16880", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12J", + "hgnc_id": "HGNC:17778", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE13", + "hgnc_id": "HGNC:29081", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE2E", + "hgnc_id": "HGNC:31960", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE12B", + "hgnc_id": "HGNC:26779", + "reasons": [ + "segdup" + ], + "share": 0.4957, + "status": "not read" + }, + { + "gene": "GAGE12C", + "hgnc_id": "HGNC:28402", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GAGE12D", + "hgnc_id": "HGNC:31904", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GAGE12F", + "hgnc_id": "HGNC:31906", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "GAGE12G", + "hgnc_id": "HGNC:31907", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "GAGE12H", + "hgnc_id": "HGNC:31908", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "GAGE2A", + "hgnc_id": "HGNC:4099", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "GAGE1", + "hgnc_id": "HGNC:4098", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CENPVL3", + "hgnc_id": "HGNC:43880", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CENPVL1", + "hgnc_id": "HGNC:31851", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "MAGED4B", + "hgnc_id": "HGNC:22880", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "MAGED4", + "hgnc_id": "HGNC:23793", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE1A", + "hgnc_id": "HGNC:4111", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "XAGE1B", + "hgnc_id": "HGNC:25400", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SSX7", + "hgnc_id": "HGNC:19653", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SSX2B", + "hgnc_id": "HGNC:22263", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN5", + "hgnc_id": "HGNC:33178", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "XAGE5", + "hgnc_id": "HGNC:30930", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "XAGE3", + "hgnc_id": "HGNC:14618", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "FAM156B", + "hgnc_id": "HGNC:31962", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FAM156A", + "hgnc_id": "HGNC:30114", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPIN2B", + "hgnc_id": "HGNC:33147", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPIN2A", + "hgnc_id": "HGNC:20694", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "ZXDA", + "hgnc_id": "HGNC:13198", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CXorf49", + "hgnc_id": "HGNC:30891", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CXorf49B", + "hgnc_id": "HGNC:34229", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "DMRTC1B", + "hgnc_id": "HGNC:31686", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "FAM236D", + "hgnc_id": "HGNC:52642", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "DMRTC1", + "hgnc_id": "HGNC:13910", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FAM236C", + "hgnc_id": "HGNC:52641", + "reasons": [ + "segdup" + ], + "share": 0.3329, + "status": "not read" + }, + { + "gene": "FAM236A", + "hgnc_id": "HGNC:44268", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "PABPC1L2B", + "hgnc_id": "HGNC:31852", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PABPC1L2A", + "hgnc_id": "HGNC:27989", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TGIF2LX", + "hgnc_id": "HGNC:18570", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "PCDH11X", + "hgnc_id": "HGNC:8656", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "TCEAL6", + "hgnc_id": "HGNC:24553", + "reasons": [ + "segdup" + ], + "share": 0.3331, + "status": "not read" + }, + { + "gene": "TCP11X1", + "hgnc_id": "HGNC:48369", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "NXF2", + "hgnc_id": "HGNC:8072", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "NXF2B", + "hgnc_id": "HGNC:23984", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "RAB40AL", + "hgnc_id": "HGNC:25410", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "TMSB15B", + "hgnc_id": "HGNC:28612", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "H2BW1", + "hgnc_id": "HGNC:27252", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "H2BW2", + "hgnc_id": "HGNC:27867", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "TMSB15C", + "hgnc_id": "HGNC:55173", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PSMD10", + "hgnc_id": "HGNC:9555", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "CT47C1", + "hgnc_id": "HGNC:53820", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "RHOXF2B", + "hgnc_id": "HGNC:33519", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "RHOXF2", + "hgnc_id": "HGNC:30011", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT47B1", + "hgnc_id": "HGNC:33293", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A12", + "hgnc_id": "HGNC:33292", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT47A11", + "hgnc_id": "HGNC:27397", + "reasons": [ + "segdup" + ], + "share": 0.8331, + "status": "not read" + }, + { + "gene": "CT47A9", + "hgnc_id": "HGNC:33290", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A8", + "hgnc_id": "HGNC:33289", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "CT47A7", + "hgnc_id": "HGNC:33288", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT47A6", + "hgnc_id": "HGNC:33287", + "reasons": [ + "segdup" + ], + "share": 0.4998, + "status": "not read" + }, + { + "gene": "CT47A5", + "hgnc_id": "HGNC:33286", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "CT47A4", + "hgnc_id": "HGNC:33285", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT47A2", + "hgnc_id": "HGNC:33283", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT47A1", + "hgnc_id": "HGNC:33282", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "RTL8B", + "hgnc_id": "HGNC:33156", + "reasons": [ + "segdup" + ], + "share": 0.333, + "status": "not read" + }, + { + "gene": "RTL8C", + "hgnc_id": "HGNC:2569", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "RTL8A", + "hgnc_id": "HGNC:24514", + "reasons": [ + "segdup" + ], + "share": 0.6664, + "status": "not read" + }, + { + "gene": "SMIM10L2B", + "hgnc_id": "HGNC:34500", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT55", + "hgnc_id": "HGNC:26047", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "ETDA", + "hgnc_id": "HGNC:53449", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "SMIM10L2A", + "hgnc_id": "HGNC:34499", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "CT45A1", + "hgnc_id": "HGNC:33267", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A3", + "hgnc_id": "HGNC:33268", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CT45A5", + "hgnc_id": "HGNC:33270", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "CT45A2", + "hgnc_id": "HGNC:28400", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CT45A7", + "hgnc_id": "HGNC:51260", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CT45A8", + "hgnc_id": "HGNC:51261", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CT45A9", + "hgnc_id": "HGNC:51262", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "CT45A10", + "hgnc_id": "HGNC:51263", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "SAGE1", + "hgnc_id": "HGNC:30369", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SPANXC", + "hgnc_id": "HGNC:14331", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXA1", + "hgnc_id": "HGNC:11218", + "reasons": [ + "segdup" + ], + "share": 0.1662, + "status": "not read" + }, + { + "gene": "SPANXA2", + "hgnc_id": "HGNC:14328", + "reasons": [ + "segdup" + ], + "share": 0.3327, + "status": "not read" + }, + { + "gene": "SPANXD", + "hgnc_id": "HGNC:14332", + "reasons": [ + "segdup" + ], + "share": 0.4996, + "status": "not read" + }, + { + "gene": "MAGEC1", + "hgnc_id": "HGNC:6812", + "reasons": [ + "segdup" + ], + "share": 0.6667, + "status": "not read" + }, + { + "gene": "SPANXN3", + "hgnc_id": "HGNC:33176", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UBE2NL", + "hgnc_id": "HGNC:31710", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SPANXN1", + "hgnc_id": "HGNC:33174", + "reasons": [ + "segdup" + ], + "share": 0.1667, + "status": "not read" + }, + { + "gene": "CXorf51B", + "hgnc_id": "HGNC:42787", + "reasons": [ + "segdup" + ], + "share": 0.3323, + "status": "not read" + }, + { + "gene": "CXorf51A", + "hgnc_id": "HGNC:30533", + "reasons": [ + "segdup" + ], + "share": 0.4992, + "status": "not read" + }, + { + "gene": "EOLA1", + "hgnc_id": "HGNC:28089", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "HSFX3", + "hgnc_id": "HGNC:52395", + "reasons": [ + "segdup" + ], + "share": 0.8332, + "status": "not read" + }, + { + "gene": "HSFX2", + "hgnc_id": "HGNC:32701", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "HSFX1", + "hgnc_id": "HGNC:29603", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "MAGEA9", + "hgnc_id": "HGNC:6807", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "EOLA2", + "hgnc_id": "HGNC:17402", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "HSFX4", + "hgnc_id": "HGNC:52398", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "MAGEA3", + "hgnc_id": "HGNC:6801", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MAGEA2B", + "hgnc_id": "HGNC:19340", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CSAG1", + "hgnc_id": "HGNC:24294", + "reasons": [ + "segdup" + ], + "share": 0.1665, + "status": "not read" + }, + { + "gene": "MAGEA12", + "hgnc_id": "HGNC:6799", + "reasons": [ + "segdup" + ], + "share": 0.3332, + "status": "not read" + }, + { + "gene": "MAGEA2", + "hgnc_id": "HGNC:6800", + "reasons": [ + "segdup" + ], + "share": 0.4999, + "status": "not read" + }, + { + "gene": "CSAG3", + "hgnc_id": "HGNC:26237", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "MAGEA6", + "hgnc_id": "HGNC:6804", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "PWWP4", + "hgnc_id": "HGNC:55197", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "SLC6A8", + "hgnc_id": "HGNC:11055", + "reasons": [ + "segdup" + ], + "share": 0.1666, + "status": "not read" + }, + { + "gene": "BCAP31", + "hgnc_id": "HGNC:16695", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "OPN1LW", + "hgnc_id": "HGNC:9936", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "OPN1MW", + "hgnc_id": "HGNC:4206", + "reasons": [ + "segdup" + ], + "share": 0.6666, + "status": "not read" + }, + { + "gene": "OPN1MW2", + "hgnc_id": "HGNC:26952", + "reasons": [ + "segdup" + ], + "share": 0.8333, + "status": "not read" + }, + { + "gene": "TEX28", + "hgnc_id": "HGNC:2563", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "CTAG1A", + "hgnc_id": "HGNC:24198", + "reasons": [ + "segdup" + ], + "share": 0.1663, + "status": "not read" + }, + { + "gene": "CTAG1B", + "hgnc_id": "HGNC:2491", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CTAG2", + "hgnc_id": "HGNC:2492", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "H2AB1", + "hgnc_id": "HGNC:22516", + "reasons": [ + "segdup" + ], + "share": 0.6661, + "status": "not read" + }, + { + "gene": "F8A1", + "hgnc_id": "HGNC:3547", + "reasons": [ + "segdup" + ], + "share": 0.833, + "status": "not read" + }, + { + "gene": "F8A2", + "hgnc_id": "HGNC:31849", + "reasons": [ + "segdup" + ], + "share": 0.0, + "status": "not read" + }, + { + "gene": "F8A3", + "hgnc_id": "HGNC:31850", + "reasons": [ + "segdup" + ], + "share": 0.1664, + "status": "not read" + }, + { + "gene": "H2AB3", + "hgnc_id": "HGNC:14455", + "reasons": [ + "segdup" + ], + "share": 0.3333, + "status": "not read" + }, + { + "gene": "IL9R", + "hgnc_id": "HGNC:6030", + "reasons": [ + "segdup" + ], + "share": 0.5, + "status": "not read" + }, + { + "gene": "WASH6P", + "hgnc_id": "HGNC:31685", + "reasons": [ + "segdup" + ], + "share": 0.6665, + "status": "not read" + }, + { + "gene": "TGIF2LY", + "hgnc_id": "HGNC:18569", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "PCDH11Y", + "hgnc_id": "HGNC:15813", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY2", + "hgnc_id": "HGNC:23924", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY4", + "hgnc_id": "HGNC:37287", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY8", + "hgnc_id": "HGNC:37471", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY3", + "hgnc_id": "HGNC:33876", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY1", + "hgnc_id": "HGNC:12381", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY9", + "hgnc_id": "HGNC:37472", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TSPY10", + "hgnc_id": "HGNC:37473", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY", + "hgnc_id": "HGNC:12668", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "VCY1B", + "hgnc_id": "HGNC:31751", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2B", + "hgnc_id": "HGNC:23921", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY2A", + "hgnc_id": "HGNC:1810", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY1", + "hgnc_id": "HGNC:18568", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HSFY2", + "hgnc_id": "HGNC:23950", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1B", + "hgnc_id": "HGNC:23914", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1A1", + "hgnc_id": "HGNC:9912", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1D", + "hgnc_id": "HGNC:23915", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1E", + "hgnc_id": "HGNC:23916", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1F", + "hgnc_id": "HGNC:23974", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "RBMY1J", + "hgnc_id": "HGNC:23917", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2", + "hgnc_id": "HGNC:13508", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ1", + "hgnc_id": "HGNC:2682", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ2", + "hgnc_id": "HGNC:15964", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1B", + "hgnc_id": "HGNC:23920", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2B", + "hgnc_id": "HGNC:25449", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ3", + "hgnc_id": "HGNC:15965", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "DAZ4", + "hgnc_id": "HGNC:15966", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "BPY2C", + "hgnc_id": "HGNC:18225", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "CDY1", + "hgnc_id": "HGNC:1809", + "reasons": [ + "segdup" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "C4B_2", + "hgnc_id": "HGNC:42398", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTT1", + "hgnc_id": "HGNC:4641", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GSTTP2", + "hgnc_id": "HGNC:33606", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "GTF2H2C_2", + "hgnc_id": "HGNC:35418", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB3", + "hgnc_id": "HGNC:4951", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "HLA-DRB4", + "hgnc_id": "HGNC:4952", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL2", + "hgnc_id": "HGNC:6330", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5A", + "hgnc_id": "HGNC:16345", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DL5B", + "hgnc_id": "HGNC:16346", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS1", + "hgnc_id": "HGNC:6333", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS2", + "hgnc_id": "HGNC:6334", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS3", + "hgnc_id": "HGNC:6335", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR2DS5", + "hgnc_id": "HGNC:6337", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "KIR3DS1", + "hgnc_id": "HGNC:6340", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "LILRA3", + "hgnc_id": "HGNC:6604", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U8", + "hgnc_id": "HGNC:27538", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR8U9", + "hgnc_id": "HGNC:29166", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "OR9G9", + "hgnc_id": "HGNC:31940", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + }, + { + "gene": "TAS2R45", + "hgnc_id": "HGNC:18876", + "reasons": [ + "ALT-only" + ], + "share": null, + "status": "not on this reference" + } + ], + "not_read_owned": [ + { + "gene": "RHCE", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "AMY1A", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "AMY1B", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "AMY1C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GSTM1", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "NOTCH2", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "FCGR1A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "GBA1", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "FCGR2A", + "owner": "parascopy", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "FCGR2C", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "FCGR3B", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR3", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "CFHR1", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "PROS1", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "UGT2B17", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "UGT2B15", + "owner": "pypgx", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPE", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "GYPB", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "GTF2H2C", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "SMN2", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "SMN1", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2H2", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HSPA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "C4A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "C4B", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CYP21A2", + "owner": "cn_loci:CYP21A2", + "share": 0.6666, + "status": "not read" + }, + { + "gene": "LPA", + "owner": "cn_loci:LPA_KIV2", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "PMS2", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "GTF2I", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "DEFA1B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CEL", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "MUC5B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "KLRC2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "KRT6B", + "owner": "parascopy", + "share": 0.0, + "status": "not read" + }, + { + "gene": "KRT6C", + "owner": "parascopy", + "share": 0.1665, + "status": "not read" + }, + { + "gene": "KRT6A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CHRFAM7A", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "STRC", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "SORD", + "owner": "parascopy", + "share": 0.8333, + "status": "not read" + }, + { + "gene": "HBA2", + "owner": "cn_loci:HBA", + "share": 0.5, + "status": "not read" + }, + { + "gene": "HBA1", + "owner": "cn_loci:HBA", + "share": 0.6659, + "status": "not read" + }, + { + "gene": "TPSB2", + "owner": "parascopy", + "share": 0.1666, + "status": "not read" + }, + { + "gene": "TPSAB1", + "owner": "parascopy", + "share": 0.3331, + "status": "not read" + }, + { + "gene": "OTOA", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "SULT1A1", + "owner": "pypgx", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "HYDIN", + "owner": "parascopy", + "share": 0.1667, + "status": "not read" + }, + { + "gene": "HP", + "owner": "cn_loci:HP", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CCL3L3", + "owner": "cn_loci:CCL3L1", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "CYP2A6", + "owner": "pypgx", + "share": 0.0, + "status": "not read" + }, + { + "gene": "CYP2B6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + }, + { + "gene": "CD177", + "owner": "parascopy", + "share": 0.3333, + "status": "not read" + }, + { + "gene": "APOBEC3A", + "owner": "parascopy", + "share": 0.5, + "status": "not read" + }, + { + "gene": "APOBEC3B", + "owner": "parascopy", + "share": 0.6667, + "status": "not read" + }, + { + "gene": "CYP2D6", + "owner": "pypgx", + "share": 0.5, + "status": "not read" + } + ], + "not_on_reference": [ + "TGIF2LY", + "PCDH11Y", + "TSPY2", + "TSPY4", + "TSPY8", + "TSPY3", + "TSPY1", + "TSPY9", + "TSPY10", + "VCY", + "VCY1B", + "CDY2B", + "CDY2A", + "HSFY1", + "HSFY2", + "RBMY1B", + "RBMY1A1", + "RBMY1D", + "RBMY1E", + "RBMY1F", + "RBMY1J", + "BPY2", + "DAZ1", + "DAZ2", + "CDY1B", + "BPY2B", + "DAZ3", + "DAZ4", + "BPY2C", + "CDY1", + "C4B_2", + "GSTT1", + "GSTTP2", + "GTF2H2C_2", + "HLA-DRB3", + "HLA-DRB4", + "KIR2DL2", + "KIR2DL5A", + "KIR2DL5B", + "KIR2DS1", + "KIR2DS2", + "KIR2DS3", + "KIR2DS5", + "KIR3DS1", + "LILRA3", + "OR8U8", + "OR8U9", + "OR9G9", + "TAS2R45" + ] + } + }, + "sex_check": { + "title": "Sex check (indexcov)", + "step": "16", + "state": "ok", + "source": "indexcov/indexcov-indexcov.ped", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "inferred_sex": "male", + "cn_chrX": "1.01", + "cn_chrY": "0.98", + "karyotype": null + } + }, + "sample_qc": { + "title": "Sample identity and contamination (somalier, VerifyBamID2)", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sample": "S", + "inferred_sex": "male", + "sex_check": "ok", + "x_depth_ratio": "0.51", + "y_depth_ratio": "0.49", + "x_sites": "400", + "freemix": "0.0010", + "declared_sex": "male", + "mosaic_call": "not_detected", + "mosaic_depth_ratio": "0.98", + "mosaic_depth_ratio_expected": "1", + "mosaic_loss_depth": "0.02", + "mosaic_loss_depth_low": "-0.01", + "mosaic_loss_depth_high": "0.05", + "mosaic_loss_af": "NA", + "mosaic_floor": "0.1", + "mosaic_x_het_sites": "0", + "karyotype": null + } + }, + "mosaic_sex_loss": { + "title": "Mosaic loss of chromosome Y or X", + "step": "33", + "state": "ok", + "source": "qc/S_sample_qc.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "call": "not_detected", + "depth_ratio": 0.98, + "depth_ratio_expected": 1.0, + "loss_depth": 0.02, + "loss_depth_low": -0.01, + "loss_depth_high": 0.05, + "loss_af": null, + "floor": 0.1, + "x_het_sites": 0, + "loss_af_low": null, + "loss_af_high": null + } + }, + "read_count": { + "title": "Reads kept by alignment", + "step": "02", + "state": "ok", + "source": "qc/S_read_count.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "input_reads": 1000, + "primary_records": 1000, + "missing_reads": 0, + "read_count_check": "ok" + } + }, + "variants": { + "title": "Variant calling", + "step": "03", + "state": "ok", + "source": "vcf/S.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1, + "snps": 0, + "indels": 1 + } + }, + "clinvar": { + "title": "ClinVar screen", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 8, + "by_stars": { + "4": 0, + "3": 2, + "2": 3, + "1": 1, + "0": 2 + }, + "hits": [ + { + "stars": 4, + "chrom": "chr7", + "pos": 117559590, + "ref": "ATCT", + "alt": "A", + "genotype": "het", + "gene": "CFTR", + "significance": "Pathogenic", + "review_status": "practice guideline", + "combined_allele": ".", + "dp": 30, + "gq": 15, + "low_confidence": true, + "gnomad_exome_af": 0.008, + "gnomad_genome_af": 0.007, + "rsids": [ + "rs113993960" + ], + "af": 0.008, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr17", + "pos": 43057100, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "BRCA1,NBR2", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": "part of a combined allele: chr17:43057100 C>T, chr17:43057102 A>G; unphased, so the parts may sit on different haplotypes", + "dp": 35, + "gq": 60, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 3, + "chrom": "chr6", + "pos": 26092913, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HFE", + "significance": "Pathogenic", + "review_status": "reviewed by expert panel", + "combined_allele": ".", + "dp": 40, + "gq": 99, + "low_confidence": false, + "gnomad_exome_af": 0.06, + "gnomad_genome_af": 0.05, + "rsids": [ + "rs99", + "rs1800562" + ], + "af": 0.06, + "common": true, + "cdna": { + "check": "clean", + "note": "", + "alt_reads": 20, + "alt_clipped": 0, + "ref_reads": 20, + "ref_clipped": 0, + "construct": null + } + }, + { + "stars": 2, + "chrom": "chr1", + "pos": 45331556, + "ref": "C", + "alt": "T", + "genotype": "het", + "gene": "MUTYH", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 30, + "gq": 50, + "low_confidence": false, + "gnomad_exome_af": 0.002, + "gnomad_genome_af": 0.0005, + "rsids": [ + "rs36053993" + ], + "af": 0.002, + "common": false, + "cdna": { + "check": "flagged", + "note": "possible cDNA/transgene contamination: check the reads", + "alt_reads": 12, + "alt_clipped": 6, + "ref_reads": 10, + "ref_clipped": 0, + "construct": "yes" + } + }, + { + "stars": 2, + "chrom": "chr16", + "pos": 176700, + "ref": "G", + "alt": "A", + "genotype": "het", + "gene": "HBA1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 2, + "chrom": "chr5", + "pos": 70951946, + "ref": "C", + "alt": "T", + "genotype": "hom", + "gene": "SMN1", + "significance": "Pathogenic", + "review_status": "criteria provided, multiple submitters, no conflicts", + "combined_allele": ".", + "dp": 31, + "gq": 45, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr13", + "pos": 20189473, + "ref": "C", + "alt": "T", + "genotype": "one copy, other copy not called ref (1/.)", + "gene": "GJB2", + "significance": "Likely pathogenic", + "review_status": "criteria provided, single submitter", + "combined_allele": "combined allele of chr13:20189473 C>T + chr13:20189475 G>A", + "dp": 5, + "gq": 40, + "low_confidence": true, + "gnomad_exome_af": 0.03, + "gnomad_genome_af": null, + "rsids": [ + "rs80338939" + ], + "af": 0.03, + "common": true, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 1, + "chrom": "chr3", + "pos": 15645186, + "ref": "G", + "alt": "C", + "genotype": "het", + "gene": "BTD", + "significance": "Pathogenic/Likely pathogenic", + "review_status": "criteria provided, conflicting classifications", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chr2", + "pos": 100, + "ref": "A", + "alt": "G", + "genotype": "het", + "gene": ".", + "significance": ".", + "review_status": ".", + "combined_allele": ".", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "stars": 0, + "chrom": "chrX", + "pos": 101398000, + "ref": "C", + "alt": "T", + "genotype": "1", + "gene": "GLA", + "significance": "Pathogenic", + "review_status": "no assertion criteria provided", + "combined_allele": "possibly one allele with chrX:101398004 G>A: check ClinVar for a combined record", + "dp": null, + "gq": null, + "low_confidence": false, + "gnomad_exome_af": null, + "gnomad_genome_af": null, + "rsids": [], + "af": null, + "common": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "low_confidence": 2, + "call_floor": { + "GQ": 20, + "DP": 8 + }, + "frequency": { + "source": "vep/S_vep.vcf.gz", + "note": null, + "common_af": 0.01 + }, + "cdna_flagged": 1, + "construct_sites": 1, + "cdna_scan": { + "note": "2 genes with spliced reads", + "loci": [ + { + "chrom": "chr1", + "start": 100, + "end": 200, + "junctions": 2, + "reads": 5, + "junction_list": "A:1-2,A:3-4" + } + ] + }, + "deep_intronic": [ + { + "chrom": "chr7", + "pos": 117639961, + "ref": "C", + "alt": "T", + "gene": "CFTR", + "hgvs": "c.3718-2477C>T", + "clinvar_id": "7129", + "call": "no call", + "detail": "" + } + ] + } + }, + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "carrier": { + "title": "Carrier list", + "step": "06", + "state": "ok", + "source": "clinvar/S_clinvar_hits.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_on_list": 3292, + "list": "assets/carrier_genes.tsv", + "carriers": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "MUTYH-related polyposis" + }, + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Symptomatic form of HFE-related hemochromatosis" + }, + { + "gene": "CFTR", + "variant": "chr7:117559590 ATCT>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 4, + "inheritance": "AR", + "condition": "Cystic fibrosis" + }, + { + "gene": "GJB2", + "variant": "chr13:20189473 C>T", + "genotype": "one copy, other copy not called ref (1/.)", + "significance": "Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Nonsyndromic hearing loss, recessive (DFNB1A)" + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "inheritance": "AR", + "condition": "Fanconi anemia" + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 0, + "inheritance": "XL", + "condition": "Fabry disease" + }, + { + "gene": "HBA1", + "variant": "chr16:176700 G>A", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Alpha-thalassemia" + }, + { + "gene": "SMN1", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "significance": "Pathogenic", + "stars": 2, + "inheritance": "AR", + "condition": "Spinal muscular atrophy", + "hemizygous": "called homozygous inside one SMN1 copy (Parascopy, step 35)" + } + ], + "set_aside": [ + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "significance": "Pathogenic/Likely pathogenic", + "stars": 1, + "inheritance": "AR", + "condition": "Biotinidase deficiency", + "kind": "hypomorph", + "name": "p.Asp444His (p.Asp424His on NM_001370658.1), partial biotinidase deficiency" + } + ], + "two_hits": { + "genes": [ + { + "gene": "SMN1", + "verdict": "biallelic", + "why": "the variant is called homozygous inside the deletion: it is hemizygous, on the one copy the deletion leaves", + "deletion": "one SMN1 copy (Parascopy, step 35)", + "variant": "chr5:70951946 C>T", + "genotype": "hemizygous", + "allele": null, + "condition": "Spinal muscular atrophy", + "inheritance": "AR" + } + ], + "deletions_read": [ + "AnnotSV class 4 and 5 deletions (step 05)", + "step 35's interval table (read depth)", + "Parascopy's SMN1 count (step 35)" + ], + "deletions_not_read": [ + "step 35's HBA1/HBA2 count: not called" + ] + } + } + }, + "pharmcat": { + "title": "PharmCAT", + "step": "07", + "state": "ok", + "source": "pharmcat/S.report.json", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "version": "3.4.0", + "html_report": true + } + }, + "cpic": { + "title": "CPIC drug-gene recommendations", + "step": "27", + "state": "ok", + "source": "cpic/S_phenotypes.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "gene": "UGT1A1", + "diplotype": "*1/*28", + "phenotype": "Intermediate", + "status": "non-normal" + } + ], + "non_normal": 1, + "ambiguous": 0, + "not_called": 0, + "unclassified": 0, + "parse_failed": false, + "warnings": [ + "CYP2C19 has two possible phenotypes", + "last" + ] + } + }, + "pypgx": { + "title": "pypgx", + "step": "32", + "state": "ok", + "source": "pypgx/S_pypgx_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes_total": 6, + "genes_called": 3, + "cyp2d6": "*1/*2", + "cnv_call": { + "CYP2D6": "Normal", + "CYP2A6": "not called: pypgx failed", + "CYP2B6": "not called: the CYP2D6 depth check failed", + "SLCO1B1": "not called: pypgx gave no copy-number call" + }, + "not_callable": [ + "CYP2E1" + ], + "low_gq": [ + { + "gene": "UGT1A1", + "diplotype": "Indeterminate", + "min_gq": 12, + "weakest": "chr2:1:A:G" + }, + { + "gene": "SLCO1B1", + "diplotype": "*1/*5", + "min_gq": null, + "weakest": "" + } + ], + "cyp2d6_evidence": { + "phenotype": "Normal Metabolizer", + "defining_filter": "PASS", + "quality": "ok" + }, + "depth_check": { + "status": "ok", + "gene_mapq1_share": "0.99", + "body_mapq20_share": "0.95" + }, + "comparison": { + "conflicts": 1, + "one_tool_only": 2, + "concordant": 1 + } + } + }, + "cyrius": { + "title": "Cyrius (CYP2D6)", + "step": "21", + "state": "ok", + "source": "cyrius/S_cyp2d6.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genotype": "*2/*1", + "filter": "PASS" + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": 1, + "text": "1 (quality 31, filter PASS)", + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": 3, + "text": "3 (quality 25, filter PASS)" + }, + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "one SMN1 copy: the pattern of an SMA carrier", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "called", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality 50, filter PASS); chr7:5980001-5990000 3 (quality 12, filter LowQual)" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": 1, + "text": "1 (quality 31, filter PASS)" + }, + "agree": "disagree" + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01:01", + "A*01:01:01" + ], + "quality": [ + "60", + "50" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-B", + "alleles": [ + "B*57:01:01:02", + "B*08:01:01,B*08:01:02,B*08:01:03" + ], + "quality": [ + "0", + "0" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1, + 5 + ], + "one_allele_type": false + }, + { + "gene": "HLA-C", + "alleles": [ + "C*07:01:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*05:01:01", + "DQA1*02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01:01", + "DQB1*02:02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*03:01:01", + "DRB1*07:01:01" + ], + "quality": [ + "40", + "nan" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HFE", + "alleles": [ + "HFE*001" + ], + "quality": [ + "20" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true, + "label": "T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the variant calls (ClinVar section)" + } + ], + "derived": { + "coeliac": { + "status": "derived", + "reason": "", + "dq2_5": "present", + "dq2_5_phase": "likely cis (DRB1*03 present)", + "dq2_2": "present", + "dq8": "absent", + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": 0.95, + "low": 0.93, + "high": 0.97 + }, + { + "code": "AFR", + "group": "Africa", + "fraction": 0.05, + "low": null, + "high": null + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": 0.6, + "low": 0.5, + "high": 0.7 + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": null, + "low": null, + "high": null + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": null, + "low": null, + "high": null + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 8, + "deletions": 5, + "duplications": 3, + "significant": 7, + "significant_filtered": 4, + "filtered_chry": 1, + "dropped": { + "q0": 1, + "gap": 1, + "band": 1 + }, + "declared_sex": "male" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "depth": 30.0, + "read_floor": 2000, + "low_reads": true + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 3, + "total_mb": 37.5, + "largest_mb": 27.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + }, + { + "region": "chr15:23000001-50000000", + "mb": 27.0 + } + ], + "masked_mb": 46.5, + "sex_chromosome_segments": 1, + "long_mb": 34.0, + "f_roh": 0.0128, + "relationship": null, + "upd_flag": { + "chrom": "chr15", + "mb": 27.0, + "share": 0.79, + "imprinted": [ + { + "region": "15q11-q13 (SNRPN, UBE3A)", + "maternal_upd": "Prader-Willi syndrome", + "paternal_upd": "Angelman syndrome", + "pmid": "22237428;27615419" + } + ] + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*03:01:01/DRB1*07:01:01" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32): *1/*2; Cyrius (step 21): *2/*1" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + }, + { + "hgvs": "m.8344A>G", + "gene": "MT-TK", + "disease": "MERRF (myoclonic epilepsy with ragged-red fibres)", + "clinvar_id": "9579", + "pmids": "2112427", + "note": "Usually heteroplasmic; the level differs between tissues.", + "pos": 8344, + "filter": "weak_evidence", + "af": "0.30" + }, + { + "hgvs": "m.11778G>A", + "gene": "MT-ND4", + "disease": "Leber hereditary optic neuropathy (LHON)", + "clinvar_id": "9708", + "pmids": "3201231", + "note": "Usually homoplasmic; not everyone who carries it loses vision.", + "pos": 11778, + "filter": "PASS", + "af": "0.99" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": 0.5, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + }, + { + "bkp5": 100, + "bkp3": 200, + "length": null, + "level_pct": 2.5 + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 7, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [ + { + "variant": "chrM:302 A>AC,ACC", + "af": 0.3 + }, + { + "variant": "chrM:16189 T>C", + "af": 0.5 + } + ], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3", + "gene": "X", + "trait": "x", + "genotype": "C/T", + "status": "low_gq", + "reading": "A low-quality call at this position (genotype quality 5, below 20): not read; check the reads.", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "low_gq": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "30.5", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TET2", + "mean_depth": "40", + "detection_limit_vaf": "0.08", + "status": "covered" + }, + { + "gene": "ASXL1", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + }, + { + "gene": "JAK2", + "mean_depth": "35", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TP53", + "mean_depth": "25", + "detection_limit_vaf": "0.1", + "status": "covered" + } + ], + "median_depth": 32.75, + "median_limit": 0.12901665405107993, + "not_covered": [ + "ASXL1" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + }, + "cpsr": { + "title": "Cancer predisposition (CPSR)", + "step": "17", + "state": "ok", + "source": "cpsr/S.cpsr.grch38.classification.tsv.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "html_report": true, + "classification_column": "CLASSIFICATION", + "classification": { + "Pathogenic": 1, + "Likely_Pathogenic": 1, + "VUS": 1, + "Benign": 1, + ".": 1 + }, + "plp": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "13:32338000 G>A", + "gene": "BRCA2", + "final": "Likely_Pathogenic", + "cpsr": "VUS", + "clinvar": null, + "authority": "CPSR", + "acmg_codes": null, + "dp": 40, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + }, + { + "variant": "5:112839000 A>T", + "gene": "APC", + "final": ".", + "cpsr": "Pathogenic", + "clinvar": null, + "authority": null, + "acmg_codes": null, + "dp": 30, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "disagreements": [ + { + "variant": "17:43057100 C>T", + "gene": "BRCA1", + "final": "Pathogenic", + "cpsr": "Likely_Benign", + "clinvar": "Pathogenic", + "authority": "ClinVar", + "acmg_codes": "PVS1", + "dp": 6, + "low_confidence": true, + "cdna": { + "check": "flagged", + "note": "splice", + "alt_reads": 5, + "alt_clipped": 5, + "ref_reads": 9, + "ref_clipped": 0, + "construct": null + } + }, + { + "variant": "2:47403000 A>G", + "gene": "MSH2", + "final": "VUS", + "cpsr": "LP", + "clinvar": "B", + "authority": null, + "acmg_codes": null, + "dp": null, + "low_confidence": false, + "cdna": { + "check": "not_checked", + "note": "not in the read-check table", + "alt_reads": null, + "alt_clipped": null, + "ref_reads": null, + "ref_clipped": null + } + } + ], + "final_plp_own_benign": 1, + "own_class_column": "CPSR_CLASSIFICATION", + "low_confidence": 1, + "call_floor_dp": 8, + "cdna_flagged": 1 + } + }, + "clinical": { + "title": "Clinical variant filter", + "step": "23", + "state": "ok", + "source": "clinical/S_clinical_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "variants": 5, + "by_impact": { + "HIGH": 3, + "MODERATE": 1, + ".": 1 + }, + "genes": 3, + "acmg_sf": { + "version": "ACMG SF v3.3", + "genes_on_list": 84, + "rules": "assets/acmg_sf.tsv", + "clinvar_hits": [ + { + "gene": "MUTYH", + "variant": "chr1:45331556 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 2, + "low_confidence": false, + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "BRCA1", + "variant": "chr17:43057100 C>T", + "genotype": "het", + "significance": "Pathogenic", + "stars": 3, + "low_confidence": false, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "GLA", + "variant": "chrX:101398000 C>T", + "genotype": "1", + "significance": "Pathogenic", + "stars": 0, + "low_confidence": false, + "rule": "All hemi, het, homozygous P and LP", + "rule_note": null, + "severity": null + } + ], + "clinvar_checked": true, + "high_impact": [ + { + "gene": "MUTYH", + "variant": "chr1:45332000 G>A", + "genotype": "het", + "consequence": "stop_gained", + "transcript": "MANE", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": 40, + "dp": 30, + "ad": "15,15", + "rule": "P and LP (2 variants)", + "rule_note": null, + "severity": null + }, + { + "gene": "MYBPC3", + "variant": "chr11:47332000 G>A", + "genotype": "het", + "consequence": "splice_donor_variant,intron_variant", + "transcript": "ANY", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": false, + "call_note": null, + "gq": null, + "dp": null, + "ad": null, + "rule": "All P and LP", + "rule_note": null, + "severity": null + }, + { + "gene": "MSH2", + "variant": "chr2:47414000 A>AT", + "genotype": "het", + "consequence": "frameshift_variant", + "transcript": "CANONICAL", + "frequency": "rare", + "frequency_unknown": false, + "weak_call": true, + "call_note": "alt reads 0.07 of the depth on a heterozygote", + "gq": 50, + "dp": 30, + "ad": "28,2", + "rule": "All P and LP", + "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", + "severity": null + } + ], + "high_impact_checked": true, + "high_impact_rare": 2, + "high_impact_weak": 1, + "high_impact_frequency_unknown": 0, + "structural": [], + "structural_checked": true, + "structural_note": null, + "not_reported": [ + { + "gene": "HFE", + "variant": "chr6:26092913 G>A", + "genotype": "het", + "source": "ClinVar", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + }, + { + "gene": "BTD", + "variant": "chr3:15645186 G>C", + "genotype": "het", + "source": "ClinVar", + "reason": "one heterozygous variant; BTD is reported with P and LP (2 variants)" + }, + { + "gene": "HFE", + "variant": "chr6:26093000 C>T", + "genotype": "hom", + "source": "HIGH impact", + "reason": "HFE is reported only for: HFE p.C282Y homozygotes only" + } + ], + "two_hit_note": null + }, + "frequency_unknown": 1, + "weak_calls": 2 + } + }, + "slivar": { + "title": "Variant prioritization (slivar)", + "step": "31", + "state": "ok", + "source": "slivar/S_slivar_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "prioritized": 2, + "frequency_unknown": 1, + "compound_het_variants": 4, + "compound_het_genes": 2, + "compound_het_pairs": 26, + "compound_het_phase": { + "trans": 23, + "cis": 1, + "unknown": 1 + }, + "compound_het_trans": [ + { + "gene": "G1", + "variants": "chr1:10 A>G + chr1:15 C>T", + "reads": "1 of 2" + }, + { + "gene": "G2", + "variants": "chr1:20 A>G + chr1:25 C>T", + "reads": "2 of 3" + }, + { + "gene": "G3", + "variants": "chr1:30 A>G + chr1:35 C>T", + "reads": "3 of 4" + }, + { + "gene": "G4", + "variants": "chr1:40 A>G + chr1:45 C>T", + "reads": "4 of 5" + }, + { + "gene": "G5", + "variants": "chr1:50 A>G + chr1:55 C>T", + "reads": "5 of 6" + }, + { + "gene": "G6", + "variants": "chr1:60 A>G + chr1:65 C>T", + "reads": "6 of 7" + }, + { + "gene": "G7", + "variants": "chr1:70 A>G + chr1:75 C>T", + "reads": "7 of 8" + }, + { + "gene": "G8", + "variants": "chr1:80 A>G + chr1:85 C>T", + "reads": "8 of 9" + }, + { + "gene": "G9", + "variants": "chr1:90 A>G + chr1:95 C>T", + "reads": "9 of 10" + }, + { + "gene": "G10", + "variants": "chr1:100 A>G + chr1:105 C>T", + "reads": "10 of 11" + }, + { + "gene": "G11", + "variants": "chr1:110 A>G + chr1:115 C>T", + "reads": "11 of 12" + }, + { + "gene": "G12", + "variants": "chr1:120 A>G + chr1:125 C>T", + "reads": "12 of 13" + }, + { + "gene": "G13", + "variants": "chr1:130 A>G + chr1:135 C>T", + "reads": "13 of 14" + }, + { + "gene": "G14", + "variants": "chr1:140 A>G + chr1:145 C>T", + "reads": "14 of 15" + }, + { + "gene": "G15", + "variants": "chr1:150 A>G + chr1:155 C>T", + "reads": "15 of 16" + }, + { + "gene": "G16", + "variants": "chr1:160 A>G + chr1:165 C>T", + "reads": "16 of 17" + }, + { + "gene": "G17", + "variants": "chr1:170 A>G + chr1:175 C>T", + "reads": "17 of 18" + }, + { + "gene": "G18", + "variants": "chr1:180 A>G + chr1:185 C>T", + "reads": "18 of 19" + }, + { + "gene": "G19", + "variants": "chr1:190 A>G + chr1:195 C>T", + "reads": "19 of 20" + }, + { + "gene": "G20", + "variants": "chr1:200 A>G + chr1:205 C>T", + "reads": "20 of 21" + } + ] + } + } + }, + "cyp2d6": { + "calls": { + "PharmCAT": null, + "pypgx": "*1/*2", + "Cyrius": "*2/*1" + }, + "agree": true, + "consensus": { + "result": "*1/*2", + "passed_to_pharmcat": true, + "reason": "pypgx and Cyrius agree", + "source": "pgx_consensus/S_pgx_consensus.tsv" + }, + "probable": null + }, + "not_run": [], + "not_assessed": [ + "Mosaic and low-fraction small variants (only chrM heteroplasmy is reported, and with step 49 the CHIP genes)", + "Methylation and imprinting", + "Repeat expansions outside the ExpansionHunter catalog", + "Phase: two variants in one gene are compound-het candidates, not confirmed", + "Variants of uncertain significance (not interpreted)" + ] +} diff --git a/tests/parity/golden/summary-collect/c-float--inf/float--inf.json b/tests/parity/golden/summary-collect/c-float--inf/float--inf.json new file mode 100644 index 000000000..164a5e060 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-float--inf/float--inf.json @@ -0,0 +1,1993 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:21 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": null, + "text": "1 (quality -inf, filter PASS), not resolved", + "agcn": "4", + "agcn_qual": "-inf", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": null, + "text": "3 (quality *, filter PASS), not resolved" + }, + "agcn": "4", + "agcn_qual": "-inf", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "SMN1 not resolved (quality under 20 or filter not PASS): no carrier reading", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "not resolved", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality -inf, filter PASS); chr7:5980001-5990000 3 (quality -inf, filter LowQual): no region with a whole copy number, filter PASS and quality 20 or more" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": null, + "text": "1 (quality -inf, filter PASS), not resolved" + }, + "agree": null + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01", + "A*01:01" + ], + "quality": [ + "-inf", + "-inf" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 not typed; HLA-DQB1 not typed", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": null + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A low confidence" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DQB1 not typed" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": -Infinity, + "low": -Infinity, + "high": -Infinity + }, + { + "code": "AFR", + "group": "Africa", + "fraction": -Infinity, + "low": -Infinity, + "high": -Infinity + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": -Infinity, + "low": -Infinity, + "high": -Infinity + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": -Infinity, + "low": -Infinity, + "high": -Infinity + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": -Infinity, + "low": -Infinity, + "high": -Infinity + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "deletions": 1, + "duplications": 0, + "significant": 1, + "significant_filtered": 1, + "filtered_chry": 0, + "dropped": { + "q0": 0, + "gap": 0, + "band": 0 + }, + "declared_sex": null + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_roh.txt has a run of length -inf", + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08) did not run" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32) and Cyrius (step 21) did not run" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "-inf" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": -Infinity, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 1, + "heteroplasmic": 0, + "heteroplasmic_sites": [], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3", + "gene": "X", + "trait": "x", + "genotype": "C/T", + "status": "low_gq", + "reading": "A low-quality call at this position (genotype quality 5, below 20): not read; check the reads.", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "low_gq": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "A", + "mean_depth": "30", + "detection_limit_vaf": ".", + "status": "covered" + }, + { + "gene": "B", + "mean_depth": "-inf", + "detection_limit_vaf": ".", + "status": "covered" + }, + { + "gene": "C", + "mean_depth": "40", + "detection_limit_vaf": ".", + "status": "covered" + } + ], + "median_depth": 30.0, + "median_limit": 0.13728764032080973, + "not_covered": [] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + } + } +} diff --git a/tests/parity/golden/summary-collect/c-float-inf/float-inf.json b/tests/parity/golden/summary-collect/c-float-inf/float-inf.json new file mode 100644 index 000000000..70e93b621 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-float-inf/float-inf.json @@ -0,0 +1,2005 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:21 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": 1, + "text": "1 (quality inf, filter PASS)", + "agcn": "4", + "agcn_qual": "inf", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": null, + "text": "3 (quality *, filter PASS), not resolved" + }, + "agcn": "4", + "agcn_qual": "inf", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "one SMN1 copy: the pattern of an SMA carrier", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "called", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality inf, filter PASS); chr7:5980001-5990000 3 (quality inf, filter LowQual)" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": 1, + "text": "1 (quality inf, filter PASS)" + }, + "agree": "disagree" + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01", + "A*01:01" + ], + "quality": [ + "inf", + "inf" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 not typed; HLA-DQB1 not typed", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": null + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DQB1 not typed" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": Infinity, + "low": Infinity, + "high": Infinity + }, + { + "code": "AFR", + "group": "Africa", + "fraction": Infinity, + "low": Infinity, + "high": Infinity + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": Infinity, + "low": Infinity, + "high": Infinity + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": Infinity, + "low": Infinity, + "high": Infinity + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": Infinity, + "low": Infinity, + "high": Infinity + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "deletions": 1, + "duplications": 0, + "significant": 0, + "significant_filtered": 0, + "filtered_chry": 0, + "dropped": { + "q0": 0, + "gap": 0, + "band": 0 + }, + "declared_sex": null + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_roh.txt has a run of length inf", + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08) did not run" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32) and Cyrius (step 21) did not run" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "inf" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": Infinity, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + }, + { + "bkp5": 100, + "bkp3": 200, + "length": null, + "level_pct": Infinity + }, + { + "bkp5": 300, + "bkp3": 400, + "length": 99, + "level_pct": Infinity + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 1, + "heteroplasmic": 0, + "heteroplasmic_sites": [], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3", + "gene": "X", + "trait": "x", + "genotype": "C/T", + "status": "low_gq", + "reading": "A low-quality call at this position (genotype quality 5, below 20): not read; check the reads.", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "low_gq": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "A", + "mean_depth": "30", + "detection_limit_vaf": ".", + "status": "covered" + }, + { + "gene": "B", + "mean_depth": "inf", + "detection_limit_vaf": ".", + "status": "covered" + }, + { + "gene": "C", + "mean_depth": "40", + "detection_limit_vaf": ".", + "status": "covered" + } + ], + "median_depth": 40.0, + "median_limit": 0.10395846911039092, + "not_covered": [] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + } + } +} diff --git a/tests/parity/golden/summary-collect/c-float-nan/float-nan.json b/tests/parity/golden/summary-collect/c-float-nan/float-nan.json new file mode 100644 index 000000000..4e6448ba6 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-float-nan/float-nan.json @@ -0,0 +1,1957 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:21 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": null, + "text": "1 (quality nan, filter PASS), not resolved", + "agcn": "4", + "agcn_qual": "nan", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": null, + "text": "3 (quality *, filter PASS), not resolved" + }, + "agcn": "4", + "agcn_qual": "nan", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "SMN1 not resolved (quality under 20 or filter not PASS): no carrier reading", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "not resolved", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality nan, filter PASS); chr7:5980001-5990000 3 (quality nan, filter LowQual): no region with a whole copy number, filter PASS and quality 20 or more" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": null, + "text": "1 (quality nan, filter PASS), not resolved" + }, + "agree": null + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01", + "A*01:01" + ], + "quality": [ + "nan", + "nan" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 not typed; HLA-DQB1 not typed", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": null + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DQB1 not typed" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "not_determined", + "reason": "HLA-DRB1 not typed" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": null, + "low": null, + "high": null + }, + { + "code": "AFR", + "group": "Africa", + "fraction": null, + "low": null, + "high": null + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": null, + "low": null, + "high": null + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": null, + "low": null, + "high": null + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": null, + "low": null, + "high": null + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "deletions": 1, + "duplications": 0, + "significant": 1, + "significant_filtered": 1, + "filtered_chry": 0, + "dropped": { + "q0": 0, + "gap": 0, + "band": 0 + }, + "declared_sex": null + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "S_roh.txt has a run of length nan", + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08) did not run" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32) and Cyrius (step 21) did not run" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "nan" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": NaN, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 1, + "heteroplasmic": 0, + "heteroplasmic_sites": [], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3", + "gene": "X", + "trait": "x", + "genotype": "C/T", + "status": "low_gq", + "reading": "A low-quality call at this position (genotype quality 5, below 20): not read; check the reads.", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "low_gq": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "unreadable", + "source": null, + "file_date": null, + "note": "ValueError: cannot convert float NaN to integer", + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + } + } +} diff --git a/tests/parity/golden/summary-collect/c-full/full.json b/tests/parity/golden/summary-collect/c-full/full.json new file mode 100644 index 000000000..d5206b6d1 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-full/full.json @@ -0,0 +1,2188 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:14 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "ok", + "source": "couple/S_couple_carrier.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "partner": "P2", + "shared": [ + { + "gene": "CFTR", + "condition": "Cystic fibrosis", + "risk": "1 in 4", + "own": "c.1521_1523del (P)", + "partner_findings": "c.350G>A (LP)", + "explanation": "Both carry a change." + } + ], + "x_linked": [ + { + "gene": "DMD", + "condition": "Duchenne", + "risk": "1 in 4 sons", + "own": "c.10G>T (P)", + "partner_findings": ".", + "explanation": "On X." + } + ], + "one_partner": 2 + } + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": 1, + "text": "1 (quality 31, filter PASS)", + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS" + }, + "smn2": { + "cn": 3, + "text": "3 (quality 25, filter PASS)" + }, + "agcn": "4", + "agcn_qual": "45.5", + "agcn_filter": "PASS", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not seen", + "alt_reads": "0", + "depth": "31", + "fraction": "0.000", + "copies": "0", + "message": "no read carries the G" + }, + "rows": 2, + "hba": { + "status": "called", + "copies": "4", + "reading": "two copies of each gene" + }, + "cn_intervals": { + "rows": 4, + "by_status": { + "called": 3, + "not assessed": 1 + }, + "changes": [ + { + "locus": "AZFc", + "name": "AZFc", + "chrom": "chrY", + "start": "23000000", + "end": "26000000", + "copies": "0", + "expected": "1", + "change": "deletion" + }, + { + "locus": "X1", + "name": "dup", + "chrom": "chr1", + "start": "1", + "end": "2", + "copies": "3", + "expected": "2", + "change": "duplication" + } + ] + }, + "cyp21a2": { + "status": "called", + "copies": "2", + "module_total": "4", + "finding": "none" + }, + "dosage": [ + { + "locus": "HP", + "status": "called", + "copies": "2", + "estimate": "2.1", + "reason": ".", + "reading": "usual" + }, + { + "locus": "LCE3BC", + "status": "called", + "copies": "1", + "estimate": "1.0", + "reason": ".", + "reading": "one deleted" + }, + { + "locus": "CCL3L1", + "status": "not assessed", + "copies": ".", + "estimate": ".", + "reason": "low depth", + "reading": "." + } + ], + "reading": "one SMN1 copy: the pattern of an SMA carrier", + "loci": [ + { + "table": "gba1", + "genes": "GBA1,GBAP1", + "tag": "carrier", + "reading": "agCN of GBA1 and its pseudogene GBAP1 together, usually 4; under 4 means a copy is missing, as a recombinant GBA1-GBAP1 allele (Gaucher disease) can leave; over 4, an extra copy", + "status": "called", + "cn": 4, + "text": "4 (2 region(s), quality 35.25 or more, filter PASS)" + }, + { + "table": "strc", + "genes": "STRC,STRCP1", + "tag": "carrier", + "reading": "agCN of STRC and STRCP1 together, usually 4; under 4 means a deletion, and an STRC deletion is a common cause of recessive hearing loss (DFNB16): 3 can be a carrier, 2 can be affected or a pseudogene loss", + "status": "unreadable", + "cn": null, + "text": "unreadable: S_strc_copy_number.tsv lacks the copy-number columns" + }, + { + "table": "mytable", + "genes": "mytable", + "tag": ".", + "reading": ".", + "status": "not assessed", + "cn": null, + "text": "not assessed: no read over the locus" + }, + { + "table": "pms2", + "genes": "pms2", + "tag": ".", + "reading": ".", + "status": "called", + "cn": null, + "text": "chr7:5970001-5980000 4 (quality 50, filter PASS); chr7:5980001-5990000 3 (quality 12, filter LowQual)" + } + ] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "2", + "status": "called", + "detail": "ok" + }, + { + "region": "smn1", + "gene": "SMN2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA1", + "copy_number": "1", + "status": "called", + "detail": "." + }, + { + "region": "hba", + "gene": "HBA2", + "copy_number": "2", + "status": "called", + "detail": "." + }, + { + "region": "pms2", + "gene": "PMS2", + "copy_number": "2", + "status": "called", + "detail": "." + } + ], + "other_regions": 2, + "other_regions_called": 1, + "variants": { + "smn1": 2, + "pms2": 1 + }, + "smn1": { + "paraphase": 2, + "parascopy": { + "cn": 1, + "text": "1 (quality 31, filter PASS)" + }, + "agree": "disagree" + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla_t1k/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-A", + "alleles": [ + "A*02:01:01", + "A*01:01:01" + ], + "quality": [ + "60", + "50" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-B", + "alleles": [ + "B*57:01:01:02", + "B*08:01:01,B*08:01:02,B*08:01:03" + ], + "quality": [ + "0", + "0" + ], + "low_confidence": true, + "withheld_from_pharmcat": true, + "candidates": [ + 1, + 5 + ], + "one_allele_type": false + }, + { + "gene": "HLA-C", + "alleles": [ + "C*07:01:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*05:01:01", + "DQA1*02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01:01", + "DQB1*02:02:01" + ], + "quality": [ + "40", + "30" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*03:01:01", + "DRB1*07:01:01" + ], + "quality": [ + "40", + "nan" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HFE", + "alleles": [ + "HFE*001" + ], + "quality": [ + "20" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true, + "label": "T1K sequence allele, not the HFE C282Y/H63D genotype: that genotype comes from the variant calls (ClinVar section)" + } + ], + "derived": { + "coeliac": { + "status": "derived", + "reason": "", + "dq2_5": "present", + "dq2_5_phase": "likely cis (DRB1*03 present)", + "dq2_2": "present", + "dq8": "absent", + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "absent", + "reason": "" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B low confidence" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [ + { + "gene": "HLA-A", + "t1k": "A*02:01", + "optitype": "A*02:01", + "agree": "yes", + "note": "" + }, + { + "gene": "HLA-B", + "t1k": "B*57:01", + "optitype": "B*57:01", + "agree": "yes", + "note": "ok" + } + ], + "reason": "" + }, + "kir": { + "typed": [ + { + "gene": "KIR2DL1", + "alleles": [ + "KIR2DL1*001", + "KIR2DL1*002" + ], + "quality": [ + "30", + "20" + ] + } + ], + "not_called": [ + "KIR3DL3" + ], + "reason": "" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "ok", + "source": "prs/S_prs_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "scores": [ + { + "condition": "CAD", + "pgs_id": "PGS000018", + "score": "0.5", + "matched": "100", + "total": "120", + "chrx": "0", + "ambiguous": "12 dropped", + "applies_to": "all", + "sex_check": "ok", + "percentile": "75", + "group": "EUR", + "evaluated_in": [ + "EUR", + "AFR" + ], + "effect": "HR 1.5 (EUR)" + }, + { + "condition": "T2D", + "pgs_id": "PGS000014", + "score": "0.1", + "matched": "10", + "total": "20", + "chrx": "0", + "ambiguous": "0", + "applies_to": "all", + "sex_check": "ok", + "percentile": "", + "group": "", + "evaluated_in": [], + "effect": "" + }, + { + "condition": "Short", + "pgs_id": "PGS1", + "score": null, + "matched": null, + "total": null, + "chrx": null, + "ambiguous": null, + "applies_to": null, + "sex_check": null, + "percentile": "", + "group": null, + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": true, + "ancestry": { + "population": "EUR", + "panel": "HGDP+1kGP", + "low_confidence": null + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "ok", + "source": "ancestry_fine/S_ancestry_fine.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "continent": [ + { + "code": "WEA", + "group": "West Eurasia", + "fraction": 0.95, + "low": 0.93, + "high": 0.97 + }, + { + "code": "AFR", + "group": "Africa", + "fraction": 0.05, + "low": null, + "high": null + } + ], + "region": [ + { + "code": "IBE", + "group": "Iberia", + "fraction": 0.6, + "low": 0.5, + "high": 0.7 + } + ], + "painting": [], + "left_out": [ + { + "code": "OCE", + "group": "Oceania", + "fraction": null, + "low": null, + "high": null + } + ], + "not_painted": [ + { + "code": "chr21", + "group": ".", + "fraction": null, + "low": null, + "high": null + } + ], + "min_references": 10, + "painting_png": "iVBORw0KGgpub3QgYSByZWFsIHBpY3R1cmUA/w==", + "legend": [ + { + "code": "AFR", + "group": "Africa", + "colour": "#e69f00" + }, + { + "code": "WEA", + "group": "West Eurasia (Europe, Middle East, North Africa)", + "colour": "#0072b2" + }, + { + "code": "CSA", + "group": "Central and South Asia", + "colour": "#009e73" + }, + { + "code": "EAS", + "group": "East Asia", + "colour": "#cc79a7" + }, + { + "code": "AMR", + "group": "Americas (Native American)", + "colour": "#d55e00" + }, + { + "code": "OCE", + "group": "Oceania", + "colour": "#56b4e9" + } + ] + } + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/results/variants/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 3, + "pass": 1 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "ok", + "source": "delly/S_sv.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 1, + "pass": 1 + } + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "ok", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 8, + "deletions": 5, + "duplications": 3, + "significant": 7, + "significant_filtered": 4, + "filtered_chry": 1, + "dropped": { + "q0": 1, + "gap": 1, + "band": 1 + }, + "declared_sex": null + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "ok", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "consensus": 3 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 3, + "rows": [ + { + "sv": "1:1100-5100", + "type": "DEL", + "gt": "0/1", + "ps": "77", + "length": "-4000", + "acmg_class": "5", + "genes": [ + "GENE1", + "GENE2" + ], + "caller": "manta", + "support": 2, + "support_callers": [ + "manta", + "delly" + ], + "exon_edge": "yes", + "exon_edge_detail": "exon 2", + "population_af": 0.001, + "population_source": "gnomAD", + "common": false, + "annotsv_class": "5", + "rule": "", + "rule_detail": "" + }, + { + "sv": "2:150-880", + "type": "DUP", + "gt": "1/1", + "ps": null, + "length": "730", + "acmg_class": "4", + "genes": [ + "GENE3" + ], + "caller": "delly", + "support": 3, + "support_callers": [ + "manta", + "delly", + "tiddit", + "caller 4" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": null, + "population_source": "", + "common": false, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + }, + { + "sv": "9:1-100", + "type": "DEL", + "gt": null, + "ps": null, + "length": "99", + "acmg_class": "4", + "genes": [ + "G9" + ], + "caller": "tiddit", + "support": 1, + "support_callers": [ + "tiddit" + ], + "exon_edge": "no", + "exon_edge_detail": "", + "population_af": 0.2, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "", + "rule_detail": "" + } + ], + "downgraded": [ + { + "sv": "7:1-100", + "type": "INV", + "gt": "0/1", + "ps": null, + "length": "99", + "acmg_class": "3", + "genes": [], + "caller": "manta", + "support": 1, + "support_callers": [ + "manta" + ], + "exon_edge": "not checked", + "exon_edge_detail": "", + "population_af": null, + "population_source": "gnomAD", + "common": true, + "annotsv_class": "4", + "rule": "mhc", + "rule_detail": "the MHC" + } + ], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 1, + "exon_edge": 1, + "exon_edge_checked": true, + "common": 1 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh_stranger.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 9, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "17/20" + }, + { + "locus": "FMR1", + "repeat_count": "58" + }, + { + "locus": "C9ORF72", + "repeat_count": "." + }, + { + "locus": "ATXN1", + "repeat_count": "30/37" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "HTT", + "repeat_count": "17/20", + "status": "normal,normal", + "normal_max": "26", + "pathologic_min": "40", + "support": "SPANNING/SPANNING", + "adsp": "10/12", + "ci": "17-17/20-20", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "normal_max": "44", + "pathologic_min": "200", + "support": "SPANNING", + "adsp": "1", + "ci": "55-60", + "sizing": "low_confidence", + "allele_sizing": "low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "C9ORF72", + "repeat_count": ".", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": ".", + "adsp": ".", + "ci": ".", + "sizing": "no_call", + "allele_sizing": "no_call", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "ATXN1", + "repeat_count": "30/37", + "status": "normal", + "normal_max": "35", + "pathologic_min": "39", + "support": "SPANNING/FLANKING", + "adsp": "4/0", + "ci": "30-30/35-40", + "sizing": "flanking_only", + "allele_sizing": "sized/flanking_only", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_T", + "repeat_count": "5/7", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/7-7", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "11/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "11-11/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "EXTRA1", + "repeat_count": "8/9", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING", + "adsp": "1", + "ci": "8-8/9-9", + "sizing": "low_confidence", + "allele_sizing": "low_confidence/low_confidence", + "zone": "", + "zone_label": "", + "zone_citation": "", + "not_graded": "no published range" + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "normal_max": "", + "pathologic_min": "400", + "support": "INREPEAT/INREPEAT", + "adsp": "0/0", + "ci": "400-700/500-800", + "sizing": "in_repeat", + "allele_sizing": "in_repeat/in_repeat", + "zone": "", + "zone_label": "", + "zone_citation": "", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + }, + { + "locus": "UGT1A1", + "repeat_count": "7/8", + "status": "normal", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "7-7/8-8", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": true, + "stranger_outdated": false, + "unsized": [ + "FMR1", + "C9ORF72", + "ATXN1", + "EXTRA1", + "RFC1" + ], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG11/TG12 5T/7T", + "pairings": [ + "TG11-5T/TG12-7T", + "TG12-5T/TG11-7T" + ], + "tg": [ + 11, + 12 + ], + "poly_t": [ + 5, + 7 + ], + "five_t": true, + "phase": "unknown", + "r117h": "G/A", + "reading": "5T: 1 copy with the 5T poly-T tract, TG11 or TG12 on the 5T copy, not told by the counts (TG11-5T/TG12-7T or TG12-5T/TG11-7T). 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. One R117H copy and one 5T copy: phase unknown. R117H lies 17.6 kb from the tract, farther than a read pair spans, so whether they share a copy (in cis, a CF-causing allele of variable penetrance) or not (in trans) is not read; testing a parent tells them apart." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*1/*28", + "ta_units": [ + 7, + 8 + ], + "pharmcat": "*1/*28", + "agreement": "agrees", + "reading": "*1/*28 (TA6/TA7, 7/8 TA units as ExpansionHunter counts them): 1 decreased-function allele (*28 and *37 are decreased function). PharmCAT (step 7) called *1/*28: the same number of *28 and *37 copies." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ], + "flagged": [ + { + "locus": "FMR1", + "repeat_count": "58", + "status": "premutation", + "sizing": "low_confidence", + "motif": { + "verdict": "interrupted", + "note": "AGG", + "dominant_motif": "CGG", + "dominant_class": ".", + "dominant_share": ".", + "motif_shares": ".", + "repeat_reads": "." + } + }, + { + "locus": "RFC1", + "repeat_count": "500/600", + "status": "full_mutation", + "sizing": "in_repeat", + "rfc1_motif": { + "verdict": "pathogenic", + "note": "AAGGG dominant", + "dominant_motif": "AAGGG", + "dominant_class": "pathogenic", + "dominant_share": "0.9", + "motif_shares": "AAGGG:0.9", + "repeat_reads": "40" + } + } + ], + "no_status": 3, + "not_graded": { + "EXTRA1": "no published range" + } + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/S/S/S_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "1.5", + "intratelomeric_reads": 40, + "depth": 30.0, + "read_floor": 2000, + "low_reads": true + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "vcf/S_roh.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "segments": 3, + "total_mb": 37.5, + "largest_mb": 27.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + }, + { + "region": "chr15:23000001-50000000", + "mb": 27.0 + } + ], + "masked_mb": 46.5, + "sex_chromosome_segments": 1, + "long_mb": 34.0, + "f_roh": 0.0128, + "relationship": null, + "upd_flag": { + "chrom": "chr15", + "mb": 27.0, + "share": 0.79, + "imprinted": [ + { + "region": "15q11-q13 (SNRPN, UBE3A)", + "maternal_upd": "Prader-Willi syndrome", + "paternal_upd": "Angelman syndrome", + "pmid": "22237428;27615419" + } + ] + } + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "ok", + "source": "mito/S_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "H5a1", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/H5a1" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/H5a1/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ], + "contamination_status": "NO", + "contamination_level": "." + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R1b1a1b1a1a2*(xR-L21)", + "valid_markers": "40", + "qc_score": "0.98", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R1b1a1b1a1a2" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R1b1a1b1a1a2/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "R-Z2105", + "level": "12", + "tree": "ISOGG 2026", + "path_derived": "30", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": "400", + "min_depth": ".", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/y-dna/R-Z2105" + }, + { + "name": "YFull YTree", + "url": "https://www.yfull.com/tree/R-Z2105/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/37-y-haplogroup.md#explore-your-branch-further" + } + ], + "path": [ + { + "Level": "1", + "Branch": "A0-T", + "SNPs": "M42", + "On_tables": "", + "Derived": "1", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + }, + { + "Level": "2", + "Branch": "R", + "SNPs": "", + "On_tables": "", + "Derived": "", + "Ancestral": "", + "No_call": "", + "Ancestral_SNPs": "" + } + ] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "ok", + "source": "y_str/S_y_str.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "markers": [ + { + "order": "1", + "panel": "Y-12", + "marker": "DYS393", + "count": "13", + "confidence": "high" + }, + { + "order": "2", + "panel": "Y-12", + "marker": "DYS390", + "count": "24", + "confidence": "low" + }, + { + "order": "3", + "panel": "Y-12", + "marker": "DYS19", + "count": "", + "confidence": "no call" + }, + { + "order": "4", + "panel": "Y-25", + "marker": "DYS464", + "count": ".", + "confidence": "not callable" + } + ], + "high": 1, + "low": 1, + "no_call": 1, + "not_callable": 1 + } + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 600000, + "called": 590000, + "no_call": 10000, + "from_reference_blocks": 0, + "template_sites": "631000", + "file_23andme": "S_23andme.txt", + "file_ancestrydna": ".", + "no_call_reasons": { + "low depth": 9000, + "filtered": 1000 + } + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "ok", + "source": "bloodtype/S_bloodtype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "systems": [ + { + "system": "ABO", + "phenotype": "A", + "genotype": "A1/O1", + "confidence": "high" + }, + { + "system": "RhD", + "phenotype": "RhD positive", + "genotype": ".", + "confidence": "medium" + }, + { + "system": "Kell", + "phenotype": ".", + "genotype": "K-/k+", + "confidence": "." + } + ], + "abo": "A", + "rhd": "RhD positive" + } + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "estimated", + "kiv2_copies": "35", + "ci95_low": "30", + "ci95_high": "40", + "class": "low", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": "ok", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "present", + "alt_reads": "4", + "depth": "200", + "fraction": ".", + "copies": ".", + "message": "." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": "A/G", + "status": "called", + "risk_alleles": 1 + }, + { + "rsid": "rs3798220", + "genotype": "T|C", + "status": "called", + "risk_alleles": 1 + } + ], + "risk_alleles": 2, + "flag": "high", + "flag_text": "Genetic signs of a high Lp(a): 2 risk alleles at rs10455872 and rs3798220 and a low KIV-2 repeat count. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "negative", + "confidence": ".", + "variant": ".", + "flag": "none", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "none" + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "ok", + "source": "locityper/S_locityper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "chr6:1-2", + "status": "called", + "genotype": "h1,h2", + "haplotype_1": "h1", + "haplotype_2": "h2", + "quality": "30", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*03:01:01/DRB1*07:01:01" + }, + { + "locus": "KIR", + "region": "chr19:1-2", + "status": "called", + "genotype": "k1,k2", + "haplotype_1": "k1", + "haplotype_2": "k2", + "quality": "20", + "total_reads": "100", + "unexplained_reads": "1", + "weighted_distance": "0.1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K KIR pass (step 08): 1 genes typed" + }, + { + "locus": "CYP2D6", + "region": "chr22:1-2", + "status": "no call", + "genotype": ".", + "haplotype_1": ".", + "haplotype_2": ".", + "quality": ".", + "total_reads": ".", + "unexplained_reads": ".", + "weighted_distance": ".", + "warnings": "low", + "filter": "fail", + "reason": "low depth", + "primary": "pypgx (step 32) and Cyrius (step 21) did not run" + }, + { + "locus": "FCGR", + "region": "chr1:1-2", + "status": "called", + "genotype": "f", + "haplotype_1": "f", + "haplotype_2": "f", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "fail", + "reason": "q", + "primary": "no other step calls FCGR" + }, + { + "locus": "ZZZ", + "region": "chr1:1-2", + "status": "called", + "genotype": "z", + "haplotype_1": "z", + "haplotype_2": "z", + "quality": "9", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "no other step calls this locus" + } + ], + "called": 4, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + }, + { + "hgvs": "m.8344A>G", + "gene": "MT-TK", + "disease": "MERRF (myoclonic epilepsy with ragged-red fibres)", + "clinvar_id": "9579", + "pmids": "2112427", + "note": "Usually heteroplasmic; the level differs between tissues.", + "pos": 8344, + "filter": "weak_evidence", + "af": "0.30" + }, + { + "hgvs": "m.11778G>A", + "gene": "MT-ND4", + "disease": "Leber hereditary optic neuropathy (LHON)", + "clinvar_id": "9708", + "pmids": "3201231", + "note": "Usually homoplasmic; not everyone who carries it loses vision.", + "pos": 11778, + "filter": "PASS", + "af": "0.99" + } + ], + "deletions": { + "source": "mito/S_chrM_deletions.tsv", + "total": 3, + "listed": [ + { + "bkp5": 8469, + "bkp3": 13447, + "length": 4977, + "level_pct": 0.5, + "known": { + "hgvs": "m.8483_13459del", + "gene": "MT-ATP6,MT-ATP8,MT-CO3,MT-ND3,MT-ND4,MT-ND4L,MT-ND5,MT-TG,MT-TH,MT-TL2", + "disease": "The common 4977 bp deletion, between two 13 bp direct repeats: Kearns-Sayre syndrome, chronic progressive external ophthalmoplegia (CPEO), Pearson syndrome", + "clinvar_id": "430680", + "pmids": "2711184;2541333;20301382", + "note": "A single large deletion is heteroplasmic and usually arises new rather than inherited; in Kearns-Sayre syndrome and CPEO it is often found in muscle and not in blood, in Pearson syndrome in blood." + } + }, + { + "bkp5": 100, + "bkp3": 200, + "length": null, + "level_pct": 2.5 + } + ], + "list_floor_pct": 1.0 + }, + "copies": { + "status": "estimated", + "reason": null, + "chrm_reads": "100000", + "chrm_length": null, + "autosomal_reads": null, + "autosomal_length": null, + "depth_ratio": null, + "mtdna_copies": "250" + }, + "pass": 7, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [ + { + "variant": "chrM:302 A>AC,ACC", + "af": 0.3 + }, + { + "variant": "chrM:16189 T>C", + "af": 0.5 + } + ], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "source": "mito/S_chrM_annotated.tsv", + "clinvar": "2026-10-01", + "gnomad": "not annotated: no file", + "variants": [ + { + "pos": "3243", + "ref": "A", + "alt": "G", + "af": "0.40", + "clinvar_id": "9589", + "clinvar_significance": "Pathogenic", + "clinvar_review": "criteria", + "clinvar_condition": "MELAS", + "gnomad_af_hom": "0", + "gnomad_af_het": "0", + "gnomad_max_hl": "0", + "polyc": false + }, + { + "pos": "302", + "ref": "A", + "alt": "AC", + "af": "0.3", + "clinvar_id": ".", + "clinvar_significance": ".", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": true + }, + { + "pos": "500", + "ref": "C", + "alt": "T", + "af": "0.99", + "clinvar_id": "2", + "clinvar_significance": "Uncertain_significance", + "clinvar_review": ".", + "clinvar_condition": ".", + "gnomad_af_hom": ".", + "gnomad_af_het": ".", + "gnomad_max_hl": ".", + "polyc": false + } + ], + "clinvar_plp": 1 + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "A/G", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3798220", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": "T|C", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs78655421", + "gene": "CFTR", + "trait": "R117H", + "genotype": "G/A", + "status": "called", + "reading": "one copy", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs2", + "gene": "X", + "trait": "x", + "genotype": "C/C", + "status": "other_allele", + "reading": "other", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rsid": "rs3", + "gene": "X", + "trait": "x", + "genotype": "C/T", + "status": "low_gq", + "reading": "A low-quality call at this position (genotype quality 5, below 20): not read; check the reads.", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [ + { + "rsid": "rs429358", + "gene": "APOE", + "trait": "APOE", + "genotype": "T/C", + "status": "called", + "reading": "e3/e4", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "called": 3, + "not_covered": 1, + "other_allele": 1, + "low_gq": 1, + "rules": [ + { + "rule": "TAS2R38", + "gene": "TAS2R38", + "trait": "bitter", + "rsids": "rs1", + "chrom": "chr7", + "pos": "1", + "genotypes": "C/G", + "gq": "40", + "phase": "unphased", + "result": "PAV/AVI", + "status": "called", + "reading": "taster", + "ancestry": "all", + "note": ".", + "pmids": "1" + }, + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs73885319,rs60910145", + "chrom": "chr22", + "pos": "1,2", + "genotypes": "A/G,T/T", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "one G1", + "ancestry": "African", + "note": ".", + "pmids": "20647424", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "ok", + "source": "archaic/S_archaic.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "population": "CEU", + "longest": [ + "a", + "b", + "j" + ], + "neanderthal_segments": 120, + "neanderthal_bp": 50000000, + "denisovan_segments": 3, + "denisovan_bp": 200000, + "analysed_bp": 2500000000, + "neanderthal_percent": "2.00", + "denisovan_percent": "0.01" + } + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "ok", + "source": "ancient/S_ancient.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "nearest": [ + { + "rank": "1", + "group": "Spain_BA", + "n": "10", + "distance": "0.01", + "dates": "2000 BCE", + "country": "Spain", + "locality": "x" + }, + { + "rank": "2", + "group": "France_N", + "n": "5", + "distance": "0.02", + "dates": "", + "country": "France", + "locality": "" + } + ], + "release": "v62", + "sites_used": "500000", + "panel_sites": ".", + "ancient_people": ".", + "ancient_groups": ".", + "present_day_people": ".", + "plot_svg": "PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPjwvc3ZnPgo=" + } + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "inherited integrated HHV-6B likely", + "human_depth": "30", + "found": [ + { + "virus": "phiX", + "accession": "NC_001422", + "reads": "500", + "breadth": "1.0", + "relative_depth": ".", + "copies_per_cell": ".", + "call": "spike-in", + "human_depth": "30", + "note": "." + }, + { + "virus": "HHV-6B", + "accession": "NC_000898", + "reads": "2000", + "breadth": "0.99", + "relative_depth": "1.0", + "copies_per_cell": "1.0", + "call": "inherited integrated HHV-6 likely", + "human_depth": "30", + "note": "." + } + ], + "genomes": 3 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [ + { + "id": "HP:0001250", + "label": "Seizure" + }, + { + "id": "HP:0000001", + "label": null + } + ], + "genes": [ + { + "rank": "1", + "gene": "GENE1", + "moi": "AD", + "exomiser_score": "0.9", + "phenotype_score": "0.8", + "variants": [ + { + "variant": "chr1-1-A-G", + "genotype": "0/1", + "consequence": "missense", + "hgvs": "p.X", + "acmg": "LP", + "disease": "D1" + }, + { + "variant": "chr1-2-A-G", + "genotype": "0/1", + "consequence": "stop", + "hgvs": "p.Y", + "acmg": "P", + "disease": "D1" + } + ] + }, + { + "rank": "2", + "gene": "GENE2", + "moi": "AR", + "exomiser_score": "0.5", + "phenotype_score": "0.4", + "variants": [ + { + "variant": "chr2-1-A-G", + "genotype": "1/1", + "consequence": "splice", + "hgvs": "c.1", + "acmg": "VUS", + "disease": "D2" + } + ] + } + ] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [ + { + "gene": "DNMT3A", + "protein_change": "R882H", + "vaf": "0.05", + "alt_reads": "3", + "depth": "60", + "status": "chip", + "note": "." + } + ], + "other": 1, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "30.5", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TET2", + "mean_depth": "40", + "detection_limit_vaf": "0.08", + "status": "covered" + }, + { + "gene": "ASXL1", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + }, + { + "gene": "JAK2", + "mean_depth": "35", + "detection_limit_vaf": "0.1", + "status": "covered" + }, + { + "gene": "TP53", + "mean_depth": "25", + "detection_limit_vaf": "0.1", + "status": "covered" + } + ], + "median_depth": 32.75, + "median_limit": 0.12901665405107993, + "not_covered": [ + "ASXL1" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 3, + "by_element": { + "ALU": 2, + "L1": 1 + }, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "Fukuyama", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "not seen" + }, + { + "gene": "TAF1", + "element": "SVA", + "condition": "XDP", + "inheritance": "XL", + "status": "not callable", + "mean_depth": "2", + "call": "." + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "assessed", + "message": ".", + "depth": "30", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": "0.05", + "calls_mocha": ".", + "calls_reported": "1", + "phasing": ".", + "calls": [ + { + "chrom": "chr20", + "start": "1", + "end": "2", + "length_mb": "10", + "arms": "q", + "type": "Loss", + "cell_fraction": "0.1", + "n_hets": "50", + "lod_baf_phase": "20", + "reported": "yes", + "reason": "." + } + ], + "not_reported": 1 + } + } + } +} diff --git a/tests/parity/golden/summary-collect/c-g2-gvcf/g2-gvcf.json b/tests/parity/golden/summary-collect/c-g2-gvcf/g2-gvcf.json new file mode 100644 index 000000000..83d5f9f11 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-g2-gvcf/g2-gvcf.json @@ -0,0 +1,353 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:14 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": "A/A", + "status": "called", + "reading": "r", + "ancestry": "a", + "note": "n", + "pmids": "p" + } + ], + "apoe": [], + "called": 1, + "not_covered": 0, + "other_allele": 0, + "low_gq": 0, + "rules": [ + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs1", + "chrom": "chr22", + "pos": "1", + "genotypes": "A/G", + "gq": "40", + "phase": ".", + "result": ".", + "status": ".", + "reading": "r", + "ancestry": "African", + "note": ".", + "pmids": "1", + "g2": { + "status": "called", + "copies": 0, + "gq": "50", + "text": "G2 (rs71785313): the reference over the whole deletion span, no G2 copy.", + "source": "vcf/S.g.vcf.gz", + "result": ".", + "reading": "Not read: the G1 sites of the APOL1 rule were not read." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + } +} diff --git a/tests/parity/golden/summary-collect/c-g2-left/g2-left.json b/tests/parity/golden/summary-collect/c-g2-left/g2-left.json new file mode 100644 index 000000000..5f2c5f905 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-g2-left/g2-left.json @@ -0,0 +1,353 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:14 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": "A/A", + "status": "called", + "reading": "r", + "ancestry": "a", + "note": "n", + "pmids": "p" + } + ], + "apoe": [], + "called": 1, + "not_covered": 0, + "other_allele": 0, + "low_gq": 0, + "rules": [ + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs1", + "chrom": "chr22", + "pos": "1", + "genotypes": "A/G", + "gq": "40", + "phase": ".", + "result": "G0/G1", + "status": "called", + "reading": "r", + "ancestry": "African", + "note": ".", + "pmids": "1", + "g2": { + "status": "called", + "copies": 1, + "gq": "42", + "text": "G2 (rs71785313): 1 copy of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G1/G2", + "reading": "Two APOL1 risk copies (G1/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + } +} diff --git a/tests/parity/golden/summary-collect/c-g2-nocall/g2-nocall.json b/tests/parity/golden/summary-collect/c-g2-nocall/g2-nocall.json new file mode 100644 index 000000000..723199efa --- /dev/null +++ b/tests/parity/golden/summary-collect/c-g2-nocall/g2-nocall.json @@ -0,0 +1,353 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:14 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": "A/A", + "status": "called", + "reading": "r", + "ancestry": "a", + "note": "n", + "pmids": "p" + } + ], + "apoe": [], + "called": 1, + "not_covered": 0, + "other_allele": 0, + "low_gq": 0, + "rules": [ + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs1", + "chrom": "chr22", + "pos": "1", + "genotypes": "A/G", + "gq": "40", + "phase": ".", + "result": "G0/G2", + "status": "called", + "reading": "r", + "ancestry": "African", + "note": ".", + "pmids": "1", + "g2": { + "status": "not_read", + "copies": null, + "gq": ".", + "text": "G2 (rs71785313) not read: a no-call at chr22:36265995.", + "source": "vcf/S.g.vcf.gz", + "result": ".", + "reading": "Not read: the G1 sites of the APOL1 rule were not read." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + } +} diff --git a/tests/parity/golden/summary-collect/c-g2-none/g2-none.json b/tests/parity/golden/summary-collect/c-g2-none/g2-none.json new file mode 100644 index 000000000..39565b96f --- /dev/null +++ b/tests/parity/golden/summary-collect/c-g2-none/g2-none.json @@ -0,0 +1,353 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:14 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": "A/A", + "status": "called", + "reading": "r", + "ancestry": "a", + "note": "n", + "pmids": "p" + } + ], + "apoe": [], + "called": 1, + "not_covered": 0, + "other_allele": 0, + "low_gq": 0, + "rules": [ + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs1", + "chrom": "chr22", + "pos": "1", + "genotypes": "A/G", + "gq": "40", + "phase": ".", + "result": "G0/G0", + "status": "called", + "reading": "r", + "ancestry": "African", + "note": ".", + "pmids": "1", + "g2": { + "status": "no_call_record", + "copies": 0, + "gq": ".", + "text": "G2 (rs71785313): no G2 deletion among the VCF's calls.", + "source": "vcf/S.vcf.gz", + "result": "G0/G0 (no G2 call)", + "reading": "No G1 and no G2 copy: no APOL1 kidney risk copy. No G2 deletion was called, but this VCF has no reference blocks, so a G2 copy at a position with no reads would not show." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + } +} diff --git a/tests/parity/golden/summary-collect/c-g2-novcf/g2-novcf.json b/tests/parity/golden/summary-collect/c-g2-novcf/g2-novcf.json new file mode 100644 index 000000000..f7f525cf5 --- /dev/null +++ b/tests/parity/golden/summary-collect/c-g2-novcf/g2-novcf.json @@ -0,0 +1,353 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:14 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": "A/A", + "status": "called", + "reading": "r", + "ancestry": "a", + "note": "n", + "pmids": "p" + } + ], + "apoe": [], + "called": 1, + "not_covered": 0, + "other_allele": 0, + "low_gq": 0, + "rules": [ + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs1", + "chrom": "chr22", + "pos": "1", + "genotypes": "A/G", + "gq": "40", + "phase": ".", + "result": "G1/G1", + "status": "called", + "reading": "r", + "ancestry": "African", + "note": ".", + "pmids": "1", + "g2": { + "status": "not_read", + "copies": null, + "gq": ".", + "source": null, + "text": "G2 (rs71785313) not read: no VCF of the sample in this folder.", + "result": ".", + "reading": "The G1 result above stands on its own." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + } +} diff --git a/tests/parity/golden/summary-collect/c-g2-other/g2-other.json b/tests/parity/golden/summary-collect/c-g2-other/g2-other.json new file mode 100644 index 000000000..d7059490c --- /dev/null +++ b/tests/parity/golden/summary-collect/c-g2-other/g2-other.json @@ -0,0 +1,353 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:14 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": "A/A", + "status": "called", + "reading": "r", + "ancestry": "a", + "note": "n", + "pmids": "p" + } + ], + "apoe": [], + "called": 1, + "not_covered": 0, + "other_allele": 0, + "low_gq": 0, + "rules": [ + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs1", + "chrom": "chr22", + "pos": "1", + "genotypes": "A/G", + "gq": "40", + "phase": ".", + "result": "G1/G1", + "status": "called", + "reading": "r", + "ancestry": "African", + "note": ".", + "pmids": "1", + "g2": { + "status": "not_read", + "copies": null, + "gq": ".", + "text": "G2 (rs71785313) not read: REF ATTATA at chr22:36265998 is not the GRCh38 base (is the VCF on GRCh38?).", + "source": "vcf/S.vcf.gz", + "result": ".", + "reading": "The G1 result above stands on its own." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + } +} diff --git a/tests/parity/golden/summary-collect/c-g2-shifted/g2-shifted.json b/tests/parity/golden/summary-collect/c-g2-shifted/g2-shifted.json new file mode 100644 index 000000000..dd9dc036a --- /dev/null +++ b/tests/parity/golden/summary-collect/c-g2-shifted/g2-shifted.json @@ -0,0 +1,353 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:14 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": "A/A", + "status": "called", + "reading": "r", + "ancestry": "a", + "note": "n", + "pmids": "p" + } + ], + "apoe": [], + "called": 1, + "not_covered": 0, + "other_allele": 0, + "low_gq": 0, + "rules": [ + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs1", + "chrom": "chr22", + "pos": "1", + "genotypes": "A/G", + "gq": "40", + "phase": ".", + "result": "G0/G0", + "status": "called", + "reading": "r", + "ancestry": "African", + "note": ".", + "pmids": "1", + "g2": { + "status": "called", + "copies": 2, + "gq": "30", + "text": "G2 (rs71785313): 2 copies of the deletion called.", + "source": "vcf/S.vcf.gz", + "result": "G2/G2", + "reading": "Two APOL1 risk copies (G2/G2): the high-risk genotype of Genovese 2010, which went with focal segmental glomerulosclerosis and hypertension-attributed kidney failure in African Americans. A risk factor, not a diagnosis." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + } +} diff --git a/tests/parity/golden/summary-collect/c-g2-two/g2-two.json b/tests/parity/golden/summary-collect/c-g2-two/g2-two.json new file mode 100644 index 000000000..f9f11c80f --- /dev/null +++ b/tests/parity/golden/summary-collect/c-g2-two/g2-two.json @@ -0,0 +1,353 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:14 UTC", + "run": { + "status_file": false, + "started_utc": null, + "declared_sex": null + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs1", + "gene": "X", + "trait": "x", + "genotype": "A/A", + "status": "called", + "reading": "r", + "ancestry": "a", + "note": "n", + "pmids": "p" + } + ], + "apoe": [], + "called": 1, + "not_covered": 0, + "other_allele": 0, + "low_gq": 0, + "rules": [ + { + "rule": "APOL1", + "gene": "APOL1", + "trait": "kidney", + "rsids": "rs1", + "chrom": "chr22", + "pos": "1", + "genotypes": "A/G", + "gq": "40", + "phase": ".", + "result": "G1/G1", + "status": "called", + "reading": "r", + "ancestry": "African", + "note": ".", + "pmids": "1", + "g2": { + "status": "not_read", + "copies": null, + "gq": "42", + "text": "G2 (rs71785313) not read: more than one record calls the G2 deletion.", + "source": "vcf/S.vcf.gz", + "result": ".", + "reading": "The G1 result above stands on its own." + } + } + ] + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + } + } +} diff --git a/tests/parity/golden/summary-collect/c-nf/nf.json b/tests/parity/golden/summary-collect/c-nf/nf.json new file mode 100644 index 000000000..63c61673a --- /dev/null +++ b/tests/parity/golden/summary-collect/c-nf/nf.json @@ -0,0 +1,1005 @@ +{ + "schema_version": 2, + "sample": "S", + "generated_utc": "2026-11-15 13:14 UTC", + "run": { + "status_file": true, + "started_utc": "2020-01-01 00:00 UTC", + "declared_sex": "female" + }, + "databases": { + "clinvar_release": null, + "cpsr_clinvar_release": null, + "hla_database": null + }, + "manifest": [], + "sections": { + "couple": { + "title": "Couple carrier screen", + "step": "39", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "parascopy": { + "title": "SMN1/SMN2 copy number (Parascopy, short reads)", + "step": "35", + "state": "ok", + "source": "paralogs/S_smn_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "smn1": { + "cn": null, + "text": "no row covers exon 7", + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": "." + }, + "smn2": { + "cn": null, + "text": "no row covers exon 7" + }, + "agcn": ".", + "agcn_qual": ".", + "agcn_filter": ".", + "smn1_marker": { + "marker": "SMN1 g.27134T>G", + "rsid": "rs143838139", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "SMN1 g.27134T>G: not assessed, step 35 wrote no S_smn1_marker.tsv (a run from before this reader, or the check did not run)." + }, + "rows": 1, + "hba": { + "status": "not assessed", + "message": "not assessed: S_hba_copy_number.tsv has no status row" + }, + "cn_intervals": null, + "cyp21a2": null, + "dosage": null, + "reading": "SMN1 and SMN2 not assessed: no model: no carrier reading", + "loci": [] + } + }, + "paralogs": { + "title": "Paralog genes (Paraphase, PacBio HiFi)", + "step": "35b", + "state": "ok", + "source": "paralogs/S_paraphase_copy_number.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "genes": [ + { + "region": "smn1", + "gene": "SMN1", + "copy_number": "x", + "status": "called", + "detail": "." + } + ], + "other_regions": 0, + "other_regions_called": 0, + "variants": {}, + "smn1": { + "paraphase": null, + "parascopy": { + "cn": null, + "text": "no row covers SMN1 exon 7" + }, + "agree": null + } + } + }, + "hla": { + "title": "HLA typing (T1K)", + "step": "08", + "state": "ok", + "source": "hla/S_hla_genotype.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "loci": [ + { + "gene": "HLA-DQA1", + "alleles": [ + "DQA1*03:01", + "DQA1*01:01" + ], + "quality": [ + "0", + "30" + ], + "low_confidence": true, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + }, + { + "gene": "HLA-DQB1", + "alleles": [ + "DQB1*02:01" + ], + "quality": [ + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1 + ], + "one_allele_type": true + }, + { + "gene": "HLA-DRB1", + "alleles": [ + "DRB1*04:01", + "DRB1*07:01" + ], + "quality": [ + "40", + "40" + ], + "low_confidence": false, + "withheld_from_pharmcat": false, + "candidates": [ + 1, + 1 + ], + "one_allele_type": false + } + ], + "derived": { + "coeliac": { + "status": "not_determined", + "reason": "HLA-DQA1 low confidence", + "dq2_5": null, + "dq2_5_phase": null, + "dq2_2": null, + "dq8": null, + "dqb1_02_dose": 2 + }, + "alleles": [ + { + "allele": "A*31:01", + "kind": "drug", + "association": "carbamazepine hypersensitivity reactions in Europeans", + "pmids": "21428769", + "citation": "McCormack 2011, N Engl J Med", + "state": "not_determined", + "reason": "HLA-A not typed" + }, + { + "allele": "B*15:02", + "kind": "drug", + "association": "carbamazepine Stevens-Johnson syndrome in Han Chinese", + "pmids": "15057820", + "citation": "Chung 2004, Nature", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*57:01", + "kind": "drug", + "association": "abacavir hypersensitivity; flucloxacillin liver injury", + "pmids": "18256392,19483685", + "citation": "Mallal 2008, N Engl J Med; Daly 2009, Nat Genet", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*58:01", + "kind": "drug", + "association": "allopurinol severe skin reactions", + "pmids": "15743917", + "citation": "Hung 2005, Proc Natl Acad Sci U S A", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "B*27", + "kind": "disease", + "association": "ankylosing spondylitis", + "pmids": "4123836,4688372", + "citation": "Brewerton 1973, Lancet; Schlosstein 1973, N Engl J Med", + "state": "not_determined", + "reason": "HLA-B not typed" + }, + { + "allele": "C*06:02", + "kind": "disease", + "association": "psoriasis", + "pmids": "16642438", + "citation": "Nair 2006, Am J Hum Genet", + "state": "not_determined", + "reason": "HLA-C not typed" + }, + { + "allele": "DQB1*06:02", + "kind": "disease", + "association": "narcolepsy with cataplexy", + "pmids": "11179016", + "citation": "Mignot 2001, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*15:01", + "kind": "disease", + "association": "multiple sclerosis", + "pmids": "12557126", + "citation": "Barcellos 2003, Am J Hum Genet", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*01:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:01", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "present", + "reason": "" + }, + { + "allele": "DRB1*04:04", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:05", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*04:08", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + }, + { + "allele": "DRB1*14:02", + "kind": "disease", + "association": "rheumatoid arthritis, shared epitope allele", + "pmids": "2446635", + "citation": "Gregersen 1987, Arthritis Rheum", + "state": "absent", + "reason": "" + } + ] + }, + "optitype": { + "genes": [], + "reason": "OptiType did not run: no class I reads" + }, + "kir": { + "typed": [], + "not_called": [], + "reason": "too few KIR reads" + } + } + }, + "prs": { + "title": "Polygenic risk scores", + "step": "25", + "state": "stale", + "source": "prs/S_prs_summary.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 25 was failed (exit 2) in the run of 2020-01-01 00:00 UTC", + "data": { + "scores": [ + { + "condition": "X", + "pgs_id": "PGS1", + "score": "", + "matched": "", + "total": "", + "chrx": "", + "ambiguous": "", + "applies_to": "", + "sex_check": "", + "percentile": "", + "group": "", + "evaluated_in": null, + "effect": "" + } + ], + "adjusted": false, + "ancestry": { + "population": "", + "panel": "", + "low_confidence": "" + } + } + }, + "ancestry_fine": { + "title": "Fine-grained ancestry (ADMIXTURE, RFMix)", + "step": "26b", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "manta": { + "title": "Structural variants (Manta)", + "step": "04", + "state": "ok", + "source": "manta/x/diploidSV.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "total": 0, + "pass": 0 + } + }, + "delly": { + "title": "Structural variants (Delly)", + "step": "19", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "cnvpytor": { + "title": "Copy number (CNVpytor)", + "step": "18", + "state": "stale", + "source": "cnvpytor/S_cnvs.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 18 was skipped (opt-in) in the run of 2020-01-01 00:00 UTC", + "data": { + "total": 2, + "deletions": 2, + "duplications": 0, + "significant": 2, + "significant_filtered": 1, + "filtered_chry": 1, + "dropped": { + "q0": 0, + "gap": 0, + "band": 0, + "chry_female": 1 + }, + "declared_sex": "female" + } + }, + "sv_consensus": { + "title": "SV consensus merge", + "step": "22", + "state": "stale", + "source": "sv_merged/S_sv_consensus.vcf.gz", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 22 was failed (exit 1) in the run of 2020-01-01 00:00 UTC", + "data": { + "consensus": 1 + } + }, + "annotsv": { + "title": "Structural variants, class 4 and 5 (AnnotSV)", + "step": "05", + "state": "ok", + "source": "annotsv/S_sv_class4_5_checked.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "count": 0, + "rows": [], + "downgraded": [], + "consensus": "sv_merged/S_sv_consensus.vcf.gz", + "consensus_note": null, + "single_caller": 0, + "exon_edge": 0, + "exon_edge_checked": true, + "common": 0 + } + }, + "expansions": { + "title": "Repeat expansions (ExpansionHunter)", + "step": "09", + "state": "ok", + "source": "expansion_hunter/S_eh.vcf", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "records": 3, + "key_loci": [ + { + "locus": "HTT", + "repeat_count": "not in output" + }, + { + "locus": "FMR1", + "repeat_count": "not in output" + }, + { + "locus": "C9ORF72", + "repeat_count": "not in output" + }, + { + "locus": "ATXN1", + "repeat_count": "not in output" + }, + { + "locus": "DMPK", + "repeat_count": "not in output" + } + ], + "loci": [ + { + "locus": "CFTR_T", + "repeat_count": "5/5", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "5-5/5-5", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "CFTR_TG", + "repeat_count": "12/12", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "8/9", + "ci": "12-12/12-12", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + }, + { + "locus": "UGT1A1", + "repeat_count": "8/10", + "status": "", + "normal_max": "", + "pathologic_min": "", + "support": "SPANNING/SPANNING", + "adsp": "5/6", + "ci": "8-8/10-10", + "sizing": "sized", + "allele_sizing": "sized/sized", + "zone": "", + "zone_label": "", + "zone_citation": "" + } + ], + "stranger": false, + "stranger_outdated": true, + "unsized": [], + "intermediate": [], + "readings": [ + { + "locus": "CFTR", + "repeat": "intron 9 (legacy IVS8) TG and poly-T tract", + "source": "Deignan et al. 2020, PMID 32404922", + "status": "read", + "genotype": "TG12-5T/TG12-5T", + "pairings": [ + "TG12-5T/TG12-5T" + ], + "tg": [ + 12, + 12 + ], + "poly_t": [ + 5, + 5 + ], + "five_t": true, + "phase": "not_read", + "r117h": ".", + "reading": "5T: 2 copys with the 5T poly-T tract, TG12 on the 5T copies. 5T lowers the splicing of exon 10 and is a variable-penetrance allele: with TG12 or TG13 on the same copy it more often causes CBAVD or another CFTR-related disorder when the other copy carries a CF-causing variant. R117H was not read (no step 42 traits row for rs78655421 was called), so whether it sits on the 5T copy is not known." + }, + { + "locus": "UGT1A1", + "repeat": "promoter TA repeat", + "source": "Gammal et al. 2016, PMID 26417955", + "status": "read", + "genotype": "*28/TA9", + "ta_units": [ + 8, + 10 + ], + "pharmcat": ".", + "agreement": "not_compared", + "reading": "*28/TA9 (TA7/TA9, 8/10 TA units as ExpansionHunter counts them): a TA length outside TA5 to TA8 is not a named allele here, so its function is not read and it is not compared with PharmCAT (step 7)." + }, + { + "locus": "SLC6A4", + "repeat": "5-HTTLPR promoter length polymorphism", + "status": "not_tested", + "genotype": ".", + "reading": "not tested by any step. CPIC's 2023 SSRI guideline (Bousman et al. 2023, PMID 37032427) found that the data on SLC6A4 do not support its use in antidepressant prescribing, so a result would carry no recommendation.", + "source": "Bousman et al. 2023, PMID 37032427" + } + ] + } + }, + "telomere": { + "title": "Telomere content (TelomereHunter)", + "step": "10", + "state": "ok", + "source": "telomere/x/deep/T_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "tel_content": "334", + "intratelomeric_reads": 61873, + "read_check": "depth unknown (no total_reads or read_lengths in the summary): not checked" + } + }, + "roh": { + "title": "Runs of homozygosity", + "step": "11", + "state": "ok", + "source": "roh/S_roh.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": null, + "data": { + "segments": 1, + "total_mb": 7.0, + "largest_mb": 7.0, + "autosomal_over_5mb": [ + { + "region": "chr2:10000001-17000000", + "mb": 7.0 + } + ], + "masked_mb": 0.0, + "sex_chromosome_segments": 0, + "long_mb": 7.0, + "f_roh": 0.0026, + "relationship": "no sign of related parents", + "upd_flag": null + } + }, + "haplogroup": { + "title": "Mitochondrial haplogroup", + "step": "12", + "state": "stale", + "source": "mito/S_haplogroup.txt", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 12 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "haplogroup": "U5a", + "explore": [ + { + "name": "FamilyTreeDNA Discover (free, no upload)", + "url": "https://discover.familytreedna.com/mtdna/U5a" + }, + { + "name": "YFull MTree", + "url": "https://www.yfull.com/mtree/U5a/" + }, + { + "name": "More routes, and the files to share", + "url": "https://github.com/GeiserX/OwnGenome/blob/main/docs/12-mito-haplogroup.md#explore-your-branch-further" + } + ] + } + }, + "y_haplogroup": { + "title": "Y-chromosome haplogroup (Yleaf)", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_haplogroup.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "insufficient markers", + "valid_markers": ".", + "qc_score": ".", + "explore": [] + } + }, + "y_tree": { + "title": "Y placement on the current public tree", + "step": "37", + "state": "ok", + "source": "y_haplogroup/S_y_tree.txt", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "haplogroup": "no branch has a derived call", + "level": ".", + "tree": ".", + "path_derived": ".", + "path_ancestral": ".", + "path_no_call": ".", + "ancestral_in_terminal": ".", + "outvoted_ancestral": ".", + "snps_called": ".", + "min_depth": ".", + "explore": [], + "path": [] + } + }, + "y_str": { + "title": "Y-STR markers (HipSTR)", + "step": "45", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "consumer_export": { + "title": "Raw-data export (23andMe v5 layout)", + "step": "40", + "state": "ok", + "source": "consumer_export/S_consumer_export.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "sites": 10, + "called": 10, + "no_call": 0, + "from_reference_blocks": 0, + "template_sites": ".", + "file_23andme": ".", + "file_ancestrydna": ".", + "no_call_reasons": {} + } + }, + "bloodtype": { + "title": "Blood groups (predicted)", + "step": "41", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "lpa": { + "title": "Lp(a): LPA KIV-2 repeat count", + "step": "46", + "state": "ok", + "source": "lpa/S_lpa_kiv2.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "too little depth", + "kiv2_copies": ".", + "ci95_low": ".", + "ci95_high": ".", + "class": ".", + "kiv2_depth": ".", + "flank_depth": ".", + "mapq0_share": ".", + "message": ".", + "advice": ".", + "null_allele": { + "marker": "LPA KIV-2 4925G>A", + "rsid": "rs1484919544", + "status": "not assessed", + "alt_reads": ".", + "depth": ".", + "fraction": ".", + "copies": ".", + "message": "LPA KIV-2 4925G>A: not assessed, step 46 wrote no S_lpa_kiv2_4925.tsv (a run from before this reader, or the check did not run)." + }, + "variants": [ + { + "rsid": "rs10455872", + "genotype": ".", + "status": "not_covered", + "risk_alleles": null + } + ], + "risk_alleles": 0, + "flag": ".", + "flag_text": "Not determined: neither the KIV-2 count nor the two risk variants could be read. Measure Lp(a) in blood." + } + }, + "vntyper": { + "title": "MUC1 VNTR frameshifts (VNtyper)", + "step": "52", + "state": "ok", + "source": "vntyper/S_vntyper.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "confidence": ".", + "variant": ".", + "flag": ".", + "alt_depth": ".", + "vntr_mean_depth": ".", + "coverage_qc": ".", + "sensitivity": ".", + "message": ".", + "advice": "." + } + }, + "locityper": { + "title": "Locityper second opinion at hard loci", + "step": "53", + "state": "stale", + "source": "locityper/S_locityper.tsv", + "file_date": "2019-06-01 12:30 UTC", + "note": "from an earlier run (2019-06-01 12:30 UTC); step 53 was skipped in the run of 2020-01-01 00:00 UTC", + "data": { + "loci": [ + { + "locus": "HLA-DRB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DRB1*04:01/DRB1*07:01" + }, + { + "locus": "HLA-DQB1", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "T1K (step 08): DQB1*02:01" + }, + { + "locus": "CYP2D6", + "region": "x", + "status": "called", + "genotype": "h", + "haplotype_1": "h", + "haplotype_2": "h", + "quality": "1", + "total_reads": "1", + "unexplained_reads": "1", + "weighted_distance": "1", + "warnings": ".", + "filter": "pass", + "reason": ".", + "primary": "pypgx (step 32) and Cyrius (step 21) did not run" + } + ], + "called": 3, + "passed": 3 + } + }, + "mito": { + "title": "Mitochondrial variants", + "step": "20", + "state": "ok", + "source": "mito/S_chrM_filtered.vcf.gz", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "confirmed": [ + { + "hgvs": "m.3243A>G", + "gene": "MT-TL1", + "disease": "MELAS (mitochondrial encephalomyopathy, lactic acidosis and stroke-like episodes); maternally inherited diabetes and deafness", + "clinvar_id": "9589", + "pmids": "2102678", + "note": "Usually heteroplasmic; the level differs between tissues and is often lower in blood than in muscle or urine.", + "pos": 3243, + "filter": "PASS", + "af": "0.40" + } + ], + "deletions_note": "no deletion table (mito/_chrM_deletions.tsv): step 20 ran before it looked for large deletions, eKLIPse wrote no table (see the step's log), or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "copies_note": "no copy-number table (mito/_mtdna_copies.tsv): step 20 ran before it counted mtDNA copies, or the Nextflow MITO_DELETIONS task failed (see the Nextflow log)", + "pass": 1, + "heteroplasmic": 1, + "heteroplasmic_sites": [ + { + "pos": 3243, + "ref": "A", + "alt": "G", + "af": 0.4 + } + ], + "heteroplasmy_floor": 0.05, + "polyc_heteroplasmic": [], + "polyc_tracts": [ + "m.303-315", + "m.16184-16193" + ], + "annotation": { + "note": "mito/S_chrM_annotated.tsv is older than the chrM calls: rerun step 20" + } + } + }, + "traits": { + "title": "Traits panel", + "step": "42", + "state": "ok", + "source": "traits/S_traits.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "rows": [ + { + "rsid": "rs10455872", + "gene": "LPA", + "trait": "Lp(a)", + "genotype": ".", + "status": "not_covered", + "reading": "no reads", + "ancestry": "all", + "note": ".", + "pmids": "1" + } + ], + "apoe": [], + "called": 0, + "not_covered": 1, + "other_allele": 0, + "low_gq": 0, + "rules": null + } + }, + "archaic": { + "title": "Archaic ancestry (IBDmix)", + "step": "43", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "ancient": { + "title": "Ancient populations (AADR PCA)", + "step": "44", + "state": "missing", + "source": null, + "file_date": null, + "note": null, + "data": {} + }, + "viral": { + "title": "Viral DNA in the unmapped reads", + "step": "47", + "state": "ok", + "source": "viral/S_viral.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "hhv6": "not assessed (no human depth)", + "human_depth": ".", + "found": [], + "genomes": 1 + } + }, + "exomiser": { + "title": "Symptom-driven ranking (Exomiser)", + "step": "48", + "state": "ok", + "source": "exomiser/S_exomiser_top.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "terms": [], + "genes": [] + } + }, + "chip": { + "title": "Clonal haematopoiesis (CHIP)", + "step": "49", + "state": "ok", + "source": "chip/S_chip.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": [], + "other": 0, + "genes": [ + { + "gene": "DNMT3A", + "mean_depth": "2", + "detection_limit_vaf": ".", + "status": "low depth" + } + ], + "median_depth": null, + "median_limit": null, + "not_covered": [ + "DNMT3A" + ] + } + }, + "mei": { + "title": "Mobile element insertions (MEGAnE)", + "step": "50", + "state": "ok", + "source": "mei/S_mei_sites.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "calls": 0, + "by_element": {}, + "sites": [ + { + "gene": "FKTN", + "element": "SVA", + "condition": "F", + "inheritance": "AR", + "status": "callable", + "mean_depth": "30", + "call": "seen" + } + ] + } + }, + "mosaic": { + "title": "Mosaic chromosomal alterations (MoChA)", + "step": "51", + "state": "ok", + "source": "mosaic/S_mocha_summary.tsv", + "file_date": "2021-03-04 05:06 UTC", + "note": null, + "data": { + "status": "not assessed", + "message": "too shallow", + "depth": ".", + "het_sites": ".", + "het_per_10mb": ".", + "chromosomes": ".", + "floor_read": ".", + "floor_min": ".", + "floor": ".", + "calls_mocha": ".", + "calls_reported": ".", + "phasing": ".", + "calls": [], + "not_reported": 0 + } + } + } +} From 11aa0770e0234113644f2c45f38e1a10567a216b Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:30:00 +0100 Subject: [PATCH 197/255] test(e2e): cover each engine limit alone and the user's overrides The engine-size section now also checks an engine with less memory only (THREADS=3 and a user -c, which must come after executor.config) and one with fewer CPUs only (an explicit --max_memory stays the only one). Bead: plat-18 --- tests/e2e/run-all-launcher.sh | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/e2e/run-all-launcher.sh b/tests/e2e/run-all-launcher.sh index d9915896a..6e0be0559 100644 --- a/tests/e2e/run-all-launcher.sh +++ b/tests/e2e/run-all-launcher.sh @@ -25,7 +25,8 @@ # Docker Desktop and colima have (a docker shim that reports 8 GB and # 2 CPUs, and a stand-in nextflow that records its arguments): run-all.sh # passes the engine's as --max_memory and --max_cpus, and the pool -# Nextflow fills, executor.config, is the engine's. +# Nextflow fills, executor.config, is the engine's. Each limit also +# alone; THREADS, an explicit --max_memory and a -c of the user's win. # Each comparison gets a negative control in the same run: a planted missing # file must be reported, the first run's trace must show the alignment and # calling tasks, and the first run's trace (tasks COMPLETED) must fail the @@ -111,11 +112,13 @@ chmod +x "${ENG}/bin/docker" "${ENG}/bin/nextflow" HC=$(getconf _NPROCESSORS_ONLN) HM=$(awk '/^MemTotal:/ {print int($2 / 1048576)}' /proc/meminfo 2>/dev/null || echo $(( $(sysctl -n hw.memsize) / 1073741824 ))) EG=$(( HM > 8 ? 8 : HM / 2 )) ECPU=$(( HC > 2 ? 2 : 1 )) -engine_run() { # engine_run NAME 'MEMTOTAL NCPU': run-all.sh with docker info answering that; its log is logs/.engine-NAME.log - rm -f "${ENG}/${1}".* - echo "+ scripts/run-all.sh ${E} female, docker info reporting '${2}' (${1})" - env -u THREADS PATH="${ENG}/bin:${PATH}" ENGINE_INFO="$2" ENGINE_REC="${ENG}/${1}" GENOME_DIR="$G3" SKIP_VALIDATION=true \ - TOOLS=roh EH_CATALOG="${G3}/none.json" "${REPO}/scripts/run-all.sh" "$E" female > "${E2E_WORK}/logs/${CASE_NAME}.engine-${1}.log" 2>&1 +engine_run() { # engine_run NAME 'MEMTOTAL NCPU' [OPTION...]: run-all.sh with docker info answering that, THREADS=$ET when + local n=$1 info=$2; shift 2 # ET is set; its log is logs/.engine-NAME.log + rm -f "${ENG}/${n}".* + echo "+ ${ET:+THREADS=${ET} }scripts/run-all.sh ${E} female $*, docker info reporting '${info}' (${n})" + env -u THREADS ${ET:+"THREADS=${ET}"} PATH="${ENG}/bin:${PATH}" ENGINE_INFO="$info" ENGINE_REC="${ENG}/${n}" GENOME_DIR="$G3" \ + SKIP_VALIDATION=true TOOLS=roh EH_CATALOG="${G3}/none.json" "${REPO}/scripts/run-all.sh" "$E" female "$@" \ + > "${E2E_WORK}/logs/${CASE_NAME}.engine-${n}.log" 2>&1 echo "+ exit $? (the stand-in nextflow exits 1)" } EXC="${G3}/${E}/nextflow/executor.config" @@ -137,6 +140,24 @@ check "negative control: an engine as big as the host gives the host's --max_mem check "negative control: and the host's --max_cpus ${HC}" has " --max_cpus ${HC} " "$BIG" check "negative control: and no executor.config" lacks "executor\.config" "$BIG" check "negative control: the small engine's executor.config is not left behind" test ! -e "$EXC" +# Each limit alone, and what the user passes wins: THREADS and an explicit --max_memory stay, and a +# -c of the user's comes after the engine's executor.config, so its executor block replaces the pool. +UC="${G3}/user.config" +ET=3 engine_run memory "$((EG * 1073741824)) $((HC + 4))" -c "$UC" +MEMO=$(cat "${ENG}/memory.args" 2>/dev/null) MEMC=$(cat "${ENG}/memory.config" 2>/dev/null) +echo "+ nextflow${MEMO}" +check "less memory alone: --max_memory ${EG}.GB" has " --max_memory ${EG}\.GB " "$MEMO" +check "less memory alone: THREADS=3 stays --max_cpus 3" has " --max_cpus 3 " "$MEMO" +check "less memory alone: the pool keeps the host's ${HC} CPUs" has "^ *cpus += ${HC}\$" "$MEMC" +check "less memory alone: the pool has ${EG} GB" has "^ *memory = '${EG} GB'\$" "$MEMC" +check "the user's -c comes after the engine's executor.config" has " -c ${EXC} (.* )?-c ${UC} " "$MEMO" +engine_run cpus "$(( (HM + 4) * 1073741824 )) ${ECPU}" --max_memory 6.GB +CPUO=$(cat "${ENG}/cpus.args" 2>/dev/null) CPUC=$(cat "${ENG}/cpus.config" 2>/dev/null) +echo "+ nextflow${CPUO}" +check "fewer CPUs alone: --max_cpus ${ECPU}" has " --max_cpus ${ECPU} " "$CPUO" +check_eq "fewer CPUs alone: the user's --max_memory 6.GB is the only one" "$(grep -o ' --max_memory [^ ]*' <<<"$CPUO" | tr -d '\n')" " --max_memory 6.GB" +check "fewer CPUs alone: the pool has ${ECPU} CPUs" has "^ *cpus += ${ECPU}\$" "$CPUC" +check "fewer CPUs alone: the pool keeps the host's ${HM} GB" has "^ *memory = '${HM} GB'\$" "$CPUC" rm -rf "$G3" "$ENG" # --- first run, with fs.protected_hardlinks 0 --------------------------------------------- From 6706eb78521867870b2336cec7b4130b2907f57c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:35:01 +0100 Subject: [PATCH 198/255] refactor(live): name the metrics asker's login flag login_required A field spelled password_set read as a hard-coded password to a secret scanner; the flag says whether the site asks for a login. Bead: D1 --- rust/crates/og-live/src/ops.rs | 6 +++--- rust/crates/og-live/tests/ops.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-live/src/ops.rs b/rust/crates/og-live/src/ops.rs index d9595a074..1dca60578 100644 --- a/rust/crates/og-live/src/ops.rs +++ b/rust/crates/og-live/src/ops.rs @@ -997,8 +997,8 @@ pub fn read_token(path: Option<&Path>) -> io::Result> { /// Who asks for `/metrics`. #[derive(Debug, Clone, Copy, Default)] pub struct Asker<'a> { - /// The site has a password. - pub password_set: bool, + /// The site asks for a login (it has a password hash). + pub login_required: bool, /// The request carries a valid session. pub logged_in: bool, /// Its `Authorization` header. @@ -1019,7 +1019,7 @@ pub fn metrics_allowed( who: &Asker<'_>, token: impl FnOnce() -> io::Result>, ) -> io::Result { - if !who.password_set || who.logged_in { + if !who.login_required || who.logged_in { return Ok(true); } let auth = who.authorization.unwrap_or(""); diff --git a/rust/crates/og-live/tests/ops.rs b/rust/crates/og-live/tests/ops.rs index 8a4ddf80c..5e2823658 100644 --- a/rust/crates/og-live/tests/ops.rs +++ b/rust/crates/og-live/tests/ops.rs @@ -540,9 +540,9 @@ fn no_path_in_the_card_and_the_metrics_carry_only_run_and_fs_in_a_valid_text() { // --- who may read /metrics ------------------------------------------------------------------ -fn who(password: bool, logged_in: bool, auth: Option<&str>) -> Asker<'_> { +fn who(login: bool, logged_in: bool, auth: Option<&str>) -> Asker<'_> { Asker { - password_set: password, + login_required: login, logged_in, authorization: auth, } From 49cfc383a80377cfed21205295550e8ad82a04fb Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:38:21 +0100 Subject: [PATCH 199/255] fix(release): credit the Rust standard library and musl, and stop on an unreadable licence file Neither is a cargo package, so the notices left them out. og dev licences notices takes --extra NAME=FILE (HTML notices become text); og.sh passes the toolchain's COPYRIGHT-library.html and musl's COPYRIGHT from the builder image, and check-release.sh dist wants both. A licence file that exists but cannot be read is now an error instead of a silent gap. Bead: lic-02 --- .github/workflows/release.yml | 2 +- CHANGELOG.md | 2 +- docs/rust.md | 6 +- rust/crates/og-dev/src/licences.rs | 148 ++++++++++++++++++---- rust/crates/og-dev/tests/licences/main.rs | 50 +++++++- rust/og.sh | 10 +- scripts/ci/check-release.sh | 11 +- 7 files changed, 190 insertions(+), 39 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7d3e728f..c8c3b385c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -112,7 +112,7 @@ jobs: echo "" echo "## og binaries" echo "" - echo "og-linux-x86_64 and og-linux-aarch64 are this release's Rust binaries, with their SHA256SUMS and a build attestation. THIRD-PARTY-LICENSES.txt holds the licences of the crates and C libraries (htslib, htscodecs) the binaries link. og-darwin-arm64 and og-darwin-x86_64 run the live site's genome map on a Mac; they leave out the tools that read BAMs. \`./scripts/setup.sh --og\` on a checkout of this tag downloads and checks all four." + echo "og-linux-x86_64 and og-linux-aarch64 are this release's Rust binaries, with their SHA256SUMS and a build attestation. THIRD-PARTY-LICENSES.txt holds the licences of the Rust standard library, the crates and the C libraries (musl, htslib, htscodecs) the binaries link. og-darwin-arm64 and og-darwin-x86_64 run the live site's genome map on a Mac; they leave out the tools that read BAMs. \`./scripts/setup.sh --og\` on a checkout of this tag downloads and checks all four." } >> "${RUNNER_TEMP}/notes.md" # Only a commit that main's CI passed is released: it must be on main, diff --git a/CHANGELOG.md b/CHANGELOG.md index c1af0a35f..b8d0afd94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The variant calling page names the Genome Reference Consortium as the source of the PAR intervals in `assets/par_grch38.bed`. - Every verb the 1.0.0 release still has to port now exists in the `og` binary, each in its own module, as a stub that prints `og : not ported yet` and exits 2. The new crates are og-render, og-live, og-couple and og-admix. Each port then fills one module and never edits `rust/og/src/main.rs` or a crate's module list, so the ports can land in parallel. No step calls these verbs yet, and no output changes. - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. -- Each release now ships `THIRD-PARTY-LICENSES.txt` beside the og binaries. The binaries link about seventy crates and the C libraries htslib and htscodecs statically, and their licences (MIT, BSD-3-Clause, Unicode-3.0 and others) ask that a binary carry their notices; no release carried any. `rust/og.sh build` writes the file with the new `og dev licences notices`, from `cargo metadata` for each of the four targets and the licence files each crate ships, including htslib's and htscodecs' own from the `hts-sys` crate. The build stops when a linked crate's licence is outside the allow list in `docs/rust.md`. The release adds the file to `SHA256SUMS`, which `sha256sum og-*` alone left out, and to the build attestation, and `scripts/ci/check-release.sh dist` refuses a release whose files or sums do not match. No pipeline output changes. +- Each release now ships `THIRD-PARTY-LICENSES.txt` beside the og binaries. The binaries link about seventy crates, the Rust standard library and the C libraries musl, htslib and htscodecs statically, and their licences (MIT, BSD-3-Clause, Unicode-3.0 and others) ask that a binary carry their notices; no release carried any. `rust/og.sh build` writes the file with the new `og dev licences notices`, from `cargo metadata` for each of the four targets and the licence files each crate ships, including htslib's and htscodecs' own from the `hts-sys` crate, plus the notices of the Rust standard library and of musl, the C library of the Linux pair. The build stops when a linked crate's licence is outside the allow list in `docs/rust.md`. The release adds the file to `SHA256SUMS`, which `sha256sum og-*` alone left out, and to the build attestation, and `scripts/ci/check-release.sh dist` refuses a release whose files or sums do not match. No pipeline output changes. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. diff --git a/docs/rust.md b/docs/rust.md index fec096372..f8bbd0085 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -59,7 +59,7 @@ The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, ` - `og-linux-x86_64` and `og-linux-aarch64`, two static musl builds for the pipeline's containers. Both are kept because an Apple Silicon host runs amd64 images under emulation, where `uname -m` says `x86_64`. - `og-darwin-arm64` and `og-darwin-x86_64`, for a Mac that hosts the live site (`scripts/owngenome-live.sh`), whose genome map runs `og map` on the host. They are built with `--no-default-features`, which leaves out the cargo feature `bam`: that host never reads a BAM, and BAM reading needs htslib, which is C and would have to cross-compile for Apple. The BAM tools (`og pgx cyp2d6-*`, `og str motif`, `og cdna check|scan`, `og comphet phase`, `og bam contigs`, `og bam-fetch-probe`) say so and exit 2 there. Every Linux build has the feature, since it is on by default. -Beside them the build writes `bin/.og/THIRD-PARTY-LICENSES.txt`, which every release ships. The binaries link about seventy crates statically (MIT, Apache-2.0, BSD-3-Clause, ISC, Zlib, Unicode-3.0 and others), and `hts-sys` builds the C libraries htslib and htscodecs into them; those licences ask that a binary carry their notices. The new `og` writes the file itself inside `OG_BUILDER`: `og dev licences notices` asks `cargo metadata --filter-platform` for each of the four targets (the macOS pair without default features, as it is built) and follows the normal dependencies from `og`, so build and dev dependencies and procedural macros, which leave no code in the binary, are not listed. The file lists every crate with its version and licence, and then each distinct licence text once. The texts come from each crate's own LICENSE, COPYING, NOTICE and similar files; a crate that builds C (it sets `links`) also gives the ones in its folders, which is how htslib's `htslib/LICENSE` and htscodecs' `htslib/htscodecs/LICENSE.md` get in without a copy that could go stale. A crate is credited for every target whose dependency graph reaches it, even where the linker then drops its unused code. The build stops when a crate's licence is outside the allow list (`ALLOWED` in `rust/crates/og-dev/src/licences.rs`: 0BSD, Apache-2.0, Apache-2.0 WITH LLVM-exception, BSD-3-Clause, ISC, MIT, Unicode-3.0, Unlicense, Zlib), so a new dependency under another licence needs a decision first. A crate that ships no licence file gets the MIT text with its authors when MIT is among its licences, and stops the build otherwise. +Beside them the build writes `bin/.og/THIRD-PARTY-LICENSES.txt`, which every release ships. The binaries link about seventy crates statically (MIT, Apache-2.0, BSD-3-Clause, ISC, Zlib, Unicode-3.0 and others), and `hts-sys` builds the C libraries htslib and htscodecs into them; those licences ask that a binary carry their notices. The new `og` writes the file itself inside `OG_BUILDER`: `og dev licences notices` asks `cargo metadata --filter-platform` for each of the four targets (the macOS pair without default features, as it is built) and follows the normal dependencies from `og`, so build and dev dependencies and procedural macros, which leave no code in the binary, are not listed. The file lists every crate with its version and licence, and then each distinct licence text once. The texts come from each crate's own LICENSE, COPYING, NOTICE and similar files; a crate that builds C (it sets `links`) also gives the ones in its folders, which is how htslib's `htslib/LICENSE` and htscodecs' `htslib/htscodecs/LICENSE.md` get in without a copy that could go stale. Two parts of the binaries are no crate, so `rust/og.sh` names their notices with `--extra NAME=FILE`: the Rust standard library (the toolchain's `COPYRIGHT-library.html`, turned into text) and musl, the C library of the Linux pair (musl's `COPYRIGHT`, which the builder image's zig ships). A licence file that cannot be read stops the build. A crate is credited for every target whose dependency graph reaches it, even where the linker then drops its unused code. The build stops when a crate's licence is outside the allow list (`ALLOWED` in `rust/crates/og-dev/src/licences.rs`: 0BSD, Apache-2.0, Apache-2.0 WITH LLVM-exception, BSD-3-Clause, ISC, MIT, Unicode-3.0, Unlicense, Zlib), so a new dependency under another licence needs a decision first. A crate that ships no licence file gets the MIT text with its authors when MIT is among its licences, and stops the build otherwise. - **From a checkout of `main` or a branch,** it builds them from `rust/` inside `OG_BUILDER`, a pinned [cargo-zigbuild](https://github.com/rust-cross/cargo-zigbuild) image (`versions.env`). Docker is the only thing it needs; no Rust on the host. The cargo cache lives in `rust/target/og-build/`. - **From a checkout of a release tag** (`vX.Y.Z`), it downloads the release's four binaries, checks them against the release's `SHA256SUMS` and, when `gh` is installed and logged in, against the build attestation (`gh attestation verify`). A release without binaries falls back to a build. @@ -115,8 +115,8 @@ To add a primitive: write it in `pycompat` with unit tests, add an operation to - **Guard, job `rust`** (every pull request, required through `ci-ok`): `cargo fmt --check`; `cargo clippy -D warnings`; a planted failing test (`--features planted-failure`) that must fail; `cargo test`; the differential test at 100,000 cases per operation; the four builds through `scripts/setup.sh --og` (both musl ones and the macOS pair); and the stale check on the real binaries, which must refuse them after an edit to `rust/`, in `rust/og.sh check` and in `nextflow run main.nf -stub`, before any task. Then the fake-docker case `reports-stale-step` runs with `bin/og` and must check its `summary.json` with `og dev schema`, and every test that carries the line `# og-leg: OG_REQUIRED` (the tests that run `og`, such as the `og sv` and `og pgx` filter tests, the step 05 tests and `tests/test_ancestry_fine_report.py`) runs with `bin/og` and `OG_REQUIRED=1`; in `unit-tests`, which has no `og`, they skip with a note. A new test that needs `og` adds that line; the job has no list to edit. Before the real run the step plants a marked test with a failing assert, which must be picked and fail. The planned-verb test must fail once one verb's arm is dropped from `og/src/main.rs`. `cargo test` includes `og dev contract --static` and `og dev report-only` on the checkout (`rust/crates/og-dev/tests/repo.rs`). It also runs `og dev helpers --static` and `--list` on the checkout (`rust/crates/og-dev/tests/helpers_repo.rs`); the `helper-binaries` job runs the same verb through `scripts/ci/check-container-helpers.sh`, with the og the `og` job builds. - **Guard, job `unit-tests`**: `tests/test_og_check.sh` runs the check on planted trees (no binary, a stale one, an edit that is uncommitted, untracked or committed, a file git ignores, a binary stamped `unknown`, a fresh and a stale macOS pair, a tree hash that cannot be computed, a release download with a bad checksum or attestation, without the macOS pair, with `gh` not logged in, with a network error, without binaries, or whose copy into `bin/.og` fails), and checks that `run-all.sh`, `validate-setup.sh` and `write_manifest.sh` act on it. - **Rust** (`rust.yml`, on a pull request that touches the Rust code, the shim or `versions.env`): times a cold and a cached build, runs `og` by bare name in the four images on amd64 and arm64 (QEMU), runs that check once more with a wrong-arch binary planted, which must fail, runs the fetch smoke with both binaries (arm64 under QEMU) and with pysam in `PYPGX_IMAGE`, which must all match `fetch.pysam.tsv`, and keeps the binaries as the artifact `og-binaries`. Every night it runs the differential test at 10,000,000 cases per operation with a new seed. -- **Release** (`release.yml`): a tag builds the four binaries and `THIRD-PARTY-LICENSES.txt`, writes `SHA256SUMS` over all five, checks them with `scripts/ci/check-release.sh dist` (each file there, listed once and matching its sum, and the htslib and htscodecs licences in the notices), attests them and attaches all six files to the release. A pull request that changes `rust/` or the workflow builds them as a dry run, with the same check. -- **`og dev licences notices`**: `cargo test -p og-dev --test licences` runs it through real `cargo metadata` on a planted workspace. The crates the binary links must all be credited, each text once; build, dev, procedural-macro, other-target and feature-off crates must not be; and a crate under SSPL-1.0, or one with no licence text that MIT cannot cover, must stop it. +- **Release** (`release.yml`): a tag builds the four binaries and `THIRD-PARTY-LICENSES.txt`, writes `SHA256SUMS` over all five, checks them with `scripts/ci/check-release.sh dist` (each file there, listed once and matching its sum, and the notices of the Rust standard library, musl, htslib and htscodecs in the licence file), attests them and attaches all six files to the release. A pull request that changes `rust/` or the workflow builds them as a dry run, with the same check. +- **`og dev licences notices`**: `cargo test -p og-dev --test licences` runs it through real `cargo metadata` on a planted workspace. The crates the binary links must all be credited, each text once; build, dev, procedural-macro, other-target and feature-off crates must not be; an extra part's HTML notices must come out as text; and a crate under SSPL-1.0, one with no licence text that MIT cannot cover, a licence file it cannot read and a missing extra file must each stop it. [Testing](testing.md) has the rest of CI. diff --git a/rust/crates/og-dev/src/licences.rs b/rust/crates/og-dev/src/licences.rs index 74458dbae..9bff4a307 100644 --- a/rust/crates/og-dev/src/licences.rs +++ b/rust/crates/og-dev/src/licences.rs @@ -2,7 +2,8 @@ //! //! ```text //! og dev licences notices [--manifest-path FILE] [--package NAME] -//! (--target TRIPLE | --target-no-default TRIPLE)... [--out FILE] +//! (--target TRIPLE | --target-no-default TRIPLE)... [--extra NAME=FILE]... +//! [--out FILE] //! ``` //! //! The og binaries link about seventy crates statically, and the Linux pair @@ -25,8 +26,16 @@ //! A crate that ships no text, under a licence that MIT satisfies, gets the //! MIT text with its authors; any other crate without a text is an error. //! -//! The file lists every crate with its version, licence and, when it is not -//! in every binary, the targets it is in; then each distinct licence text +//! The binary also holds code that is no cargo package: the Rust standard +//! library and, in a musl build, the musl C library. `--extra NAME=FILE` +//! adds such a part with its licence file (an `.html` file, as the Rust +//! toolchain ships its notices, is turned into plain text); `rust/og.sh` +//! passes both from the builder image. A licence file that cannot be read, +//! a crate's or an extra one, is an error. +//! +//! The file lists the extra parts, then every crate with its version, +//! licence and, when it is not in every binary, the targets it is in; then +//! each distinct licence text //! once, headed by the crates and files it covers. Errors name every crate //! at fault and exit 1; usage errors exit 2. @@ -51,7 +60,7 @@ pub const ALLOWED: &[&str] = &[ "Zlib", ]; -const USAGE: &str = "usage: og dev licences notices [--manifest-path FILE] [--package NAME] (--target TRIPLE | --target-no-default TRIPLE)... [--out FILE]"; +const USAGE: &str = "usage: og dev licences notices [--manifest-path FILE] [--package NAME] (--target TRIPLE | --target-no-default TRIPLE)... [--extra NAME=FILE]... [--out FILE]"; /// The MIT text, for a crate under MIT that ships no licence file. const MIT_TEXT: &str = @@ -95,6 +104,8 @@ struct Opts { manifest: Option, package: String, builds: Vec, + /// Parts that are no cargo package: (name, licence file). + extras: Vec<(String, String)>, out: Option, } @@ -103,6 +114,7 @@ fn parse(args: &[String]) -> Result { manifest: None, package: "og".into(), builds: Vec::new(), + extras: Vec::new(), out: None, }; let mut it = args.iter(); @@ -116,6 +128,15 @@ fn parse(args: &[String]) -> Result { "--manifest-path" => o.manifest = Some(value()?), "--package" => o.package = value()?, "--out" => o.out = Some(value()?), + "--extra" => { + let v = value()?; + match v.split_once('=') { + Some((name, file)) if !name.is_empty() && !file.is_empty() => { + o.extras.push((name.into(), file.into())); + } + _ => return Err(format!("--extra wants NAME=FILE, got {v:?}")), + } + } "--target" | "--target-no-default" => o.builds.push(Build { default_features: a == "--target", triple: value()?, @@ -185,9 +206,33 @@ fn notices(opts: &Opts) -> Result> { } } let mut problems = Vec::new(); - // Each distinct text once, in the order its first crate comes. + // Each distinct text once, in the order its first part comes. let mut texts: Vec<(String, Vec)> = Vec::new(); let mut index: HashMap = HashMap::new(); + let mut add = |who: String, text: String| match index.get(&text) { + Some(&i) => texts[i].1.push(who), + None => { + index.insert(text.clone(), texts.len()); + texts.push((text, vec![who])); + } + }; + for (name, file) in &opts.extras { + match std::fs::read(file) { + Ok(bytes) => { + let raw = String::from_utf8_lossy(&bytes); + let text = if file.ends_with(".html") { + html_text(&raw) + } else { + raw.into_owned() + }; + let base = Path::new(file) + .file_name() + .map_or(file.as_str(), |n| n.to_str().unwrap_or(file.as_str())); + add(format!("{name}: {base}"), text); + } + Err(e) => problems.push(format!("{name}: cannot read {file}: {e}")), + } + } for c in crates.values() { let id = format!("{} {}", c.name, c.version); let Some(expr) = &c.license else { @@ -208,7 +253,13 @@ fn notices(opts: &Opts) -> Result> { continue; } } - let mut found = licence_files(c); + let mut found = match licence_files(c) { + Ok(f) => f, + Err(e) => { + problems.push(format!("{id}: {e}")); + continue; + } + }; if found.is_empty() { if satisfied(expr, &|l| l == "MIT") == Ok(true) { let who = if c.authors.is_empty() { @@ -228,14 +279,7 @@ fn notices(opts: &Opts) -> Result> { } } for (label, text) in found { - let who = format!("{id}: {label}"); - match index.get(&text) { - Some(&i) => texts[i].1.push(who), - None => { - index.insert(text.clone(), texts.len()); - texts.push((text, vec![who])); - } - } + add(format!("{id}: {label}"), text); } } if !problems.is_empty() { @@ -254,16 +298,23 @@ fn render( s.push_str("Third-party licences of the og binaries\n"); s.push_str("=======================================\n\n"); s.push_str("OwnGenome is GPL-3.0-or-later (LICENSE in its source). The og binaries\n"); - s.push_str("also hold the code of the crates below, linked statically. A crate that\n"); - s.push_str("builds C carries that library's licence in its own folder: hts-sys gives\n"); - s.push_str("htslib/LICENSE and htslib/htscodecs/LICENSE.md, the licences of htslib and\n"); - s.push_str("htscodecs. Each licence text comes once below, after the crates and files\n"); - s.push_str("it covers. `og dev licences notices` writes this file from cargo metadata\n"); - s.push_str("and the files each crate ships.\n\n"); + s.push_str("also hold the code of the parts and crates below, linked statically. A\n"); + s.push_str("crate that builds C carries that library's licence in its own folder:\n"); + s.push_str("hts-sys gives htslib/LICENSE and htslib/htscodecs/LICENSE.md, the licences\n"); + s.push_str("of htslib and htscodecs. Each licence text comes once below, after the\n"); + s.push_str("parts, crates and files it covers. `og dev licences notices` writes this\n"); + s.push_str("file from cargo metadata and the files each crate ships.\n\n"); s.push_str(&format!( "Targets: {}\n\n", all.iter().cloned().collect::>().join(", ") )); + if !opts.extras.is_empty() { + s.push_str("Parts that are no crate\n-----------------------\n"); + for (name, _) in &opts.extras { + s.push_str(&format!("{name}\n")); + } + s.push('\n'); + } let title = format!("Crates ({})", crates.len()); s.push_str(&format!("{title}\n{}\n", "-".repeat(title.len()))); for c in crates.values() { @@ -441,8 +492,9 @@ fn is_licence_name(name: &str) -> bool { .any(|p| n.starts_with(p)) } -/// (label, text) of every licence file of a crate, label = path in the crate. -fn licence_files(c: &Crate) -> Vec<(String, String)> { +/// (label, text) of every licence file of a crate, label = path in the +/// crate. A file found but not readable is an error, never a silent gap. +fn licence_files(c: &Crate) -> Result, String> { let mut paths: BTreeSet = BTreeSet::new(); collect(&c.dir, c.builds_c, &mut paths); if let Some(f) = &c.license_file { @@ -453,14 +505,53 @@ fn licence_files(c: &Crate) -> Vec<(String, String)> { } paths .into_iter() - .filter_map(|p| { - let bytes = std::fs::read(&p).ok()?; + .map(|p| { let label = p.strip_prefix(&c.dir).unwrap_or(&p).display().to_string(); - Some((label, String::from_utf8_lossy(&bytes).into_owned())) + let bytes = std::fs::read(&p).map_err(|e| format!("cannot read {label}: {e}"))?; + Ok((label, String::from_utf8_lossy(&bytes).into_owned())) }) .collect() } +/// The text of an HTML notice file: tags dropped, the five escapes HTML +/// writes undone, runs of blank lines folded to one. +fn html_text(html: &str) -> String { + let mut bare = String::with_capacity(html.len()); + let mut in_tag = false; + for ch in html.chars() { + match ch { + '<' => in_tag = true, + '>' if in_tag => in_tag = false, + c if !in_tag => bare.push(c), + _ => {} + } + } + let bare = bare + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("'", "'") + .replace("&", "&"); + let mut out = String::with_capacity(bare.len()); + let mut blank = true; + for line in bare.lines().map(str::trim_end) { + if line.is_empty() { + if !blank { + out.push('\n'); + } + blank = true; + } else { + out.push_str(line); + out.push('\n'); + blank = false; + } + } + out.truncate(out.trim_end().len()); + out.push('\n'); + out +} + /// Licence files in `dir`, and below it when `deep` (a crate that builds C). fn collect(dir: &Path, deep: bool, out: &mut BTreeSet) { let Ok(entries) = std::fs::read_dir(dir) else { @@ -589,6 +680,13 @@ mod tests { } } + #[test] + fn html_notices_become_text() { + let html = + "

Notices

\n

A & B <x> "q"

\n\n\n
line\n
\n"; + assert_eq!(html_text(html), "Notices\nA & B \"q\"\n\nline\n"); + } + #[test] fn an_unreadable_expression_is_an_error() { for e in ["", "MIT OR", "(MIT", "MIT)", "AND MIT", "MIT WITH"] { diff --git a/rust/crates/og-dev/tests/licences/main.rs b/rust/crates/og-dev/tests/licences/main.rs index ae7fdcad4..7feaa0267 100644 --- a/rust/crates/og-dev/tests/licences/main.rs +++ b/rust/crates/og-dev/tests/licences/main.rs @@ -91,6 +91,7 @@ devonly = { path = "../../vendor/devonly" } [target.'cfg(windows)'.dependencies] bad = { path = "../../vendor/bad" } notext = { path = "../../vendor/notext" } +locked = { path = "../../vendor/locked" } "#, ); t.write("ws/app/src/lib.rs", ""); @@ -132,6 +133,21 @@ notext = { path = "../../vendor/notext" } vendor(&t, "devonly", "license = \"SSPL-1.0\"", &sspl); vendor(&t, "bad", "license = \"SSPL-1.0\"", &sspl); vendor(&t, "notext", "license = \"Apache-2.0\"", &[]); + // A licence file that cannot be read must stop the run, not drop out. + vendor( + &t, + "locked", + "license = \"MIT\"", + &[("LICENSE", "unreadable\n")], + ); + use std::os::unix::fs::PermissionsExt; + let locked = t.0.join("vendor/locked/LICENSE"); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o000)).unwrap(); + // A part that is no crate, its notices in HTML as the Rust toolchain ships them. + t.write( + "std/COPYRIGHT-library.html", + "

Std notices

\n

A & B

\n", + ); let cargo = std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()); let lock = std::process::Command::new(cargo) .args(["generate-lockfile", "--offline", "--manifest-path"]) @@ -173,7 +189,11 @@ const LINUX: &str = "x86_64-unknown-linux-gnu"; #[test] fn every_linked_crate_is_credited_and_nothing_else() { let t = workspace(); - let (code, text, err) = notices(&t, &["--target", LINUX]); + let extra = format!( + "Rust standard library={}", + t.path("std/COPYRIGHT-library.html") + ); + let (code, text, err) = notices(&t, &["--target", LINUX, "--extra", &extra]); assert_eq!(code, 0, "{err}"); let listed: Vec<&str> = text .lines() @@ -202,6 +222,15 @@ fn every_linked_crate_is_credited_and_nothing_else() { text.contains("For csys 0.1.0: vendored/clib/COPYING\n\nthe C library's own notice"), "{text}" ); + // A part that is no crate: listed, and its HTML notices as text. + assert!( + text.contains("Parts that are no crate\n-----------------------\nRust standard library\n"), + "{text}" + ); + assert!( + text.contains("For Rust standard library: COPYRIGHT-library.html\n\nStd notices\nA & B\n"), + "{text}" + ); // No file, MIT allowed: the MIT text with the crate's authors. assert!(text.contains("For dual 0.1.0: MIT, no licence file in the crate\n\nCopyright (c) Dual Author \n\nPermission is hereby granted"), "{text}"); for absent in [ @@ -212,6 +241,7 @@ fn every_linked_crate_is_credited_and_nothing_else() { "devonly", "bad 0.1.0", "notext", + "locked", "server side text", ] { assert!( @@ -248,9 +278,17 @@ fn a_build_without_default_features_leaves_the_optional_crate_out() { #[test] fn a_crate_outside_the_allow_list_or_without_a_text_fails() { let t = workspace(); + let missing = format!("musl libc={}", t.path("no/COPYRIGHT")); let (code, text, err) = notices( &t, - &["--target", LINUX, "--target", "x86_64-pc-windows-gnu"], + &[ + "--target", + LINUX, + "--target", + "x86_64-pc-windows-gnu", + "--extra", + &missing, + ], ); assert_eq!(code, 1, "{err}"); assert!(text.is_empty(), "no file on an error, got:\n{text}"); @@ -264,7 +302,13 @@ fn a_crate_outside_the_allow_list_or_without_a_text_fails() { ), "{err}" ); - assert_eq!(err.lines().count(), 2, "{err}"); + assert!(err.contains("musl libc: cannot read "), "{err}"); + // Root reads a mode-000 file anyway; everyone else must be stopped by it. + let root_reads = std::fs::read(t.0.join("vendor/locked/LICENSE")).is_ok(); + if !root_reads { + assert!(err.contains("locked 0.1.0: cannot read LICENSE: "), "{err}"); + } + assert_eq!(err.lines().count(), if root_reads { 3 } else { 4 }, "{err}"); } #[test] diff --git a/rust/og.sh b/rust/og.sh index 8cfbd6031..c190044e1 100755 --- a/rust/og.sh +++ b/rust/og.sh @@ -192,15 +192,21 @@ build() { # The notices of everything the four link, written by the new og itself in # the builder, where cargo and the crate sources are: the Linux binary of # the builder's own arch, with the targets and feature sets built above. + # Two parts are no crate, so the builder names their notices: the Rust + # standard library (the toolchain's COPYRIGHT-library.html) and musl, the + # C library of the Linux pair (musl's COPYRIGHT, which zig ships). local lic=() notices="${cache}/THIRD-PARTY-LICENSES.txt" for a in $ARCHES; do lic+=(--target "$(triple "$a")"); done for a in $DARWIN_ARCHES; do lic+=(--target-no-default "$(darwin_triple "$a")"); done rm -f "$notices" - # shellcheck disable=SC2016 # $(uname -m) and $@ expand in the builder's sh + # shellcheck disable=SC2016 # the sh -c script expands in the builder "$ENGINE" run --rm --user "$(id -u):$(id -g)" \ -e HOME=/cache/home -e CARGO_HOME=/cache/cargo -e CARGO_TARGET_DIR=/cache/target \ -v "${ROOT}/rust:/src:ro" -v "${cache}:/cache" -w /src "$OG_BUILDER" \ - sh -c 'exec "/cache/target/$(uname -m)-unknown-linux-musl/release/og" "$@"' og \ + sh -c 'sr=$(rustc --print sysroot) || exit + set -- "$@" --extra "Rust standard library=${sr}/share/doc/rust/COPYRIGHT-library.html" + for m in /usr/local/zig*/lib/libc/musl/COPYRIGHT; do set -- "$@" --extra "musl libc=${m}"; done + exec "/cache/target/$(uname -m)-unknown-linux-musl/release/og" "$@"' og \ dev licences notices "${lic[@]}" --out /cache/THIRD-PARTY-LICENSES.txt || return [ -s "$notices" ] || { echo "ERROR: og dev licences notices wrote no ${notices}" >&2; return 1; } put "${outs[@]}" || return 1 diff --git a/scripts/ci/check-release.sh b/scripts/ci/check-release.sh index c88ca0ecc..557607ed8 100755 --- a/scripts/ci/check-release.sh +++ b/scripts/ci/check-release.sh @@ -31,9 +31,10 @@ # DIR is what a release uploads: the four og binaries, # THIRD-PARTY-LICENSES.txt (rust/og.sh build) and SHA256SUMS. Each file # must be there, not empty, listed once in SHA256SUMS and match its -# line. The licence file must list its crates and carry the htslib and -# htscodecs licences (hts-sys's htslib/LICENSE and -# htslib/htscodecs/LICENSE.md). Prints each file with its checksum. +# line. The licence file must list its crates and carry the notices of +# the Rust standard library, musl, htslib and htscodecs (hts-sys's +# htslib/LICENSE and htslib/htscodecs/LICENSE.md). Prints each file +# with its checksum. # # --repo DIR runs against another git work tree (the tests plant one); the # default is the checkout this script is in. @@ -180,9 +181,11 @@ case "$MODE" in done lic="${DIST}/THIRD-PARTY-LICENSES.txt" grep -qE '^Crates \([1-9][0-9]*\)$' "$lic" || die "THIRD-PARTY-LICENSES.txt lists no crates" + grep -qE '^For Rust standard library: ' "$lic" || die "THIRD-PARTY-LICENSES.txt does not carry the Rust standard library's notices" + grep -qE '^For musl libc: COPYRIGHT$' "$lic" || die "THIRD-PARTY-LICENSES.txt does not carry musl's licence" grep -qE '^For hts-sys [^:]+: htslib/LICENSE$' "$lic" || die "THIRD-PARTY-LICENSES.txt does not carry htslib's licence (hts-sys htslib/LICENSE)" grep -qE '^For hts-sys [^:]+: htslib/htscodecs/LICENSE\.md$' "$lic" || die "THIRD-PARTY-LICENSES.txt does not carry htscodecs' licence (hts-sys htslib/htscodecs/LICENSE.md)" - echo "OK: the og binaries and THIRD-PARTY-LICENSES.txt match SHA256SUMS, and the licences of htslib and htscodecs are in it" + echo "OK: the og binaries and THIRD-PARTY-LICENSES.txt match SHA256SUMS, and the notices of the Rust standard library, musl, htslib and htscodecs are in it" ;; *) die "usage: $0 [--repo DIR] {tag TAG [--notes-out FILE] | checks RUNS.TSV [--ignore NAME]... | declared | dist DIR}" From b9ee18c2ed12c4a45895521437ecc8151a1b1af4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:39:38 +0100 Subject: [PATCH 200/255] test(e2e): build genotype and repeat e2e inputs without inline Python The cases now make their inputs with og dev fixture kinds, read values with jq or awk, and compare with awk or og. Each case keeps its name, its PASS lines and what it asserts. The paths rows name the fixture source and bin/og so a change to either reruns the case. Bead: D1 --- scripts/ci/e2e-case-paths.tsv | 26 ++--- tests/e2e/bash-step-09-str.sh | 24 ++-- tests/e2e/expansions-and-rfc1-1-motif.sh | 53 ++++----- tests/e2e/fixitfelix-chr21-1.sh | 63 ++++++----- tests/e2e/genotype-rule-rows-1-str-extra.sh | 105 +++--------------- tests/e2e/genotype-rules-1-phase.sh | 101 +---------------- tests/e2e/genotype-rules-2-g2.sh | 57 ++++------ .../hba-short-read-accuracy-1-nf-report.sh | 10 +- tests/e2e/mito-extended-1-deletions.sh | 17 ++- tests/e2e/paralog-catalog-1-gba.sh | 13 +-- .../e2e/paralog-checks-and-doc-fixes-1-hba.sh | 39 +------ tests/e2e/paralog-rows-1.sh | 67 ++++------- tests/e2e/paralogs-paraphase-1.sh | 105 +++++------------- ...alogs-report-section-1-nf-report-tables.sh | 11 +- tests/e2e/pgx-4-parascopy.sh | 13 +-- tests/e2e/str-catalog-1-strchive.sh | 98 ++++++---------- tests/e2e/str-extra-repeats-1-readings.sh | 98 ++-------------- tests/e2e/traits-panel-1.sh | 37 ++---- 18 files changed, 237 insertions(+), 700 deletions(-) diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 2a5417769..260cc4741 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -87,18 +87,18 @@ couple-common-sv-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrie cpic-phenoconversion-1-medications assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/og modules/local/parascopy/* scripts/35-paralogs.sh exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh +expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh rust/crates/og-dev/src/fixture.rs families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh bin/og followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv bin/og scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh followups-cn-1-couple-report assets/carrier_genes.tsv assets/par_grch38.bed assets/stub/pedigree.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/couple/* modules/local/html_report/* scripts/39-couple.sh scripts/ci/fake-docker/lib.sh followups-nf-1-nf-wiring assets/stub/* assets/test_samplesheet.csv assets/steps.tsv bin/collect_summary.py bin/render_report.py bin/results_index.py bin/run_state.py conf/test.config conf/test_all.config conf/containers.config modules/local/html_report/* modules/local/couple/* modules/local/parascopy/* modules/local/deepvariant/* modules/local/cpsr/* modules/local/clinvar_screen/* modules/local/locityper/* workflows/* tests/fixtures/consumer_export/sites.tsv.gz bin/og genome-lookup-1 bin/og scripts/ci/fake-docker/lib.sh scripts/owngenome-lookup.sh -genotype-rule-rows-1-str-extra assets/str_extra_loci.json assets/stub/db/catalog.json scripts/09-expansion-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -genotype-rules-1-phase assets/genotype_rules.tsv assets/traits.tsv modules/local/traits/* modules/local/vcf_precheck/* scripts/42-traits.sh scripts/ci/check-rule-alleles.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -genotype-rules-2-g2 assets/genotype_rules.tsv bin/collect_summary.py +genotype-rule-rows-1-str-extra assets/str_extra_loci.json assets/stub/db/catalog.json scripts/09-expansion-hunter.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh bin/og rust/crates/og-dev/src/fixture.rs +genotype-rules-1-phase assets/genotype_rules.tsv assets/traits.tsv modules/local/traits/* modules/local/vcf_precheck/* scripts/42-traits.sh scripts/ci/check-rule-alleles.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh bin/og rust/crates/og-dev/src/fixture.rs +genotype-rules-2-g2 assets/genotype_rules.tsv bin/collect_summary.py bin/og genotype-rules-2-lpa-null bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py modules/local/lpa/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/generate-report.sh genotype-rules-2-smn1-marker bin/collect_summary.py bin/og bin/render_report.py modules/local/parascopy/* scripts/24-html-report.sh scripts/35-paralogs.sh scripts/generate-report.sh haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/viewer/index.html bin/collect_summary.py bin/og bin/yleaf_run.py modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/vcf_precheck/* modules/local/y_haplogroup/* scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh @@ -109,7 +109,7 @@ ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh -mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh scripts/generate-report.sh rust/crates/og-dev/src/fixture.rs mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/og scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh @@ -121,14 +121,14 @@ nextflow-report-parity-1-sections assets/acmg_sf.tsv assets/callability_genes.ts nf-wiring-1-report-tables assets/acmg_sf.tsv assets/test_samplesheet.csv assets/stub/* bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py conf/test_all.config conf/test.config modules/local/align_minimap2/* modules/local/clinical_filter/* modules/local/clinvar_screen/* modules/local/cpsr/* modules/local/html_report/* modules/local/slivar/* modules/local/vep/* scripts/24-html-report.sh tests/fixtures/consumer_export/sites.tsv.gz nf-wiring-2-sv-str-tools bin/cnvpytor_to_vcf.py bin/family_report.py bin/og modules/local/cnvpytor/* modules/local/duphold/* modules/local/expansion_hunter/* modules/local/indexcov/* modules/local/manta/* modules/local/vcf_precheck/* output-contract-1-hg002 assets/* bin/* modules/* scripts/* -paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -paralogs-paraphase-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/html_report/* modules/local/paraphase/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/24-html-report.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs +paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs +paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs +paralogs-paraphase-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/paraphase_report.py bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/html_report/* modules/local/paraphase/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/24-html-report.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz pgx-1-depth-check bin/og scripts/21-cyrius.sh scripts/32-pypgx.sh scripts/ci/alt-depth-ab.sh scripts/ci/fake-docker/lib.sh pgx-2-outside-calls assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/og scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +pgx-4-parascopy assets/paralog_loci.tsv bin/cn_loci.tsv bin/og scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs pgx-5-kir assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh pgx-6-gvcf-column assets/live/index.html assets/par_grch38.bed assets/pgs_scores.tsv assets/viewer/index.html bin/collect_summary.py bin/og modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/indexcov/* modules/local/pharmcat/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/16-indexcov.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh pgx-inhibitor-rows-1-contraindication assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/cn_loci.tsv bin/collect_summary.py bin/og modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/pharmcat/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh @@ -158,8 +158,8 @@ run-all-sample-qc-and-stale-index-1-index-times assets/par_grch38.bed modules/lo setup-gencode-exons-and-nextflow-clip-check-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/clinvar_screen/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh short-sample-id-stub assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/cpsr_sample_id bin/og bin/render_report.py bin/results_index.py modules/local/cpic_lookup/* modules/local/cpsr/* modules/local/hla_typing/* modules/local/html_report/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/17-cpsr.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh single-source-lib-ownership scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh -str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/expansion_hunter/* modules/local/stranger/* modules/local/vcf_precheck/* scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh +str-catalog-1-strchive assets/str_extra_loci.json assets/stub/db/catalog.json assets/stub/sample.bam assets/stub/sample.bam.bai bin/og modules/local/expansion_hunter/* modules/local/stranger/* modules/local/vcf_precheck/* scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs +str-extra-repeats-1-readings assets/str_extra_loci.json bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/expansion_hunter/* modules/local/html_report/* scripts/09-expansion-hunter.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs sv-mito-telomere-steps-1-sv-merge scripts/22-survivor-merge.sh scripts/ci/fake-docker/lib.sh tests/fixtures/sv/cnvpytor.vcf.in tests/fixtures/sv/delly.vcf.in tests/fixtures/sv/manta.vcf.in sv-mito-telomere-steps-2-mito assets/callability_genes.tsv bin/collect_summary.py scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh sv-mito-telomere-steps-3-y-haplogroup assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh diff --git a/tests/e2e/bash-step-09-str.sh b/tests/e2e/bash-step-09-str.sh index ba7a456e5..fbd9f2c5d 100755 --- a/tests/e2e/bash-step-09-str.sh +++ b/tests/e2e/bash-step-09-str.sh @@ -11,25 +11,15 @@ CAT="${GENOME_DIR}/reference/eh_catalog_fixture.json" docker run --rm "$EXPANSIONHUNTER_IMAGE" cat /usr/local/share/ExpansionHunter/variant_catalog/grch38/variant_catalog.json \ > "${CASE_TMP}/catalog.json" -python3 - "${CASE_TMP}/catalog.json" "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta.fai" "$CAT" <<'PY2' -import json, sys -loci = json.load(open(sys.argv[1])) -contigs = {line.split("\t")[0] for line in open(sys.argv[2])} -def regions(locus): - out = [] - for key in ("ReferenceRegion", "OfftargetRegions"): - r = locus.get(key, []) - out += r if isinstance(r, list) else [r] - return out # The bundled catalog names contigs without "chr" (X, 14); ExpansionHunter # matches them to chrX, chr14. It stops on the first one the reference lacks. -def known(c): - return c in contigs or "chr" + c in contigs -kept = [l for l in loci if all(known(r.split(":")[0]) for r in regions(l))] -print(f"catalog loci: {len(loci)}, on the fixture's contigs: {len(kept)}") -json.dump(kept, open(sys.argv[3], "w"), indent=1) -PY2 -N_CAT=$(python3 -c 'import json, sys; print(len(json.load(open(sys.argv[1]))))' "$CAT" 2>/dev/null || echo 0) +# shellcheck disable=SC2016 # jq program +jq --rawfile fai "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta.fai" ' + ($fai | split("\n") | map(split("\t")[0] | select(. != null and . != "")) | map({(.): true}) | add) as $c + | [.[] | select([(.ReferenceRegion, .OfftargetRegions) | select(. != null) | if type == "array" then .[] else . end + | split(":")[0] | $c[.] or $c["chr" + .]] | all)]' "${CASE_TMP}/catalog.json" > "$CAT" +N_CAT=$(jq length "$CAT" 2>/dev/null || echo 0) +echo "catalog loci: $(jq length "${CASE_TMP}/catalog.json"), on the fixture's contigs: ${N_CAT}" check_ge "catalog loci on the fixture's contigs" "$N_CAT" 5 EH_CATALOG="$CAT" run_step 09-expansion-hunter.sh "$SAMPLE" male diff --git a/tests/e2e/expansions-and-rfc1-1-motif.sh b/tests/e2e/expansions-and-rfc1-1-motif.sh index 1db4791fc..ae5c24ea2 100755 --- a/tests/e2e/expansions-and-rfc1-1-motif.sh +++ b/tests/e2e/expansions-and-rfc1-1-motif.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -# Step 09b's RFC1 motif count (bin/rfc1_motif.py) with the real Stranger and +# Step 09b's RFC1 motif count (og str motif) with the real Stranger and # PYPGX_IMAGE. The fixture has no reads at RFC1, so the case makes three # samples of its own: an ExpansionHunter VCF with one RFC1 record (the header # of tests/smoke/stranger_input.vcf.in) and a BAM of made-up reads at the -# locus, the reads of tests/test_rfc1_motif.py. +# locus (og dev fixture rfc1). # path RFC1 at 100 and 150 units, in-repeat; AAGGG reads (soft-clipped, unmapped # and placed at the mate, mates on another contig, reverse strand) # and three reference reads: Stranger flags full_mutation, the @@ -17,39 +17,26 @@ # with the mate, need a real genome. . "$(dirname "$0")/lib.sh" -make_sample() { # make_sample - local s=$1 kind=$2 cn=$3 so=$4 ci=$5 +make_sample() { # make_sample + local s=$1 kind=$2 cn=$3 so=$4 ci=$5 in="${REPO}/tests/smoke/stranger_input.vcf.in" a gt=1/1 alts="" rm -rf "${GENOME_DIR:?}/${s}" mkdir -p "${GENOME_DIR}/${s}/expansion_hunter" "${GENOME_DIR}/${s}/aligned" - python3 - "$REPO" "$kind" "$cn" "$so" "$ci" "${GENOME_DIR}/${s}/expansion_hunter/${s}_eh.vcf" \ - "${GENOME_DIR}/${s}/aligned/reads.sam" <<'PY' -import os, sys -repo, kind, cn, so, ci, vcf, sam = sys.argv[1:8] -sys.path.insert(0, os.path.join(repo, "tests")) -import test_rfc1_motif as t -head = [l for l in open(os.path.join(repo, "tests", "smoke", "stranger_input.vcf.in")) if l.startswith("#")] -alts = sorted({int(x) for x in cn.split("/")}) -head[-1:-1] = [f'##ALT=\n' for a in alts] -gt = "1/2" if len(alts) == 2 else "1/1" -sp, fl, ir = ("/".join([x, x]) for x in ("0", "2", "20")) # spanning, flanking and in-repeat reads -with open(vcf, "w") as fh: - fh.writelines(head) - fh.write(f"{t.CHROM}\t{t.START + 1}\tRFC1\tA\t" + ",".join(f"" for a in alts) - + f"\t.\tPASS\tSVTYPE=STR;END={t.END};REF=11;RL=55;RU=AARRG;VARID=RFC1;REPID=RFC1" - "\tGT:SO:REPCN:REPCI:ADSP:ADFL:ADIR:LC\t" + ":".join([gt, so, cn, ci, sp, fl, ir]) + ":30.0\n") -at, elsewhere = t.case(kind) -with open(sam, "w") as fh: - fh.write("@HD\tVN:1.6\tSO:unsorted\n") - fh.write(f"@SQ\tSN:{t.CHROM}\tLN:190214555\n@SQ\tSN:{t.OTHER}\tLN:248956422\n") - for r in at + list(elsewhere.values()): - flag = (1 | (64 if r.is_read1 else 128) | (4 if r.is_unmapped else 0) | (16 if r.is_reverse else 0) - | (256 if r.is_secondary else 0) | (512 if r.is_qcfail else 0) | (1024 if r.is_duplicate else 0)) - mate = "=" if r.mate_contig == r.contig else r.mate_contig - fh.write("\t".join(map(str, [r.query_name, flag, r.contig, r.reference_start + 1, - 0 if r.is_unmapped else 60, r.cigar_text or "*", mate, - r.next_reference_start + 1, 0, r.query_sequence, - "I" * len(r.query_sequence)])) + "\n") -PY + # The header of the smoke VCF with an ALT line per allele size, and one RFC1 + # record: spanning 0/0, flanking 2/2 and in-repeat 20/20 reads. + local sizes + sizes=$(tr '/' '\n' <<< "$cn" | sort -nu) + [ "$(wc -l <<< "$sizes")" -eq 2 ] && gt=1/2 + { + grep '^#' "$in" | sed '$d' + for a in $sizes; do + printf '##ALT=\n' "$a" "$a" + alts="${alts:+${alts},}" + done + grep '^#' "$in" | tail -n 1 + printf 'chr4\t39348425\tRFC1\tA\t%s\t.\tPASS\tSVTYPE=STR;END=39348479;REF=11;RL=55;RU=AARRG;VARID=RFC1;REPID=RFC1' "$alts" + printf '\tGT:SO:REPCN:REPCI:ADSP:ADFL:ADIR:LC\t%s:%s:%s:%s:0/0:2/2:20/20:30.0\n' "$gt" "$so" "$cn" "$ci" + } > "${GENOME_DIR}/${s}/expansion_hunter/${s}_eh.vcf" + "${REPO}/bin/og" dev fixture rfc1 "$kind" "${GENOME_DIR}/${s}/aligned/reads.sam" docker run --rm -u "$(id -u):$(id -g)" -v "${GENOME_DIR}:/genome" -w "/genome/${s}/aligned" "$SAMTOOLS_IMAGE" \ sh -c "samtools sort -o ${s}_sorted.bam reads.sam && samtools index ${s}_sorted.bam" rm -f "${GENOME_DIR}/${s}/aligned/reads.sam" diff --git a/tests/e2e/fixitfelix-chr21-1.sh b/tests/e2e/fixitfelix-chr21-1.sh index 15751896f..00bd6c278 100644 --- a/tests/e2e/fixitfelix-chr21-1.sh +++ b/tests/e2e/fixitfelix-chr21-1.sh @@ -132,32 +132,32 @@ norm reference/ffx/cmrg.vcf.gz "${CASE_TMP}/cmrg.txt" for side in stock masked; do norm "ffx-${side}/vcf/ffx-${side}.vcf.gz" "${CASE_TMP}/${side}.txt" done -# recall SIDE: "gene matched benchmark recall" lines +# recall SIDE: "gene matched benchmark recall" lines. A benchmark variant (a +# non-reference genotype of the truth) counts for a gene when it lies in the +# gene's span and in a CMRG region, and is matched when the side has a PASS +# (or ".") call at the same position and alleles with as many alternate alleles. recall() { - python3 - "${CASE_TMP}/cmrg.txt" "${RD}/cmrg.bed" "${CASE_TMP}/$1.txt" "$GENES" <<'PY' -import sys -truth, bed, calls, genes = sys.argv[1:5] -regions = [(int(c[1]), int(c[2])) for c in (l.split("\t") for l in open(bed)) if c[0] == "chr21"] -def alts(gt): - return sum(1 for a in gt.replace("|", "/").split("/") if a not in ("0", ".")) -def rows(path, pass_only): - out = {} - for line in open(path): - c = line.rstrip("\n").split("\t") - if pass_only and c[6] not in ("PASS", "."): - continue - n = alts(c[9].split(":")[0]) - if n: - out[(int(c[1]), c[3], c[4])] = n - return out -t, k = rows(truth, False), rows(calls, True) -for g in genes.split(): - name, s, e = g.split(":") - s, e = int(s), int(e) - keys = [x for x in t if s < x[0] <= e and any(a < x[0] <= b for a, b in regions)] - hit = sum(1 for x in keys if k.get(x) == t[x]) - print(name, hit, len(keys), f"{hit / len(keys):.3f}" if keys else "nan") -PY + # shellcheck disable=SC2016 # awk program + awk -F'\t' -v genes="$GENES" ' + function alts(gt, n, a, i, k) { gsub(/\|/, "/", gt); n = split(gt, a, "/"); k = 0 + for (i = 1; i <= n; i++) if (a[i] != "0" && a[i] != ".") k++; return k } + FILENAME == ARGV[1] { if ($1 == "chr21") { nr++; rs[nr] = $2 + 0; re[nr] = $3 + 0 }; next } + FILENAME == ARGV[2] { split($10, g, ":"); n = alts(g[1]); if (n) { t[$2 SUBSEP $4 SUBSEP $5] = n; tp[$2 SUBSEP $4 SUBSEP $5] = $2 + 0 }; next } + $7 == "PASS" || $7 == "." { split($10, g, ":"); n = alts(g[1]); if (n) k[$2 SUBSEP $4 SUBSEP $5] = n } + END { + ng = split(genes, gl, " ") + for (j = 1; j <= ng; j++) { + split(gl[j], f, ":"); s = f[2] + 0; e = f[3] + 0; keys = 0; hit = 0 + for (x in t) { + p = tp[x] + if (!(s < p && p <= e)) continue + inr = 0; for (i = 1; i <= nr; i++) if (rs[i] < p && p <= re[i]) { inr = 1; break } + if (!inr) continue + keys++; if ((x in k) && k[x] == t[x]) hit++ + } + if (keys) printf "%s %d %d %.3f\n", f[1], hit, keys, hit / keys; else printf "%s %d %d nan\n", f[1], hit, keys + } + }' "${RD}/cmrg.bed" "${CASE_TMP}/cmrg.txt" "${CASE_TMP}/$1.txt" } R_MASKED=$(recall masked 2>&1); echo "masked chr21: ${R_MASKED}" R_STOCK=$(recall stock 2>&1); echo "stock chr21: ${R_STOCK}" @@ -189,13 +189,12 @@ reading() { cp "$CAT" "${t}/assets/callability_genes.tsv" ln -s "$1" "${t}/S/mosdepth/S_callability.bed.gz" "${REPO}/bin/og" summary collect --sample S --sample-dir "${t}/S" --assets "${t}/assets" --out "${t}/summary.json" > /dev/null - python3 - "$CAT" "${t}/summary.json" <<'PY' -import csv, json, sys -rows = list(csv.DictReader(open(sys.argv[1]), delimiter="\t")) -d = json.load(open(sys.argv[2]))["sections"]["callability"]["data"] -bad = {g["gene"] for g in d["not_read"]} -print(" ".join(f"{r['gene']}={'not_read' if r['gene'] in bad else 'read'}" for r in rows)) -PY + # Each gene of the table, in its order, read or not_read. + # shellcheck disable=SC2016 # jq program + jq -r --arg genes "$(awk -F'\t' 'NR > 1 {print $1}' "$CAT")" \ + '[.sections.callability.data.not_read[].gene] as $bad + | [$genes | split("\n")[] | select(. != "") | . as $g | "\($g)=\(if any($bad[]; . == $g) then "not_read" else "read" end)"] + | join(" ")' "${t}/summary.json" } for side in stock masked; do S="ffx-${side}" diff --git a/tests/e2e/genotype-rule-rows-1-str-extra.sh b/tests/e2e/genotype-rule-rows-1-str-extra.sh index e96ece9e0..2591d79da 100644 --- a/tests/e2e/genotype-rule-rows-1-str-extra.sh +++ b/tests/e2e/genotype-rule-rows-1-str-extra.sh @@ -25,7 +25,7 @@ SETUP="${CASE_TMP}/setup" rm -rf "$SETUP" mkdir -p "$SETUP" CAT="${SETUP}/reference/strchive-${STRCHIVE_VERSION}/catalog.json" -count() { python3 -c 'import json, sys; print(len(json.load(open(sys.argv[1]))))' "$1" 2>/dev/null || echo 0; } +count() { jq length "$1" 2>/dev/null || echo 0; } # --- 1. setup, with and without the opt-in ------------------------------------------------ STR_EXTRA_LOCI=yes "${REPO}/scripts/setup.sh" --strchive-data "$SETUP" > "${CASE_TMP}/setup.log" 2>&1 @@ -34,13 +34,8 @@ cat "${CASE_TMP}/setup.log" check_eq "the catalog has STRchive's 82 entries and the 2 extra loci" "$(count "$CAT")" 84 check "catalog.stamp names the extra catalog's sha256" has "extra=$(sha256sum "$EXTRA" | cut -d' ' -f1)" \ "$(cat "${CAT%/*}/catalog.stamp" 2>/dev/null)" -python3 - "$CAT" "${G}/reference-str-extra.json" <<'PY' -import json, sys -json.dump([e for e in json.load(open(sys.argv[1])) if e.get("Source") != "STRchive"], open(sys.argv[2], "w"), indent=1) -PY -check_eq "the extra loci of the merged catalog" \ - "$(python3 -c 'import json, sys; print(",".join(e["LocusId"] for e in json.load(open(sys.argv[1]))))' "${G}/reference-str-extra.json")" \ - "CFTR,UGT1A1" +jq 'map(select(.Source != "STRchive"))' "$CAT" > "${G}/reference-str-extra.json" +check_eq "the extra loci of the merged catalog" "$(jq -r 'map(.LocusId) | join(",")' "${G}/reference-str-extra.json")" "CFTR,UGT1A1" "${REPO}/scripts/setup.sh" --strchive-data "$SETUP" > "${CASE_TMP}/setup-off.log" 2>&1 check_eq "control: setup.sh again without the opt-in exits 0" "$?" 0 check_eq "control: without the opt-in the catalog is back to 82 entries" "$(count "$CAT")" 82 @@ -51,80 +46,7 @@ RD="${G}/reference-str-extra" rm -rf "$RD" mkdir -p "$RD" for s in EXA EXB; do rm -rf "${G:?}/${s}"; mkdir -p "${G}/${s}/aligned"; done -python3 - "$RD" "$G" <<'PY' -import random, sys -rd, g = sys.argv[1], sys.argv[2] -rng = random.Random(11) -W = 1500 # random bases each side of a tract -# chrom: (contig length, 0-based tract start, left bases, reference tract units, right bases) -LOCI = {"chr7": (117551000, 117548606, "TTTTGA", [("TG", 11), ("T", 7)], "AACAGG"), - "chr2": (233763000, 233760234, "TTGCCA", [("TA", 7)], "AGTAGG")} -win = {} -for c, (length, start, left, units, right) in LOCI.items(): - lf = "".join(rng.choice("ACGT") for _ in range(W - len(left))) + left - rf = right + "".join(rng.choice("ACGT") for _ in range(W - len(right))) - win[c] = (start - W, lf, rf) - tract = "".join(m * n for m, n in units) - with open(f"{rd}/ref.fa", "a") as f: - f.write(f">{c}\n") - seq_start = start - W - body = "N" * seq_start + lf + tract + rf - body += "N" * (length - len(body)) - for i in range(0, len(body), 60): - f.write(body[i:i + 60] + "\n") - -def reads(c, haps, tag): - """Read pairs from each haplotype's tract; the CIGAR puts the tract's length change at its end.""" - w0, lf, rf = win[c] - ref_tract = "".join(m * n for m, n in LOCI[c][3]) - out = [] - for h, units in enumerate(haps): - tract = "".join(m * n for m, n in units) - seq = lf + tract + rf - t0, t1 = len(lf), len(lf) + len(tract) - d = len(tract) - len(ref_tract) - - def place(a, n=150): - """(1-based POS, CIGAR) of hap bases [a, a+n), or None when the read starts in the tract.""" - if a + n <= t0: - return w0 + a + 1, f"{n}M" - if a >= t1: - return w0 + a - d + 1, f"{n}M" - if a >= t0: - return None - if a + n <= t1: - return None - pre = t0 - a + min(len(tract), len(ref_tract)) - post = n - (t0 - a) - len(tract) - mid = f"{d}I" if d > 0 else (f"{-d}D" if d < 0 else "") - cig = f"{pre}M{mid}{post}M" if mid else f"{n}M" - return w0 + a + 1, cig - k = 0 - for a in range(0, len(seq) - 400, 4): - f_len = 300 + (a % 7) * 10 - b = a + f_len - 150 - p1, p2 = place(a), place(b) - if not p1 or not p2: - continue - k += 1 - name = f"{tag}{c}h{h}r{k}" - r1, r2 = seq[a:a + 150], seq[b:b + 150] - tlen = p2[0] + 150 - p1[0] - out.append(f"{name}\t99\t{c}\t{p1[0]}\t60\t{p1[1]}\t=\t{p2[0]}\t{tlen}\t{r1}\t{'I' * 150}") - out.append(f"{name}\t147\t{c}\t{p2[0]}\t60\t{p2[1]}\t=\t{p1[0]}\t{-tlen}\t{r2}\t{'I' * 150}") - return out - -SAMPLES = {"EXA": ([[("TG", 12), ("T", 5)], [("TG", 11), ("T", 7)]], [[("TA", 7)], [("TA", 8)]]), - "EXB": ([[("TG", 11), ("T", 7)]] * 2, [[("TA", 7)]] * 2)} -for s, (cftr, ugt) in SAMPLES.items(): - with open(f"{g}/{s}/aligned/{s}.sam", "w") as f: - f.write("@HD\tVN:1.6\tSO:unsorted\n") - for c, (length, *_r) in LOCI.items(): - f.write(f"@SQ\tSN:{c}\tLN:{length}\n") - f.write(f"@RG\tID:{s}\tSM:{s}\n") - for line in reads("chr7", cftr, s) + reads("chr2", ugt, s): - f.write(line + f"\tRG:Z:{s}\n") -PY +"${REPO}/bin/og" dev fixture str-extra "${RD}/ref.fa" EXA "${G}/EXA/aligned/EXA.sam" EXB "${G}/EXB/aligned/EXB.sam" check "the made-up reference was written" test -s "${RD}/ref.fa" sam faidx reference-str-extra/ref.fa check_eq "ref.fa.fai: chr7 and chr2" "$(cut -f1 "${RD}/ref.fa.fai" 2>/dev/null | tr '\n' ' ')" "chr7 chr2 " @@ -138,16 +60,15 @@ done # --- 3. step 09 on the extra loci ----------------------------------------------------------- repcn() { # repcn : REPCN of that record - python3 - "${G}/$1/expansion_hunter/$1_eh.vcf" "$2" <<'PY' 2>/dev/null -import sys -for line in open(sys.argv[1]): - if line.startswith("#"): - continue - c = line.rstrip("\n").split("\t") - info = dict(kv.split("=", 1) for kv in c[7].split(";") if "=" in kv) - if info.get("VARID") == sys.argv[2]: - print(dict(zip(c[8].split(":"), c[9].split(":"))).get("REPCN", ".")) -PY + # shellcheck disable=SC2016 # awk program + awk -F'\t' -v id="$2" '!/^#/ { + n = split($8, kv, ";"); v = "" + for (i = 1; i <= n; i++) if (kv[i] ~ /^VARID=/) v = substr(kv[i], 7) + if (v != id) next + nk = split($9, k, ":"); split($10, x, ":"); r = "." + for (i = 1; i <= nk; i++) if (k[i] == "REPCN") r = x[i] + print r + }' "${G}/$1/expansion_hunter/$1_eh.vcf" 2>/dev/null } export REF_FASTA="${RD}/ref.fa" for s in EXA EXB; do diff --git a/tests/e2e/genotype-rules-1-phase.sh b/tests/e2e/genotype-rules-1-phase.sh index 72fb40c6a..0e5f58c69 100644 --- a/tests/e2e/genotype-rules-1-phase.sh +++ b/tests/e2e/genotype-rules-1-phase.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Step 42's genotype rules (assets/genotype_rules.tsv) on synthetic data, with -# the real bcftools, WhatsHap and Python images, and the Nextflow mirror: +# the real bcftools, WhatsHap and Python images, and the Nextflow mirror +# (the made-up chr7, gVCFs and reads are og dev fixture tas2r38): # - scripts/ci/check-rule-alleles.sh --online: every rule site matches # dbSNP's GRCh38 record, and the dbSNP table pinned in the script is what # dbSNP's VCF of the pinned build holds today; controls: a rules copy with @@ -58,103 +59,7 @@ RD="${G}/reference-genotype-rules" rm -rf "$RD" mkdir -p "$RD" for s in GRSPAN GRSWAP GRNOSPAN GRREF; do rm -rf "${G:?}/${s}"; mkdir -p "${G}/${s}/vcf" "${G}/${s}/aligned"; done -python3 - "$RD" "$G" <<'PY' -import random, sys -rd, g = sys.argv[1], sys.argv[2] -random.seed(42) -LEN, W0, W1 = 159345973, 141972000, 141974600 # GRCh38 chr7 length; the window [W0, W1), 1-based -S3, S2, S1 = 141972804, 141972905, 141973545 # rs10246939 (T), rs1726866 (G), rs713598 (C) -REF = {S3: "T", S2: "G", S1: "C"} -win = [random.choice("ACGT") for _ in range(W1 - W0)] -for p, b in REF.items(): - win[p - W0] = b -seq = lambda p: win[p - W0] # noqa: E731 -with open(f"{rd}/chr7.fa", "w") as f: - f.write(">chr7\n") - line = 60 - buf = [] - for start in range(1, LEN + 1, line): - end = min(start + line, LEN + 1) - if end <= W0 or start >= W1: - buf.append("N" * (end - start)) - else: - buf.append("".join(win[p - W0] if W0 <= p < W1 else "N" for p in range(start, end))) - if len(buf) >= 100000: - f.write("\n".join(buf) + "\n"); buf = [] - f.write("\n".join(buf) + "\n") - -def gvcf(path, het): - hdr = ("##fileformat=VCFv4.2\n##contig=\n" % LEN - + '##ALT=\n' - + '##INFO=\n' - + '##FORMAT=\n' - + '##FORMAT=\n' - + '##FORMAT=\n' - + '##FORMAT=\n' - + '##FORMAT=\n' - + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n") - rec, start = [], W0 - for p in (sorted(het) if het else []): - rec.append(f"chr7\t{start}\t.\t{seq(start)}\t<*>\t0\t.\tEND={p - 1}\tGT:GQ:MIN_DP:PL\t0/0:42:30:0,90,900") - rec.append(f"chr7\t{p}\t.\t{seq(p)}\t{het[p]},<*>\t40\tPASS\t.\tGT:GQ:DP:PL\t0/1:35:40:400,0,400,990,990,990") - start = p + 1 - rec.append(f"chr7\t{start}\t.\t{seq(start)}\t<*>\t0\t.\tEND={W1 - 1}\tGT:GQ:MIN_DP:PL\t0/0:42:30:0,90,900") - with open(path, "w") as f: - f.write(hdr + "\n".join(rec) + "\n") - -HET = {S3: "C", S2: "A", S1: "G"} -for s in ("GRSPAN", "GRSWAP", "GRNOSPAN"): - gvcf(f"{g}/{s}/vcf/{s}.g.vcf", HET) - with open(f"{g}/{s}/vcf/{s}.vcf", "w") as f: # the variant-only VCF the samplesheet needs - f.write("##fileformat=VCFv4.2\n##contig=\n" % LEN - + '##FORMAT=\n' - + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - + "".join(f"chr7\t{p}\t.\t{seq(p)}\t{HET[p]}\t40\tPASS\t.\tGT\t0/1\n" for p in sorted(HET))) -gvcf(f"{g}/GRREF/vcf/GRREF.g.vcf", {}) - -def hap(alleles): - """The window with these alleles at S1, S2, S3.""" - h = list(win) - for p, b in zip((S1, S2, S3), alleles): - h[p - W0] = b - return h - -def comp(s): - return s[::-1].translate(str.maketrans("ACGT", "TGCA")) - -def sam(path, frags): - with open(path, "w") as f: - f.write(f"@HD\tVN:1.6\tSO:unsorted\n@SQ\tSN:chr7\tLN:{LEN}\n") - for line in frags: - f.write(line + "\n") - -def pairs(h, tag, n=20): - """Read pairs: read 1 covers S3 and S2, its mate S1.""" - out = [] - for i in range(n): - a, b = S3 - 40 + i, S1 - 60 + i - s1, s2 = "".join(h[a - W0:a - W0 + 150]), "".join(h[b - W0:b - W0 + 150]) - tlen = b + 150 - a - name = f"{tag}{i}" - out.append(f"{name}\t99\tchr7\t{a}\t60\t150M\t=\t{b}\t{tlen}\t{s1}\t{'I' * 150}") - out.append(f"{name}\t147\tchr7\t{b}\t60\t150M\t=\t{a}\t{-tlen}\t{s2}\t{'I' * 150}") - return out - -def singles(h, tag, n=20): - """Single 60-base reads centred on one site each: none covers two.""" - out = [] - for p in (S3, S2, S1): - for i in range(n): - a = p - 30 + (i % 10) - 5 - s = "".join(h[a - W0:a - W0 + 60]) - out.append(f"{tag}{p}_{i}\t{16 if i % 2 else 0}\tchr7\t{a}\t60\t60M\t*\t0\t0\t{s}\t{'I' * 60}") - return out - -PAV, AVI = hap("GGC"), hap("CAT") -sam(f"{g}/GRSPAN/aligned/GRSPAN.sam", pairs(PAV, "pav") + pairs(AVI, "avi")) -sam(f"{g}/GRSWAP/aligned/GRSWAP.sam", pairs(hap("GAC"), "gac") + pairs(hap("CGT"), "cgt")) -sam(f"{g}/GRNOSPAN/aligned/GRNOSPAN.sam", singles(PAV, "pav") + singles(AVI, "avi")) -PY +"${REPO}/bin/og" dev fixture tas2r38 "$RD" "$G" check "the synthetic chr7 was written" test -s "${RD}/chr7.fa" sam faidx reference-genotype-rules/chr7.fa check_eq "chr7.fa.fai: one contig of GRCh38 chr7's length" "$(cut -f1,2 "${RD}/chr7.fa.fai" 2>/dev/null | tr '\t' ' ')" "chr7 159345973" diff --git a/tests/e2e/genotype-rules-2-g2.sh b/tests/e2e/genotype-rules-2-g2.sh index ee560b54b..d35327008 100644 --- a/tests/e2e/genotype-rules-2-g2.sh +++ b/tests/e2e/genotype-rules-2-g2.sh @@ -42,33 +42,28 @@ in_genome "$BCFTOOLS_IMAGE" rm -rf genotype-rules-g2 mkdir -p "$D" for s in G2HET G2REF; do mkdir -p "${D}/${s}/vcf" "${D}/${s}/traits" - python3 - "$REPO" "${D}/${s}" "$s" <<'PY' -import os, subprocess, sys -repo, d, s = sys.argv[1:] -sys.path.insert(0, os.path.join(repo, "bin")) -g = os.path.join(d, "g.tsv") -with open(g, "w") as f: # G0/G1 at the two G1 sites - f.write("chr22\t36265860\tA\tG\tA/G\t40\nchr22\t36265988\tT\tG\tT/G\t40\n") -subprocess.run([os.environ.get("OG", os.path.join(repo, "bin", "og")), "traits", "table", - "--table", os.path.join(repo, "assets", "traits.tsv"), - "--rules", os.path.join(repo, "assets", "genotype_rules.tsv"), - "--genotypes", g, "--sample", s, - "--out", os.path.join(d, "traits", f"{s}_traits.tsv"), - "--rules-out", os.path.join(d, "traits", f"{s}_genotype_rules.tsv")], check=True) -os.remove(g) -body = ["chr22\t36200000\t.\tA\t<*>\t0\t.\tEND=36265994\tGT:GQ\t0/0:50"] -if s == "G2HET": - body.append("chr22\t36265995\trs71785313\tAATAATT\tA,<*>\t30\tPASS\t.\tGT:GQ\t0/1:30") - body.append("chr22\t36266002\t.\tT\t<*>\t0\t.\tEND=36300000\tGT:GQ\t0/0:50") -else: - body.append("chr22\t36265995\t.\tA\t<*>\t0\t.\tEND=36300000\tGT:GQ\t0/0:50") -with open(os.path.join(d, "vcf", f"{s}.g.vcf"), "w") as f: - f.write("##fileformat=VCFv4.2\n##contig=\n" - "##INFO=\n" - "##FORMAT=\n" - "##FORMAT=\n" - f"#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{s}\n" + "\n".join(body) + "\n") -PY + # The rule line: og traits table on G0/G1 at the two G1 sites. + printf 'chr22\t36265860\tA\tG\tA/G\t40\nchr22\t36265988\tT\tG\tT/G\t40\n' > "${D}/${s}/g.tsv" + "${OG:-${REPO}/bin/og}" traits table --table "${REPO}/assets/traits.tsv" --rules "${REPO}/assets/genotype_rules.tsv" \ + --genotypes "${D}/${s}/g.tsv" --sample "$s" --out "${D}/${s}/traits/${s}_traits.tsv" \ + --rules-out "${D}/${s}/traits/${s}_genotype_rules.tsv" + rm -f "${D}/${s}/g.tsv" + # The gVCF: a reference block up to G2, then G2 heterozygous (G2HET) or a + # reference block over it (G2REF). + { + printf '##fileformat=VCFv4.2\n##contig=\n' + printf '##INFO=\n' + printf '##FORMAT=\n' + printf '##FORMAT=\n' + printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t%s\n' "$s" + printf 'chr22\t36200000\t.\tA\t<*>\t0\t.\tEND=36265994\tGT:GQ\t0/0:50\n' + if [ "$s" = G2HET ]; then + printf 'chr22\t36265995\trs71785313\tAATAATT\tA,<*>\t30\tPASS\t.\tGT:GQ\t0/1:30\n' + printf 'chr22\t36266002\t.\tT\t<*>\t0\t.\tEND=36300000\tGT:GQ\t0/0:50\n' + else + printf 'chr22\t36265995\t.\tA\t<*>\t0\t.\tEND=36300000\tGT:GQ\t0/0:50\n' + fi + } > "${D}/${s}/vcf/${s}.g.vcf" in_genome "$BCFTOOLS_IMAGE" bash -c "bcftools view -Oz -o 'genotype-rules-g2/${s}/vcf/${s}.g.vcf.gz' \ 'genotype-rules-g2/${s}/vcf/${s}.g.vcf' && bcftools index -t 'genotype-rules-g2/${s}/vcf/${s}.g.vcf.gz' \ && rm 'genotype-rules-g2/${s}/vcf/${s}.g.vcf'" @@ -78,12 +73,8 @@ done read_result() { "${REPO}/bin/og" summary collect --sample "$1" --sample-dir "${D}/$1" --assets "${REPO}/assets" \ --out "${CASE_TMP}/$1.summary.json" > /dev/null - python3 - "${CASE_TMP}/$1.summary.json" <<'PY' -import json, sys -rules = json.load(open(sys.argv[1]))["sections"]["traits"]["data"]["rules"] -r = next(x for x in rules if x["rule"] == "APOL1") -print(r["result"], r["g2"]["result"], r["g2"]["source"], r["g2"]["status"], sep="\t") -PY + jq -r '.sections.traits.data.rules[] | select(.rule == "APOL1") | [.result, .g2.result, .g2.source, .g2.status] | @tsv' \ + "${CASE_TMP}/$1.summary.json" } HET=$(read_result G2HET) REF=$(read_result G2REF) diff --git a/tests/e2e/hba-short-read-accuracy-1-nf-report.sh b/tests/e2e/hba-short-read-accuracy-1-nf-report.sh index ef465eec9..26f4dd774 100755 --- a/tests/e2e/hba-short-read-accuracy-1-nf-report.sh +++ b/tests/e2e/hba-short-read-accuracy-1-nf-report.sh @@ -74,14 +74,8 @@ LINE='HBA1 and HBA2 together (alpha-globin genes, from depth)' html_count() { grep -Fc "$2" "${CASE_TMP}/$1/results/${SAMPLE}/${SAMPLE}_report.html" 2>/dev/null || true; } # hba_not_assessed NAME: 1 when summary.json lists HBA1/HBA2 as not assessed, else 0 hba_not_assessed() { - python3 - "${CASE_TMP}/$1/results/${SAMPLE}/summary.json" <<'PY' -import json, sys -try: - d = json.load(open(sys.argv[1])) -except (OSError, ValueError): - print("no summary"); sys.exit(0) -print(int(any(x.startswith("HBA1/HBA2 copy number") for x in d.get("not_assessed", [])))) -PY + jq 'if any((.not_assessed // [])[]; startswith("HBA1/HBA2 copy number")) then 1 else 0 end' \ + "${CASE_TMP}/$1/results/${SAMPLE}/summary.json" 2>/dev/null || echo "no summary" } report_run with 1 diff --git a/tests/e2e/mito-extended-1-deletions.sh b/tests/e2e/mito-extended-1-deletions.sh index 67f0b91e1..b1049de7b 100755 --- a/tests/e2e/mito-extended-1-deletions.sh +++ b/tests/e2e/mito-extended-1-deletions.sh @@ -36,7 +36,7 @@ check_eq "the image is Bioconda's build 1.8--hdfd78af_6" "$TAG" "1.8--hdfd78af_6 check "Bioconda's package of that build downloads" \ retry curl -fsSL -o "$PKG" "https://conda.anaconda.org/bioconda/noarch/eklipse-1.8-hdfd78af_6.conda" rm -rf "${T}/pkg"; mkdir -p "${T}/pkg" -python3 -c 'import sys, zipfile; zipfile.ZipFile(sys.argv[1]).extractall(sys.argv[2])' "$PKG" "${T}/pkg" +"${REPO}/bin/og" setup unzip "$PKG" "${T}/pkg" zstd -dc "${T}"/pkg/info-*.tar.zst | tar -x -C "${T}/pkg" info/recipe/meta.yaml zstd -dc "${T}"/pkg/pkg-*.tar.zst | tar -x -C "${T}/pkg" bin/eKLIPse_sc.py bin/eKLIPse.py URL=$(awk '$1 == "url:" {print $2; exit}' "${T}/pkg/info/recipe/meta.yaml" 2>/dev/null) @@ -72,7 +72,7 @@ DEL="${SAMPLE}mtdel" WT="${SAMPLE}mtwt" for N in "$DEL" "$WT"; do args=() [ "$N" = "$DEL" ] && args=(--deletion 8470-13446 --level 0.10) - python3 "${REPO}/tests/test_mito_extended.py" synth-sam --fasta "${T}/chrM.fa" --header "${T}/header.sam" \ + "${REPO}/bin/og" dev fixture mito-sam --fasta "${T}/chrM.fa" --header "${T}/header.sam" \ --sample "$N" --depth 2000 "${args[@]}" --out "${T}/${N}.sam" dk "$SAMTOOLS_IMAGE" samtools sort -o "${T}/${N}.bam" "${T}/${N}.sam" rm -rf "${G:?}/${N}"; mkdir -p "${G}/${N}/aligned" @@ -87,14 +87,10 @@ check_eq "no soft-clipped read in the control BAM" \ # near N: deletions of N's table within 50 bp of the common deletion's breakpoints. near() { awk -F'\t' 'NR > 1 && ($1 - 8469) ^ 2 <= 2500 && ($2 - 13447) ^ 2 <= 2500' "${G}/$1/mito/$1_chrM_deletions.tsv" 2>/dev/null | wc -l | tr -d ' '; } +# summary_text N: the text report generate-report.sh writes for N. summary_text() { - python3 - "${REPO}/bin" "${G}/$1" "$1" <<'PY' -import sys -sys.path.insert(0, sys.argv[1]) -import collect_summary, render_report -s = collect_summary.collect(sys.argv[3], sys.argv[2]) -print(render_report.text_report(s)) -PY + run_step generate-report.sh "$1" > /dev/null + cat "${G}/$1/$1_report.txt" 2>/dev/null } for N in "$DEL" "$WT"; do run_step 20-mtoolbox.sh "$N" @@ -106,8 +102,9 @@ done check_eq "one deletion within 50 bp of both breakpoints, with 10% of the molecules deleted" "$(near "$DEL")" 1 LEVEL=$(awk -F'\t' 'NR > 1 && ($1 - 8469) ^ 2 <= 2500 && ($2 - 13447) ^ 2 <= 2500 {print $4}' \ "${G}/${DEL}/mito/${DEL}_chrM_deletions.tsv" 2>/dev/null | head -n 1) +# shellcheck disable=SC2016 # awk program check "its level is above 0 and at most 20% (${LEVEL:-none})" \ - python3 -c "import sys; v = float(sys.argv[1]); sys.exit(not 0 < v <= 20)" "${LEVEL:-x}" + awk -v v="${LEVEL:-x}" 'BEGIN { exit !(v ~ /^[0-9]+(\.[0-9]+)?$/ && v + 0 > 0 && v + 0 <= 20) }' check_eq "red control: no deletion in the BAM without one" \ "$(awk 'NR > 1' "${G}/${WT}/mito/${WT}_chrM_deletions.tsv" 2>/dev/null | wc -l | tr -d ' ')" 0 check_eq "the copy number of a BAM with no autosomal reads is not assessed" \ diff --git a/tests/e2e/paralog-catalog-1-gba.sh b/tests/e2e/paralog-catalog-1-gba.sh index cd548aef5..04f074c61 100755 --- a/tests/e2e/paralog-catalog-1-gba.sh +++ b/tests/e2e/paralog-catalog-1-gba.sh @@ -34,17 +34,8 @@ check_eq "models on the list" "$(printf '%s\n' "$COMMITTED" | grep -c .)" 167 # --- the stand-in reference (as pgx-4-parascopy.sh) ------------------------------- REF="${GENOME_DIR}/reference/paralog_catalog_ref.fasta" -python3 - "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" "$REF" <<'PY' -import shutil, sys -with open(sys.argv[2], "wb") as out, open(sys.argv[1], "rb") as src: - shutil.copyfileobj(src, out) - out.write(b">chr11\n") - left = 135086622 # GRCh38 chr11, as the homology table lists it - while left: - n = min(left, 60) - out.write(b"N" * n + b"\n") - left -= n -PY +# GRCh38 chr11's length, as the homology table lists it. +"${REPO}/bin/og" dev fixture append-n "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" "$REF" chr11 135086622 sam faidx "reference/$(basename "$REF")" check "the stand-in reference has chr11" grep -q $'^chr11\t135086622\t' "${REF}.fai" diff --git a/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh b/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh index 7e203c434..1c7ded99e 100755 --- a/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh +++ b/tests/e2e/paralog-checks-and-doc-fixes-1-hba.sh @@ -36,17 +36,8 @@ echo "+ scripts/setup.sh --parascopy-data" "${REPO}/scripts/setup.sh" --parascopy-data "$GENOME_DIR" REF="${GENOME_DIR}/reference/parascopy_hba_ref.fasta" -python3 - "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" "$REF" <<'PY' -import shutil, sys -with open(sys.argv[2], "wb") as out, open(sys.argv[1], "rb") as src: - shutil.copyfileobj(src, out) - out.write(b">chr11\n") - left = 135086622 # GRCh38 chr11, as the homology table lists it - while left: - n = min(left, 60) - out.write(b"N" * n + b"\n") - left -= n -PY +# GRCh38 chr11's length, as the homology table lists it. +"${REPO}/bin/og" dev fixture append-n "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" "$REF" chr11 135086622 sam faidx "reference/$(basename "$REF")" W="hba_e2e" @@ -58,31 +49,7 @@ sam view -H "${W}/slice.bam" > "${GENOME_DIR}/${W}/header.sam" sam faidx reference/GRCh38_no_alt_analysis_set.fasta chr16:95001-235000 > "${GENOME_DIR}/${W}/chr16.fa" for P in HG002hba4 HG002hba3; do - python3 - "${GENOME_DIR}/${W}/chr16.fa" "${GENOME_DIR}/${W}/header.sam" "$P" "${GENOME_DIR}/${W}/${P}_syn.sam" <<'PY' -import sys -fa, header, sample, out = sys.argv[1:] -seq = "".join(l.strip() for l in open(fa) if not l.startswith(">")).upper() -start0 = 95000 # 0-based start of chr16:95001-235000 -rg = next((l.split("\t")[1][3:] for l in open(header) if l.startswith("@RG")), None) -hba2 = (172875, 173710) # the HBA row's HBA2 (og cn bed --locus HBA), 0-based -L, step, n = 150, 5, 0 -comp = str.maketrans("ACGTN", "TGCAN") -seen = {} -for i in range(len(seq) - L + 1): - seen[seq[i:i + L]] = seen.get(seq[i:i + L], 0) + 1 -def mapq(r): # 0 for a read that fits more than one place - return 0 if seen.get(r, 0) + seen.get(r.translate(comp)[::-1], 0) > 1 else 60 -with open(out, "w") as f: - f.write(open(header).read()) - for i in range(0, len(seq) - L + 1, step): - s = start0 + i - if sample.endswith("3") and s < hba2[1] and s + L > hba2[0]: - n += 1 - if n % 2: - continue - f.write(f"syn{s}\t0\tchr16\t{s + 1}\t{mapq(seq[i:i + L])}\t{L}M\t*\t0\t0\t{seq[i:i + L]}\t{'I' * L}" - + (f"\tRG:Z:{rg}" if rg else "") + "\n") -PY + "${REPO}/bin/og" dev fixture hba-tiles "${GENOME_DIR}/${W}/chr16.fa" "${GENOME_DIR}/${W}/header.sam" "$P" "${GENOME_DIR}/${W}/${P}_syn.sam" mkdir -p "${GENOME_DIR}/${P}/aligned" sam sort -o "${W}/${P}_syn.bam" "${W}/${P}_syn.sam" sam merge -f "${P}/aligned/${P}_sorted.bam" "${W}/slice_chr5_chr20.bam" "${W}/${P}_syn.bam" diff --git a/tests/e2e/paralog-rows-1.sh b/tests/e2e/paralog-rows-1.sh index 0646a968a..dc36534be 100755 --- a/tests/e2e/paralog-rows-1.sh +++ b/tests/e2e/paralog-rows-1.sh @@ -26,17 +26,8 @@ LOCI_ALL="${REPO}/assets/paralog_loci.tsv" # --- the stand-in reference (as paralog-catalog-1-gba) ---------------------------- REF="${GENOME_DIR}/reference/paralog_rows_ref.fasta" -python3 - "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" "$REF" <<'PY' -import shutil, sys -with open(sys.argv[2], "wb") as out, open(sys.argv[1], "rb") as src: - shutil.copyfileobj(src, out) - out.write(b">chr11\n") - left = 135086622 # GRCh38 chr11, as the homology table lists it - while left: - n = min(left, 60) - out.write(b"N" * n + b"\n") - left -= n -PY +# GRCh38 chr11's length, as the homology table lists it. +"${REPO}/bin/og" dev fixture append-n "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" "$REF" chr11 135086622 sam faidx "reference/$(basename "$REF")" check "the stand-in reference has chr11" grep -q $'^chr11\t135086622\t' "${REF}.fai" @@ -45,33 +36,20 @@ P=HG002rows mkdir -p "${GENOME_DIR}/${P}/aligned" # Every region of the catalog's models on a chromosome of the fixture's reference, 1 kb padded, BED. BEDR="${GENOME_DIR}/${P}/model_regions.bed" -python3 - "$LOCI_ALL" "${REPO}/assets/parascopy_models_v1.7.txt" "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta.fai" "$BEDR" <<'PY' -import re, sys -loci, models, fai, out = sys.argv[1:] -with open(loci) as f: - lines = [x.rstrip("\n").split("\t") for x in f if x.strip() and not x.startswith("#")] -want = {r[lines[0].index("model")][:-3] for r in lines[1:]} -contigs = {x.split("\t")[0] for x in open(fai)} -spans = [] -for line in open(models): - if line.startswith("#") or not line.strip(): - continue - g, r, _ = line.rstrip("\n").split("\t") - m = re.fullmatch(r"([^:]+):(\d+)-(\d+)", r) - if g in want and m.group(1) in contigs: - spans.append((m.group(1), max(0, int(m.group(2)) - 1001), int(m.group(3)) + 1000)) -spans.sort() -merged = [] -for c, s, e in spans: - if merged and merged[-1][0] == c and s <= merged[-1][2]: - merged[-1][2] = max(merged[-1][2], e) - else: - merged.append([c, s, e]) -with open(out, "w") as f: - for c, s, e in merged: - f.write(f"{c}\t{s}\t{e}\n") -print(f"{len(merged)} regions, {sum(e - s for _, s, e in merged)} bp") -PY +# The catalog's model names (its model column without .gz), then each model +# region on a reference contig padded by 1 kb, sorted and merged. +# shellcheck disable=SC2016 # awk programs +awk -F'\t' 'FILENAME == ARGV[1] { if (/^#/ || !NF) next; if (!h) { for (i = 1; i <= NF; i++) if ($i == "model") m = i; h = 1; next } + want[substr($m, 1, length($m) - 3)] = 1; next } + FILENAME == ARGV[2] { contig[$1] = 1; next } + /^#/ || !NF { next } + ($1 in want) && match($2, /^[^:]+:[0-9]+-[0-9]+$/) { c = $2; sub(/:.*/, "", c); r = substr($2, length(c) + 2) + split(r, x, "-"); if (c in contig) { s = x[1] - 1001; print c "\t" (s < 0 ? 0 : s) "\t" x[2] + 1000 } }' \ + "$LOCI_ALL" "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta.fai" "${REPO}/assets/parascopy_models_v1.7.txt" \ + | LC_ALL=C sort -k1,1 -k2,2n -k3,3n \ + | awk -F'\t' -v OFS='\t' '$1 == c && $2 <= e { if ($3 > e) e = $3; next } { if (n++) print c, s, e; c = $1; s = $2; e = $3 } + END { if (n) print c, s, e }' > "$BEDR" +echo "$(grep -c . "$BEDR") regions, $(awk '{n += $3 - $2} END {print n + 0}' "$BEDR") bp" check_ge "regions to read from GIAB" "$(grep -c . "$BEDR" 2>/dev/null || echo 0)" 20 GIAB_BAM="https://giab.s3.amazonaws.com/data/AshkenazimTrio/HG002_NA24385_son/NIST_HiSeq_HG002_Homogeneity-10953946/NHGRI_Illumina300X_AJtrio_novoalign_bams/HG002.GRCh38.60x.1.bam" sam_https() { # the image's libcurl has no CA bundle, so lend it the host's @@ -154,10 +132,7 @@ REP="${CASE_TMP}/report_all" mkdir -p "$REP" python3 "${REPO}/bin/render_report.py" --assets "${REPO}/assets" --sample "$P" --sample-dir "${GENOME_DIR}/${P}" \ --json "${REP}/summary.json" -o "${REP}/report.html" > "${REP}/render.log" 2>&1 -loci_in_summary() { - python3 -c "import json, sys; d = json.load(open(sys.argv[1]))['sections']['parascopy']['data']; print(len(d.get('loci', [])))" \ - "${REP}/summary.json" 2>/dev/null || echo 0 -} +loci_in_summary() { jq '.sections.parascopy.data.loci // [] | length' "${REP}/summary.json" 2>/dev/null || echo 0; } check_eq "summary.json has the 36 loci other than SMN1" "$(loci_in_summary)" 36 check "the HTML report lists the GBA locus" grep -q '
' "${REP}/report.html" @@ -203,11 +178,11 @@ cid=$(docker create "$PARAPHASE_IMAGE" true) pbin=$(docker run --rm --entrypoint sh "$PARAPHASE_IMAGE" -c 'command -v paraphase' 2>/dev/null) docker cp -L "${cid}:${pbin:-/usr/local/bin/paraphase}" "$BIN" >/dev/null 2>&1 docker rm "$cid" >/dev/null 2>&1 -embedded() { python3 -c "import sys; sys.exit(0 if open(sys.argv[2], 'rb').read() in open(sys.argv[1], 'rb').read() else 1)" "$1" "$2"; } -check "the pinned image's paraphase embeds that config byte for byte" embedded "$BIN" "$CFG" +check "the pinned image's paraphase embeds that config byte for byte" "${REPO}/bin/og" dev fixture contains "$BIN" "$CFG" printf 'smn1:\n genes: X\n' > "${CASE_TMP}/other.yaml" -check "control: another config is not found in the binary" sh -c "! python3 -c \"import sys; sys.exit(0 if open(sys.argv[2], 'rb').read() in open(sys.argv[1], 'rb').read() else 1)\" '$BIN' '${CASE_TMP}/other.yaml'" -N=$(python3 -c "import re, sys; print(len(re.findall(r'^[A-Za-z0-9_]+:[ \t]*$', open(sys.argv[1]).read(), re.M)))" "$CFG") +check "control: another config is not found in the binary" \ + test "$("${REPO}/bin/og" dev fixture contains "$BIN" "${CASE_TMP}/other.yaml"; echo $?)" = 1 +N=$(grep -cE '^[A-Za-z0-9_]+:[[:blank:]]*$' "$CFG") check_eq "regions in the pinned config" "$N" "$(sed -n 's/^PARAPHASE_REGIONS = \([0-9]*\)$/\1/p' "${REPO}/tests/test_paraphase_report.py")" check "docs/35-paralogs.md states that count" grep -q "It covers ${N} GRCh38 regions" "${REPO}/docs/35-paralogs.md" rm -f "$BIN" "$REF" "${REF}.fai" diff --git a/tests/e2e/paralogs-paraphase-1.sh b/tests/e2e/paralogs-paraphase-1.sh index 3f429e681..353a66dfe 100755 --- a/tests/e2e/paralogs-paraphase-1.sh +++ b/tests/e2e/paralogs-paraphase-1.sh @@ -69,38 +69,32 @@ D="${GENOME_DIR}/${P}/aligned_longread" mkdir -p "$D" curl -fsSL --retry 5 -o "${CASE_TMP}/config.yaml" "${RAW}/config.yaml" curl -fsSL --retry 5 -o "${CASE_TMP}/genome_region.bed" "${RAW}/genome_region.bed" -python3 - "${CASE_TMP}/config.yaml" "${CASE_TMP}/genome_region.bed" "$REGIONS" > "${D}/regions.bed" <<'PY' -import re, sys -cfg, depth, wanted = open(sys.argv[1]).read(), sys.argv[2], set(sys.argv[3].split(",")) -iv, block, chrom = [], None, None -for line in cfg.splitlines(): - m = re.match(r"^(\S+):\s*$", line) - if m: - block = m.group(1) - continue - if block not in wanted: - continue - for c, s, e in re.findall(r"(chr[0-9XY]+):(\d+)-(\d+)", line): - iv.append((c, int(s), int(e))) - chrom = c - if "depth_region" in line and chrom: # [[start, end]] on the family's chromosome - for s, e in re.findall(r"\[(\d+),\s*(\d+)\]", line): - iv.append((chrom, int(s), int(e))) -iv = [(c, max(0, s - 30000), e + 30000) for c, s, e in iv] -for line in open(depth): # positions of 1.6 kb depth windows: 2 kb either side - f = line.split() - if len(f) >= 2: - iv.append((f[0], max(0, int(f[1]) - 2000), int(f[1]) + 2000)) -iv.sort() -out = [] -for c, s, e in iv: - if out and out[-1][0] == c and s <= out[-1][2]: - out[-1][2] = max(out[-1][2], e) - else: - out.append([c, s, e]) -for c, s, e in out: - print(f"{c}\t{s}\t{e}") -PY +# The regions of the wanted families in the config ("chrN:start-end", and each +# depth_region's [start, end] pairs on the family's chromosome) 30 kb padded, +# and the depth windows 2 kb either side, sorted and merged. +# shellcheck disable=SC2016 # awk programs +{ + awk -v wanted=",${REGIONS}," ' + /^[^[:space:]]+:[[:space:]]*$/ { block = $0; sub(/:[[:space:]]*$/, "", block); next } + index(wanted, "," block ",") == 0 { next } + { line = $0 + while (match(line, /chr[0-9XY]+:[0-9]+-[0-9]+/)) { + r = substr(line, RSTART, RLENGTH); line = substr(line, RSTART + RLENGTH) + c = r; sub(/:.*/, "", c); split(substr(r, length(c) + 2), x, "-"); chrom = c + s = x[1] - 30000; print c "\t" (s < 0 ? 0 : s) "\t" x[2] + 30000 + } + if (index($0, "depth_region") && chrom != "") { + line = $0 + while (match(line, /\[[0-9]+,[[:space:]]*[0-9]+\]/)) { + r = substr(line, RSTART + 1, RLENGTH - 2); line = substr(line, RSTART + RLENGTH) + split(r, x, ","); s = x[1] - 30000; print chrom "\t" (s < 0 ? 0 : s) "\t" x[2] + 30000 + } + } + }' "${CASE_TMP}/config.yaml" + awk 'NF >= 2 { s = $2 - 2000; print $1 "\t" (s < 0 ? 0 : s) "\t" $2 + 2000 }' "${CASE_TMP}/genome_region.bed" +} | LC_ALL=C sort -k1,1 -k2,2n -k3,3n \ + | awk -F'\t' -v OFS='\t' '$1 == c && $2 <= e { if ($3 > e) e = $3; next } { if (n++) print c, s, e; c = $1; s = $2; e = $3 } + END { if (n) print c, s, e }' > "${D}/regions.bed" echo "HiFi slice: $(wc -l < "${D}/regions.bed") regions, $(awk '{n += $3 - $2} END {print n}' "${D}/regions.bed") bp" check_ge "the slice has the families' regions and the depth windows" "$(wc -l < "${D}/regions.bed" | tr -d ' ')" 20 retry sam_https view -b -M -L "${P}/aligned_longread/regions.bed" --keep-tag RG,NM,SA,rq,HP,PS \ @@ -154,53 +148,8 @@ DEL_START=169742 DEL_END=173999 # 1-based: bases DEL_START..DEL_END-1 go # spans them is joined across the gap, one that ends or starts inside keeps # the part outside (at least 1 kb, or it is dropped). They are written as # FASTQ, their tags in the comment for minimap2 -y, and listed for pass 2. -cat > "${CASE_TMP}/cut.py" <<'PY' -import re, sys -a, b, fq, names = int(sys.argv[1]), int(sys.argv[2]), open(sys.argv[3], "w"), open(sys.argv[4], "w") -spanning = trimmed = dropped = 0 -for line in sys.stdin: - f = line.rstrip("\n").split("\t") - tags = f[11:] - if "HP:i:1" not in tags or int(f[1]) & 0x900: - continue - pos = ref = int(f[3]) - qry, qa, qb = 0, None, None # query offsets of the first base at or after a, and b - for n, op in re.findall(r"(\d+)([MIDNSHP=X])", f[5]): - n = int(n) - if op in "M=XDN": - for x, attr in ((a, "qa"), (b, "qb")): - if (qa if attr == "qa" else qb) is None and ref + n > x: - off = qry + max(0, x - ref) if op in "M=X" else qry - if attr == "qa": - qa = off - else: - qb = off - ref += n - if op in "M=X": - qry += n - elif op in "IS": - qry += n - if ref <= a or pos >= b: - continue - names.write(f[0] + "\n") - left = f[9][:qa] if pos < a else "" - right = f[9][qb:] if ref > b else "" - lq = f[10][:qa] if pos < a else "" - rq = f[10][qb:] if ref > b else "" - if left and right: - spanning += 1 - elif len(left + right) >= 1000: - trimmed += 1 - else: - dropped += 1 - continue - keep = [t for t in tags if t[:2] in ("RG", "rq", "HP", "PS")] - fq.write(f"@{f[0]}\t" + "\t".join(keep) + f"\n{left + right}\n+\n{lq + rq}\n") -print(f"haplotype-1 reads joined across the deletion: {spanning}, trimmed: {trimmed}, dropped: {dropped}", file=sys.stderr) -print(spanning) -PY SPAN=$(sam view "${P}/aligned_longread/${P}_sorted.bam" "chr16:$((DEL_START - 50000))-$((DEL_END + 50000))" \ - | python3 "${CASE_TMP}/cut.py" "$DEL_START" "$DEL_END" "${QD}/cut.fq" "${QD}/names.txt") + | "${REPO}/bin/og" dev fixture cut-haplotype "$DEL_START" "$DEL_END" "${QD}/cut.fq" "${QD}/names.txt") check_ge "haplotype-1 reads joined across the planted deletion" "${SPAN:-0}" 5 # Pass 2: every other read as it was, and the cut reads aligned again to chr16. sam view -b -N "^${Q}/aligned_longread/names.txt" -o "${Q}/aligned_longread/kept.bam" "${P}/aligned_longread/${P}_sorted.bam" diff --git a/tests/e2e/paralogs-report-section-1-nf-report-tables.sh b/tests/e2e/paralogs-report-section-1-nf-report-tables.sh index d51f31f7a..3bb87dce7 100755 --- a/tests/e2e/paralogs-report-section-1-nf-report-tables.sh +++ b/tests/e2e/paralogs-report-section-1-nf-report-tables.sh @@ -80,13 +80,10 @@ check_ge "the cDNA read check column of the ClinVar hits" "$(html_count with "$C check_ge "the flagged count of the cDNA read check, 1 for the planted hit" \ "$(html_count with "$C" 'Flagged by the cDNA read check1<')" 1 check_ge "the compound-het pairs by phase from step 31's table" "$(html_count with CHPH 'Pairs in trans / cis / unknown')" 1 -check "the summary of CHPH counts the pairs table's rows" python3 - "${CASE_TMP}/with/results/CHPH/summary.json" "$SL_PAIRS" <<'PY' -import json, sys -d = json.load(open(sys.argv[1]))["sections"]["slivar"]["data"] -rows = sum(1 for _ in open(sys.argv[2])) - 1 -print(d.get("compound_het_pairs"), rows) -sys.exit(d.get("compound_het_pairs") != rows) -PY +# shellcheck disable=SC2016 # jq program +check "the summary of CHPH counts the pairs table's rows" \ + jq -e --argjson rows "$(($(wc -l < "$SL_PAIRS") - 1))" '.sections.slivar.data.compound_het_pairs == $rows' \ + "${CASE_TMP}/with/results/CHPH/summary.json" report_run without 0 check_eq "control: without the depth check there is no depth-check line" \ diff --git a/tests/e2e/pgx-4-parascopy.sh b/tests/e2e/pgx-4-parascopy.sh index 21adf5cc8..86b50b1bd 100755 --- a/tests/e2e/pgx-4-parascopy.sh +++ b/tests/e2e/pgx-4-parascopy.sh @@ -26,17 +26,8 @@ mkdir -p "${GENOME_DIR}/${P}/aligned" cp "${FIXTURE_DIR}/${SAMPLE}_slice.bam" "${GENOME_DIR}/${P}/aligned/${P}_sorted.bam" cp "${FIXTURE_DIR}/${SAMPLE}_slice.bam.bai" "${GENOME_DIR}/${P}/aligned/${P}_sorted.bam.bai" REF="${GENOME_DIR}/reference/parascopy_test_ref.fasta" -python3 - "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" "$REF" <<'PY' -import shutil, sys -with open(sys.argv[2], "wb") as out, open(sys.argv[1], "rb") as src: - shutil.copyfileobj(src, out) - out.write(b">chr11\n") - left = 135086622 # GRCh38 chr11, as the homology table lists it - while left: - n = min(left, 60) - out.write(b"N" * n + b"\n") - left -= n -PY +# GRCh38 chr11's length, as the homology table lists it. +"${REPO}/bin/og" dev fixture append-n "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" "$REF" chr11 135086622 sam faidx "reference/$(basename "$REF")" check "the stand-in reference has chr11" grep -q $'^chr11\t135086622\t' "${REF}.fai" diff --git a/tests/e2e/str-catalog-1-strchive.sh b/tests/e2e/str-catalog-1-strchive.sh index eca065644..a8c09f762 100644 --- a/tests/e2e/str-catalog-1-strchive.sh +++ b/tests/e2e/str-catalog-1-strchive.sh @@ -12,7 +12,7 @@ # (FAME1) TTTCA reads, with an ExpansionHunter VCF that sizes both # above the catalog's normal maximum. Step 9b flags both with the real # Stranger, the RFC1 count reads the catalog's RFC1 (pathogenic_motif, -# the shares of tests/test_rfc1_motif.py) and the SAMD12 count +# the shares of og dev fixture rfc1 pathogenic) and the SAMD12 count # pathogenic_motif; the same with TTTTA reads at SAMD12 gives # benign_motif. # 5. STRANGER and STRANGER_RFC1 in Nextflow, given the same catalog, write @@ -35,7 +35,7 @@ CAT="${SETUP}/reference/strchive-${STRCHIVE_VERSION}/catalog.json" check_eq "setup.sh --strchive-data exits 0" "$?" 0 cat "${CASE_TMP}/setup.log" check "setup.sh names the catalog it wrote" has "\[OK\] STRchive ${STRCHIVE_VERSION//./\\.} catalog" "$(cat "${CASE_TMP}/setup.log")" -check_eq "the catalog has 82 loci" "$(python3 -c 'import json, sys; print(len(json.load(open(sys.argv[1]))))' "$CAT" 2>/dev/null)" 82 +check_eq "the catalog has 82 loci" "$(jq length "$CAT" 2>/dev/null)" 82 # --- 2. ExpansionHunter on the loci of the fixture's contigs ------------------------------------ S="${SAMPLE}strc" @@ -45,16 +45,17 @@ mkdir -p "${SD}/aligned" ln -f "${GENOME_DIR}/${SAMPLE}/aligned/${SAMPLE}_sorted.bam" "${SD}/aligned/${S}_sorted.bam" ln -f "${GENOME_DIR}/${SAMPLE}/aligned/${SAMPLE}_sorted.bam.bai" "${SD}/aligned/${S}_sorted.bam.bai" SUB="${GENOME_DIR}/reference/strchive_fixture.json" -python3 - "$CAT" "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta.fai" "$SUB" > "${CASE_TMP}/subset.txt" <<'PY' -import json, sys -loci = json.load(open(sys.argv[1])) -contigs = {line.split("\t")[0] for line in open(sys.argv[2])} -regions = lambda l: l["ReferenceRegion"] if isinstance(l["ReferenceRegion"], list) else [l["ReferenceRegion"]] -kept = [l for l in loci if all(r.split(":")[0] in contigs for r in regions(l))] -json.dump(kept, open(sys.argv[3], "w"), indent=1) -print(len(kept), sum(len(regions(l)) for l in kept), sum("NormalMax" in l for l in kept), - ",".join(l["LocusId"] for l in kept if "NoThresholds" in l)) -PY +# The loci whose every reference region is on a fixture contig; then their +# count, their repeats, the graded ones (with NormalMax) and the ids of the +# ones the catalog leaves ungraded (NoThresholds). +# shellcheck disable=SC2016 # jq programs +jq --rawfile fai "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta.fai" ' + ($fai | split("\n") | map(split("\t")[0] | select(. != null and . != "")) | map({(.): true}) | add) as $c + | map(select([.ReferenceRegion | if type == "array" then .[] else . end | split(":")[0] | $c[.]] | all))' \ + "$CAT" > "$SUB" +jq -r '"\(length) \(map(.ReferenceRegion | if type == "array" then length else 1 end) | add // 0)" + + " \(map(select(has("NormalMax"))) | length) \(map(select(has("NoThresholds")) | .LocusId) | join(","))"' \ + "$SUB" > "${CASE_TMP}/subset.txt" read -r N_LOCI N_VAR N_GRADED UNGRADED < "${CASE_TMP}/subset.txt" echo "loci on the fixture's contigs: ${N_LOCI} (${N_VAR} repeats, ${N_GRADED} graded; not graded: ${UNGRADED})" check_ge "catalog loci on the fixture's contigs" "$N_LOCI" 40 @@ -62,9 +63,8 @@ EH_CATALOG="$SUB" run_step 09-expansion-hunter.sh "$S" male check_step_exit 09-expansion-hunter.sh if [ "$STEP_RC" != 0 ]; then echo "ExpansionHunter stopped: each locus alone" - python3 -c 'import json, sys; [print(l["LocusId"]) for l in json.load(open(sys.argv[1]))]' "$SUB" | while read -r id; do - python3 -c 'import json, sys; json.dump([l for l in json.load(open(sys.argv[1])) if l["LocusId"] == sys.argv[2]], open(sys.argv[3], "w"))' \ - "$SUB" "$id" "${GENOME_DIR}/reference/one_locus.json" + jq -r '.[].LocusId' "$SUB" | while read -r id; do + jq --arg id "$id" 'map(select(.LocusId == $id))' "$SUB" > "${GENOME_DIR}/reference/one_locus.json" docker run --rm -u "$(id -u):$(id -g)" -v "${GENOME_DIR}:/genome" "$EXPANSIONHUNTER_IMAGE" ExpansionHunter \ --reads "/genome/${S}/aligned/${S}_sorted.bam" --reference /genome/reference/GRCh38_no_alt_analysis_set.fasta \ --variant-catalog /genome/reference/one_locus.json --output-prefix "/genome/${S}/one" --sex male \ @@ -96,47 +96,21 @@ make_sample() { # make_sample rm -rf "${GENOME_DIR:?}/${s}" mkdir -p "${GENOME_DIR}/${s}/expansion_hunter" "${GENOME_DIR}/${s}/aligned" cp "$CAT" "${GENOME_DIR}/${s}/expansion_hunter/${s}_eh_catalog.json" - python3 - "$REPO" "$unit" "${GENOME_DIR}/${s}/expansion_hunter/${s}_eh.vcf" "${GENOME_DIR}/${s}/aligned/reads.sam" <<'PY' -import os, random, sys -repo, unit, vcf, sam = sys.argv[1:5] -sys.path.insert(0, os.path.join(repo, "tests")) -import test_rfc1_motif as t -head = [l for l in open(os.path.join(repo, "tests", "smoke", "stranger_input.vcf.in")) if l.startswith("##") - and not l.startswith("##contig")] -head += [f'##ALT=\n' for a in (40, 100, 150)] -head += ["##contig=\n", "##contig=\n", - "##contig=\n"] -fmt = "GT:SO:REPCN:REPCI:ADSP:ADFL:ADIR:LC" -expanded = "1/2:INREPEAT/INREPEAT:100/150:80-120/130-170:0/0:2/2:20/20:30.0" # two long alleles -with open(vcf, "w") as fh: - fh.writelines(head) - fh.write("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n") - # The catalog's RFC1 record is its AAGGG repeat, and SAMD12's its TGAAA one. - fh.write(f"chr4\t39348480\tRFC1\tA\t,\t.\tPASS\tSVTYPE=STR;END=39348483;REF=0;RL=4;RU=AAGGG;" - f"VARID=RFC1;REPID=RFC1\t{fmt}\t{expanded}\n") - fh.write(f"chr8\t118366913\tSAMD12\tA\t\t.\tPASS\tSVTYPE=STR;END=118366918;REF=1;RL=6;RU=TGAAA;" - f"VARID=SAMD12;REPID=SAMD12\t{fmt}\t0/1:SPANNING/INREPEAT:1/40:1-1/30-50:10/0:2/2:0/10:30.0\n") -at, elsewhere = t.case("pathogenic") -rng = random.Random(3) -start = 118366812 -fame = [(f"f{i}", "chr8", start, "150M", unit * 30) for i in range(5)] -left = "".join(rng.choice("ACGT") for _ in range(60)) -ref_allele = left + "TAAAA" * 20 + "TGAAA" + left[:30] -fame += [(f"fr{i}", "chr8", start - 60, f"{len(ref_allele)}M", ref_allele) for i in range(6)] -with open(sam, "w") as fh: - fh.write("@HD\tVN:1.6\tSO:unsorted\n") - fh.write(f"@SQ\tSN:{t.CHROM}\tLN:190214555\n@SQ\tSN:{t.OTHER}\tLN:248956422\n@SQ\tSN:chr8\tLN:145138636\n") - for r in at + list(elsewhere.values()): - flag = (1 | (64 if r.is_read1 else 128) | (4 if r.is_unmapped else 0) | (16 if r.is_reverse else 0) - | (256 if r.is_secondary else 0) | (512 if r.is_qcfail else 0) | (1024 if r.is_duplicate else 0)) - mate = "=" if r.mate_contig == r.contig else r.mate_contig - fh.write("\t".join(map(str, [r.query_name, flag, r.contig, r.reference_start + 1, - 0 if r.is_unmapped else 60, r.cigar_text or "*", mate, - r.next_reference_start + 1, 0, r.query_sequence, - "I" * len(r.query_sequence)])) + "\n") - for name, chrom, pos, cig, seq in fame: - fh.write("\t".join(map(str, [name, 0, chrom, pos + 1, 60, cig, "*", 0, 0, seq, "I" * len(seq)])) + "\n") -PY + # The smoke VCF's header with its contigs replaced, and two expanded records: + # the catalog's RFC1 record is its AAGGG repeat, and SAMD12's its TGAAA one. + local in="${REPO}/tests/smoke/stranger_input.vcf.in" fmt=GT:SO:REPCN:REPCI:ADSP:ADFL:ADIR:LC a + { + grep '^##' "$in" | grep -v '^##contig' + for a in 40 100 150; do printf '##ALT=\n' "$a" "$a"; done + printf '##contig=\n##contig=\n##contig=\n' + printf '#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n' + printf 'chr4\t39348480\tRFC1\tA\t,\t.\tPASS\tSVTYPE=STR;END=39348483;REF=0;RL=4;RU=AAGGG;VARID=RFC1;REPID=RFC1' + printf '\t%s\t1/2:INREPEAT/INREPEAT:100/150:80-120/130-170:0/0:2/2:20/20:30.0\n' "$fmt" + printf 'chr8\t118366913\tSAMD12\tA\t\t.\tPASS\tSVTYPE=STR;END=118366918;REF=1;RL=6;RU=TGAAA;VARID=SAMD12;REPID=SAMD12' + printf '\t%s\t0/1:SPANNING/INREPEAT:1/40:1-1/30-50:10/0:2/2:0/10:30.0\n' "$fmt" + } > "${GENOME_DIR}/${s}/expansion_hunter/${s}_eh.vcf" + # The RFC1 reads of og dev fixture rfc1 pathogenic, and SAMD12 reads of the unit. + "${REPO}/bin/og" dev fixture rfc1 pathogenic "${GENOME_DIR}/${s}/aligned/reads.sam" --samd12 "$unit" docker run --rm -u "$(id -u):$(id -g)" -v "${GENOME_DIR}:/genome" -w "/genome/${s}/aligned" "$SAMTOOLS_IMAGE" \ sh -c "samtools sort -o ${s}_sorted.bam reads.sam && samtools index ${s}_sorted.bam" rm -f "${GENOME_DIR}/${s}/aligned/reads.sam" @@ -202,12 +176,12 @@ NF [ "$rc" = 0 ] || grep -E 'ERROR|Error executing process|Command error' -A 8 "${CASE_TMP}/nf.log" | head -40 NF_DIR="${CASE_TMP}/nf/results/${P}/expansion_hunter" records() { # records : the records with their INFO keys sorted - python3 -c 'import sys -for line in open(sys.argv[1]): - if not line.startswith("#"): - f = line.rstrip("\n").split("\t") - f[7] = ";".join(sorted(f[7].split(";"))) - print("\t".join(f))' "$1" + # shellcheck disable=SC2016 # awk program + LC_ALL=C awk -F'\t' -v OFS='\t' '!/^#/ { + n = split($8, kv, ";") + for (i = 2; i <= n; i++) { v = kv[i]; for (j = i - 1; j >= 1 && (kv[j] "") > (v ""); j--) kv[j + 1] = kv[j]; kv[j + 1] = v } + $8 = kv[1]; for (i = 2; i <= n; i++) $8 = $8 ";" kv[i] + print }' "$1" } check_ge "step 9b graded records to compare" "$(records "${GENOME_DIR}/${P}/expansion_hunter/${P}_eh_stranger.vcf" | grep -c 'STR_STATUS=')" 2 check "STRANGER grades as step 9b, the same records" \ diff --git a/tests/e2e/str-extra-repeats-1-readings.sh b/tests/e2e/str-extra-repeats-1-readings.sh index 65981889f..f2ee2ffae 100644 --- a/tests/e2e/str-extra-repeats-1-readings.sh +++ b/tests/e2e/str-extra-repeats-1-readings.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash # The counts of the repeats STRchive leaves out, read into results, with the # real tools: ExpansionHunter 5.0.0 (step 09) sizes the CFTR TG/poly-T tract -# and the UGT1A1 TA repeat of assets/str_extra_loci.json on made-up reads, +# and the UGT1A1 TA repeat of assets/str_extra_loci.json on made-up reads +# (og dev fixture str-extra), # and the reports (step 24 and generate-report.sh, bin/collect_summary.py and # bin/render_report.py) read them: # SXA CFTR TG12-5T on one copy and TG11-7T on the other, UGT1A1 7 and 8 TA @@ -29,80 +30,7 @@ rm -rf "$RD" mkdir -p "$RD" cp "${REPO}/assets/str_extra_loci.json" "${G}/reference-str-readings.json" for s in SXA SXB; do rm -rf "${G:?}/${s}"; mkdir -p "${G}/${s}/aligned"; done -python3 - "$RD" "$G" <<'PY' -import random, sys -rd, g = sys.argv[1], sys.argv[2] -rng = random.Random(11) -W = 1500 # random bases each side of a tract -# chrom: (contig length, 0-based tract start, left bases, reference tract units, right bases) -LOCI = {"chr7": (117551000, 117548606, "TTTTGA", [("TG", 11), ("T", 7)], "AACAGG"), - "chr2": (233763000, 233760234, "TTGCCA", [("TA", 7)], "AGTAGG")} -win = {} -for c, (length, start, left, units, right) in LOCI.items(): - lf = "".join(rng.choice("ACGT") for _ in range(W - len(left))) + left - rf = right + "".join(rng.choice("ACGT") for _ in range(W - len(right))) - win[c] = (start - W, lf, rf) - tract = "".join(m * n for m, n in units) - with open(f"{rd}/ref.fa", "a") as f: - f.write(f">{c}\n") - seq_start = start - W - body = "N" * seq_start + lf + tract + rf - body += "N" * (length - len(body)) - for i in range(0, len(body), 60): - f.write(body[i:i + 60] + "\n") - -def reads(c, haps, tag): - """Read pairs from each haplotype's tract; the CIGAR puts the tract's length change at its end.""" - w0, lf, rf = win[c] - ref_tract = "".join(m * n for m, n in LOCI[c][3]) - out = [] - for h, units in enumerate(haps): - tract = "".join(m * n for m, n in units) - seq = lf + tract + rf - t0, t1 = len(lf), len(lf) + len(tract) - d = len(tract) - len(ref_tract) - - def place(a, n=150): - """(1-based POS, CIGAR) of hap bases [a, a+n), or None when the read starts in the tract.""" - if a + n <= t0: - return w0 + a + 1, f"{n}M" - if a >= t1: - return w0 + a - d + 1, f"{n}M" - if a >= t0: - return None - if a + n <= t1: - return None - pre = t0 - a + min(len(tract), len(ref_tract)) - post = n - (t0 - a) - len(tract) - mid = f"{d}I" if d > 0 else (f"{-d}D" if d < 0 else "") - cig = f"{pre}M{mid}{post}M" if mid else f"{n}M" - return w0 + a + 1, cig - k = 0 - for a in range(0, len(seq) - 400, 4): - f_len = 300 + (a % 7) * 10 - b = a + f_len - 150 - p1, p2 = place(a), place(b) - if not p1 or not p2: - continue - k += 1 - name = f"{tag}{c}h{h}r{k}" - r1, r2 = seq[a:a + 150], seq[b:b + 150] - tlen = p2[0] + 150 - p1[0] - out.append(f"{name}\t99\t{c}\t{p1[0]}\t60\t{p1[1]}\t=\t{p2[0]}\t{tlen}\t{r1}\t{'I' * 150}") - out.append(f"{name}\t147\t{c}\t{p2[0]}\t60\t{p2[1]}\t=\t{p1[0]}\t{-tlen}\t{r2}\t{'I' * 150}") - return out - -SAMPLES = {"SXA": ([[("TG", 12), ("T", 5)], [("TG", 11), ("T", 7)]], [[("TA", 7)], [("TA", 8)]]), - "SXB": ([[("TG", 11), ("T", 7)]] * 2, [[("TA", 7)]] * 2)} -for s, (cftr, ugt) in SAMPLES.items(): - with open(f"{g}/{s}/aligned/{s}.sam", "w") as f: - f.write("@HD\tVN:1.6\tSO:unsorted\n") - for c, (length, *_r) in LOCI.items(): - f.write(f"@SQ\tSN:{c}\tLN:{length}\n") - f.write(f"@RG\tID:{s}\tSM:{s}\n") - for line in reads("chr7", cftr, s) + reads("chr2", ugt, s): - f.write(line + f"\tRG:Z:{s}\n") -PY +"${REPO}/bin/og" dev fixture str-extra "${RD}/ref.fa" SXA "${G}/SXA/aligned/SXA.sam" SXB "${G}/SXB/aligned/SXB.sam" check "the made-up reference was written" test -s "${RD}/ref.fa" sam faidx reference-str-readings/ref.fa for s in SXA SXB; do @@ -117,19 +45,13 @@ for s in SXA SXB; do check_step_exit 09-expansion-hunter.sh done unset REF_FASTA -python3 - "$G" "$REPO" <<'PY' -import os, sys -g, repo = sys.argv[1], sys.argv[2] -# The columns of step 42's traits table (og traits table). -OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", "note", "pmids"] -os.makedirs(f"{g}/SXA/traits", exist_ok=True) -os.makedirs(f"{g}/SXA/cpic", exist_ok=True) -with open(f"{g}/SXA/traits/SXA_traits.tsv", "w") as f: - f.write("\t".join(OUT_COLUMNS) + "\n") - f.write("rs78655421\tCFTR\tchr7\t117530975\tCFTR R117H\tG/G\tcalled\tNo R117H copy.\tno\t-\t32404922\n") -with open(f"{g}/SXA/cpic/SXA_phenotypes.tsv", "w") as f: - f.write("Gene\tDiplotype\tPhenotype\tStatus\nUGT1A1\t*1/*28\tIntermediate Metabolizer\tnon-normal\n") -PY +# The columns of step 42's traits table (og traits table), with one R117H row. +mkdir -p "${G}/SXA/traits" "${G}/SXA/cpic" +{ + printf 'rsid\tgene\tchrom\tpos\ttrait\tgenotype\tstatus\treading\tancestry\tnote\tpmids\n' + printf 'rs78655421\tCFTR\tchr7\t117530975\tCFTR R117H\tG/G\tcalled\tNo R117H copy.\tno\t-\t32404922\n' +} > "${G}/SXA/traits/SXA_traits.tsv" +printf 'Gene\tDiplotype\tPhenotype\tStatus\nUGT1A1\t*1/*28\tIntermediate Metabolizer\tnon-normal\n' > "${G}/SXA/cpic/SXA_phenotypes.tsv" # --- the reports ------------------------------------------------------------------------------ for s in SXA SXB; do diff --git a/tests/e2e/traits-panel-1.sh b/tests/e2e/traits-panel-1.sh index e6cb56765..9ca18dae9 100644 --- a/tests/e2e/traits-panel-1.sh +++ b/tests/e2e/traits-panel-1.sh @@ -95,31 +95,18 @@ mkdir -p "${G}/${P2}/vcf" PL_POS=$(awk -F'\t' '$1 == "rs1801272" { print $3 }' "${CASE_TMP}/where.tsv") check "rs1801272 lies in a fixture slice" test "$(awk -F'\t' '$1 == "rs1801272" { print $6 }' "${CASE_TMP}/where.tsv")" = in # The gVCF without the records over the position, plus one hom-alt record there. -python3 - "${G}/${GVCF}" "${G}/${P2}/vcf/planted.vcf" "$PL_POS" "$SAMPLE" <<'PY' -import gzip, sys -src, dst, pos, sample = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4] -rec = f"chr19\t{pos}\t.\tA\tT,<*>\t40\tPASS\t.\tGT:GQ:DP:AD:VAF:PL\t1/1:40:30:0,30,0:1,0:990,90,0,990,990,990\n" -done = False -with gzip.open(src, "rt") as f, open(dst, "w") as o: - for line in f: - if line.startswith("#"): - o.write(line) - continue - c = line.split("\t", 8) - p = int(c[1]) - end = p + len(c[3]) - 1 - for kv in c[7].split(";"): - if kv.startswith("END="): - end = int(kv[4:]) - if c[0] == "chr19" and p <= pos <= end: - continue - if not done and c[0] == "chr19" and p > pos: - o.write(rec) - done = True - o.write(line) - if not done: - sys.exit("no chr19 record after the planted position") -PY +# shellcheck disable=SC2016 # awk program +gzip -dc "${G}/${GVCF}" | awk -F'\t' -v pos="$PL_POS" ' + /^#/ { print; next } + { p = $2 + 0; end = p + length($4) - 1; n = split($8, kv, ";") + for (i = 1; i <= n; i++) if (kv[i] ~ /^END=/) end = substr(kv[i], 5) + 0 + if ($1 == "chr19" && p <= pos && pos <= end) next + if (!done && $1 == "chr19" && p > pos) { + print "chr19\t" pos "\t.\tA\tT,<*>\t40\tPASS\t.\tGT:GQ:DP:AD:VAF:PL\t1/1:40:30:0,30,0:1,0:990,90,0,990,990,990" + done = 1 } + print } + END { if (!done) { print "no chr19 record after the planted position" > "/dev/stderr"; exit 1 } }' \ + > "${G}/${P2}/vcf/planted.vcf" in_genome "$BCFTOOLS_IMAGE" bash -c "bcftools view -Oz -o '${P2}/vcf/${P2}.g.vcf.gz' '${P2}/vcf/planted.vcf' && bcftools index -t '${P2}/vcf/${P2}.g.vcf.gz'" check_eq "the planted gVCF has the hom-alt record" \ "$(bcf query -r "chr19:${PL_POS}" -f '[%GT]\n' "${P2}/vcf/${P2}.g.vcf.gz" 2>/dev/null | head -n 1)" 1/1 From bd6bf888470821408a79ca2a787022668125d3d6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:39:53 +0100 Subject: [PATCH 201/255] docs(results): correct the step 23 filter rule and the CYP2D6 row The clinical filter paragraph said step 23 keeps every HIGH-impact variant with no frequency filter. scripts/23-clinical-filter.sh keeps HIGH and MODERATE variants below 1% MAX_AF, those of unknown frequency in their own tier, and ClinVar pathogenic ones at any frequency. The CYP2D6 row said poor metabolizers get a toxic buildup of codeine, tramadol and tamoxifen. Those three need CYP2D6 to work, so a poor metabolizer gets less effect, and an ultrarapid one risks opioid toxicity. The root .gitignore drops every results/ folder, so git add refused the chapter pages even once tracked. It now keeps docs/results/. --- .gitignore | 2 ++ CHANGELOG.md | 1 + docs/results/pharmacogenomics.md | 2 +- docs/results/secondary-findings.md | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 7f135ae3a..c9d40d5df 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,8 @@ work/ .nextflow/ .nextflow.log* results/ +# docs/results/ is the results guide, one page per report chapter, not run output +!docs/results/ results_*/ .nf-test/ /nextflow diff --git a/CHANGELOG.md b/CHANGELOG.md index 371624eee..d5f8b3ef2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. - The guide to interpreting results now has one page per report chapter, under `docs/results/`: secondary findings and carrier status, ClinVar and variant annotation, pharmacogenomics and HLA, structural variants and repeats, ancestry and lineage, telomeres and somatic variants, and data quality. The text moved word for word. `docs/interpreting-results.md` keeps the "Before You Look" bullets, the "Before You Panic" table, the example outputs and the general advice, and lists every chapter page. Each old link to a section of that page still works: it lands on a link to the section's new place. - The repeat table moved with its chapter, so `tests/test_str_ranges.py` now reads it from `docs/results/structural-and-repeats.md`. + - Two moved lines were wrong and are corrected. The clinical filter paragraph said step 23 keeps every HIGH-impact variant with no frequency filter; step 23 keeps HIGH and MODERATE variants below 1% `MAX_AF`, those of unknown frequency in their own tier, and ClinVar pathogenic ones at any frequency. The CYP2D6 row said poor metabolizers get a toxic buildup of codeine, tramadol and tamoxifen; for those three a poor metabolizer gets less effect, and an ultrarapid metabolizer risks opioid toxicity. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. diff --git a/docs/results/pharmacogenomics.md b/docs/results/pharmacogenomics.md index b0f0d15f4..a2cdec9af 100644 --- a/docs/results/pharmacogenomics.md +++ b/docs/results/pharmacogenomics.md @@ -13,7 +13,7 @@ Part of [Interpreting Your Results](../interpreting-results.md). Read [Before Yo | Gene | Affects | Common Impact | |---|---|---| | CYP2C19 | PPIs, clopidogrel, SSRIs, voriconazole | Rapid metabolizers burn through drugs too fast | -| CYP2D6 | Codeine, tramadol, tamoxifen, many psych meds | Poor metabolizers get toxic buildup | +| CYP2D6 | Codeine, tramadol, tamoxifen, many psych meds | Poor metabolizers get little pain relief from codeine and tramadol, less benefit from tamoxifen, and a toxic buildup of many psych meds; ultrarapid metabolizers risk opioid toxicity from codeine and tramadol | | CYP2C9 | Warfarin, NSAIDs, phenytoin | Dose adjustment needed | | DPYD | 5-fluorouracil (cancer drug) | Poor metabolizers can die from standard doses | | SLCO1B1 | Statins (simvastatin, atorvastatin) | Increased myopathy risk | diff --git a/docs/results/secondary-findings.md b/docs/results/secondary-findings.md index 5f355c585..5afe3222e 100644 --- a/docs/results/secondary-findings.md +++ b/docs/results/secondary-findings.md @@ -51,7 +51,7 @@ The TTN rule (truncating variants only) and the LMNA rule (missense only with ca **Where to look:** `${SAMPLE}/clinical/${SAMPLE}_clinical_summary.tsv`, one row per variant with gene, impact, scores and gnomAD constraint. -Step 23 keeps the HIGH-impact variants, the MODERATE ones below 1% in gnomAD exomes, the ClinVar pathogenic ones and, when step 30 ran, the variants that CADD, SpliceAI, REVEL or AlphaMissense flag. Expect a few hundred rows. Most are heterozygous variants in genes that tolerate one broken copy. Read the rows in constrained genes (low LOEUF, see [gene constraint](clinvar.md#gnomad-gene-constraint-step-23-summary)) first, then the homozygous ones. The HIGH-impact rows have no frequency filter, so check gnomAD for each before you worry. [Step 23](../23-clinical-filter.md#what-gets-filtered) gives the exact rules. +Step 23 keeps the HIGH-impact and MODERATE variants below 1% in every population VEP reports (`MAX_AF`), the HIGH and MODERATE ones with no known frequency (in a tier of their own), the ClinVar pathogenic ones at any frequency and, when step 30 ran, the variants that CADD, SpliceAI, REVEL or AlphaMissense flag. Expect a few hundred rows. Most are heterozygous variants in genes that tolerate one broken copy. Read the rows in constrained genes (low LOEUF, see [gene constraint](clinvar.md#gnomad-gene-constraint-step-23-summary)) first, then the homozygous ones. A row of unknown frequency is not rare, so check gnomAD for it before you worry. [Step 23](../23-clinical-filter.md#what-gets-filtered) gives the exact rules. ## Variant Prioritization (Step 31) From 2b3ac476ec03935360ada0a524990e9ef6c2ea29 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:43:54 +0100 Subject: [PATCH 202/255] fix(clinical): keep an unsplit VEP record when any of its ALTs is a hit Step 13 writes one ALT per record, but an input it did not split would lose the hit entirely under an exact ALT match. The e2e case gains such a record. Bead: clin-22 --- docs/23-clinical-filter.md | 2 +- scripts/23-clinical-filter.sh | 8 ++++++-- tests/e2e/clinvar-tier-allele-match.sh | 14 +++++++++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/23-clinical-filter.md b/docs/23-clinical-filter.md index ab3deb059..0a6377446 100644 --- a/docs/23-clinical-filter.md +++ b/docs/23-clinical-filter.md @@ -53,7 +53,7 @@ The same impacts with no frequency value. Their own tier, not rare (see above). ### ClinVar pathogenic/likely pathogenic, at any frequency - Preferred source: the step 6 hits file, built from the ClinVar file in `clinvar/` that `setup.sh` refreshes. The tier holds the VEP records whose allele is a hit: the same CHROM, POS, REF and ALT. - Another allele at a hit's position is not in the tier. A call of T>A,C splits into two records at one position; when ClinVar classified only T>A, the T>C record stays out. -- Step 6 and step 13 both split and left-align the sample against the same reference, so one allele is written the same way in the hits and in the VEP output. +- Step 6 and step 13 both split and left-align the sample against the same reference, so one allele is written the same way in the hits and in the VEP output, one ALT per record. A VEP record with several ALTs (an input step 13 did not split) is kept when any of its ALTs is a hit. - A combined hit of step 6 (two neighbouring records that make one allele, `PGP_JOINED_FROM` in the hits) puts both of its parts in the tier. VEP annotated the parts, not the joined allele. The tier's count then counts each part as one record. - Next, when step 6 has not run: `ClinVar_CLNSIG`, the same ClinVar file step 13 annotated with `--custom` (pathogenic or likely pathogenic, not conflicting). It follows a ClinVar refresh once step 13 runs again. - Last: VEP's `CLIN_SIG`, from the VEP cache release, so a ClinVar refresh never reaches it. The step says which source it used. diff --git a/scripts/23-clinical-filter.sh b/scripts/23-clinical-filter.sh index 8f42653f8..cb71fe958 100755 --- a/scripts/23-clinical-filter.sh +++ b/scripts/23-clinical-filter.sh @@ -284,10 +284,14 @@ if [ "$HAS_HITS" -eq 1 ]; then cut -f1,2 "$ALLELES" | sort -u > "$TARGETS" if [ -s "$TARGETS" ]; then # -T narrows the input to the hits' positions; the awk keeps the records - # whose allele is one of them. + # whose allele is one of them. Step 13 writes one ALT per record; a + # record of an unsplit input is kept when any of its ALTs is a hit. run_in --cpus 4 --memory 4g "${BCFTOOLS_IMAGE}" \ bcftools view -f PASS -T "${C}/${SAMPLE}_clinvar_targets.tsv" "${CONTAINER_INPUT}" \ - | awk -F'\t' 'NR == FNR {want[$0] = 1; next} /^#/ || (($1 "\t" $2 "\t" $4 "\t" $5) in want)' "$ALLELES" - \ + | awk -F'\t' 'NR == FNR {want[$0] = 1; next} /^#/ {print; next} { + n = split($5, alt, ",") + for (i = 1; i <= n; i++) if (($1 "\t" $2 "\t" $4 "\t" alt[i]) in want) {print; next} + }' "$ALLELES" - \ | run_in -i --cpus 2 --memory 2g "${BCFTOOLS_IMAGE}" bcftools view -Oz -o "${C}/${SAMPLE}_clinvar_unranked.vcf.gz" else echo " Step 6 found no ClinVar hit." diff --git a/tests/e2e/clinvar-tier-allele-match.sh b/tests/e2e/clinvar-tier-allele-match.sh index fffa34f72..1537d15c8 100755 --- a/tests/e2e/clinvar-tier-allele-match.sh +++ b/tests/e2e/clinvar-tier-allele-match.sh @@ -6,10 +6,11 @@ # 30000100 C>T a plain hit # 30000200 T>A and T>C one position, two alleles (a split # T>A,C call); only T>A is a hit +# 30000300 G>A,T one unsplit record; only G>T is a hit # The step 6 hits are the combined allele (30000038 GCCAT>CATGGG, a position # neither part has, with PGP_JOINED_FROM naming the parts), 30000100 C>T and -# 30000200 T>A. The tier must hold the two parts, C>T and T>A: four records, -# and not T>C. Every record is common and MODIFIER, so no other tier keeps +# 30000200 T>A and 30000300 G>T. The tier must hold the two parts, C>T, T>A +# and the unsplit record: five records, and not T>C. Every record is common and MODIFIER, so no other tier keeps # one, and the summary must not list T>C either. . "$(dirname "$0")/lib.sh" @@ -35,6 +36,7 @@ row() { printf 'chr22\t%s\t.\t%s\t%s\t50\tPASS\t%s\tGT:GQ:DP:AD\t0/1:50:30:15,15 row 30000100 C T T row 30000200 T A A row 30000200 T C C + printf 'chr22\t30000300\t.\tG\tA,T\t50\tPASS\t%s\tGT:GQ:DP:AD\t1/2:50:30:0,15,15\n' "$(csq T)" } > "${GENOME_DIR}/${T}/vep/${T}_vep.vcf" { @@ -47,6 +49,7 @@ row() { printf 'chr22\t%s\t.\t%s\t%s\t50\tPASS\t%s\tGT:GQ:DP:AD\t0/1:50:30:15,15 printf 'chr22\t30000038\t9900003\tGCCAT\tCATGGG\t.\tPASS\tCLNSIG=Pathogenic;PGP_JOINED_FROM=chr22:30000037:AGC:A|chr22:30000042:T:TGGG\tGT\t0/1\n' printf 'chr22\t30000100\t9900009\tC\tT\t.\tPASS\tCLNSIG=Pathogenic\tGT\t0/1\n' printf 'chr22\t30000200\t9900011\tT\tA\t.\tPASS\tCLNSIG=Likely_pathogenic\tGT\t1/.\n' + printf 'chr22\t30000300\t9900013\tG\tT\t.\tPASS\tCLNSIG=Pathogenic\tGT\t1/.\n' } > "${GENOME_DIR}/${T}/clinvar/${T}_clinvar_hits.vcf" run_step 23-clinical-filter.sh "$T" @@ -62,12 +65,13 @@ check_eq "the other allele at that position, T>C, is not" "$(has_allele chr22:30 check_eq "the plain hit chr22:30000100 C>T is in the tier" "$(has_allele chr22:30000100:C:T)" 1 check_eq "the combined hit's first part chr22:30000037 AGC>A is in the tier" "$(has_allele chr22:30000037:AGC:A)" 1 check_eq "the combined hit's second part chr22:30000042 T>TGGG is in the tier" "$(has_allele chr22:30000042:T:TGGG)" 1 -check_eq "the tier holds exactly those four records" "$(vcf_count "$TIER")" 4 -check "the log counts four ClinVar records" has 'ClinVar pathogenic/LP: +4 ' "$(cat "$STEP_LOG")" +check_eq "the unsplit record chr22:30000300 G>A,T, with G>T a hit, is in the tier" "$(has_allele chr22:30000300:G:A,T)" 1 +check_eq "the tier holds exactly those five records" "$(vcf_count "$TIER")" 5 +check "the log counts five ClinVar records" has 'ClinVar pathogenic/LP: +5 ' "$(cat "$STEP_LOG")" SUMMARY="${GENOME_DIR}/${T}/clinical/${T}_clinical_summary.tsv" check "the summary exists" test -s "$SUMMARY" check_eq "the summary has no row for chr22:30000200 T>C" \ "$(awk -F'\t' '$1 == "chr22" && $2 == 30000200 && $3 == "T" && $4 == "C"' "$SUMMARY" | wc -l | tr -d ' ')" 0 -check_eq "the summary lists the four tier records" "$(awk 'NR > 1' "$SUMMARY" | wc -l | tr -d ' ')" 4 +check_eq "the summary lists the five tier records" "$(awk 'NR > 1' "$SUMMARY" | wc -l | tr -d ' ')" 5 finish From c62d41f17bb58a8e98bba70de1f2842785e7dbee Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:48:13 +0100 Subject: [PATCH 203/255] test(ancestry): unplant the og-admix red controls after their red run The planted-red feature is off by default again; it stays for anyone who wants to see the controls fail: cargo test -p og-admix --features planted-red --no-fail-fast. Bead: D5 --- rust/crates/og-admix/Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust/crates/og-admix/Cargo.toml b/rust/crates/og-admix/Cargo.toml index f9c378653..cf646e56e 100644 --- a/rust/crates/og-admix/Cargo.toml +++ b/rust/crates/og-admix/Cargo.toml @@ -7,8 +7,6 @@ license.workspace = true publish.workspace = true [features] -# Red-first run: every control planted, so CI shows each test fail. Reverted in the next commit. -default = ["planted-red"] # Feeds each red control its broken input in place of the good one # (a shuffled map, T doubled, a flipped byte, a swapped sample order, a # second seed), so `cargo test -p og-admix --features planted-red From 226fda359111cdf9a4fa61eab67cb39c0b273df7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:48:55 +0100 Subject: [PATCH 204/255] test(live): give the ops tests explicit warn percents With OWNGENOME_MEM_WARN_PERCENT or OWNGENOME_DISK_WARN_PERCENT set in the shell, the Rust card read them and the Python oracle did not, so four checks failed. The tests now pass 90 to both sides (or the case's own disk percent); the default path stays covered in ops_headroom.rs. Bead: D1 --- rust/crates/og-live/tests/ops.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/rust/crates/og-live/tests/ops.rs b/rust/crates/og-live/tests/ops.rs index 5e2823658..cc4b4c967 100644 --- a/rust/crates/og-live/tests/ops.rs +++ b/rust/crates/og-live/tests/ops.rs @@ -192,13 +192,15 @@ fn case(now: f64) -> Case { } } +/// The warn percents are given (90 unless the case says otherwise), so an +/// `OWNGENOME_*_WARN_PERCENT` of the shell running the tests changes nothing. fn cfg_for(t: &Tree, c: &Case) -> OpsConfig { OpsConfig { root: Some(t.root.clone()), eta_box: Some("opsbox".into()), eta_cores: Some(8), - disk_warn_percent: c.disk_warn_percent.map(str::to_owned), - mem_warn_percent: None, + disk_warn_percent: Some(c.disk_warn_percent.unwrap_or("90").to_owned()), + mem_warn_percent: Some("90".into()), stall_minutes: Some(c.stall_minutes), } } @@ -809,7 +811,7 @@ state = run_state.build(a["sample"], steps, status, None, [], run_state.event_ta if a["stopped"]: state["run"]["state"] = "stopped" cfg = types.SimpleNamespace(root=a["root"], steps=run_state.STEPS_TSV, eta_box="opsbox", eta_cores=8, - disk_warn_percent=a["dwp"], mem_warn_percent=None, stall_minutes=a["stall"]) + disk_warn_percent=a["dwp"], mem_warn_percent="90", stall_minutes=a["stall"]) o = ops.run_ops(a["run_dir"], state, events, cfg, now=a["now"], steps=steps, physical=a["physical"]) sys.stdout.write(json.dumps(o) + "\n---\n" + ops.metrics_text([(a["sample"], o)], 0.9, a["physical"], 0)) "#; @@ -832,8 +834,7 @@ fn py_ops(t: &Tree, c: &Case) -> (Value, String) { put("stall", Value::Float(c.stall_minutes)); put( "dwp", - c.disk_warn_percent - .map_or(Value::Null, |p| Value::Str(p.into())), + Value::Str(c.disk_warn_percent.unwrap_or("90").into()), ); let arg = pycompat::json::dumps(&Value::Object(args), &pycompat::json::Dump::default()).unwrap(); From e9ae6a55cf72282c67925bef3222247a9b03ee97 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:50:16 +0100 Subject: [PATCH 205/255] test(og-dev): pin which half of the HBA2 reads hba-tiles drops The pin ran on a window with no read over HBA2, so dropping the other half of them passed. The test now checks the first read goes and the second stays, as the inline Python did. Bead: D1 --- .../crates/og-dev/src/fixture/e2e/hba_tiles.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/rust/crates/og-dev/src/fixture/e2e/hba_tiles.rs b/rust/crates/og-dev/src/fixture/e2e/hba_tiles.rs index fecf78d52..a33620407 100644 --- a/rust/crates/og-dev/src/fixture/e2e/hba_tiles.rs +++ b/rust/crates/og-dev/src/fixture/e2e/hba_tiles.rs @@ -113,20 +113,22 @@ mod tests { #[test] fn a_sample_ending_in_3_loses_every_other_read_over_hba2() { let seq = PyRandom::new(9).bases(80_000); - let over = |r: &str| { + // The 0-based starts of the reads over HBA2. + let over = |r: &str| -> Vec { r.lines() - .filter(|l| { - let s: usize = l.split('\t').nth(3).unwrap().parse::().unwrap() - 1; - s < HBA2.1 && s + L > HBA2.0 - }) - .count() + .map(|l| l.split('\t').nth(3).unwrap().parse::().unwrap() - 1) + .filter(|&s| s < HBA2.1 && s + L > HBA2.0) + .collect() }; let (four, three) = ( records(&seq, "HG002hba4", None), records(&seq, "HG002hba3", None), ); - assert_eq!(over(&four), 196); - assert_eq!(over(&three), 98); + let (all, kept) = (over(&four), over(&three)); + assert_eq!(all.len(), 196); + // As the Python did: the first read over HBA2 goes, the second stays. + let even: Vec = all.iter().skip(1).step_by(2).copied().collect(); + assert_eq!(kept, even); assert_eq!(four.lines().count() - three.lines().count(), 98); assert!(!four.contains("RG:Z")); } From 45046fcf1ebc7a5a4fb6dae2dc832490db3ef119 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:50:38 +0100 Subject: [PATCH 206/255] docs(rust): say every og binary links htslib, and that a tag checkout gets the notices as a release asset --- docs/rust.md | 8 ++++---- rust/crates/og-dev/src/licences.rs | 4 ++-- rust/og.sh | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/rust.md b/docs/rust.md index f8bbd0085..4c05e0b3e 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -57,12 +57,12 @@ The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, ` `scripts/setup.sh ` does the same when `og` is missing or stale, after it pulls the images. `--og` builds only `og`. This puts four binaries in `bin/.og/` (ignored by git): - `og-linux-x86_64` and `og-linux-aarch64`, two static musl builds for the pipeline's containers. Both are kept because an Apple Silicon host runs amd64 images under emulation, where `uname -m` says `x86_64`. -- `og-darwin-arm64` and `og-darwin-x86_64`, for a Mac that hosts the live site (`scripts/owngenome-live.sh`), whose genome map runs `og map` on the host. They are built with `--no-default-features`, which leaves out the cargo feature `bam`: that host never reads a BAM, and BAM reading needs htslib, which is C and would have to cross-compile for Apple. The BAM tools (`og pgx cyp2d6-*`, `og str motif`, `og cdna check|scan`, `og comphet phase`, `og bam contigs`, `og bam-fetch-probe`) say so and exit 2 there. Every Linux build has the feature, since it is on by default. - -Beside them the build writes `bin/.og/THIRD-PARTY-LICENSES.txt`, which every release ships. The binaries link about seventy crates statically (MIT, Apache-2.0, BSD-3-Clause, ISC, Zlib, Unicode-3.0 and others), and `hts-sys` builds the C libraries htslib and htscodecs into them; those licences ask that a binary carry their notices. The new `og` writes the file itself inside `OG_BUILDER`: `og dev licences notices` asks `cargo metadata --filter-platform` for each of the four targets (the macOS pair without default features, as it is built) and follows the normal dependencies from `og`, so build and dev dependencies and procedural macros, which leave no code in the binary, are not listed. The file lists every crate with its version and licence, and then each distinct licence text once. The texts come from each crate's own LICENSE, COPYING, NOTICE and similar files; a crate that builds C (it sets `links`) also gives the ones in its folders, which is how htslib's `htslib/LICENSE` and htscodecs' `htslib/htscodecs/LICENSE.md` get in without a copy that could go stale. Two parts of the binaries are no crate, so `rust/og.sh` names their notices with `--extra NAME=FILE`: the Rust standard library (the toolchain's `COPYRIGHT-library.html`, turned into text) and musl, the C library of the Linux pair (musl's `COPYRIGHT`, which the builder image's zig ships). A licence file that cannot be read stops the build. A crate is credited for every target whose dependency graph reaches it, even where the linker then drops its unused code. The build stops when a crate's licence is outside the allow list (`ALLOWED` in `rust/crates/og-dev/src/licences.rs`: 0BSD, Apache-2.0, Apache-2.0 WITH LLVM-exception, BSD-3-Clause, ISC, MIT, Unicode-3.0, Unlicense, Zlib), so a new dependency under another licence needs a decision first. A crate that ships no licence file gets the MIT text with its authors when MIT is among its licences, and stops the build otherwise. +- `og-darwin-arm64` and `og-darwin-x86_64`, for a Mac that hosts the live site (`scripts/owngenome-live.sh`), whose genome map runs `og map` on the host. They are built with `--no-default-features`, which leaves out the cargo feature `bam`: that host never reads a BAM. The feature only turns off the BAM tools; htslib is still linked, through `og-summary`. The BAM tools (`og pgx cyp2d6-*`, `og str motif`, `og cdna check|scan`, `og comphet phase`, `og bam contigs`, `og bam-fetch-probe`) say so and exit 2 there. Every Linux build has the feature, since it is on by default. - **From a checkout of `main` or a branch,** it builds them from `rust/` inside `OG_BUILDER`, a pinned [cargo-zigbuild](https://github.com/rust-cross/cargo-zigbuild) image (`versions.env`). Docker is the only thing it needs; no Rust on the host. The cargo cache lives in `rust/target/og-build/`. -- **From a checkout of a release tag** (`vX.Y.Z`), it downloads the release's four binaries, checks them against the release's `SHA256SUMS` and, when `gh` is installed and logged in, against the build attestation (`gh attestation verify`). A release without binaries falls back to a build. +- **From a checkout of a release tag** (`vX.Y.Z`), it downloads the release's four binaries, checks them against the release's `SHA256SUMS` and, when `gh` is installed and logged in, against the build attestation (`gh attestation verify`). A release without binaries falls back to a build. A tag checkout downloads only the four binaries; `THIRD-PARTY-LICENSES.txt` is a separate asset of the release. + +Beside the binaries a build writes `bin/.og/THIRD-PARTY-LICENSES.txt`, which every release ships. The binaries link about seventy crates statically (MIT, Apache-2.0, BSD-3-Clause, ISC, Zlib, Unicode-3.0 and others), and `hts-sys` builds the C libraries htslib and htscodecs into them; those licences ask that a binary carry their notices. The new `og` writes the file itself inside `OG_BUILDER`: `og dev licences notices` asks `cargo metadata --filter-platform` for each of the four targets (the macOS pair without default features, as it is built) and follows the normal dependencies from `og`, so build and dev dependencies and procedural macros, which leave no code in the binary, are not listed. The file lists every crate with its version and licence, and then each distinct licence text once. The texts come from each crate's own LICENSE, COPYING, NOTICE and similar files; a crate that builds C (it sets `links`) also gives the ones in its folders, which is how htslib's `htslib/LICENSE` and htscodecs' `htslib/htscodecs/LICENSE.md` get in without a copy that could go stale. Two parts of the binaries are no crate, so `rust/og.sh` names their notices with `--extra NAME=FILE`: the Rust standard library (the toolchain's `COPYRIGHT-library.html`, turned into text) and musl, the C library of the Linux pair (musl's `COPYRIGHT`, which the builder image's zig ships). A licence file that cannot be read stops the build. A crate is credited for every target whose dependency graph reaches it, even where the linker then drops its unused code. The build stops when a crate's licence is outside the allow list (`ALLOWED` in `rust/crates/og-dev/src/licences.rs`: 0BSD, Apache-2.0, Apache-2.0 WITH LLVM-exception, BSD-3-Clause, ISC, MIT, Unicode-3.0, Unlicense, Zlib), so a new dependency under another licence needs a decision first. A crate that ships no licence file gets the MIT text with its authors when MIT is among its licences, and stops the build otherwise. For day-to-day work on the Rust code a host toolchain is quicker: `cd rust && cargo test`. A plain `cargo build` stamps the binary `unknown`, which the check below refuses, so use `scripts/setup.sh --og` for a binary the pipeline will run. diff --git a/rust/crates/og-dev/src/licences.rs b/rust/crates/og-dev/src/licences.rs index 9bff4a307..e3d4fe1ed 100644 --- a/rust/crates/og-dev/src/licences.rs +++ b/rust/crates/og-dev/src/licences.rs @@ -6,8 +6,8 @@ //! [--out FILE] //! ``` //! -//! The og binaries link about seventy crates statically, and the Linux pair -//! also links htslib and htscodecs (C, built by `hts-sys`). Their licences +//! The og binaries link about seventy crates statically, and all four also +//! link htslib and htscodecs (C, built by `hts-sys`). Their licences //! (MIT, BSD-3-Clause, Unicode-3.0 and others) ask that a binary carry their //! notices, so every release ships the file this verb writes //! (`rust/og.sh build`, `.github/workflows/release.yml`). diff --git a/rust/og.sh b/rust/og.sh index c190044e1..a3c4353fa 100755 --- a/rust/og.sh +++ b/rust/og.sh @@ -175,7 +175,7 @@ build() { t0=$(date +%s) # Two cargo runs, because the feature set is per run: Linux with BAM (the # default features), macOS without it (zig links against the image's - # macOS SDK; no C is compiled). + # macOS SDK). zigbuild() { "$ENGINE" run --rm --user "$(id -u):$(id -g)" \ -e HOME=/cache/home -e CARGO_HOME=/cache/cargo -e CARGO_TARGET_DIR=/cache/target -e OG_TREE_HASH="$hash" \ From 229737d927ffe1aef878a0c2898f6f1238c9e6d4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:51:48 +0100 Subject: [PATCH 207/255] fix(live): decide who is a child from the family files on each request, and keep main's views The person page now reads who is a child from state.json's family plugin, which the server computes from the family files on every request by one rule (child_of): a child in any family is one, and while a family folder cannot be read, the people it might list are closed alike on their page, home, the family page and the finding route, and reopen as soon as the files read again. An adult's page reads only the state through the gate. A child's results answer is checked on the text the gate holds, so answers reach the views in the order they arrived. The map's tooltip starts hidden="hidden" again, as on main. On the family page a released child shows summary.json's count (results.json's saved_attention), adults keep their own member line, and a gated partner keeps the partner role and the couple line. --- assets/live/core.js | 80 +++++++++++++++++----------- assets/live/family-page.js | 19 +++++-- assets/live/map.js | 16 +++--- assets/live/person.js | 6 ++- bin/live_results.py | 12 +++-- bin/owngenome_live.py | 28 ++++++---- bin/owngenome_live_family.py | 99 ++++++++++++++++++++++------------- bin/owngenome_live_results.py | 5 ++ tests/demo/build_live_demo.py | 4 ++ 9 files changed, 174 insertions(+), 95 deletions(-) diff --git a/assets/live/core.js b/assets/live/core.js index 0728fb4a8..ed73d8a2d 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -249,7 +249,8 @@ function makeRecord(url, child, r, old) { var acts = actsOf(r), rec = { url: url, child: child, acts: acts, opened: {}, order: [], perm: [], ids: r && Array.isArray(r.findings) ? r.findings.map(function (f) { return f && f.id; }) : [], - attention: r && typeof r.attention === 'number' ? r.attention : null }; + attention: r && typeof r.attention === 'number' ? r.attention : null, + saved: r && typeof r.saved_attention === 'number' ? r.saved_attention : null }; if (!acts) { return rec; } // An act whose record changed is closed again; an unchanged one keeps its opening for this page. rec.order = old ? old.order.filter(function (id) { return same(byId(old, id), byId(rec, id)); }) : []; @@ -378,13 +379,15 @@ }); }); } - // The person and finding pages: the run's state, its family's roles and, for a child, its results, read in the - // order the views read them (the finding page asks for its results at once, the person page once the state has - // answered) and with the views' options. The family is read once the page's scripts have run, so live.js has - // opened its event stream first, as it does without the gate. In live mode the person page asks for the family and - // then its results on its first state, both at once: the gate asks for both in that order before it knows whether - // the person is a child, so the two answers race as they do without the gate and reach the views in the order they - // arrived. Only a child's results are read by the gate; an adult's answer goes to the views unread. + // The person and finding pages: the run's state and, for a child, its results, read in the order the views read + // them (the finding page asks for its results at once, the person page once the state has answered) and with the + // views' options. Who is a child is the server's one rule (bin/owngenome_live.py child_of), which state.json's + // family plugin carries (child, or unknown for a run no readable family lists while one cannot be read); a state + // that cannot be read cannot say the person is not a child. An adult's page then reads nothing more through the + // gate: its views ask for the family and the results themselves. For a child the gate reads the results first. Its + // family is asked for once the page's scripts have run, so live.js has opened its event stream first, and in live + // mode its results right after, as the person page asks for both on its first state: the two answers race as they + // do without the gate and reach the views in the order they arrived. var parsed = new Promise(function (done) { if (document.readyState === 'complete') { done(); } else { document.addEventListener('DOMContentLoaded', done); window.addEventListener('load', done); } }); @@ -398,16 +401,14 @@ var raw = x[0]; var state = replaying ? raw && raw.frames && raw.frames.slice(-1)[0] : raw; state = replaying ? state && state.state : state; - var fam = state && state.plugins && state.plugins.family; + var fam = state && state.plugins ? state.plugins.family : undefined; + var child = !state || !state.plugins || !!(fam && (fam.unknown || (fam.kind === 'member' && fam.child !== false))); var entry = fam && fam.kind === 'member' && (fam.families || [])[0]; - if (!entry) { return !state || !state.plugins || !!(fam && fam.unknown); } - var famRead = sharedJSON(absolute(entry.url, url) + 'family.json', NO_STORE); - if (!replaying && !findingPath) { pre = keep(url + 'results.json', NO_STORE); } - return famRead.then(function (f) { - var member = f && f.state && (f.state.members || []).filter(function (m) { return m.sample === root.getAttribute('data-run'); })[0]; - // A family that cannot be read, or does not list this person, cannot say they are not a child. - return !member || !!(member.father || member.mother); - }); + if (child && entry) { + keep(absolute(entry.url, url) + 'family.json', NO_STORE); + if (!replaying && !findingPath) { pre = keep(url + 'results.json', NO_STORE); } + } + return child; }).then(function (child) { if (!child && !results) { return { url: url, child: false, acts: [], opened: {}, order: [], perm: [] }; } return (results || record(url, child, true, pre)).then(function (rec) { rec.child = child; return rec; }); @@ -434,11 +435,12 @@ var PROGRESS = ['name', 'kind', 'url', 'state', 'started_utc', 'last_heard_utc', 'finished_utc', 'planned', 'done', 'running', 'failed']; function progressOf(r) { var out = {}; PROGRESS.forEach(function (k) { if (r[k] !== undefined) { out[k] = r[k]; } }); return out; } // A released child's flagged-card count: family.json leaves it out (null) while the server keeps the child's report - // closed, so the page takes it from the results it read, once every result to act on is opened. + // closed, so once every result to act on is opened the page takes the count family.json gives without the gate, + // that of the child's summary.json, which the child's results.json carries as saved_attention. function released(raw, recs) { Object.keys(recs).forEach(function (k) { var r = raw.runs && raw.runs[k]; - if (r && r.attention === null && !shut(recs[k]) && recs[k].attention !== null) { r.attention = recs[k].attention; } + if (r && r.attention === null && !shut(recs[k])) { r.attention = recs[k].saved; } }); return raw; } @@ -455,17 +457,23 @@ layout: raw.layout ? { width: raw.layout.width, height: raw.layout.height, size: raw.layout.size, lines: copy(raw.layout.lines), people: (raw.layout.people || []).map(function (m) { return { sample: m.sample, sex: m.sex, sequenced: m.sequenced, x: m.x, y: m.y }; }) } : null, couple_note: raw.couple_note, chrom_lengths: copy(raw.chrom_lengths), gated: closed, runs: {} }; - // A gated person this family lists as a parent or partner (a child in another family) gets their buttons here too. - f.state.children = (raw.state.members || []).filter(function (m) { return m.father || m.mother || gated(m.sample); }).map(function (m) { + // Each person keeps the role this family gives them: its children are the ones it lists with a father or a + // mother. A gated person it lists as a parent or partner (a child in another family) is in gated alone, and the + // view gives them their buttons beside the children. + f.state.children = (raw.state.members || []).filter(function (m) { return m.father || m.mother; }).map(function (m) { return gated(m.sample) ? { child: m.sample, gated: true } : copy((raw.state.children || []).filter(function (c) { return c.child === m.sample; })[0]); }).filter(Boolean); // A row belongs to a gated child when it names that child as one of its members, by exact name. function names(v) { return String(v || '').split(/[\s,;]+/); } - f.state.couples = copy((raw.state.couples || []).filter(function (c) { return !gated(c.a) && !gated(c.b); })); + // A couple with a gated partner keeps its names and whether it was screened; its screen is that partner's data. + f.state.couples = (raw.state.couples || []).map(function (c) { + if (!gated(c.a) && !gated(c.b)) { return copy(c); } + return c.screened ? { a: c.a, b: c.b, closed: true } : { a: c.a, b: c.b, report: null, table: null, screened: false, shared: 0, x_linked: 0 }; + }); // The parents' own items stay: the plan rows that name no gated child, and the couple reports of couples with // no gated partner. Any other report of the family folder may hold a child's data and stays out. f.state.plan = copy((raw.state.plan || []).filter(function (r) { return !names(r.members).some(gated); })); - var reports = f.state.couples.map(function (c) { return c.report; }).filter(Boolean); + var reports = f.state.couples.map(function (c) { return c.closed ? null : c.report; }).filter(Boolean); f.state.reports = copy((raw.state.reports || []).filter(function (r) { return reports.indexOf(r.url) >= 0; })); function free(r) { return !gated(r.sample_a) && !gated(r.sample_b) && !gated(r.a) && !gated(r.b); } f.relationships = copy((raw.relationships || []).filter(free)); @@ -499,14 +507,13 @@ } return f; } - // Who is a child, one rule for the person page (preparePerson: the first of state.json's families, then its - // family.json), this home page and the server (bin/owngenome_live.py child_of): the first family, by name, that - // lists the person decides, by a father or a mother there; a person no readable family lists is a child while - // some family's members cannot be read. + // Who is a child, the server's one rule (bin/owngenome_live.py child_of) over the families of runs.json: a run any + // readable family lists with a father or a mother; while a family cannot be read, every run while its members + // cannot be read (an empty member list) and the runs it lists while another of its files cannot be read. function childIn(fams, name) { - function me(f) { return (f.members || []).filter(function (m) { return m.sample === name; })[0]; } - var f = fams.filter(me)[0]; - return f ? !!(me(f).father || me(f).mother) : fams.some(function (x) { return !(x.members || []).length; }); + function listed(f) { return (f.members || []).filter(function (m) { return m.sample === name; }); } + if (fams.some(function (f) { return listed(f).some(function (m) { return m.father || m.mother; }); })) { return true; } + return fams.some(function (f) { return !(f.members || []).length || (f.unreadable && listed(f).length > 0); }); } // The home page decides from runs.json alone: a child's summary is closed when runs.json counts or names a result // to act on (the server counts a child's from the summary its results.json reads). @@ -554,7 +561,7 @@ if (ownsPage) { return new Promise(function () {}); } var path = new URL(requestURL).pathname; if (response.ok && current && current.child && path === new URL(current.url + 'results.json').pathname) { - return response.clone().json().then(function (r) { + var check = function (r) { var next = makeRecord(current.url, true, r, current); if (!same(next.acts, current.acts)) { current = records[current.url] = next; @@ -562,7 +569,14 @@ if (shut(current)) { ownsPage = true; eventSources.forEach(function (es) { es.close(); }); cleanPage(); transfer(); return new Promise(function () {}); } } return viewResponse(response); - }); + }; + // A kept answer is checked on the text the gate read, in this task, so it reaches the view in its turn + // and no answer handed after it overtakes it. + var text = kept && kept.text; + if (typeof text === 'string') { + try { return check(JSON.parse(text)); } catch (e) { /* not JSON: read as the view would */ } + } + return response.clone().json().then(check); } if (response.ok && projected(path)) { return response.json().then(function (data) { return loadedJSON(requestURL, data); }).then(function (data) { @@ -605,6 +619,8 @@ window.EventSource.CLOSED = NativeEvents.CLOSED; window.PGPUI.ready = ready; window.PGPUI.gateOwnsPage = function () { return ownsPage; }; + // Whether the gate treats this page's person as a child (the server's one rule), for person.js's closed page. + window.PGPUI.gateChild = function () { return !!(current && current.child); }; window.PGPUI.closedText = CLOSED_TEXT; // The acts the gate released, for person.js's map switches: every act of a child's open report, else none. window.PGPUI.openedActs = function () { return current && !shut(current) ? copy(current.opened) : {}; }; diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 26a10a130..a3a8072bb 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -52,6 +52,7 @@ var seq = (st.members || []).filter(function (m) { return m.sequenced; }).length; out.push([seq + ' genome' + (seq === 1 ? '' : 's') + ' of this family are read together here. ', false]); (st.couples || []).forEach(function (c) { + if (c.closed) { out.push([c.a + ' and ' + c.b + '’s carrier screen stays closed in chapter III. ', false]); return; } if (!c.screened) { out.push([c.a + ' and ' + c.b + ' are not screened as a couple yet. ', false]); return; } if (c.shared) { out.push([c.a + ' and ' + c.b + ' both carry a change in ' + c.shared + ' gene' + (c.shared > 1 ? 's' : '') + ': chapter III gives the chance per pregnancy for each. ', true]); } else { out.push([c.a + ' and ' + c.b + ' share no carrier gene in what the screen sees; ', false], ['it does not see everything. ', false]); } @@ -138,11 +139,11 @@ var state = !m.sequenced ? 'Not sequenced' : !r ? 'No run on this server' : r.state === 'finished' ? 'Finished' : (r.state === 'stopped' ? 'Stopped, ' : '') + r.done + ' of ' + r.planned + ' steps'; add(txt, el('span', { 'class': 'person-state' }, state)); - // A child whose report is closed gets the fixed closed line. A count left out (null: the server keeps the - // child's report closed and the page has not opened it) is never read as nothing flagged. + // A person whose report is closed gets the fixed closed line, never a count; every other member the line + // it has without the gate. if (r && r.gated) { add(txt, el('span', { 'class': 'person-look' }, 'Report closed')); - } else if (r && r.results && r.attention !== null && r.attention !== undefined) { + } else if (r && r.results) { add(txt, el('span', { 'class': 'person-look' + (r.attention ? ' attn' : '') }, r.attention ? r.attention + ' worth a look' : 'Nothing flagged; see the caveats')); } @@ -173,6 +174,12 @@ if (!f.state.couples.length) { add(s, el('p', { 'class': 'dim' }, 'The pedigree names no partners.')); } f.state.couples.forEach(function (c) { var p = el('p', { 'data-couple': c.a + ' ' + c.b, 'class': 'couple-line' }); + // A couple with a partner whose report is closed (core.js): the screen is that partner's data. + if (c.closed) { + p.textContent = c.a + ' and ' + c.b + ': the carrier screen stays closed while a partner’s report is closed.'; + add(s, p); + return; + } if (!c.screened) { p.textContent = c.a + ' and ' + c.b + ': not screened yet (step 39).'; add(s, p); @@ -398,6 +405,12 @@ } add(s, d); }); + // A person whose report is closed and whom this family lists as a parent or partner (a child in another family) + // gets their buttons here too, after the children. + (f.gated || []).forEach(function (name) { + if (f.state.children.some(function (c) { return c.child === name; })) { return; } + add(s, el('h3', {}, name), U.familyControls(name, fetchFamily)); + }); return s; } diff --git a/assets/live/map.js b/assets/live/map.js index d66dd078c..3ec9e53df 100644 --- a/assets/live/map.js +++ b/assets/live/map.js @@ -8,11 +8,13 @@ // stretch, the runs in view, how to move, and why a layer is not on the map. // // The veil. The document is the same for every reader: each row a closed result could reveal carries the id -// of that result (veil). Whether it is closed is the page's question (person.js veil()): a closed row is not -// drawn, has no hit target, no tooltip, no line in the margin and never reaches the address. The closed -// results are counted in a dashed chip that names no gene and no place, with an Open button that asks once, -// then opens the result through the page's own function, so the map and the card open together. A reload -// closes them again: the open state lives in the page's memory only. +// of that result (veil). Whether it is closed is the page's question (person.js mapVeil): a closed row is not +// drawn, has no hit target, no tooltip, no line in the margin and never reaches the address. A child's report +// never reaches this figure while one of its results to act on is unopened: the page's one gate (core.js) holds +// the whole page. So on a page the gate has released, a result to act on is opened, with a chip that closes it +// again through the page (which hands the page back to the gate), and the rows closed here are a gated result +// (APOE, behind its warning) until its own chip opens it, and a row whose result is not among the page's results. +// A reload closes them again: the open state lives in the page's memory only. // // No library and no build step: one canvas, the camera {ci, c, ls, k} (open chromosome, centre bp, natural // log of the span, 0 genome .. 1 open) eased toward its target each frame, one level() by bp per CSS pixel. @@ -165,9 +167,7 @@ add(head, kicker, add(el('div', { 'class': 'gm-titlerow' }), title, gauge, zoom), crumb, finder, chips); var stage = el('div', { 'class': 'gm-stage' }); var canvas = el('canvas', { 'class': 'gm-canvas', tabindex: '0', role: 'img', 'aria-describedby': 'gm-narration' }); - // hidden as hideTip() writes it (tip.hidden = true gives hidden=""), so the drawing is the same whether or not a - // refresh hid the tooltip before the document arrived. - var tip = el('div', { 'class': 'gm-tip', role: 'tooltip', hidden: '' }); + var tip = el('div', { 'class': 'gm-tip', role: 'tooltip', hidden: 'hidden' }); var toast = el('div', { 'class': 'gm-toast', role: 'status', hidden: 'hidden' }); add(stage, canvas, tip, toast); // The marks drawn, as links: the keyboard and screen-reader path to each finding (Tab steps through them). diff --git a/assets/live/person.js b/assets/live/person.js index 1a05b90fc..252563771 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -108,7 +108,11 @@ function isChild() { var m = me(); return !!(m && (m.father || m.mother)); } // A page whose results to act on were opened on purpose (core.js releases a child's report, or a person whose // family roles are not known, only once each is opened): the map marks them opened, with a chip to close them. - function closedPage() { return isChild() || familyStatus === 'pending' || familyStatus === 'failed' || (familyStatus === 'known' && !me()); } + // The gate's own answer counts too: a child in any family, by the server's one rule, not only in the first. + function closedPage() { + return isChild() || familyStatus === 'pending' || familyStatus === 'failed' || (familyStatus === 'known' && !me()) || + !!(U.gateChild && U.gateChild()); + } function memberUrl(s) { var r = family && family.data.runs && family.data.runs[s]; return r && r.url ? r.url : null; diff --git a/bin/live_results.py b/bin/live_results.py index 3f0a3a603..e9e1af597 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -1204,16 +1204,18 @@ def veil_id(spans, text="", chrom=None, pos=None, end=None): def veil(summary, found, chapters, lead_parts=()): - """The rows a child's page closes with a result to act on, by the result's id: each card's table rows + """Which rows outside a result's own card name a result to act on, by the result's id: each card's table rows (rows: card -> one id or None per row; the ClinVar card's by each hit's own position), each card's viewer links (viewer: card -> one per link), each card's lines (lines: card -> one per line; a known deep intronic site is printed whether hit or not), the marks of the report's charts (marks: chart -> {mark id: id}), every other finding that names a closed result (findings: finding id -> id; a symptom ranking can rank the gene) and the lead's segments that name one (lead: one per segment of LEAD_PARTS, the "act" - names left out: the page closes those itself), each card's facts (facts: card -> one per fact) and the items - of its drawing (widget: card -> one per CHIP call or copy-number dot). spans is closable() itself: the family - page (assets/live/family-page.js) reads which results are closed from it, and closes every place of the child - that could carry one while one is. Empty when nothing can be closed.""" + names left out), each card's facts (facts: card -> one per fact) and the items of its drawing (widget: card + -> one per CHIP call or copy-number dot); spans is closable() itself. A child's page no longer closes these + rows one by one: its one gate (assets/live/core.js) holds the whole report while a result to act on is + unopened and reads the findings of level "act" for it. Of this field only marks is still read, by person.js, + for a chart mark on a page the gate released (the search index and the genome map carry their own tags); the + other subfields stay so that no field of results.json is removed. Empty when nothing can be closed.""" spans = closable(found) if not spans: return {} diff --git a/bin/owngenome_live.py b/bin/owngenome_live.py index 05125276b..7986fe719 100755 --- a/bin/owngenome_live.py +++ b/bin/owngenome_live.py @@ -106,16 +106,20 @@ def valid_run(name): def child_of(families, name): - """Whether NAME's pages treat it as a child, the one rule of assets/live/core.js (childIn, preparePerson): - the first family, by name, whose members list NAME decides, by a father or a mother there; a run no - readable family lists is a child while some family's members cannot be read (an empty member list). - families: [{"name", "members": [{"sample", "father", "mother"}]}], as /runs.json lists them.""" - fams = sorted(families or [], key=lambda f: f.get("name") or "") - for f in fams: - for m in f.get("members") or []: - if m.get("sample") == name: - return bool(m.get("father") or m.get("mother")) - return any(not (f.get("members") or []) for f in fams) + """Whether NAME's pages treat it as a child, the one rule of the person page (state.json's plugins.family.child, + from bin/owngenome_live_family.py member_state), the home page (assets/live/core.js childIn), the family page + (family.json's child) and the finding route: a run that any readable family lists with a father or a mother is a + child, whatever the other families say. While a family cannot be read, a run it might list is treated as a child + too (fail closed): every run while its members cannot be read (an empty member list), and the members it lists + while another of its files cannot be read (unreadable). Computed from the files on each request. + families: [{"name", "members": [{"sample", "father", "mother"}], "unreadable"}], as /runs.json lists them.""" + fams = families or [] + + def listed(f): + return [m for m in f.get("members") or [] if m.get("sample") == name] + if any(m.get("father") or m.get("mother") for f in fams for m in listed(f)): + return True + return any(not (f.get("members") or []) or (f.get("unreadable") and listed(f)) for f in fams) def utc_now(): @@ -924,8 +928,10 @@ def runs_index(self): screened=o.get("screened", False)) else: item["members"] = r.get("members") or [] + item["unreadable"] = bool(r.get("unreadable")) out.append(item) - fams = [{"name": x["name"], "url": x["url"], "members": x["members"]} for x in out if x["kind"] == "family"] + fams = [{"name": x["name"], "url": x["url"], "members": x["members"], "unreadable": x["unreadable"]} + for x in out if x["kind"] == "family"] return {"runs": [x for x in out if x["kind"] != "family"], "families": fams} diff --git a/bin/owngenome_live_family.py b/bin/owngenome_live_family.py index a55c0f616..47725f49c 100644 --- a/bin/owngenome_live_family.py +++ b/bin/owngenome_live_family.py @@ -484,23 +484,71 @@ def member_summary(live, d): return got if isinstance(got, dict) else None -def member_state(root, sample): - """The families that list the sample. A family whose pedigree cannot be read may be the one that - lists it as a child: with no readable family listing it, the answer is unknown, never "in no - family" ({"kind": "member", "families": [], "unknown": true}), and its pages keep results to act - on closed, as the home page does for a run no readable family lists.""" - fams, unread = [], False - for name, d in family_dirs(root): +_BUILT = {} + + +def folder_stamp(d): + """What a read of the family folder depends on: each entry's name, mode, size, mtime and ctime (a chmod changes + the ctime alone). None when the folder cannot be listed.""" + try: + names = sorted(os.listdir(d)) + except OSError: + return None + out = [] + for n in names: try: - order, _ = members_of(d) + st = os.stat(os.path.join(d, n)) + out.append((n, st.st_mode, st.st_size, st.st_mtime_ns, st.st_ctime_ns)) + except OSError: + out.append((n, None)) + return tuple(out) + + +def family_readable(name, d): + """Whether family.json's family part (family_page) builds from the folder's files as they are now, with its + pedigree read. Cached by folder_stamp, so a file that reads again counts at once, with no request needed.""" + stamp = folder_stamp(d) + hit = _BUILT.get(d) + if stamp is not None and hit and hit[0] == stamp: + return hit[1] + try: + ok = stamp is not None and not family_page(name, d)["state"].get("error") + except Exception: # any file of the folder that cannot be read + ok = False + _BUILT[d] = (stamp, ok) + return ok + + +def families(root): + """Each family folder under root as /runs.json lists it: {"name", "dir", "kind", "members", "unreadable"}. + members is [] while the pedigree cannot be read; unreadable while family.json cannot be built from the folder.""" + out = [] + for n, d in family_dirs(root): + try: + order, members = members_of(d) + people = [{"sample": s, "sex": members[s]["sex"], "father": members[s]["father"], + "mother": members[s]["mother"], "partner": members[s]["partner"], + "sequenced": members[s]["sequenced"]} for s in order] except (OSError, ValueError): - unread = True - continue - if sample in order: - fams.append({"name": name, "url": "/runs/%s/" % name}) - if fams: - return {"kind": "member", "families": fams} - return {"kind": "member", "families": [], "unknown": True} if unread else None + people = [] + out.append({"name": n, "dir": d, "kind": "family", "members": people, + "unreadable": not people or not family_readable(n, d)}) + return out + + +def member_state(root, sample): + """The families that list the sample, and child: whether its pages treat it as a child, by the one rule over + every family (bin/owngenome_live.py child_of). A family that cannot be read may be the one that lists it as a + child: with no readable family listing it, the answer is unknown, never "in no family" ({"kind": "member", + "families": [], "unknown": true}), and its pages keep results to act on closed, as the home page does.""" + from owngenome_live import child_of + fams = families(root) + child = child_of(fams, sample) + mine = [{"name": f["name"], "url": "/runs/%s/" % f["name"]} for f in fams + if any(m["sample"] == sample for m in f["members"])] + if mine: + return {"kind": "member", "families": mine, "child": child} + return {"kind": "member", "families": [], "unknown": True} if child else None def register(app): @@ -508,24 +556,7 @@ def root(): return os.path.realpath(app.config.root) def runs(app_): - out = [] - for n, d in family_dirs(root()): - try: - order, members = members_of(d) - people = [{"sample": s, "sex": members[s]["sex"], "father": members[s]["father"], - "mother": members[s]["mother"], "partner": members[s]["partner"], - "sequenced": members[s]["sequenced"]} for s in order] - except (OSError, ValueError): - people = [] - # A family whose family.json could not be built has no readable members either: the server then decides - # who is a child as the page does when family.json fails (bin/owngenome_live.py child_of). - if n in unreadable: - people = [] - out.append({"name": n, "dir": d, "kind": "family", "members": people}) - return out - - # Families whose last family.json failed: listed with no members until one is built again. - unreadable = set() + return families(root()) def page_data(req, m): name = m.group(1) @@ -536,11 +567,9 @@ def page_data(req, m): try: out = page_json(req, name, d) except Exception as e: # an unreadable file is a fixed error, never a dropped connection - unreadable.add(name) print(f"owngenome_live: family.json of {name} not built: {type(e).__name__}", file=sys.stderr) app.log("family", name, "family.json", 500) return req.send_body(500, "text/plain; charset=utf-8", "family not readable\n") - unreadable.discard(name) app.log("family", name, "family.json", 200) return req.send_json(out) diff --git a/bin/owngenome_live_results.py b/bin/owngenome_live_results.py index f8f8db57a..e0caaf594 100644 --- a/bin/owngenome_live_results.py +++ b/bin/owngenome_live_results.py @@ -44,6 +44,11 @@ def results(req, m): if summary: files = live.run_files(run) if run["dir"] else set() out = live_results.build(name, summary, files) + # For a child whose report the server keeps closed (family.json leaves its count out): the + # flagged-card count of summary.json, as family.json counts it otherwise. The family page shows it + # once every result to act on is opened there (assets/live/core.js released). + if live.gated(name, d, st): + out["saved_attention"] = live.attention(d)[0] else: out["note"] = "No step has finished yet: the results appear here as the steps end." except Exception as e: # a reader bug must not take the page down diff --git a/tests/demo/build_live_demo.py b/tests/demo/build_live_demo.py index 8327144f7..d7c17fe07 100644 --- a/tests/demo/build_live_demo.py +++ b/tests/demo/build_live_demo.py @@ -287,6 +287,8 @@ def snap(t): results = live_results.build(SAMPLE, live.summary(SAMPLE, d, live.base_state(run)), owngenome_live.Live.run_files(None, {"dir": d, "name": SAMPLE})) results["generated_utc"] = checked + if live.gated(SAMPLE, d, live.base_state(run)): # as the results plugin adds it + results["saved_attention"] = live.attention(d)[0] # The other people: one frame each, their state at the end, and their results. others = {} for name, _, _ in OTHERS: @@ -297,6 +299,8 @@ def snap(t): res = live_results.build(name, live.summary(name, od, live.base_state(r)), owngenome_live.Live.run_files(None, {"dir": od, "name": name})) res["generated_utc"] = checked + if live.gated(name, od, live.base_state(r)): + res["saved_attention"] = live.attention(od)[0] others[name] = {"frames": [{"t": checked, "state": st}], "results": res} finally: time.time, run_headroom._statvfs, run_headroom.physical_memory = real From 60ddbacc430d8b71ff62f7a6434b1cf432f4012f Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:51:56 +0100 Subject: [PATCH 208/255] test(live): check section addresses against main's outcomes, the file-based child rule and the family page's roles The live section-address check now asserts the outcomes main's own view files give, for an adult and a no-act child, with the family first and with the results first, and the map tooltip's hidden="hidden". New checks: a family file that cannot be read closes the people that family lists on the server, in state.json, on the finding route, on their page and home tile, before any family.json request, and all reopen once it reads; a released child shows summary.json's count; an adult's member line is unchanged when its summary.json cannot be read; a gated partner is not counted as a child and keeps the couple line; a child in any family stays one when the family where it is a partner sorts first. --- tests/test_owngenome_live_family.py | 246 +++++++++++++++++++++------- 1 file changed, 187 insertions(+), 59 deletions(-) diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index 277003634..ba7320ff6 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -9,9 +9,10 @@ chromosome map (child_a has one, child_b none), the relationship check, the shared segments, the couple screen counted from step 39's table, the plan, and exactly the result files that exist; - 3. child_a's state links to family_a; a sample in no family gets null; while a - family's pedigree cannot be read, a sample no readable family lists gets - "unknown" (it may be that family's child), and child_a still its family; + 3. child_a's state links to family_a and says it is a child, gf_a's that it is + not; a sample in no family gets null; while a family's pedigree cannot be + read, a sample no readable family lists gets "unknown" (it may be that + family's child), and every listed sample is treated as a child; 4. GET /family//pedigree.svg is the drawing of the declared parents; a listed file is served as written; notes.txt (in the folder, written by no step), a name with '..', a symlink out of the folder, a sample's @@ -31,17 +32,21 @@ gate_checks on the family page), here so that browser time is not added to test_live_demo.py's; 8. the real server answers every finding id of a child with a result to act on with the same page, and keeps the 404 for an adult's id that is not a finding. - 9. the report gate on the real server, in live mode: one rule of who is a child; the home page counts a + 9. the report gate on the real server, in live mode: one rule of who is a child (a child in any family is one); + the home page counts a child's results to act on from the summary its results.json reads (a stale summary.json too); family.json has no flagged-card count for a child whose report is closed, so a parent's page is the same whichever card holds the child's results; and an adult's and a no-act child's pages make the requests of the page without the gate, in its order, the event stream included, and are the same page (the genome map drawn with OG); an - adult's page at a section address is the page without the gate whichever of the family and the results - answers first; each member run of family.json says whether it is a child by the one rule; a no-act child - with a stale summary.json keeps its numbers on the home page; a family file that cannot be read gives a - fixed 500 and the server then reads that family as unreadable; on the family page a gated child reads - "Report closed" and, once opened there, the real count; a child of one family is closed on another family's - page where it is a partner. + adult's and a no-act child's page at a section address give the outcome main's view files give in each order + (marks kept when the family answers first, cleared when the results do, the map's tooltip hidden="hidden"); + each member run of family.json says whether it is a child by the one rule; a no-act child with a stale + summary.json keeps its numbers on the home page; a family file that cannot be read gives a fixed 500, and + while it cannot be read the people that family lists are closed alike on the server, in state.json, on the + finding route, on their page and home tile, and reopen once it reads, all with no family.json request; on + the family page a gated child reads "Report closed" and, once opened there, the count summary.json gives; an + adult's member line is as without the gate; a child of one family is closed on another family's page where + it is a partner, keeps that role there, and stays a child when that family sorts first. Checks 2 and 4 are run again on a copy of the fixture with two parents swapped and must fail there: a check that cannot fail is no check. @@ -246,8 +251,10 @@ def main(): # 3. a sample's page hook = dict(app.hooks)["family"] - check("child_a's page links to family_a", hook(os.path.join(root, "child_a"), {}, []) == - {"kind": "member", "families": [{"name": "family_a", "url": "/runs/family_a/"}]}) + check("child_a's page links to family_a and says it is a child", hook(os.path.join(root, "child_a"), {}, []) == + {"kind": "member", "families": [{"name": "family_a", "url": "/runs/family_a/"}], "child": True}) + check("gf_a's page links to family_a and says it is not a child", hook(os.path.join(root, "gf_a"), {}, []) == + {"kind": "member", "families": [{"name": "family_a", "url": "/runs/family_a/"}], "child": False}) check("a sample in no family gets null", hook(os.path.join(root, "loner"), {}, []) is None) # 4. the route @@ -288,7 +295,10 @@ def main(): got = (hook(os.path.join(root, "loner"), {}, []), hook(os.path.join(root, "child_a"), {}, [])) check("while a family cannot be read, a sample no readable family lists is unknown, never in no family", got == ({"kind": "member", "families": [], "unknown": True}, - {"kind": "member", "families": [{"name": "family_a", "url": "/runs/family_a/"}]}), got) + {"kind": "member", "families": [{"name": "family_a", "url": "/runs/family_a/"}], "child": True}), got) + got = hook(os.path.join(root, "gf_a"), {}, []) + check("...and an adult a readable family lists is treated as a child too: the unreadable family may list it as one", + got == {"kind": "member", "families": [{"name": "family_a", "url": "/runs/family_a/"}], "child": True}, got) shutil.rmtree(empty) check("...and once it reads again, in no family", hook(os.path.join(root, "loner"), {}, []) is None) @@ -475,9 +485,14 @@ def gate_server_checks(live, get, root, port, work): {"name": "a", "members": [{"sample": "x", "father": "p", "mother": None}]}, {"name": "c", "members": []}] child_of = getattr(owngenome_live, "child_of", lambda *a: None) - got = [child_of(fams, "x"), child_of(fams[:1], "x"), child_of(fams, "y"), child_of(fams[:2], "y")] - check("one rule of who is a child: the first family by name decides; a run no readable family lists is a child " - "while a family cannot be read", got == [True, False, True, False], got) + got = [child_of(fams, "x"), child_of(fams[:1], "x"), child_of(fams, "y"), child_of(fams[:2], "y"), + child_of([fams[1], fams[0]], "x"), child_of([dict(fams[0], name="0")] + fams[1:2], "x")] + check("one rule of who is a child: a child in any readable family is one, whichever family sorts first; every run " + "is one while a family's members cannot be read", got == [True, False, True, False, True, True], got) + unread = [dict(fams[0], unreadable=True)] + got = [child_of(unread, "x"), child_of(unread, "y")] + check("...and while another file of a family cannot be read, the runs it lists are, and no other run", + got == [True, False], got) check("the finding route asks the same rule", __import__("owngenome_live_results").child_run(live, "father_a") and not __import__("owngenome_live_results").child_run(live, "gf_a")) made = [] @@ -553,26 +568,100 @@ def index(): run("child_a") +def failure_state(live, get): + """What the server says of gf_a (an adult family_a lists) and loner (an adult in no family), both with a result + to act on: child_of, state.json's family plugin (child, unknown) and a guessed finding address; and family_a as + /runs.json lists it (unreadable, how many members).""" + def fam(n): + f = (json.loads(get("/runs/%s/state.json" % n)[2]).get("plugins") or {}).get("family") or {} + return f.get("child"), f.get("unknown") + fa = [f for f in json.loads(get("/runs.json")[2])["families"] if f["name"] == "family_a"][0] + return {n: (live.child(n), fam(n), get("/runs/%s/finding/two-hits-cftr/" % n)[0]) for n in ("gf_a", "loner")} | { + "family_a": (fa.get("unreadable"), len(fa["members"]))} + + def family_failure_checks(live, get, root): - """A family file that cannot be read: family.json answers a fixed error, never a dropped connection, and the server - then reads that family as unreadable, as the page does when family.json fails (one rule of who is a child).""" + """A family file that cannot be read: family.json answers a fixed error, never a dropped connection. Who is a child + is computed from the files on each request, never from which requests came before: while a file of family_a cannot + be read, the runs it lists are closed alike on their pages, home and the finding route (fail closed), runs in no + family stay as they were, and all reopen as soon as the file reads again, with no family.json request.""" + status = os.path.join(root, "loner", "logs", "run_status.tsv") + with open(status, "w") as f: + f.write("meta\tstarted_epoch\t1767347940\nstep\t06\tplanned\n") + real = live.summary + live.summary = lambda name, d, st: ACT_SUMMARY if name in ("gf_a", "loner") else real(name, d, st) + rel, mem = os.path.join(root, "family_a", "relationships.tsv"), os.path.join(root, "family_a", "members.tsv") + mode = os.stat(rel).st_mode + try: + before = failure_state(live, get) + os.chmod(rel, 0) + try: + closed = failure_state(live, get) # no family.json request yet + try: + code, _, body = get("/runs/family_a/family.json") + except (OSError, http.client.HTTPException) as e: + code, body = type(e).__name__, b"" + finally: + os.chmod(rel, mode) + reopened = failure_state(live, get) # no family.json request since the file reads again + os.chmod(mem, 0) + try: + unread = failure_state(live, get) + finally: + os.chmod(mem, mode) + finally: + live.summary = real + os.remove(status) + adult = {"gf_a": (False, (False, None), 404), "loner": (False, (None, None), 404), "family_a": (False, 7)} + check("with every family file readable: gf_a and loner are adults on the server, in state.json and on the finding " + "route", before == adult, before) + check("family.json with an unreadable file: a fixed error", code == 500 and body == b"family not readable\n", + (code, body[:80])) + check("...and while it cannot be read, before any family.json request, the runs family_a lists are closed alike " + "(child_of, state.json, the finding route) and keep their member list; a run in no family is as it was", + closed == {"gf_a": (True, (True, None), 200), "loner": (False, (None, None), 404), "family_a": (True, 7)}, closed) + check("...and once it reads again, all reopen with no family.json request", reopened == adult, reopened) + check("while family_a's members cannot be read, every run is closed, and one no family lists is unknown", + unread == {"gf_a": (True, (None, True), 200), "loner": (True, (None, True), 200), "family_a": (True, 0)}, unread) + + +def failure_browser_checks(b, base, live, root): + """The pages agree with the server while a file of family_a cannot be read: gf_a's page and home tile are closed, + loner's are not, and gf_a's page opens again once the file reads, with no family.json request.""" + status = os.path.join(root, "loner", "logs", "run_status.tsv") + with open(status, "w") as f: + f.write("meta\tstarted_epoch\t1767347940\nstep\t06\tplanned\n") + prev = live.summary + live.summary = lambda name, d, st: ACT_SUMMARY if name in ("gf_a", "loner") else prev(name, d, st) rel = os.path.join(root, "family_a", "relationships.tsv") mode = os.stat(rel).st_mode - os.chmod(rel, 0) + + def page(run): + b.go("about:blank") + assert b.wait("location.href === 'about:blank'") + b.go(base + "/runs/%s/" % run) + return b.wait("document.documentElement.dataset.pgpGated === 'true' ? 'closed' : " + "document.documentElement.dataset.pgpStatus === 'ready' && !!document.querySelector('#results .card') ? 'open' : ''", 15) + + def home(): + b.go("about:blank") + assert b.wait("location.href === 'about:blank'") + b.go(base + "/") + return b.wait("var t = function (n) { var x = document.querySelector('.ptile[data-sample=' + n + ']'); " + "return x ? (x.textContent.indexOf(PGPUI.closedText) >= 0 ? 'closed' : 'open') : null; }; " + "t('gf_a') && t('loner') && [t('gf_a'), t('loner')].join()", 15) try: + os.chmod(rel, 0) try: - code, _, body = get("/runs/family_a/family.json") - except (OSError, http.client.HTTPException) as e: - code, body = type(e).__name__, b"" - check("family.json with an unreadable file: a fixed error", code == 500 and body == b"family not readable\n", - (code, body[:80])) - check("...and the server then reads that family as unreadable: an adult it lists is a child until it reads again", - live.child("gf_a"), live.child("gf_a")) + got = [page("gf_a"), page("loner"), home()] + finally: + os.chmod(rel, mode) + got.append(page("gf_a")) finally: - os.chmod(rel, mode) - code = get("/runs/family_a/family.json")[0] - check("...and once the file reads again, family.json answers and the adult is an adult", code == 200 and not live.child("gf_a"), - (code, live.child("gf_a"))) + live.summary = prev + os.remove(status) + check("browser: while a file of family_a cannot be read, gf_a's page and home tile are closed and loner's open; once " + "it reads again gf_a's page opens", got == ["closed", "open", "closed,open", "open"], got) # The map's per-load timings, the live free-disk figure and the run's elapsed time change between two loads of the @@ -660,13 +749,18 @@ def capture(route, text): return b.eval("[%s, location.href, (window.ogRequests || []).map(function (r) { " "return r.join(' '); }), (document.getElementById('genome-map') || {dataset: {}}).dataset.pgpMap || '']" % PAGE_JS) pages = {} + # The family and the map's document race when nothing is held: the map's tooltip ends hidden="hidden" when the + # family answers first and hidden="" when it answers after the map was drawn, with the gate or without. Held + # back a little, the map comes last, so the two captures can be compared whole. for label, route, summary in (("an adult", "/runs/gf_a/", None), ("a no-act child", "/runs/father_a/", NO_ACT)): # The child's results come from its summary as results.json reads it (a step file read now). live.summary = lambda name, d, st: summary if summary and name == "father_a" else adult(name, d, st) + keep_routes = slow_routes(live, {"/map.json": 0.5}) try: pair = [capture(route, demo.bare_core(source)), capture(route, source)] finally: live.summary = adult + live.app.routes[:] = keep_routes pages[label] = pair check("live mode: the page of %s makes the requests of the page without the gate, in its order, and is the " "same page" % label, pair[0][:3] == pair[1][:3], @@ -680,27 +774,32 @@ def capture(route, text): print(" live mode, an adult's reads without the gate: " + ", ".join(pages["an adult"][0][2])) check("live mode: without the gate the page asks for state.json, then the event stream, then family.json", first == ["state.json", "events", "family.json"], first) - # A section address on an adult's page: the marks it sets (the contents' aria-current, the card's .target) - # depend on whether the family or the results answer first, without the gate as with it. Each order is forced - # by holding the other answer back, twice each; the map's document, asked for once the results are in, comes - # last, as it does when nothing is held. + # A section address on an adult's and a no-act child's page: the marks it sets (the contents' aria-current, the + # card's .target) depend on whether the family or the results answer first. Each order is forced by holding the + # other answer back, twice each, and the map's document comes last. The outcomes asserted are the ones main's + # own view files give on this server (no gate, its person.js and map.js): the marks stay when the family answers + # first and the family's redraw clears them when the results answer first, and the map's tooltip starts and + # stays hidden="hidden". So a change to a shared view file shows here, not only a change the gate makes. + live.summary = lambda name, d, st: NO_ACT if name == "father_a" else adult(name, d, st) marks = [] for first_, held in (("results.json", "/family.json"), ("family.json", "/results.json")): - keep_routes = slow_routes(live, {held: 0.6, "/map.json": 1.5}) + keep_routes = slow_routes(live, {held: 0.5, "/map.json": 1.0}) try: - for route in ("/runs/gf_a/#s3", "/runs/gf_a/#s/clinvar"): - for _ in range(2): - pair = [capture(route, demo.bare_core(source)), capture(route, source)] - marks.append((first_, route, pair[0][0].count('aria-current="true"'), pair[0][:3] == pair[1][:3])) - if pair[0][:3] != pair[1][:3]: - print(" %s first, %s: %r" % (first_, route, differences(pair[0][0], pair[1][0]) or - [p[1:3] for p in pair])) + for who in ("gf_a", "father_a"): + for route in ("/runs/%s/#s3" % who, "/runs/%s/#s/clinvar" % who): + for _ in range(2): + page = capture(route, source)[0] + tip = re.search(r'
]*>', page) + got = (page.count('aria-current="true"'), bool(re.search(r'
0) for m in marks}) == 2 and len({m[2] > 0 for m in marks}) == 2, marks) + live.summary = adult + check("live mode: an adult's and a no-act child's page at a section address give main's outcome in each order: " + "the marks stay when the family answers first, clear when the results do, and the map's tooltip stays " + "hidden=\"hidden\"", all(m[3] for m in marks), [m for m in marks if not m[3]] or marks[:2]) # A parent's page, whichever card holds the child's two results to act on. seen = [] for summary in (TWO_IN_ONE_CARD, TWO_CARDS): @@ -709,6 +808,7 @@ def capture(route, text): check("live mode: a parent's page is the same whichever card holds a child's results to act on", seen[0] == seen[1], "the pages differ") b.call("Page.removeScriptToEvaluateOnNewDocument", {"identifier": log}, b.session) + failure_browser_checks(b, base, live, live.root) family_gate_checks(b, base, live, run) finally: live.summary = real @@ -722,22 +822,32 @@ def capture(route, text): def family_gate_checks(b, base, live, run): """The family page: a gated child's member line is the fixed closed line, never "Nothing flagged"; once the child's - results are opened there, the line and the pedigree's mark give the real count, as without the gate. A person who - is a child in one family and a partner in another is closed on both family pages (one rule of who is a child).""" + results are opened there, the line and the pedigree's mark give the count family.json gives without the gate (the + child's summary.json, here older than the results the page reads); an adult's line is as without the gate, also when + its summary.json cannot be read. A person who is a child in one family and a partner in another is closed on both + family pages, keeps the partner role there (no child counted, the couple line kept), and stays a child when the + family where it is a partner sorts first (a child in any family is one).""" import live_results - run("father_a", TWO_CARDS) + # summary.json is older than the results the page reads: its count is the one family.json gives without the gate. + saved = {"sections": dict(TWO_CARDS["sections"], **STALE_SUMMARY["sections"])} + run("father_a", saved) prev = live.summary live.summary = lambda name, d, st: TWO_CARDS if name == "father_a" else prev(name, d, st) - n = live_results.overview(TWO_CARDS)["attention"] - assert n, "the released child needs a count to show" + n, now = live_results.overview(saved)["attention"], live_results.overview(TWO_CARDS)["attention"] + assert n and n != now, (n, now) + gf = os.path.join(live.root, "gf_a", "summary.json") + with open(gf, "w") as f: + f.write("{not json") z = os.path.join(live.root, "family_z") os.makedirs(z) with open(os.path.join(z, "members.tsv"), "w") as f: f.write("sample\tsex\tfather\tmother\tpartner\tin_pedigree\tsequenced\n" "father_a\tmale\t\t\tzara\tyes\tyes\nzara\tfemale\t\t\tfather_a\tyes\tno\n") - line = ("Array.prototype.filter.call(document.querySelectorAll('.person-card'), function (c) { " - "return c.querySelector('.person-name').textContent === 'father_a'; }).map(function (c) { " - "var l = c.querySelector('.person-look'); return l ? l.textContent : ''; })[0]") + def look(who): + return ("Array.prototype.filter.call(document.querySelectorAll('.person-card'), function (c) { " + "return c.querySelector('.person-name').textContent === '%s'; }).map(function (c) { " + "var l = c.querySelector('.person-look'); return l ? l.textContent : ''; })[0]" % who) + line = look("father_a") flag = ("Array.prototype.filter.call(document.querySelectorAll('.ped-person'), function (g) { " "return g.querySelector('.ped-name').textContent === 'father_a'; }).map(function (g) { " "var f = g.querySelector('.ped-flag'); return f ? f.textContent : ''; })[0]") @@ -747,6 +857,7 @@ def family_gate_checks(b, base, live, run): b.go(base + "/runs/family_a/") assert b.wait("!!document.querySelector('[data-gate-child=father_a] [data-opening^=closed]')"), "father_a never gated" closed = [b.eval(line), b.eval(flag)] + adult = b.eval(look("gf_a")) for _ in range(4): if not b.eval("document.querySelectorAll('[data-gate-child=father_a] [data-opening^=closed]').length"): break @@ -755,18 +866,35 @@ def family_gate_checks(b, base, live, run): "!!document.querySelector('[data-gate-child=father_a] [data-opening^=opened]')", 10) b.wait("!document.querySelector('[data-gate-child=father_a]') && /worth a look|Nothing flagged/.test(%s || '')" % line, 15) opened = [b.eval(line), b.eval(flag)] - check("family page: a gated child's member line is the fixed closed line; once opened, the real count shows on " - "the line and the pedigree", closed == ["Report closed", ""] and opened == ["%d worth a look" % n, "▲ %d" % n], - (closed, opened)) + check("family page: a gated child's member line is the fixed closed line; once opened, the count family.json gives " + "without the gate (summary.json's, not the newer results') shows on the line and the pedigree", + closed == ["Report closed", ""] and opened == ["%d worth a look" % n, "▲ %d" % n], (closed, opened, n, now)) + check("family page: an adult whose summary.json cannot be read reads as without the gate", + adult == "Nothing flagged; see the caveats", adult) b.go("about:blank") assert b.wait("location.href === 'about:blank'") b.go(base + "/runs/family_z/") got = b.wait("document.documentElement.dataset.pgpStatus === 'ready' && !!document.querySelector('.person-card') && " - "(document.querySelector('[data-gate-child=father_a]') ? 'closed' : 'open')", 15) - check("family page: a person who is a child in another family is closed on this family's page too", got == "closed", got) + "[document.querySelector('[data-gate-child=father_a]') ? 'closed' : 'open', " + "/One child|children:/.test(document.getElementById('family-lead').textContent) ? 'child counted' : 'no child', " + "/father_a and zara are not screened as a couple yet/.test(document.body.textContent) ? 'couple line' : 'no couple line'].join()", 15) + check("family page: a person who is a child in another family is closed on this family's page too, keeps the " + "partner role (no child counted) and the couple's line", got == "closed,no child,couple line", got) + first = os.path.join(live.root, "family_0") + os.rename(z, first) + z = first + b.go("about:blank") + assert b.wait("location.href === 'about:blank'") + b.go(base + "/runs/family_0/") + got = [live.child("father_a"), b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " + "!!document.querySelector('.person-card') && " + "(document.querySelector('[data-gate-child=father_a]') ? 'closed' : 'open')", 15)] + check("a child in any family is one: closed when the family where it is a partner sorts first", + got == [True, "closed"], got) finally: live.summary = prev shutil.rmtree(z, ignore_errors=True) + os.remove(gf) run("father_a", NO_ACT) From c94687e42078c738e27dd3dfea3c115c2d3ca83a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:51:56 +0100 Subject: [PATCH 209/255] docs(live): describe the child rule from the files, the fail-closed state and the new fields --- CHANGELOG.md | 2 +- docs/live-site.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b8f13116..bd646312c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -186,7 +186,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Each output is written whole or not at all: a run that stops leaves no part-written table behind. A read-only `family_plan.tsv` is replaced; a folder that cannot be written still stops the plan. - Step 38, step 39 with a families file, `run-all.sh` with `FAMILIES` and the family workflow now need `og` (`scripts/setup.sh --og`). -- A child's live report now has one gate where the page reads its data, instead of a veil on each place that could name a result. While any of a child's results to act on is unopened, the page shows a fixed explanation, the number of such results, one "Show the result" button each, and the own record of each result opened on purpose; nothing else of the child's report reaches the page, and the page is the same whichever result is closed. Which result a button opens is drawn at random on each load, and the address names buttons (`#f/closed-1`), never a result. The server answers every finding address of such a child with the same page. Opening the last result shows the full report with its own addresses. Closing one again closes the report and that result, and the others stay opened; a reload or Back closes them all. The cost: such a child sees none of the rest of their report until every result to act on is opened. The home page shows a gated child's tile with fixed text, the count and the run's progress; the family page shows the child's buttons and leaves out the child's rows, stretches, couple screen and cells (the side by side reads "closed"), while the parents' values stay. Adults and children with no result to act on see the same pages as before, with the same requests in the same order; their pages draw once the family has answered. One exception: another member's page, a parent's for example, no longer shows the flagged-card count of a child whose report is closed, because that count changes with which cards the unopened results flag. The family page reads "Report closed" for that child instead, and shows the count again once it opens the child's results. A person who is a child in one family is closed on every family page that lists them. APOE keeps its own warning. Three outputs change, and no field is renamed or removed: `family.json` gives `attention: null` for such a child and a new `child` field for each member run; `family.json` answers 500 with a fixed line when a family file cannot be read, where it dropped the connection; and `/runs.json` counts a child's results from the summary its `results.json` reads when either summary has a result to act on, also while `summary.json` is older than a finished step. `results.json` keeps its `veil` field, which the gate does not read. The gate guards the page in the browser: the static files still reach a browser that asks for them. +- A child's live report now has one gate where the page reads its data, instead of a veil on each place that could name a result. While any of a child's results to act on is unopened, the page shows a fixed explanation, the number of such results, one "Show the result" button each, and the own record of each result opened on purpose; nothing else of the child's report reaches the page, and the page is the same whichever result is closed. Which result a button opens is drawn at random on each load, and the address names buttons (`#f/closed-1`), never a result. The server answers every finding address of such a child with the same page. Opening the last result shows the full report with its own addresses. Closing one again closes the report and that result, and the others stay opened; a reload or Back closes them all. The cost: such a child sees none of the rest of their report until every result to act on is opened. The home page shows a gated child's tile with fixed text, the count and the run's progress; the family page shows the child's buttons and leaves out the child's rows, stretches, couple screen and cells (the side by side reads "closed"), while the parents' values stay. Adults and children with no result to act on see the same pages as before, with the same requests in the same order. Two things differ for them. Another member's page, a parent's for example, no longer shows the flagged-card count of a child whose report is closed, because that count changes with which cards the unopened results flag; the family page reads "Report closed" for that child instead, and shows the count `family.json` gave before once it opens the child's results. And the home tile of a child with no result to act on now, whose `summary.json` is older than a finished step and still names one, counts from the summary its `results.json` reads. A person any family lists as a child is closed on their page, the home page and every family page that lists them, whatever the other families say; a family where they are a partner keeps that role. One deliberate difference shows only while a file of a family folder cannot be read: the people that family may list are treated as children, closed on their pages, the home page, the family page and the finding addresses, until the file reads again (everyone while its members cannot be read, its own members while another of its files cannot be read). The server works this out from the files on each request. APOE keeps its own warning. These outputs change, and no field is renamed or removed: `family.json` gives `attention: null` for such a child and a new `child` field for each member run; `family.json` answers 500 with a fixed line when a family file cannot be read, where it dropped the connection; `/runs.json` counts a child's results from the summary its `results.json` reads when either summary has a result to act on, also while `summary.json` is older than a finished step, and gives each family `unreadable`; `state.json`'s family plugin adds `child` for a member; and `results.json` adds `saved_attention` for such a child, the flagged-card count of `summary.json`. `results.json` keeps its `veil` field, which the gate does not read. The gate guards the page in the browser: the static files still reach a browser that asks for them. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. diff --git a/docs/live-site.md b/docs/live-site.md index 325c49121..f6e6c7740 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -62,7 +62,7 @@ A stopped run's page says so under the name and in the summary line, with the da While the run goes, the page still opens on the report being written. A short band under the name says how far the run is ("23 of 27 are done, about 8–13 hours remain"), with the steps done as a number and a bar, the time elapsed, the time to go from the usual step times and the retries, and links to the step timeline. Each section and each line of the contents says whether it is written, being written or waiting. The run itself, below, closes the page after section 10, folded until opened (or reached through `#progress`, the band or the contents); a page with no results yet keeps it open. -**A child's report gate.** A child's report stays closed while any of the child's results to act on in adult life (the findings of level `act`) is unopened. A child is a person their family lists with a father or a mother. When several families list the person, the first by name decides. A person whose family roles cannot be read is treated as a child, and so is a person no readable family lists while some family cannot be read. When a file of a family folder cannot be read, `family.json` answers 500 with a fixed line, and the server lists that family with no members until it builds again, so the server decides as the page does when `family.json` fails. A child whose results cannot be read stays closed too. The person page, the home page and the server decide this by the same rule. While the gate is shut, the page shows only: +**A child's report gate.** A child's report stays closed while any of the child's results to act on in adult life (the findings of level `act`) is unopened. A child is a person any family lists with a father or a mother, whatever the other families that list them say. While a family folder cannot be read, a person it might list is treated as a child too, so the gate fails closed: everyone while its members cannot be read, and the people it lists while another of its files cannot be read (then `family.json` answers 500 with a fixed line). The server works this out from the files on each request, so once the files read again the people open again, with no other request needed. A person whose state cannot be read is treated as a child, and a child whose results cannot be read stays closed too. The server gives the answer to the person page in `state.json` (the family plugin's `child`, or `unknown` for a person no readable family lists), and the home page, the family page and the finding addresses decide by the same rule. While the gate is shut, the page shows only: - a fixed explanation and the number of results to act on ("1 result to act on", "2 results to act on"); - one "Show the result" button per closed result, all in one block; @@ -74,13 +74,13 @@ Which result a button opens is drawn at random on each page load, so neither a b Opening the last result loads the page again, and the full report shows, as an adult's does, with its own addresses (`#f/`). A map or section address returns to the default view. On a finding address, the finding's page opens when the results have that finding; any other id opens the run's page, as the server answers 404 for it once the report is open. The open report keeps results closed until opened, as before the gate, so its search says "Results this page keeps closed are left out of the search until they are opened." under the hits. Only which results were opened crosses that load: a one-use ticket in session storage with a digest of the results' records, bound to that one load by a random value in the window's name. The page reads and removes it as it starts, before any network read, so a slow server does not lose the opening. Closing a result again (the genome map's "Close it again") clears the page at once and loads it again closed, without its search, caches or views; that result is closed, and the others stay opened. A result to act on that appears or changes while the report is open closes it the same way. A reload, or a page shown again with Back, closes every result again. -The cost: a child with an unopened result to act on sees none of the rest of their report until every one is opened. Adults and children with no result to act on never meet the gate. The gate reads the files their page reads anyway, with the same options, and hands each view its first answer for that file, whatever its status. So each file is fetched as often, in the same order and with the same options as without the gate. In live mode the person page asks for the family and for its results at once, on its first state. The gate asks for both, in that order, as soon as the state has answered, so the two answers race as they do without the gate, and they reach the views in the order they arrived. The marks a section address sets (the contents' `aria-current`, the card's highlight) therefore come out as without the gate: they stay when the family answers first, and the redraw that the family's answer starts clears them when the results answer first. The event stream opens when `live.js` asks for it, right after `state.json`; its messages wait until the gate knows whether the page is a child's, then reach the page in order. One thing differs: the views get their first answer once the family has answered, so the page draws later by the time the family takes. +The cost: a child with an unopened result to act on sees none of the rest of their report until every one is opened. Adults and children with no result to act on never meet the gate. The gate reads the files their page reads anyway, with the same options, and hands each view its first answer for that file, whatever its status. So each file is fetched as often, in the same order and with the same options as without the gate. For an adult the gate reads only the state; the views then ask for the family and the results themselves. For a child the gate also reads the results before any view gets an answer. In live mode the person page asks for the family and for its results at once, on its first state, so for a child the gate asks for both, in that order, as soon as the state has answered. The two answers race as they do without the gate, and they reach the views in the order they arrived: the gate checks a child's results on the text it already read, so no answer overtakes another. The marks a section address sets (the contents' `aria-current`, the card's highlight) therefore come out as without the gate: they stay when the family answers first, and the redraw that the family's answer starts clears them when the results answer first. The event stream opens when `live.js` asks for it, right after `state.json`; its messages wait until the gate knows whether the page is a child's, then reach the page in order. One thing differs for a child: its views get their first answer once its results have answered, so its page draws later by the time the results take. -One count is left out for a child whose report is closed. Other members' pages, a parent's for example, list the family's members with how many cards each report flags. That count changes with which cards the child's unopened results flag, so `family.json` gives `attention: null` for that child. While the child is gated on the family page, the child's member line reads "Report closed" and the pedigree has no count, never "Nothing flagged". Once every result to act on of the child is opened there, the page takes the count from the child's `results.json` and shows it as before. For adults and children with no result to act on, the count in `family.json` is as before. Each member run in `family.json` also has `child`: whether that person's pages treat them as a child, by the rule above over every family. So a person who is a child in one family is closed on every family page that lists them, a partner's family too. +One count is left out for a child whose report is closed. Other members' pages, a parent's for example, list the family's members with how many cards each report flags. That count changes with which cards the child's unopened results flag, so `family.json` gives `attention: null` for that child. While the child is gated on the family page, the child's member line reads "Report closed" and the pedigree has no count, never "Nothing flagged". Once every result to act on of the child is opened there, the page shows the count `family.json` gives without the gate, that of the child's `summary.json`, which the child's `results.json` carries as `saved_attention`. For adults and children with no result to act on, the count in `family.json` and the member line are as before, "Nothing flagged" for a count that cannot be read included. Each member run in `family.json` also has `child`: whether that person's pages treat them as a child, by the rule above. So a person any family lists as a child is closed on every family page that lists them, a partner's family too. There they keep the role that family gives them: they are not counted as its child, and a couple they are part of keeps its line, with its screen closed. The home page decides from `/runs.json` alone. A gated child is a child (the rule above, read from the families of `/runs.json`) whose entry counts or names a result to act on. The server counts a child's results from the summary its `results.json` reads. While `summary.json` is older than a finished step, that is the step files read again, so the home page closes a child whenever the child's page does. A child with no result to act on in either summary keeps the numbers of `summary.json`, as before. Its tile and attention rows show the fixed text and the count of results to act on, and link to the run's page. The run's progress (its state, the steps done, planned, running and failed, and its times) stays, because it says how far the run is, never what it found. -On the family page, chapter IV shows a gated child's buttons only, and each child opens on its own. A gated person whom this family lists as a parent or partner, because another family lists them as a child, gets their buttons in chapter IV too. The family view gets the family without that child's report data: no de novo or compound-het rows, stretches, trio status or trio files of the child, no relationship or shared-segment row with the child, and no couple screen of a couple the child is part of. The side by side keeps a row only when an open member has a value in it, in a fixed order, and the child's cell reads "closed". Plan rows that name the child are left out. The parents' other plan rows and the couple reports of couples with no gated partner stay. Other reports of the family folder are left out while a child is gated, because they can hold the child's data. A member with no run here has no report, so the gate does not close anything of theirs. The pedigree and the members' progress stay. +On the family page, chapter IV shows a gated child's buttons only, and each child opens on its own. A gated person whom this family lists as a parent or partner, because another family lists them as a child, gets their buttons in chapter IV too, after the children. The family view gets the family without that child's report data: no de novo or compound-het rows, stretches, trio status or trio files of the child, no relationship or shared-segment row with the child, and no couple screen of a couple the child is part of (the couple keeps its names and whether it was screened; a screened couple's line says its screen stays closed). The side by side keeps a row only when an open member has a value in it, in a fixed order, and the child's cell reads "closed". Plan rows that name the child are left out. The parents' other plan rows and the couple reports of couples with no gated partner stay. Other reports of the family folder are left out while a child is gated, because they can hold the child's data. A member with no run here has no report, so the gate does not close anything of theirs. The pedigree and the members' progress stay. The gate holds the page's `fetch` and `EventSource`. A view or plugin reads a run's files through them. `XMLHttpRequest`, an `` and `import()` are not held, so the views use them only on what the gate has released. The gate guards the page in the browser: its document, attributes, accessibility text, title, address, canvas and drawings, storage and the requests its views make. The static files (`results.json`, `map.json`, the charts, the search index and `family.json`) still reach a browser that asks for them whole: the gate is not an access control for them. @@ -110,9 +110,9 @@ Per run it serves the folder's BAM or CRAM, its VCF and its SV VCF, each with it The reads cross the network to the browser that logged in: only the byte ranges igv.js asks for, but they are the sample's reads. Off loopback, use TLS (`--tls-cert` and `--tls-key`, or a proxy that ends TLS); without it the password and the reads cross the network in clear. Opening the viewer writes one access-log row; the byte ranges igv.js reads write none. -`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members. A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. For a child with a result to act on in either summary, these fields come from the summary the child's `results.json` reads, even when `summary.json` is older. For a gated child the page replaces `levels`, `top` and `lead` with fixed text before the home page draws them (the report gate above). +`GET /` lists the runs: each folder under `GENOME_DIR` holding `logs/run_status.tsv`, `nextflow/` or `run_manifest.tsv`, and each run the site has messages for. The home page reads `/runs.json`: each run's progress and attention count, and each family with its members and `unreadable` (whether a file of its folder cannot be read now). A run with results also has `levels` (its findings counted by level), `top` (up to four findings to show first, data-quality problems first: id, level, title; a finding that names a result to act on, such as a symptom ranking of its gene, is never among them), `lead` (its lead paragraph as plain text), `lead_closed` (the same with the names of the findings to act on left out, and every other part that names one, for a child's tile) and `screened` (whether ClinVar, the clinical filter or CPSR has a result; without one the home page never says that no finding needs a doctor). While a run goes these come from the summary its page last read, as long as no step has finished and `summary.json` has not changed since that read, so the home page never shows less than the person page. While a family's members cannot be read, the home page keeps the findings to act on closed for every run that no readable family lists. For a child with a result to act on in either summary, these fields come from the summary the child's `results.json` reads, even when `summary.json` is older. For a gated child the page replaces `levels`, `top` and `lead` with fixed text before the home page draws them (the report gate above). -The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases` and `veil` (the rows outside a result's own card that name a result to act on, and `spans`, the results to act on with their gene and position; the report gate reads the findings of level `act` instead; on a page that keeps results closed until opened, a child's or one whose family roles are not known, the charts and the search box still leave out a mark or hit whose tag names no finding of these results). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. +The person page reads `/runs//results.json` from `bin/owngenome_live_results.py`: `bin/live_results.py` turns the run's summary (its `summary.json` when step 24 wrote it after every finished step, else the step files read again when a step ends; a rewritten `summary.json` is read again too, and the page fetches the cards again) into the report's chapters, one card per section. Each card also has its `level`; the file adds `findings` (each with its id, level, section, title, plain text, gene, evidence, caveat, plan and related findings), `levels` (the counts), `journal` (the ten sections and the cards in each), `lead` (the lead paragraph as text pieces, the bold ones marked, and the names of the findings to act on marked `act`), `level_words`, `databases`, `saved_attention` (for a child whose report the server keeps closed: the flagged-card count of `summary.json`, as `family.json` counts it for anyone else) and `veil` (the rows outside a result's own card that name a result to act on, and `spans`, the results to act on with their gene and position; the report gate reads the findings of level `act` instead; on a page that keeps results closed until opened, a child's or one whose family roles are not known, the charts and the search box still leave out a mark or hit whose tag names no finding of these results). It computes nothing about genetics and holds only what the report prints. The family page reads `/runs//family.json` from the family plugin, and asks again every 30 seconds, so a member's progress and a new couple screen show without a reload. ## Privacy and what is stored From 8dda4b9e0aee4be78f4611bb70dfc63b75068005 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:28:00 +0100 Subject: [PATCH 210/255] docs(rust): say og-setup also links htslib on macOS, and htslib is linked on all four targets --- docs/rust.md | 4 ++-- rust/og.sh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/rust.md b/docs/rust.md index 4c05e0b3e..334555709 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -57,7 +57,7 @@ The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, ` `scripts/setup.sh ` does the same when `og` is missing or stale, after it pulls the images. `--og` builds only `og`. This puts four binaries in `bin/.og/` (ignored by git): - `og-linux-x86_64` and `og-linux-aarch64`, two static musl builds for the pipeline's containers. Both are kept because an Apple Silicon host runs amd64 images under emulation, where `uname -m` says `x86_64`. -- `og-darwin-arm64` and `og-darwin-x86_64`, for a Mac that hosts the live site (`scripts/owngenome-live.sh`), whose genome map runs `og map` on the host. They are built with `--no-default-features`, which leaves out the cargo feature `bam`: that host never reads a BAM. The feature only turns off the BAM tools; htslib is still linked, through `og-summary`. The BAM tools (`og pgx cyp2d6-*`, `og str motif`, `og cdna check|scan`, `og comphet phase`, `og bam contigs`, `og bam-fetch-probe`) say so and exit 2 there. Every Linux build has the feature, since it is on by default. +- `og-darwin-arm64` and `og-darwin-x86_64`, for a Mac that hosts the live site (`scripts/owngenome-live.sh`), whose genome map runs `og map` on the host. They are built with `--no-default-features`, which leaves out the cargo feature `bam`: that host never reads a BAM. The feature only turns off the BAM tools; htslib is still linked, through `og-summary` and `og-setup`. The BAM tools (`og pgx cyp2d6-*`, `og str motif`, `og cdna check|scan`, `og comphet phase`, `og bam contigs`, `og bam-fetch-probe`) say so and exit 2 there. Every Linux build has the feature, since it is on by default. - **From a checkout of `main` or a branch,** it builds them from `rust/` inside `OG_BUILDER`, a pinned [cargo-zigbuild](https://github.com/rust-cross/cargo-zigbuild) image (`versions.env`). Docker is the only thing it needs; no Rust on the host. The cargo cache lives in `rust/target/og-build/`. - **From a checkout of a release tag** (`vX.Y.Z`), it downloads the release's four binaries, checks them against the release's `SHA256SUMS` and, when `gh` is installed and logged in, against the build attestation (`gh attestation verify`). A release without binaries falls back to a build. A tag checkout downloads only the four binaries; `THIRD-PARTY-LICENSES.txt` is a separate asset of the release. @@ -100,7 +100,7 @@ A port must write exactly what the Python tool wrote. Most differences hide in n `og-io` reads a path or `-` as plain text, gzip or BGZF (sniffed by the magic bytes; NUL padding after a gzip member is skipped, as Python's `gzip` does), splits lines as Python's text mode does (`\n`, `\r\n` and a lone `\r`), decodes UTF-8 strictly or with `replace`, and writes a file under a temporary name before renaming it into place. -`og-bam` reads an indexed BAM through `rust-htslib`, which builds htslib from source and links it into `og` on both musl targets, so the binary stays static. Its curl, S3, GCS, bzip2 and lzma features are off: `og` reads local BAM files only, and only CRAM needs bzip2 and lzma. `fetch` is htslib's own region query, the call pysam makes, so a region returns the same reads; `reference_end`, `cigarstring` and `get_blocks` are rebuilt from the CIGAR the way pysam computes them. htslib adds about 700 KB to each binary. The fetch smoke checks it: `crates/og-bam/tests/data` holds a 2 kb slice of the test fixture's BAM at CYP2D6, a list of regions with the edge cases (a read's first and last base, a base inside a deletion, a placed unmapped read, a contig without reads) and what pysam fetches there (`tests/pysam_fetch.py` writes it). `cargo test` and `og bam-fetch-probe` must write the same lines. +`og-bam` reads an indexed BAM through `rust-htslib`, which builds htslib from source and links it statically into `og` on all four targets. Its curl, S3, GCS, bzip2 and lzma features are off: `og` reads local BAM files only, and only CRAM needs bzip2 and lzma. `fetch` is htslib's own region query, the call pysam makes, so a region returns the same reads; `reference_end`, `cigarstring` and `get_blocks` are rebuilt from the CIGAR the way pysam computes them. htslib adds about 700 KB to each binary. The fetch smoke checks it: `crates/og-bam/tests/data` holds a 2 kb slice of the test fixture's BAM at CYP2D6, a list of regions with the edge cases (a read's first and last base, a base inside a deletion, a placed unmapped read, a contig without reads) and what pysam fetches there (`tests/pysam_fetch.py` writes it). `cargo test` and `og bam-fetch-probe` must write the same lines. Known limits, each an error in Rust where Python would go on: ints beyond 128 bits, a lone surrogate in a JSON string, non-ASCII digits in `float()` and `int()`, a string `indent`, JSON nested deeper than 1000 levels (`RecursionError`; CPython 3.14 reads about 100,000 on a default stack), and `repr()` of a string that holds a character above U+00FF, which also covers `str()` of a list or dict holding one (`value` returns `Unsupported`; `str()` of the string itself works). The first five never appear in what the pipeline reads. The U+00FF limit can: a port that turns a list or dict into text must handle that error, and the PR that ports it says how. diff --git a/rust/og.sh b/rust/og.sh index a3c4353fa..87b448889 100755 --- a/rust/og.sh +++ b/rust/og.sh @@ -5,7 +5,8 @@ # host runs amd64 images under emulation, where uname -m says x86_64. On a # macOS host (the live site, scripts/owngenome-live.sh) bin/og runs # bin/.og/og-darwin- instead: built without the cargo feature -# `bam`, because that host never reads a BAM and htslib is C. +# `bam`, because that host never reads a BAM (htslib is still linked, +# through og-summary and og-setup). # # rust/og.sh install scripts/setup.sh --og: on a release tag (vX.Y.Z, # clean rust/) download the release's binaries and From c67689126c9625ff6c54d1799ee1d3f83823cc58 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:30:55 +0100 Subject: [PATCH 211/255] docs(run): say that a user's -c overrides each executor setting, not the whole block Nextflow merges -c files key by key: an executor block of the user's that sets only cpus keeps the engine's memory from executor.config. Bead: plat-18 --- CHANGELOG.md | 2 +- docs/hardware-requirements.md | 2 +- scripts/run-all.sh | 2 +- tests/e2e/run-all-launcher.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28b4388ab..7cf125bc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- `run-all.sh` now sizes Nextflow from the container engine, not only from the host. Docker Desktop and colima run the containers in a VM that is usually smaller than the Mac, but `run-all.sh` gave Nextflow the Mac's CPUs and RAM. Nextflow then started more tasks than the VM could hold, and the VM killed them (exit 137). When `docker info` reports fewer CPUs or less memory than the host, `run-all.sh` now passes the engine's figures as the default `--max_cpus` and `--max_memory`. It also writes them to `/nextflow/executor.config` and passes that file with `-c`, so Nextflow books no more than the VM has. A `-c` of your own still replaces it. On native Linux the engine reports the host's figures, and the command is unchanged. `validate-setup.sh` adds an `[INFO]` line when Docker has less memory than the host. No option or output file changes. +- `run-all.sh` now sizes Nextflow from the container engine, not only from the host. Docker Desktop and colima run the containers in a VM that is usually smaller than the Mac, but `run-all.sh` gave Nextflow the Mac's CPUs and RAM. Nextflow then started more tasks than the VM could hold, and the VM killed them (exit 137). When `docker info` reports fewer CPUs or less memory than the host, `run-all.sh` now passes the engine's figures as the default `--max_cpus` and `--max_memory`. It also writes them to `/nextflow/executor.config` and passes that file with `-c`, so Nextflow books no more than the VM has. A `-c` of your own comes after it, so each `executor` setting in it wins. On native Linux the engine reports the host's figures, and the command is unchanged. `validate-setup.sh` adds an `[INFO]` line when Docker has less memory than the host. No option or output file changes. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index 4d50ba6f5..33457eebf 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -178,7 +178,7 @@ executor { } ``` -Check the shares on a running task: `docker ps` lists the pipeline's containers as `nxf-...`, and `docker inspect -f '{{.HostConfig.CpuShares}}' ` should print 256. On Docker Desktop and colima, `run-all.sh` already writes an `executor` block with the VM's CPUs and memory to `/nextflow/executor.config`. It passes that file before your options, so a `-c` of your own replaces the block. Keep `THREADS` and `--max_memory` no larger than your block, or Nextflow refuses the tasks that ask for more. +Check the shares on a running task: `docker ps` lists the pipeline's containers as `nxf-...`, and `docker inspect -f '{{.HostConfig.CpuShares}}' ` should print 256. On Docker Desktop and colima, `run-all.sh` already writes an `executor` block with the VM's CPUs and memory to `/nextflow/executor.config`. It passes that file before your options, so each setting in an `executor` block of your own (`-c`) wins over it; a setting you leave out keeps the VM's figure. Keep `THREADS` and `--max_memory` no larger than your block, or Nextflow refuses the tasks that ask for more. ### Runtime per step diff --git a/scripts/run-all.sh b/scripts/run-all.sh index d52c88a70..8034e9051 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -289,7 +289,7 @@ MC="${G}/clinvar/clinvar_chr.vcf.gz"; if [ -s "$MC" ] && [ -s "${MC}.tbi" ]; the # The CPUs and RAM the tasks get: the host's, or the container engine's when docker info reports less. Docker Desktop # and colima run the containers in a VM, often smaller than the host; Nextflow books the host's RAM by default, and # a task the VM cannot hold is killed (exit 137). Then the default caps are the engine's, and executor.config sets the -# pool Nextflow fills to it, so a task waits for room instead. A later -c of the user's replaces that pool. +# pool Nextflow fills to it, so a task waits for room instead. Each executor setting in a later -c of the user's wins. NC=$(getconf _NPROCESSORS_ONLN) EM="" EC="" EXC="${S}/nextflow/executor.config" EX="" M=$(awk '/^MemTotal:/ {print int($2 / 1048576)}' /proc/meminfo 2>/dev/null || echo $(( $(sysctl -n hw.memsize 2>/dev/null || echo 0) / 1073741824 ))) read -r EM EC _ < <(docker info --format '{{.MemTotal}} {{.NCPU}}' 2>/dev/null) || true diff --git a/tests/e2e/run-all-launcher.sh b/tests/e2e/run-all-launcher.sh index 6e0be0559..c81117ba6 100644 --- a/tests/e2e/run-all-launcher.sh +++ b/tests/e2e/run-all-launcher.sh @@ -141,7 +141,7 @@ check "negative control: and the host's --max_cpus ${HC}" has " --max_cpus ${HC} check "negative control: and no executor.config" lacks "executor\.config" "$BIG" check "negative control: the small engine's executor.config is not left behind" test ! -e "$EXC" # Each limit alone, and what the user passes wins: THREADS and an explicit --max_memory stay, and a -# -c of the user's comes after the engine's executor.config, so its executor block replaces the pool. +# -c of the user's comes after the engine's executor.config, so each setting in its executor block wins. UC="${G3}/user.config" ET=3 engine_run memory "$((EG * 1073741824)) $((HC + 4))" -c "$UC" MEMO=$(cat "${ENG}/memory.args" 2>/dev/null) MEMC=$(cat "${ENG}/memory.config" 2>/dev/null) From 6c8f753798c1dca9f50301c8a47166c2e2dd0f00 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:33:04 +0100 Subject: [PATCH 212/255] fix(benchmark): compare a hand-placed vcf_strelka2/.vcf.gz that run-all queues the step for run-all.sh queues the benchmark step for any vcf_*/.vcf.gz, so skipping vcf_strelka2/.vcf.gz left a DeepVariant plus Strelka2 sample with one caller and exit 1. It now counts when step 03c's results/variants/variants.vcf.gz is absent. Bead: plat-71 --- scripts/benchmark-variants.sh | 5 ++++- tests/fake-docker/run-all-switches.sh | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/benchmark-variants.sh b/scripts/benchmark-variants.sh index 0ff97a8f3..9b2fba621 100755 --- a/scripts/benchmark-variants.sh +++ b/scripts/benchmark-variants.sh @@ -182,7 +182,10 @@ for vcf in "${GENOME_DIR}/${SAMPLE}"/vcf_*/"${SAMPLE}.vcf.gz"; do dir=${vcf%/*} dir=${dir##*/vcf_} case "$dir" in - gatk|freebayes|octopus|strelka2) continue ;; + gatk|freebayes|octopus) continue ;; + # Step 03c writes results/variants/variants.vcf.gz; a hand-placed + # vcf_strelka2/.vcf.gz counts only when that file is absent. + strelka2) [ -f "$SK_VCF" ] && continue; name=Strelka2 ;; clair3) name=Clair3 ;; *[!A-Za-z0-9._-]*|"") echo "WARNING: skipping vcf_${dir}/: the directory name must be letters, digits, '.', '_' or '-'" >&2; continue ;; *) name=$dir ;; diff --git a/tests/fake-docker/run-all-switches.sh b/tests/fake-docker/run-all-switches.sh index baea851cb..09a7e55da 100644 --- a/tests/fake-docker/run-all-switches.sh +++ b/tests/fake-docker/run-all-switches.sh @@ -142,6 +142,12 @@ run_expect 0 bench-clair3 env TOOLS=pharmcat BENCHMARK=true FAKE_DOCKER_RUN_HOOK grep -qE $'^DeepVariant\tClair3\t2\t0\t0\t1.0000\t1\t0.5000$' "${G}/sample1/benchmark/comparison.tsv" \ || fail "DeepVariant and Clair3 were not compared: $(cat "${G}/sample1/benchmark/comparison.tsv" 2>/dev/null)" grep -qE $'^Clair3\tmine\t' "${G}/sample1/benchmark/comparison.tsv" || fail "vcf_mine/ was not compared" +# A hand-placed vcf_strelka2/.vcf.gz also queues the step, so it is +# compared when step 03c's results/variants/variants.vcf.gz is absent. +callers vcf vcf_strelka2 +run_expect 0 bench-strelka2 env TOOLS=pharmcat BENCHMARK=true FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/isec-hook" "${SCRIPTS}/run-all.sh" sample1 male +grep -qE $'^DeepVariant\tStrelka2\t' "${G}/sample1/benchmark/comparison.tsv" \ + || fail "vcf_strelka2/sample1.vcf.gz was not compared: $(cat "${G}/sample1/benchmark/comparison.tsv" 2>/dev/null)" rm -rf "${G}/sample1"/vcf_* "${G}/sample1/benchmark" "${G}/sample1/vcf/sample1.vcf.gz" "${G}/sample1/vcf/sample1.vcf.gz.tbi" n=$(grep -c '^nextflow :: ' "$FAKE_DOCKER_LOG") From a9466db3c07beb96871e29e35ae393d8f04f99c1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:33:04 +0100 Subject: [PATCH 213/255] docs(tool-rationale): say step 03a writes a plain VCF where GVCF mode is recommended Bead: plat-55 --- docs/tool-rationale.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tool-rationale.md b/docs/tool-rationale.md index 2307e2a19..8461e0097 100644 --- a/docs/tool-rationale.md +++ b/docs/tool-rationale.md @@ -84,7 +84,7 @@ The cost is time. With 8 shards DeepVariant 1.10 took 8 h 50 min to 13 h 30 min ### GATK HaplotypeCaller: when to use it - **Closer to a clinical lab's calls.** Many clinical genetics labs and CLIA-certified pipelines call with GATK HaplotypeCaller. If you need to compare your calls with a clinical lab report, running GATK alongside DeepVariant helps identify discrepancies. `scripts/03a-gatk-haplotypecaller.sh` is not the full GATK Best Practices germline workflow, though: it calls the duplicate-marked step-02 BAM directly, without base quality score recalibration (BQSR), and writes a plain VCF (`-ERC NONE`). Its calls can differ from a lab's GATK calls for that reason alone. -- **Cohort calling.** GATK's GVCF mode (`-ERC GVCF`) produces per-sample genomic VCFs that can be joint-genotyped across a cohort with `GenomicsDBImport` + `GenotypeGVCFs`. This is the standard workflow for family studies and population-scale projects. +- **Cohort calling.** GATK's GVCF mode (`-ERC GVCF`) produces per-sample genomic VCFs that can be joint-genotyped across a cohort with `GenomicsDBImport` + `GenotypeGVCFs`. This is the standard workflow for family studies and population-scale projects. Step 03a writes a plain VCF (`-ERC NONE`), so change that argument to `-ERC GVCF` before you use it this way. - **Balanced performance.** GATK sits between DeepVariant (highest precision) and FreeBayes (highest sensitivity), offering a different precision/recall tradeoff that can complement either tool. ### FreeBayes: when to use it @@ -107,7 +107,7 @@ The cost is time. With 8 shards DeepVariant 1.10 took 8 h 50 min to 13 h 30 min | Closest to a clinical lab's calls | GATK HaplotypeCaller (without BQSR here, so not identical to a lab's GATK Best Practices calls) | | Maximum sensitivity (research) | FreeBayes (with quality filtering) | | Fastest runtime | Strelka2 (72 min on 8 threads in one historical run; align with BWA-MEM2 for it) | -| Cohort / family joint calling | GATK HaplotypeCaller (GVCF mode) | +| Cohort / family joint calling | GATK HaplotypeCaller (GVCF mode; step 03a needs `-ERC GVCF` for it) | | Consensus approach (2+ callers agree) | DeepVariant + GATK (or all three) | **References:** @@ -218,6 +218,6 @@ Stick with the defaults. The default tools were chosen for the best single-tool | Am I exploring broadly and can tolerate false positives? | Yes | Run all three SNP callers | | Do I plan to use Strelka2? | Yes | Align with BWA-MEM2 (XS tag required) | | Do I have a GPU? | Yes | Neither entry point uses it; only `call_variants` could ([DeepVariant and a GPU](troubleshooting.md#step-3-deepvariant-and-a-gpu)) | -| Am I analyzing a family/cohort? | Yes | Use GATK HC in GVCF mode for joint genotyping | +| Am I analyzing a family/cohort? | Yes | Use GATK HC in GVCF mode for joint genotyping (step 03a writes `-ERC NONE`; switch it to `-ERC GVCF`) | | Am I only looking at structural variants? | Yes | Manta + Delly + CNVpytor with consensus merge | | Do I want results as fast as possible? | Yes | minimap2 + DeepVariant + Manta (defaults) | From 1108f7e44ae845080f84fc745a80083dff5a8078 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:38:29 +0100 Subject: [PATCH 214/255] docs(changelog): give the simulator's measured time and memory Re-measured on the synthetic 1,000-person, 100,000-site panel: 0.9 to 1.0 s and 13 to 14 MB over five runs. Bead: D5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6bae1e76..b6a697f47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The variant calling page names the Genome Reference Consortium as the source of the PAR intervals in `assets/par_grch38.bed`. - Every verb the 1.0.0 release still has to port now exists in the `og` binary, each in its own module, as a stub that prints `og : not ported yet` and exits 2. The new crates are og-render, og-live, og-couple and og-admix. Each port then fills one module and never edits `rust/og/src/main.rs` or a crate's module list, so the ports can land in parallel. No step calls these verbs yet, and no output changes. - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. -- `og ancestry-fine simulate` builds an admixed genome with known truth, for testing the fine-ancestry engines that replace ADMIXTURE and RFMix in step 26b. It copies haplotypes of held-out panel people: ancestry tracts come at T events per Morgan with each ancestry drawn from the given proportions, and inside a tract the copied haplotype switches at 1 per Morgan by default. It writes an unphased VCF (sorted alleles, so nothing says which copy an allele came from) and a truth BED of each copy's ancestry that covers every chromosome from its first site to its last with no gap or overlap. The same seed gives the same bytes. On a synthetic panel of 1,000 people and 100,000 sites it takes 1.2 to 2.3 s and 12 to 15 MB. No step calls it, and no output of a run changes. The new crate `og-admix` holds it and what the engines will share: the panel reader (phased biallelic SNVs, bit-packed), the genetic map, the `model/ref.geno` file of packed reference genotypes and a seeded xoshiro256** generator. `og ancestry-fine chroms` now reads its maps with this reader. What changes on purpose: +- `og ancestry-fine simulate` builds an admixed genome with known truth, for testing the fine-ancestry engines that replace ADMIXTURE and RFMix in step 26b. It copies haplotypes of held-out panel people: ancestry tracts come at T events per Morgan with each ancestry drawn from the given proportions, and inside a tract the copied haplotype switches at 1 per Morgan by default. It writes an unphased VCF (sorted alleles, so nothing says which copy an allele came from) and a truth BED of each copy's ancestry that covers every chromosome from its first site to its last with no gap or overlap. The same seed gives the same bytes. On a synthetic panel of 1,000 people and 100,000 sites it takes about 1 s and 13 to 14 MB. No step calls it, and no output of a run changes. The new crate `og-admix` holds it and what the engines will share: the panel reader (phased biallelic SNVs, bit-packed), the genetic map, the `model/ref.geno` file of packed reference genotypes and a seeded xoshiro256** generator. `og ancestry-fine chroms` now reads its maps with this reader. What changes on purpose: - A map with a cM value that is not finite, or with cM going down as the position goes up, now stops `og ancestry-fine chroms` with an error naming the file. Before, a `nan` cM dropped the chromosome without a word, and a decreasing map gave a wrong span. Beagle's GRCh38 maps never go down. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. From 136d31bacd9874caaa27effb66be2476e56fa8c7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:41:07 +0100 Subject: [PATCH 215/255] test(live): the Python site still loads the ops plugin, and the headroom tests ignore the shell's warn percents Ports checks 14 and 15 of the deleted Python suite: while bin/owngenome_live.py lives, it must serve plugins.ops and /metrics, and a hook error must stay on the card. Two controls run the site on a plugin copy with its hook unregistered and with its catch-all cut. The ops_headroom tests pass explicit warn percents, and the environment check requires its child test to have run. Bead: D1 --- rust/crates/og-live/tests/ops.rs | 199 ++++++++++++++++++++++ rust/crates/og-live/tests/ops_headroom.rs | 35 ++-- 2 files changed, 214 insertions(+), 20 deletions(-) diff --git a/rust/crates/og-live/tests/ops.rs b/rust/crates/og-live/tests/ops.rs index cc4b4c967..97e8fc9c7 100644 --- a/rust/crates/og-live/tests/ops.rs +++ b/rust/crates/og-live/tests/ops.rs @@ -743,6 +743,205 @@ fn the_state_hook_gives_the_card_or_one_error_line() { ); } +// --- the Python live site, while it still runs the plugin ----------------------------------- + +/// A free local port. +fn free_port() -> u16 { + std::net::TcpListener::bind("127.0.0.1:0") + .and_then(|l| l.local_addr()) + .expect("a local port") + .port() +} + +/// One GET on the local site: status, content type and body; `None` while +/// it does not answer. +fn get(port: u16, path: &str) -> Option<(u16, String, String)> { + use std::io::{Read, Write}; + let mut c = std::net::TcpStream::connect(("127.0.0.1", port)).ok()?; + c.set_read_timeout(Some(Duration::from_secs(10))).ok()?; + write!(c, "GET {path} HTTP/1.0\r\nHost: 127.0.0.1\r\n\r\n").ok()?; + let mut raw = Vec::new(); + c.read_to_end(&mut raw).ok()?; + let raw = String::from_utf8(raw).ok()?; + let (head, body) = raw.split_once("\r\n\r\n")?; + let status = head.split(' ').nth(1)?.parse().ok()?; + let ctype = head + .lines() + .find_map(|l| { + let (k, v) = l.split_once(':')?; + k.eq_ignore_ascii_case("content-type") + .then(|| v.trim().to_owned()) + }) + .unwrap_or_default(); + Some((status, ctype, body.to_owned())) +} + +/// The site's process, stopped when the test ends, pass or fail. +struct Site(std::process::Child); + +impl Drop for Site { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +/// What `bin/owngenome_live.py` serves on the ops tree. +struct Served { + /// `/runs/OPS-001/state.json`. + state: Value, + /// `/metrics`: content type and text. + metrics: (String, String), + /// `state.json` again, once the history has lost its `trace_file` column. + broken: Option, +} + +fn key<'a>(v: &'a Value, k: &str) -> Option<&'a Value> { + match v { + Value::Object(m) => m.get(k), + _ => None, + } +} + +/// Starts the Python site on a new ops tree (`plugin_dir` in place of +/// `bin/` when given) and reads it. +fn python_site(plugin_dir: Option<&Path>) -> Result { + let t = make_tree(true, &[]); + let base = t.root.parent().unwrap().to_path_buf(); + let state_dir = base.join("state"); + let runs = state_dir.join("runs").join(SAMPLE); + std::fs::create_dir_all(&runs).unwrap(); + write_events(&runs.join("events.jsonl"), &t.events); + let (port, iport) = (free_port(), free_port()); + let exe = std::env::var("OG_PYTHON").unwrap_or_else(|_| "python3".into()); + let err_log = base.join("site.stderr"); + let mut cmd = Command::new(&exe); + cmd.arg(repo().join("bin/owngenome_live.py")) + .arg("--root") + .arg(&t.root) + .arg("--state-dir") + .arg(&state_dir) + .args([ + "--port", + &port.to_string(), + "--ingest-port", + &iport.to_string(), + ]) + .args(["--quiet", "--stall-minutes", "30"]) + .args(["--disk-warn-percent", "90", "--mem-warn-percent", "90"]) + .env("PGP_BOX", "opsbox") + .env_remove("OWNGENOME_LIVE_PASSWORD_HASH") + .stdout(std::process::Stdio::null()) + .stderr(std::fs::File::create(&err_log).unwrap()); + if let Some(d) = plugin_dir { + cmd.arg("--plugin-dir").arg(d); + } + let mut site = Site( + cmd.spawn() + .map_err(|e| format!("cannot run {exe}: {e}; set OG_PYTHON"))?, + ); + let why = || std::fs::read_to_string(&err_log).unwrap_or_default(); + let state_path = format!("/runs/{SAMPLE}/state.json"); + let read_state = || match get(port, &state_path) { + Some((200, _, body)) => loads(&body).map_err(|e| format!("state.json: {e:?}")), + Some((code, _, body)) => Err(format!("state.json answered {code}: {body}")), + None => Err("state.json: no answer".into()), + }; + let mut state = Err(String::new()); + for _ in 0..200 { + if let Some(code) = site.0.try_wait().unwrap() { + return Err(format!("the site exited ({code}): {}", why())); + } + state = read_state(); + if state.is_ok() { + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + let state = state.map_err(|e| format!("{e}; {}", why()))?; + let (_, ctype, text) = get(port, "/metrics").ok_or("/metrics: no answer")?; + std::fs::write( + t.root.join("pipeline_info/step_history/opsbox.tsv"), + "step\tseconds\nROH\t60\n", + ) + .unwrap(); + let broken = read_state().ok(); + Ok(Served { + state, + metrics: (ctype, text), + broken, + }) +} + +/// Check 14 and 15 of the Python suite this file replaces: the site loads +/// the plugin (state.json has `plugins.ops`, `/metrics` answers in the +/// Prometheus text format), and an error in the plugin's hook becomes the +/// card's error field while the rest of state.json stays. +fn site_serves_the_plugin(s: &Served) -> Result<(), String> { + let ops = key(&s.state, "plugins").and_then(|p| key(p, "ops")); + if ops.and_then(|o| key(o, "alerts")).is_none() { + return Err(format!("state.json has no plugins.ops card: {ops:?}")); + } + let (ctype, text) = &s.metrics; + let running = format!("pgp_run_running{{run=\"{SAMPLE}\"}} 1"); + if !ctype.contains("version=0.0.4") + || !text.lines().any(|l| l == "pgp_runs 1") + || !text.lines().any(|l| l == running) + { + return Err(format!("/metrics: {ctype} {text}")); + } + let broken = s + .broken + .as_ref() + .ok_or("state.json fails on a bad history")?; + let err = key(broken, "plugins") + .and_then(|p| key(p, "ops")) + .and_then(|o| key(o, "error")); + match err { + Some(Value::Str(e)) + if e.contains("could not be read") && key(broken, "steps").is_some() => + { + Ok(()) + } + _ => Err(format!("a hook error gave {err:?}")), + } +} + +/// While `bin/owngenome_live.py` lives, it must load the plugin and serve +/// `/metrics`; the check goes with the Python server. Two controls run the +/// site on a copy of the plugin with its hook unregistered, and with its +/// catch-all cut: both must fail the check. +#[test] +fn the_python_site_loads_the_plugin_serves_metrics_and_keeps_a_hook_error_on_the_card() { + let bin = repo().join("bin"); + let plugin = bin.join("owngenome_live_ops.py"); + if !bin.join("owngenome_live.py").is_file() || !plugin.is_file() { + eprintln!("bin/owngenome_live.py is gone: the Python site check retired with it"); + return; + } + let served = python_site(None).unwrap_or_else(|e| panic!("{e}")); + site_serves_the_plugin(&served).unwrap_or_else(|e| panic!("{e}")); + let src = std::fs::read_to_string(&plugin).unwrap(); + let catch_all = " except Exception as e: # anything else: the card says so, the rest of the run's state stays\n return {\"error\": f\"run health could not be read ({type(e).__name__})\"}\n"; + for (what, from, to) in [ + ("unregistered", " app.state_hook(\"ops\", hook)\n", ""), + ("no catch-all", catch_all, ""), + ] { + assert!( + src.contains(from), + "the control found no line to cut ({what})" + ); + let dir = tmp().join("plugins"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("owngenome_live_ops.py"), src.replacen(from, to, 1)).unwrap(); + let red = python_site(Some(&dir)).and_then(|s| site_serves_the_plugin(&s)); + assert!( + red.is_err(), + "the site check passed a plugin with its hook {what}" + ); + } +} + // --- parity with the Python plugin ---------------------------------------------------------- fn git(args: &[&str]) -> std::process::Output { diff --git a/rust/crates/og-live/tests/ops_headroom.rs b/rust/crates/og-live/tests/ops_headroom.rs index a6b61c97c..b8c25707f 100644 --- a/rust/crates/og-live/tests/ops_headroom.rs +++ b/rust/crates/og-live/tests/ops_headroom.rs @@ -48,16 +48,6 @@ fn tmp() -> PathBuf { d } -/// The warn percents the checks expect come from no environment variable. -fn no_warn_env() { - for k in ["OWNGENOME_DISK_WARN_PERCENT", "OWNGENOME_MEM_WARN_PERCENT"] { - assert!( - std::env::var_os(k).is_none(), - "unset {k}: the checks test the default warn percents" - ); - } -} - /// A host whose every file system is `used_pct`% used; its device ids are /// `devs`, else 1; existence is the real one. struct Fake { @@ -121,6 +111,9 @@ fn snapshot_of(host: &dyn Host, r: &Request<'_>) -> Snapshot { headroom::snapshot(host, &steps(), r).expect("the warn percents are good") } +/// The warn percents are given as 90, so an `OWNGENOME_*_WARN_PERCENT` of +/// the shell running the tests changes nothing; only the child test below +/// reads the environment. fn request<'a>(work: Option<&'a str>, outdir: Option<&'a str>) -> Request<'a> { Request { events: &[], @@ -128,8 +121,8 @@ fn request<'a>(work: Option<&'a str>, outdir: Option<&'a str>) -> Request<'a> { work_dir: work, outdir, max_memory: None, - disk_warn_percent: None, - mem_warn_percent: None, + disk_warn_percent: Some("90"), + mem_warn_percent: Some("90"), } } @@ -180,7 +173,6 @@ fn disk_91_warns_at_the_default_89_does_not() { /// a test may not set a variable of its own process. #[test] fn disk_warn_percent_80_and_its_environment_variable_warn_at_85() { - no_warn_env(); let d = tmp(); let host = fake(85); let plain = headroom::lines(&snapshot_of(&host, &request(Some(s(&d)), None))); @@ -202,10 +194,14 @@ fn disk_warn_percent_80_and_its_environment_variable_warn_at_85() { child.env("OG_PLANTED", f); } let out = child.output().unwrap(); + let said = String::from_utf8_lossy(&out.stdout); + // A filter that matches no test exits 0 with "0 passed": the child must have run. assert!( - !plain[0].contains("WARN") && arg.contains("WARN: at or above 80%") && out.status.success(), - "{plain:?} {arg} {}", - String::from_utf8_lossy(&out.stdout) + !plain[0].contains("WARN") + && arg.contains("WARN: at or above 80%") + && out.status.success() + && said.contains(" 1 passed;"), + "{plain:?} {arg} {said}" ); } @@ -217,7 +213,9 @@ fn child_disk_warn_from_the_environment() { og_runlog::eta::fault::plant(Box::leak(f.into_boxed_str())); } let d = std::env::var("OPS_CHILD_DIR").expect("run by its parent test"); - let text = headroom::lines(&snapshot_of(&fake(85), &request(Some(&d), None))).join("\n"); + let mut r = request(Some(&d), None); + r.disk_warn_percent = None; + let text = headroom::lines(&snapshot_of(&fake(85), &r)).join("\n"); assert!(text.contains("WARN: at or above 80%"), "{text}"); } @@ -272,7 +270,6 @@ fn f_files_0_gives_no_inode_ratio() { #[test] fn a_real_statvfs_gives_a_ratio_in_0_to_1_and_a_disk_line() { - no_warn_env(); let d = tmp(); let real = headroom::disk(&System, &[("work", Some(s(&d)))], default_warn()); let lines = headroom::lines(&snapshot_of(&System, &request(Some(s(&d)), Some(s(&d))))); @@ -391,7 +388,6 @@ fn a_running_message_after_a_finished_one_of_the_same_task_id_keeps_the_finished #[test] fn max_memory_64_gb_on_a_32_gb_host_warns_on_128_it_does_not() { - no_warn_env(); let mut r = request(None, None); r.max_memory = Some("64.GB"); let mut host = fake(50); @@ -418,7 +414,6 @@ fn strings(snap: &Snapshot) -> Vec { #[test] fn the_snapshot_holds_no_path_and_takes_its_folders_from_the_metadata() { - no_warn_env(); let d = tmp(); let (work, res) = (d.join("work"), d.join("results")); std::fs::create_dir_all(&work).unwrap(); From 4b13255fb04d4e3f281feef810aad7937635c126 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:56:33 +0100 Subject: [PATCH 216/255] test(live): hold the map's document back on the parent-page comparison too The parent's page compared whichever card holds a child's results raced the family against the map's document, so the tooltip ended hidden="hidden" or hidden="" by who won, as on main. Hold map.json back half a second as the other live comparisons do, and print where the pages differ. --- tests/test_owngenome_live_family.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index ba7320ff6..72b22f7b6 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -801,12 +801,17 @@ def capture(route, text): "the marks stay when the family answers first, clear when the results do, and the map's tooltip stays " "hidden=\"hidden\"", all(m[3] for m in marks), [m for m in marks if not m[3]] or marks[:2]) # A parent's page, whichever card holds the child's two results to act on. + # The map's document is held back, as above, so the tooltip's end state does not depend on the race. seen = [] - for summary in (TWO_IN_ONE_CARD, TWO_CARDS): - run("father_a", summary) - seen.append(capture("/runs/gf_a/", source)[0]) + keep_routes = slow_routes(live, {"/map.json": 0.5}) + try: + for summary in (TWO_IN_ONE_CARD, TWO_CARDS): + run("father_a", summary) + seen.append(capture("/runs/gf_a/", source)[0]) + finally: + live.app.routes[:] = keep_routes check("live mode: a parent's page is the same whichever card holds a child's results to act on", - seen[0] == seen[1], "the pages differ") + seen[0] == seen[1], differences(seen[0], seen[1])) b.call("Page.removeScriptToEvaluateOnNewDocument", {"identifier": log}, b.session) failure_browser_checks(b, base, live, live.root) family_gate_checks(b, base, live, run) From 931c426f29719835cbd90c74f5c2b6655faa5608 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:59:19 +0100 Subject: [PATCH 217/255] fix(ancestry): read a map's first position and its flat stretches the way the docs say cm_at took the first marker at a first position that holds two, while every other position takes the last one. pos_at returned the last position of an inner or trailing flat stretch, where its doc promises the first. Both now do what the docs say; the tests cover both cases. Bead: D5 --- rust/crates/og-admix/src/map.rs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/rust/crates/og-admix/src/map.rs b/rust/crates/og-admix/src/map.rs index 71894b8a8..602155ed7 100644 --- a/rust/crates/og-admix/src/map.rs +++ b/rust/crates/og-admix/src/map.rs @@ -97,7 +97,7 @@ impl GeneticMap { let (Some(&p_first), Some(&p_last)) = (self.pos.first(), self.pos.last()) else { return 0.0; }; - if pos <= p_first { + if pos < p_first { return self.cm[0]; } if pos >= p_last { @@ -124,11 +124,15 @@ impl GeneticMap { if cm <= c_first { return self.pos[0] as f64; } - if cm >= c_last { + if cm > c_last { return self.pos[self.pos.len() - 1] as f64; } - // The first marker above cm, and the last one at or below it. - let hi = self.cm.partition_point(|&c| c <= cm); + // The first marker at or above cm; it exists, and is not the first. + let hi = self.cm.partition_point(|&c| c < cm); + if self.cm[hi] == cm { + return self.pos[hi] as f64; + } + // Here cm[hi - 1] < cm < cm[hi], so the stretch rises. let (p0, c0, p1, c1) = (self.pos[hi - 1], self.cm[hi - 1], self.pos[hi], self.cm[hi]); p0 as f64 + (p1 - p0) as f64 * (cm - c0) / (c1 - c0) } @@ -199,6 +203,21 @@ mod tests { // A flat stretch maps back to its first position. assert_eq!(map.pos_at(1.0), 100.0); assert_eq!(map.pos_at(1.5), 250.0); + // A first position with two markers: the last one counts there too. + let map = GeneticMap::from_markers(vec![(100, 1.0), (100, 2.0), (200, 3.0)]).unwrap(); + assert_eq!(map.cm_at(99), 1.0); + assert_eq!(map.cm_at(100), 2.0); + assert_eq!(map.cm_at(150), 2.5); + // An inner and a trailing flat stretch map back to their first + // position; a cM between markers still interpolates. + let map = + GeneticMap::from_markers(vec![(0, 0.0), (10, 1.0), (20, 1.0), (30, 2.0), (40, 2.0)]) + .unwrap(); + assert_eq!(map.pos_at(1.0), 10.0); + assert_eq!(map.pos_at(2.0), 30.0); + assert_eq!(map.pos_at(0.5), 5.0); + assert_eq!(map.pos_at(1.5), 25.0); + assert_eq!(map.pos_at(3.0), 40.0); let empty = GeneticMap::default(); assert!(empty.is_empty()); assert_eq!(empty.cm_at(5), 0.0); From 669b7f39cbea3ff46599d064b906dc7482efe96e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:59:19 +0100 Subject: [PATCH 218/255] fix(ancestry): stop the simulator on a rate it cannot draw and on outputs it cannot write A T or switch rate of about 1e16 per Morgan or more left the event's place in cM unchanged, so the draw never ended; both are now capped at 10,000 per Morgan. og ancestry-fine simulate now refuses a chromosome named twice in --chroms, an --out or --truth that is a folder, and one file for both, before it writes anything; a folder as --truth used to leave the VCF in place without its truth. Bead: D5 --- CHANGELOG.md | 4 +-- rust/crates/og-admix/src/simulate.rs | 28 +++++++++++++++++-- rust/crates/og-ancestry/src/simulate.rs | 36 ++++++++++++++++++++++++- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6a697f47..3f6089eb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,8 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The variant calling page names the Genome Reference Consortium as the source of the PAR intervals in `assets/par_grch38.bed`. - Every verb the 1.0.0 release still has to port now exists in the `og` binary, each in its own module, as a stub that prints `og : not ported yet` and exits 2. The new crates are og-render, og-live, og-couple and og-admix. Each port then fills one module and never edits `rust/og/src/main.rs` or a crate's module list, so the ports can land in parallel. No step calls these verbs yet, and no output changes. - Guard's rust job runs every test that carries the line `# og-leg: OG_REQUIRED` with `og`, instead of two hand-kept lists of the same 41 tests. -- `og ancestry-fine simulate` builds an admixed genome with known truth, for testing the fine-ancestry engines that replace ADMIXTURE and RFMix in step 26b. It copies haplotypes of held-out panel people: ancestry tracts come at T events per Morgan with each ancestry drawn from the given proportions, and inside a tract the copied haplotype switches at 1 per Morgan by default. It writes an unphased VCF (sorted alleles, so nothing says which copy an allele came from) and a truth BED of each copy's ancestry that covers every chromosome from its first site to its last with no gap or overlap. The same seed gives the same bytes. On a synthetic panel of 1,000 people and 100,000 sites it takes about 1 s and 13 to 14 MB. No step calls it, and no output of a run changes. The new crate `og-admix` holds it and what the engines will share: the panel reader (phased biallelic SNVs, bit-packed), the genetic map, the `model/ref.geno` file of packed reference genotypes and a seeded xoshiro256** generator. `og ancestry-fine chroms` now reads its maps with this reader. What changes on purpose: - - A map with a cM value that is not finite, or with cM going down as the position goes up, now stops `og ancestry-fine chroms` with an error naming the file. Before, a `nan` cM dropped the chromosome without a word, and a decreasing map gave a wrong span. Beagle's GRCh38 maps never go down. +- `og ancestry-fine simulate` builds an admixed genome with known truth, for testing the fine-ancestry engines that replace ADMIXTURE and RFMix in step 26b. It copies haplotypes of held-out panel people: ancestry tracts come at T events per Morgan with each ancestry drawn from the given proportions, and inside a tract the copied haplotype switches at 1 per Morgan by default. It writes an unphased VCF (sorted alleles, so nothing says which copy an allele came from) and a truth BED of each copy's ancestry that covers every chromosome from its first site to its last with no gap or overlap. The same seed gives the same bytes. T and the switch rate are at most 10,000 per Morgan: far above that, the draw of events never ends. On a synthetic panel of 1,000 people and 100,000 sites it takes about 1 s and 13 to 14 MB. No step calls it, and no output of a run changes. The new crate `og-admix` holds it and what the engines will share: the panel reader (phased biallelic SNVs, bit-packed), the genetic map, the `model/ref.geno` file of packed reference genotypes and a seeded xoshiro256** generator. `og ancestry-fine chroms` now reads its maps with this reader. What changes on purpose: + - A map with a cM value that is not finite, or with cM going down as the position goes up, now stops `og ancestry-fine chroms` with an error naming the file. Before, a `nan` cM dropped the chromosome without a word, and a decreasing map gave a wrong span. Beagle's GRCh38 maps never go down. At a first position that holds two markers, the later marker's cM now counts, as it already did at every other position. - The genome map shows where you are in the whole genome and lists what it draws. While a chromosome is open, a strip above the map draws every chromosome end to end, with the view as a brass brush and a tick for each finding on the map; a click on the strip opens a chromosome, and a drag brushes a new view or pans (on a phone the strip is a picture only). Under the map, an "In view" table lists every mark the map draws one by one in view: kind, what, where, size, who, copy and source, sorted by place with the findings first, a click on a header to sort by it. It shows 12 rows, then all of them (a scrolled list past 200 rows, at most 5,000; on a phone at most 200 rows, then zoom in), and a row flies the map to its feature. It follows the map once the map stops, announces only its count to a screen reader, and is folded on a phone; T folds and opens it. A closed result is never a row or a tick. One rule decides what the canvas, the strip and the table draw, and `og map build --predicate-table` writes the same rule's answers so a browser check holds the two to one set. A margin update no longer runs while the map is still moving: it waits for the view the map comes to rest on. diff --git a/rust/crates/og-admix/src/simulate.rs b/rust/crates/og-admix/src/simulate.rs index be354dcba..54d690794 100644 --- a/rust/crates/og-admix/src/simulate.rs +++ b/rust/crates/og-admix/src/simulate.rs @@ -52,9 +52,15 @@ pub struct Config { pub seed: u64, } +/// The most events per Morgan T or the switch rate may ask for. A higher +/// rate gives tracts shorter than the gap between two panel sites, and at +/// about 1e16 an event's place in cM stops moving, so the draw never ends. +pub const MAX_PER_MORGAN: f64 = 10_000.0; + impl Config { - /// Checks the numbers: T above 0, a switch rate of 0 or more, and - /// proportions above 0 that sum to 1 (within 1e-6), one per ancestry. + /// Checks the numbers: T above 0, a switch rate of 0 or more, both at + /// most [`MAX_PER_MORGAN`], and proportions above 0 that sum to 1 + /// (within 1e-6), one per ancestry. pub fn check(&self) -> Result<()> { if !(self.generations.is_finite() && self.generations > 0.0) { return Err(Error(format!( @@ -62,12 +68,24 @@ impl Config { self.generations ))); } + if self.generations > MAX_PER_MORGAN { + return Err(Error(format!( + "T is {}: ancestry events per Morgan must be at most {MAX_PER_MORGAN}", + self.generations + ))); + } if !(self.switch_rate.is_finite() && self.switch_rate >= 0.0) { return Err(Error(format!( "the switch rate is {}: copying switches per Morgan must be 0 or more", self.switch_rate ))); } + if self.switch_rate > MAX_PER_MORGAN { + return Err(Error(format!( + "the switch rate is {}: copying switches per Morgan must be at most {MAX_PER_MORGAN}", + self.switch_rate + ))); + } if self.mu.is_empty() { return Err(Error("no ancestry proportions".into())); } @@ -743,9 +761,15 @@ mod tests { c.generations = 0.0; assert!(run(c, &people()).contains("must be above 0")); let mut c = cfg(1); + c.generations = 1e300; + assert!(run(c, &people()).contains("at most 10000")); + let mut c = cfg(1); c.switch_rate = -1.0; assert!(run(c, &people()).contains("must be 0 or more")); let mut c = cfg(1); + c.switch_rate = 20_000.0; + assert!(run(c, &people()).contains("at most 10000")); + let mut c = cfg(1); c.mu[1].1 = 0.5; assert!(run(c, &people()).contains("sum to 1.1, not 1")); let mut c = cfg(1); diff --git a/rust/crates/og-ancestry/src/simulate.rs b/rust/crates/og-ancestry/src/simulate.rs index 48eea7b51..661e599df 100644 --- a/rust/crates/og-ancestry/src/simulate.rs +++ b/rust/crates/og-ancestry/src/simulate.rs @@ -116,6 +116,9 @@ fn chroms(a: &Args, panel: &Path, maps: &Path) -> Result> { if c.is_empty() { return Err(Fail("--chroms names no chromosome".into())); } + if let Some((_, x)) = c.iter().enumerate().find(|&(i, x)| c[..i].contains(x)) { + return Err(Fail(format!("--chroms names {x} twice"))); + } return Ok(c); } let entries = std::fs::read_dir(panel).map_err(|e| crate::cannot_read(panel, &e))?; @@ -161,6 +164,18 @@ pub(crate) fn main(a: &Args, out: &mut dyn Write) -> Result<()> { ))); } let vcf_path = PathBuf::from(a.req("--out")); + let truth_path = PathBuf::from(a.req("--truth")); + for p in [&vcf_path, &truth_path] { + if p.is_dir() { + return Err(Fail(format!("{}: is a folder, not a file", p.display()))); + } + } + if vcf_path == truth_path { + return Err(Fail(format!( + "--out and --truth name one file: {}", + vcf_path.display() + ))); + } let mut pending = crate::Pending::new(); let file = pending.create(vcf_path.clone())?; let write_err = |e: std::io::Error| crate::cannot_write(&vcf_path, &e); @@ -195,7 +210,7 @@ pub(crate) fn main(a: &Args, out: &mut dyn Write) -> Result<()> { } let mut bed = Vec::new(); simulate::write_truth(&mut bed, &done, &cfg).map_err(|e| Fail(e.0))?; - pending.put(PathBuf::from(a.req("--truth")), &bed)?; + pending.put(truth_path, &bed)?; pending.commit_all()?; let mut bp = vec![0i64; cfg.mu.len()]; for c in &done { @@ -362,6 +377,25 @@ mod tests { err.contains("argument --generations: invalid float value: 'x'"), "{err}" ); + let (rc, _, err) = run_v(&args(&d, &["--out", &vcf, "--chroms", "chr2,chr21,chr2"])); + assert_eq!(rc, 1); + assert!(err.contains("--chroms names chr2 twice"), "{err}"); + // An output that is a folder, or one file for both outputs, stops + // the run before anything is written. + std::fs::create_dir_all(d.path("folder.bed")).unwrap(); + let mut v = args(&d, &["--out", &vcf]); + v[12] = d.path("folder.bed"); + let (rc, _, err) = run_v(&v); + assert_eq!(rc, 1); + assert!(err.contains("folder.bed: is a folder, not a file"), "{err}"); + let (rc, _, err) = run_v(&args(&d, &["--out", &d.path("folder.bed")])); + assert_eq!(rc, 1); + assert!(err.contains("folder.bed: is a folder, not a file"), "{err}"); + let mut v = args(&d, &["--out", &vcf]); + v[12] = vcf.clone(); + let (rc, _, err) = run_v(&v); + assert_eq!(rc, 1); + assert!(err.contains("--out and --truth name one file"), "{err}"); let (rc, _, err) = run(&["simulate", "--og-scaffold-probe"]); assert_eq!(rc, 2); assert!( From d36ae2ecb74ad6395e603a63a4b0062324d63e15 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:03:22 +0100 Subject: [PATCH 219/255] test(pgx): pin that a below-floor MT-RNR1 allele does not hide a called one --- .../og-filters/tests/i411_mtrnr1_het.rs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/rust/crates/og-filters/tests/i411_mtrnr1_het.rs b/rust/crates/og-filters/tests/i411_mtrnr1_het.rs index ba1bd88a5..cfc8303b8 100644 --- a/rust/crates/og-filters/tests/i411_mtrnr1_het.rs +++ b/rust/crates/og-filters/tests/i411_mtrnr1_het.rs @@ -130,3 +130,27 @@ fn red_first_a_nan_af_is_no_af_not_a_call() { "held back: m.1555A>G (chrM:1555 A>G AF nan) has no allele fraction" ); } + +#[test] +fn a_named_allele_below_the_floor_does_not_hide_a_call() { + let (call, lines) = run(&[rec(827, "A", "G", 0.03), rec(1555, "A", "G", 0.99)]); + assert_eq!(call.as_deref(), Some("m.1555A>G"), "{lines:?}"); + assert_eq!( + lines, + [ + format!( + "MT-RNR1 m.1555A>G passed to PharmCAT as an outside call (homoplasmic, chrM:1555 A>G AF 0.990; {INC})" + ), + "not passed: m.827A>G (chrM:827 A>G AF 0.030) is below the detection floor (AF below 0.05): not called, and not absent either".to_owned(), + ] + ); + // Any other held record still withholds the call. + let mut filtered = rec(1494, "C", "T", 0.50); + filtered.filter = "strand_bias".into(); + let (call, _) = run(&[ + rec(827, "A", "G", 0.03), + filtered, + rec(1555, "A", "G", 0.99), + ]); + assert_eq!(call, None); +} From 198f889dbca370dbca70531ebd3849b204de414e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:03:24 +0100 Subject: [PATCH 220/255] fix(pgx): log a below-floor MT-RNR1 allele beside a called one instead of withholding the call --- CHANGELOG.md | 2 +- docs/07-pharmacogenomics.md | 4 ++-- modules/local/pharmcat/main.nf | 4 ++-- rust/crates/og-filters/src/mito.rs | 15 ++++++++++----- scripts/07-pharmacogenomics.sh | 4 ++-- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d669004aa..36123753a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- A heteroplasmic MT-RNR1 variant now gives the aminoglycoside warning. Step 07 and the Nextflow PHARMCAT process passed a named MT-RNR1 allele to PharmCAT only at an allele fraction (AF) of 0.95 or more, so m.1555A>G or m.1494C>T at AF 0.05 to 0.95 left MT-RNR1 No Result and the warning was lost. CPIC sets no heteroplasmy level at which aminoglycosides are safe, so `og pgx mito-merge` now passes a PASS named allele from AF 0.05, the pipeline's heteroplasmy floor, and its log says `heteroplasmic` with the AF. Below AF 0.05 the allele is not passed, the log says it is below the detection floor, and the gene is not called Reference either. The default of `--min-af` moves from 0.95 to 0.05; the option stays. A named allele whose AF reads `nan` is now held back as having no allele fraction; it was passed as homoplasmic. No output file or format changes: the MT-RNR1 line of the outside calls now appears in these cases. Part of #411 (pgx-04, pgx-32). +- A heteroplasmic MT-RNR1 variant now gives the aminoglycoside warning. Step 07 and the Nextflow PHARMCAT process passed a named MT-RNR1 allele to PharmCAT only at an allele fraction (AF) of 0.95 or more, so m.1555A>G or m.1494C>T at AF 0.05 to 0.95 left MT-RNR1 No Result and the warning was lost. CPIC sets no heteroplasmy level at which aminoglycosides are safe, so `og pgx mito-merge` now passes a PASS named allele from AF 0.05, the pipeline's heteroplasmy floor, and its log says `heteroplasmic` with the AF. Below AF 0.05 the allele is not passed, the log says it is below the detection floor, and the gene is not called Reference either; beside a named allele at the floor it is only named in the log and does not withhold that call. The default of `--min-af` moves from 0.95 to 0.05; the option stays. A named allele whose AF reads `nan` is now held back as having no allele fraction; it was passed as homoplasmic. No output file or format changes: the MT-RNR1 line of the outside calls now appears in these cases. Part of #411 (pgx-04, pgx-32). - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. diff --git a/docs/07-pharmacogenomics.md b/docs/07-pharmacogenomics.md index 78ed16e08..3c50af7ca 100644 --- a/docs/07-pharmacogenomics.md +++ b/docs/07-pharmacogenomics.md @@ -97,7 +97,7 @@ docker run --rm \ - **gVCF.** PharmCAT refuses a gVCF, and decides by the file name too (`.g.vcf`, `.genomic.vcf`). Yet a gVCF is the better input: a variants-only VCF leaves about half of PharmCAT's genes Unknown, because PharmCAT cannot tell a reference call from a position that was not covered. So the script reads `vcf/${SAMPLE}.g.vcf.gz` when step 3 wrote one and expands its reference blocks into `${SAMPLE}.pgx_regions.vcf.gz` (step 0 above, deleted afterwards), which PharmCAT accepts; without a gVCF it reads `${SAMPLE}.vcf.gz`, and does not check whether that file is itself a gVCF (PharmCAT then stops on it). The Nextflow pipeline does the same with DeepVariant's gVCF, or with the gVCF a samplesheet row gives in its `gvcf` column (`run-all.sh` fills it from `vcf/${SAMPLE}.g.vcf.gz`); a gVCF given in the `vcf` column stops the run before any analysis when `pharmcat` is selected. For a vendor gVCF, remove the reference blocks and rename the file: [Starting from a Vendor VCF](vcf-first.md). - **Positions longer than one base.** The expansion writes one record per base, each with a one-base REF. 51 PharmCAT 3.4.0 positions have a REF or ALT longer than one base: the insertions and deletions of DPYD, UGT1A1 (\*28, the TA-repeat insertion at chr2:233760233), TPMT, CYP3A4, CYP3A5, CFTR, CYP2C9, NUDT15, RYR1, CYP2D6 and G6PD, and two two-base substitutions in RYR1. PharmCAT's preprocessor does not take a one-base record as a call at such a position: it sets the record aside (FILTER `PCATxINDEL`, or `PCATxREF` for the substitutions) and lists the position in `missing_pgx_var.vcf`. Where PharmCAT's REF is longer than one base (the deletions, UGT1A1 \*28, the substitutions), its matcher then discards the record ("REF in VCF (C) does not match expected reference (CAT)"), so the position has no call and the alleles defined there are uncallable, even where the gVCF shows you match the reference over the whole span. At an insertion with a one-base REF the matcher still reads the set-aside record as a reference call ("PharmCAT cannot validate this position"). So step 0 runs `og pgx refblock-fill` on the expanded file. For each of these positions in PharmCAT's positions file (`/pharmcat/pharmcat_positions.vcf.bgz` in the image), it writes a 0/0 record with PharmCAT's own ID, REF and ALT (INFO flag `PGX_REFBLOCK`) in place of the one-base record, when every base of the REF span lies in reference blocks with GQ 20 or more (GQ is Phred-scaled: 20 means a 1 in 100 chance the reference call is wrong; genotype 0/0, or 0 on a haploid chrX; FILTER `PASS` or `.`) and no other record overlaps the span: no variant call, no `./.` block, no block below GQ 20. Otherwise the expanded file stays as it was there: the position stays missing, and at an insertion the matcher reads the one-base record as before. The step log names the positions it filled (`pgx_refblock_fill: N of 51 PharmCAT positions longer than one base called 0/0 from reference blocks with GQ >= 20: ...`). The GQ threshold applies to these positions only: a one-base position takes any 0/0 block, as before. Where the expanded file has a record at such a position that is not a one-base reference call, that record stays and no 0/0 record is added beside it; the log says so. The Nextflow module does the same. -- **MT-RNR1 from the chrM calls.** PharmCAT 3.4.0 has no MT-RNR1 definition for its matcher: it reads the gene only as an outside call, one allele (chrM is not a pair), so from a VCF alone MT-RNR1 is No Result on every sample. When [step 20](20-mtoolbox.md) wrote its Mutect2 chrM calls (`mito/${SAMPLE}_chrM_filtered.vcf.gz`), the script runs `og pgx mito-merge` (step 3 above). It reads PharmCAT's own MT-RNR1 alleles from the jar (`Reference` and names such as `m.1555A>G`) and adds one MT-RNR1 line to a copy of step 36's outside calls (`${SAMPLE}.pharmcat_outside_calls.tsv`, deleted afterwards): a named single-base allele with FILTER `PASS` and an allele fraction (AF) of 0.05 or more, the pipeline's heteroplasmy floor; the one PharmCAT ranks highest (increased risk, then uncertain, then normal) when there are several, with the others named in the step log; or `Reference` when no named allele is called. A heteroplasmic allele (AF 0.05 to 0.95) is passed like a homoplasmic one: CPIC's MT-RNR1 guideline sets no heteroplasmy level at which aminoglycosides become safe, and asks for the homoplasmic guidance whenever a relevant variant is detected. So a heteroplasmic m.1555A>G or m.1494C>T gives PharmCAT's aminoglycoside recommendation, and the step log says `heteroplasmic` with the AF (the [step 20](20-mtoolbox.md) table and the report's chrM section give the level too). Mutect2 writes variant records only, so a position without reads has no record either: `Reference` needs read evidence, 20 reads or more (`samtools depth`, MAPQ and base quality 20) at every position where PharmCAT names an MT-RNR1 allele, from the sample's BAM (`aligned/${SAMPLE}_sorted.bam`, the one step 20 read). Without the BAM, or with a position below 20 reads, no `Reference` is passed and the log names the positions; a named allele needs no depth, its record is the evidence. Mutect2 writes the genotype 0/1 even at AF 1.0, so the AF decides, not the genotype. No line is added, and the gene stays No Result with the reason in the step log, when a named allele is below the detection floor (AF below 0.05; the log says so, and the gene is not called Reference either), filtered, or has no AF (or an AF that reads `nan`); when a `PASS` record at AF 0.05 or more at a position where PharmCAT names an allele (m.960 and m.961 included, named there only as deletions) has an ALT PharmCAT does not name; when a `PASS` insertion or deletion overlaps a named position or lies within 10 bases of m.960 or m.961, where PharmCAT names deletions in a C run that the script does not map; and when the file has no `PASS` chrM record. Without step 20's file the step says so and MT-RNR1 stays No Result: run step 20 before this step. The PHARMCAT module does the same with MITO_VARIANTS' calls. +- **MT-RNR1 from the chrM calls.** PharmCAT 3.4.0 has no MT-RNR1 definition for its matcher: it reads the gene only as an outside call, one allele (chrM is not a pair), so from a VCF alone MT-RNR1 is No Result on every sample. When [step 20](20-mtoolbox.md) wrote its Mutect2 chrM calls (`mito/${SAMPLE}_chrM_filtered.vcf.gz`), the script runs `og pgx mito-merge` (step 3 above). It reads PharmCAT's own MT-RNR1 alleles from the jar (`Reference` and names such as `m.1555A>G`) and adds one MT-RNR1 line to a copy of step 36's outside calls (`${SAMPLE}.pharmcat_outside_calls.tsv`, deleted afterwards): a named single-base allele with FILTER `PASS` and an allele fraction (AF) of 0.05 or more, the pipeline's heteroplasmy floor; the one PharmCAT ranks highest (increased risk, then uncertain, then normal) when there are several, with the others named in the step log; or `Reference` when no named allele is called. A heteroplasmic allele (AF 0.05 to 0.95) is passed like a homoplasmic one: CPIC's MT-RNR1 guideline sets no heteroplasmy level at which aminoglycosides become safe, and asks for the homoplasmic guidance whenever a relevant variant is detected. So a heteroplasmic m.1555A>G or m.1494C>T gives PharmCAT's aminoglycoside recommendation, and the step log says `heteroplasmic` with the AF (the [step 20](20-mtoolbox.md) table and the report's chrM section give the level too). Mutect2 writes variant records only, so a position without reads has no record either: `Reference` needs read evidence, 20 reads or more (`samtools depth`, MAPQ and base quality 20) at every position where PharmCAT names an MT-RNR1 allele, from the sample's BAM (`aligned/${SAMPLE}_sorted.bam`, the one step 20 read). Without the BAM, or with a position below 20 reads, no `Reference` is passed and the log names the positions; a named allele needs no depth, its record is the evidence. Mutect2 writes the genotype 0/1 even at AF 1.0, so the AF decides, not the genotype. No line is added, and the gene stays No Result with the reason in the step log, when a named allele is below the detection floor (AF below 0.05) and no other named allele is called (the log says so, and the gene is not called Reference either; beside a called allele it is only named in the log), filtered, or has no AF (or an AF that reads `nan`); when a `PASS` record at AF 0.05 or more at a position where PharmCAT names an allele (m.960 and m.961 included, named there only as deletions) has an ALT PharmCAT does not name; when a `PASS` insertion or deletion overlaps a named position or lies within 10 bases of m.960 or m.961, where PharmCAT names deletions in a C run that the script does not map; and when the file has no `PASS` chrM record. Without step 20's file the step says so and MT-RNR1 stays No Result: run step 20 before this step. The PHARMCAT module does the same with MITO_VARIANTS' calls. - **PharmCAT's own log file.** The jar's built-in `logback.xml` writes `pharmcat.log` into the working directory. In the image that is `/pharmcat`, which the calling user cannot write, so every run printed a logback status dump with a `pharmcat.log (Permission denied)` stack trace (the calls were not affected). The script runs the jar with `-w /tmp`, a writable directory that goes away with the container. A Nextflow task runs in its own writable work directory, so the module needs nothing: the log lands there. - **A backslash in a `##` header line.** PharmCAT up to 3.4.0 bundles vcf-parser 0.3.1, which stops with "Error parsing metadata: character to be escaped is missing" on one. The line is valid VCF; bcftools writes it for a soft filter with a quoted string (`bcftools filter -s LowDP -e 'FORMAT/DP<10 && GT!="0/0"'`). The script and the Nextflow module rewrite the header of PharmCAT's own copy (`${SAMPLE}.pharmcat_input.vcf`, deleted afterwards by the script): on `##` lines `\"` becomes `'` and any other `\` becomes `/`. PharmCAT's calls are the same with and without the rewrite. A newer PharmCAT is no fix yet: 3.4.0 still bundles vcf-parser 0.3.1 and fails the same way. @@ -120,7 +120,7 @@ docker run --rm \ ## Limitations - **HLA-A, HLA-B and CYP2D6** are not called from a VCF: PharmCAT 3.4.0 reports them with no result (`callSource` `NONE`). [Step 36](36-pgx-consensus.md) gives PharmCAT T1K's HLA types (step 8) and a CYP2D6 call only when pypgx (step 32) and Cyrius (step 21, opt-in) agree on depth that passed its check; PharmCAT reads them with `-po` (`callSource` `OUTSIDE`). Run step 36 before this step, or run this step again after it. A disagreement, or one caller alone, leaves CYP2D6 without a result, on purpose. -- **MT-RNR1** is an outside call from step 20's chrM calls (see above): PharmCAT 3.4.0 does not call it from a VCF. A heteroplasmic named allele, such as m.1555A>G at AF 0.4, is passed and gets the aminoglycoside recommendation; the PharmCAT report does not show its AF, the step log does. A named allele below AF 0.05 is not passed: the gene stays No Result and the step log names the allele and its AF as below the detection floor. At that level the call cannot be told from noise or NuMT reads, so it is not called absent either. +- **MT-RNR1** is an outside call from step 20's chrM calls (see above): PharmCAT 3.4.0 does not call it from a VCF. A heteroplasmic named allele, such as m.1555A>G at AF 0.4, is passed and gets the aminoglycoside recommendation; the PharmCAT report does not show its AF, the step log does. A named allele below AF 0.05 is not passed. With no other named allele called, the gene stays No Result and the step log names the allele and its AF as below the detection floor. At that level the call cannot be told from noise or NuMT reads, so it is not called absent either. - **CYP4F2 can come out No Result** when unphased heterozygous calls elsewhere in the gene, a weak one among them, leave no diplotype that matches. CPIC's warfarin guidance needs only the \*3 position (rs2108622). Step 27 then reads the genotype PharmCAT itself lists at that position in `report.json` (the gene's `variants`) and prints it in a section headed "Single-Position Fallback (not a PharmCAT call)", with the CPIC warfarin rule when you carry the variant. It does the same for VKORC1 at rs9923231 (-1639G>A). PharmCAT's own result for the gene stays No Result in the phenotypes table. A position with no genotype is reported as not callable. The report carries no GQ or read depth for the position: check it in your VCF before acting on it. - PharmCAT may disagree with lab reports on complex haplotypes (e.g., NAT2). Discrepancies can arise from different genome builds (hg19 vs hg38), different star allele definitions, or different variant calling pipelines. When a discrepancy matters clinically, compare both sets of raw variant calls and consult the PharmVar database for the current allele definitions — do not blindly trust either source. - PharmCAT output structure changes across releases. If you upgrade PharmCAT, re-test step 27 (`27-cpic-lookup.sh`) because it parses the JSON output directly. diff --git a/modules/local/pharmcat/main.nf b/modules/local/pharmcat/main.nf index 27e78dc8d..2575b1410 100644 --- a/modules/local/pharmcat/main.nf +++ b/modules/local/pharmcat/main.nf @@ -32,8 +32,8 @@ heteroplasmy level; the log gives the AF), or Reference when there is none and the BAM has 20 reads or more (samtools depth, MAPQ and base quality 20) at every position PharmCAT names an allele at; no line, with - the reason in the task log, for a named allele below AF 0.05 or - filtered, or a position without that depth. + the reason in the task log, for a named allele below AF 0.05 (when no + other is called) or filtered, or a position without that depth. The jar's built-in logback.xml writes pharmcat.log into the working directory. A task runs in its own writable work directory, so the log diff --git a/rust/crates/og-filters/src/mito.rs b/rust/crates/og-filters/src/mito.rs index f4512d5b0..cc72ff63c 100644 --- a/rust/crates/og-filters/src/mito.rs +++ b/rust/crates/og-filters/src/mito.rs @@ -13,11 +13,12 @@ //! homoplasmic guidance for any detected MT-RNR1 variant, so the floor is //! the pipeline's heteroplasmy floor (AF 0.05), not homoplasmy (AF 0.95), //! and the log line gives the level. A named allele below the floor is not -//! called and not called absent either. One such allele is the call; of -//! several, the one PharmCAT ranks highest (increased risk, then uncertain, -//! then normal). +//! called and not called absent either: it holds `Reference` back, not a +//! named allele at the floor, and the log names it. One such allele is the +//! call; of several, the one PharmCAT ranks highest (increased risk, then +//! uncertain, then normal). //! With none, `Reference`, but only when `--depth` shows `--min-depth` -//! reads or more at every named position. Anything that makes the answer +//! reads or more at every named position. Anything else that makes the answer //! unsure holds every call back, with a line saying why ([`Held`]). //! //! `--outside` (step 36's calls) is copied to `--out` without its MT-RNR1 @@ -502,7 +503,10 @@ pub fn decide( )); return (None, lines); } - if !held.is_empty() { + // A named allele below the floor holds `Reference` back, but not a + // named call at the floor: then it is only logged. + let only_below = held.iter().all(|h| matches!(h, Held::BelowFloor { .. })); + if !held.is_empty() && (calls.is_empty() || !only_below) { lines.extend(held.iter().map(|h| format!("held back: {h}"))); lines.push(format!( "no {GENE} call passed to PharmCAT: it stays No Result; read the records above in the chrM call set" @@ -534,6 +538,7 @@ pub fn decide( c.function )); } + lines.extend(held.iter().map(|h| format!("not passed: {h}"))); return (Some(best.name.clone()), lines); } if !a.has_reference { diff --git a/scripts/07-pharmacogenomics.sh b/scripts/07-pharmacogenomics.sh index 65f2f37f7..f5ea70149 100755 --- a/scripts/07-pharmacogenomics.sh +++ b/scripts/07-pharmacogenomics.sh @@ -110,8 +110,8 @@ run_in \ # the AF), or Reference when there is none and the BAM has 20 reads or more # (samtools depth, MAPQ and base quality 20) at every position PharmCAT names # an allele at; no line, with the reason in the log, for a named allele below -# AF 0.05 or filtered, or a position without that depth (the PHARMCAT module -# does the same). +# AF 0.05 (when no other is called) or filtered, or a position without that +# depth (the PHARMCAT module does the same). OUTSIDE="${GENOME_DIR}/${SAMPLE}/pgx_consensus/${SAMPLE}_outside_calls.tsv" MITO="${GENOME_DIR}/${SAMPLE}/mito/${SAMPLE}_chrM_filtered.vcf.gz" MERGED="${OUTPUT_DIR}/${SAMPLE}.pharmcat_outside_calls.tsv" From 66b7ece8bd08b56e328a2f0ade60f999e0f66e8c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:08:48 +0100 Subject: [PATCH 221/255] test(live): let open_gate read through the reload the last opening starts Under load the read right after the last click could meet the reload that opening starts, and Chrome answered 'Inspected target navigated or closed'. That read now counts as released and waits for the opened page, as it already did when the read came after the reload. --- tests/test_live_demo.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_live_demo.py b/tests/test_live_demo.py index 485079f6b..07d4292d1 100644 --- a/tests/test_live_demo.py +++ b/tests/test_live_demo.py @@ -337,7 +337,14 @@ def open_gate(b): if not closed: return b.eval("document.querySelector('[data-opening^=closed]').click(); true") - if not b.eval("document.querySelector('[data-opening^=closed]') !== null"): + # The last opening loads the page again: a read that meets that navigation means the gate released. + try: + left = b.eval("document.querySelector('[data-opening^=closed]') !== null") + except RuntimeError as e: + if 'navigated' not in str(e): + raise + left = False + if not left: assert b.wait("!document.documentElement.dataset.pgpGated"), 'gate did not release' return raise AssertionError('gate did not release in ten clicks') From c25e8a6a7d6cc232e4a12727955fbbcefbfb86b1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:18:55 +0100 Subject: [PATCH 222/255] docs(results): point contributors and the report at the chapter pages, and check them The contributor guides still sent output notes to the overview page, and the report's repeat note named the overview for the thresholds table that now lives in docs/results/structural-and-repeats.md. The lint link loop and the T1K/Parascopy version check globbed docs/*.md only, so the new chapter pages were outside both. A planted wrong T1K version on a chapter page now fails the version check; before, it passed. --- .github/workflows/lint.yml | 2 +- CLAUDE.md | 2 +- CONTRIBUTING.md | 4 ++-- bin/render_report.py | 2 +- tests/test_doc_versions.sh | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7c64de319..e8b0c09a4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -132,7 +132,7 @@ jobs: run: | echo "Checking internal markdown links..." rm -f /tmp/link_fail - for md in docs/*.md README.md; do + for md in docs/*.md docs/results/*.md README.md; do # Extract markdown links: [text](target) grep -oP '\]\([^)]+\)' "$md" 2>/dev/null | \ grep -oP '\(([^)]+)\)' | tr -d '()' | \ diff --git a/CLAUDE.md b/CLAUDE.md index c713c6cfb..1051482c6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -138,7 +138,7 @@ User's FASTQ/BAM/VCF 6. Add the module to the step list (a `plan` call) in `scripts/run-all.sh` (its `--tools` name and database parameter), so a default run includes it 7. Add a row for the image to `tests/smoke/commands.tsv`: a real command on the fixture and a check on what it wrote (an image with no row fails `container-test.yml`) 8. Update `docs/00-reference-setup.md` if new reference data needed -9. Update `docs/interpreting-results.md` if output needs explanation +9. Update the chapter page under `docs/results/` (listed in `docs/interpreting-results.md`) if output needs explanation 10. Test on at least one sample before committing ### Bumping a Tool diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a4929bcb3..1a8e266a3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,7 +65,7 @@ Every pipeline step needs: - `scripts/NN-tool-name.sh` — The executable script - An entry in the `nav` of [`zensical.toml`](zensical.toml), or the docs site check fails on the orphan page - A mention in the category table of [`docs/pipeline-overview.md`](docs/pipeline-overview.md) -- Section in `docs/interpreting-results.md` if the output needs explanation +- A section in the chapter page under `docs/results/` (listed in [`docs/interpreting-results.md`](docs/interpreting-results.md)) if the output needs explanation ### No Personal Data @@ -99,7 +99,7 @@ The CI pipeline enforces this with automated scanning. - `scripts/run-all.sh`: one entry in its step list (a `plan` call), so a default run passes the module's name in `--tools` (and its database parameter when the data is under `GENOME_DIR`). A bash-only step that a default run should include gets a switch there instead, run after the pipeline - [`tests/smoke/commands.tsv`](tests/smoke/commands.tsv): a row for the image, a real command on the fixture and a check on what the tool wrote. [`container-test.yml`](.github/workflows/container-test.yml) fails for an image with no row - `scripts/validate-setup.sh`: only if the step needs reference data to check - - `docs/interpreting-results.md`: add output interpretation + - the chapter page under `docs/results/` (listed in `docs/interpreting-results.md`): add output interpretation - `docs/00-reference-setup.md`: if new reference data is needed - `CLAUDE.md` — if the architecture tree changes 8. **Document failures** in `docs/lessons-learned.md` if you hit any issues during development. diff --git a/bin/render_report.py b/bin/render_report.py index 530cee487..9f73b1b41 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -53,7 +53,7 @@ def stale_note(sec): HLA_NOTE = ("HLA typing from short-read WGS is approximate. If either HLA-A or HLA-B allele has a T1K quality " "of 0 or less, that gene is not passed to PharmCAT (step 36).") EH_NOTE = ("Repeat-expansion calls from short reads can be wrong at some loci, and a long expansion " - "cannot be sized. Disease thresholds: docs/interpreting-results.md.") + "cannot be sized. Disease thresholds: docs/results/structural-and-repeats.md.") TELOMERE_UNIT = "intratelomeric reads per million reads with 48-52% GC" TELOMERE_NOTE = (f"Relative telomere content ({TELOMERE_UNIT}), not a telomere length: compare it only with " "samples of a similar age sequenced on the same platform (docs/10-telomere-analysis.md).") diff --git a/tests/test_doc_versions.sh b/tests/test_doc_versions.sh index 998c88082..0ed0421cf 100644 --- a/tests/test_doc_versions.sh +++ b/tests/test_doc_versions.sh @@ -51,7 +51,7 @@ check_docs() { n=0 # A tool name, optional Markdown emphasis or code marks, an optional "v" # or "version ", then a dotted version. One grep over every page. - hits=$(grep -noE "\b${tool}[*\`]*[[:space:]]+(v|version[[:space:]]+)?[0-9]+\.[0-9]+(\.[0-9]+)?" "$dir"/*.md || true) + hits=$(grep -noE "\b${tool}[*\`]*[[:space:]]+(v|version[[:space:]]+)?[0-9]+\.[0-9]+(\.[0-9]+)?" "$dir"/*.md "$dir"/results/*.md || true) if [ -n "$hits" ]; then while IFS= read -r h; do n=$((n + 1)) From 7f2150f984901bc92b8ad19741f77033823cfabf Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:25:21 +0100 Subject: [PATCH 223/255] test(bloodtype): a VCF with no contig lines, or a stale .csi beside its .tbi, must still read A2 Both are read once from the start. Before the fix step 41 stopped on each: bcftools writes no record on an undeclared contig, and bcftools loads the stale .csi before the current .tbi. Bead: trait-30 --- tests/e2e/bloodtype-1.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/e2e/bloodtype-1.sh b/tests/e2e/bloodtype-1.sh index 2d0482872..d13b99187 100755 --- a/tests/e2e/bloodtype-1.sh +++ b/tests/e2e/bloodtype-1.sh @@ -20,6 +20,8 @@ # - ABO c.1061delC written two bases right (GG>G) gives A2/A2 as the table's # CG>C does, because step 41 left-aligns first (og on the VCF as written # reads A1/A1, the control), and a C>A,G 1/2 at c.803 keeps its one B copy; +# - that right-shifted VCF still gives A2/A2 with no ##contig line and no +# index, and with a current .tbi beside a stale .csi (htslib loads the .csi); # - the Nextflow processes (BLOODTYPE_DEPTH, BLOODTYPE) on a VCF and BAM # write the same three files as the bash step on the same two files. . "$(dirname "$0")/lib.sh" @@ -211,6 +213,22 @@ sample "$M" "${T}/${M}.vcf.gz" "${T}/win.bam" step "$M" check_eq "C>A,G 1/2 at c.803 stays one record: one B copy" "$(ev "$M" ABO_803)" "called 1" check_eq "and ABO A1/B" "$(cell "$M" ABO genotype)" "A1/B" +# The right-shifted VCF read once from the start: with no ##contig line (bcftools +# writes no record on an undeclared contig, so step 41 adds the lines), and with a +# current .tbi beside a .csi older than the VCF (htslib loads the .csi first). +BC="${SAMPLE}btc" BI="${SAMPLE}bti" +zcat "${T}/${R}.vcf.gz" | grep -v '^##contig' > "${T}/${BC}.vcf" +dk "$BCFTOOLS_IMAGE" bgzip -f "${T}/${BC}.vcf" +sample "$BC" "${T}/${BC}.vcf.gz" "${T}/win.bam" +check_eq "control: that VCF declares no contig" "$(zcat "${G}/${BC}/vcf/${BC}.vcf.gz" | grep -c '^##contig')" 0 +step "$BC" +check_eq "no ##contig line and no index: ABO A2/A2" "$(cell "$BC" ABO phenotype)/$(cell "$BC" ABO genotype)" "A2/A2/A2" +sample "$BI" "${T}/${R}.vcf.gz" "${T}/win.bam" +dk "$BCFTOOLS_IMAGE" bcftools index -f -t "${G}/${BI}/vcf/${BI}.vcf.gz" +: > "${G}/${BI}/vcf/${BI}.vcf.gz.csi" +touch -d 2000-01-01 "${G}/${BI}/vcf/${BI}.vcf.gz.csi" +step "$BI" +check_eq "a current .tbi beside a stale .csi: ABO A2/A2" "$(cell "$BI" ABO phenotype)/$(cell "$BI" ABO genotype)" "A2/A2/A2" unset REF_FASTA rm -f "$REFBT" "${REFBT}.fai" From 1fbeabc0de6bbac726622236e22214b4571818cd Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:25:21 +0100 Subject: [PATCH 224/255] fix(bloodtype): check the index htslib loads, and declare missing contigs before streaming The freshness check now looks at the .csi when there is one, as htslib loads it first. A VCF read once from the start gets a ##contig line for each marker window contig its header lacks, so bcftools can write its records. Bead: trait-30 --- docs/41-bloodtype.md | 1 + modules/local/bloodtype/main.nf | 33 +++++++++++++++++++++------ scripts/41-bloodtype.sh | 40 ++++++++++++++++++++++++++------- 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/docs/41-bloodtype.md b/docs/41-bloodtype.md index 7f1f0a919..9515067ef 100644 --- a/docs/41-bloodtype.md +++ b/docs/41-bloodtype.md @@ -74,6 +74,7 @@ The table writes each indel left-aligned, the way `bcftools norm` writes it: ABO - A multi-allelic indel is left-aligned as a whole, so its REF can stay longer than the marker's (`TCC TC,T` against `TC T`). Its REF then differs from the table's, and the site is a no call, as the paragraph above says. - A record whose REF differs from the reference is kept as it is (`-c w`), with a warning in the log. - A record on a contig the reference lacks (a VCF named `1`, `2`, ... against the chr-named reference) cannot be aligned. It goes through unchanged. +- bcftools reads the VCF through its index when the index htslib loads (the `.csi` if there is one, else the `.tbi`) is not older than the VCF. Otherwise it reads the VCF once from the start. Read that way, a VCF whose header has no `##contig` line for a marker's contig first gets one, because bcftools writes no record on an undeclared contig. - The left-aligned file is a work file: the bash step deletes it after the rules run, and the pipeline does not publish it. ### Several records at one site diff --git a/modules/local/bloodtype/main.nf b/modules/local/bloodtype/main.nf index f6ca92ec0..71d42d34b 100644 --- a/modules/local/bloodtype/main.nf +++ b/modules/local/bloodtype/main.nf @@ -126,9 +126,12 @@ process BLOODTYPE { // indel can shift along its repeat) are left-aligned first. A // multi-allelic record stays whole; -c w keeps a record whose REF differs // from the reference as it is. Records on a contig the reference lacks go - // through unchanged. The VCF is read through its index when it has a - // current one, else once from the start. The same commands as - // scripts/41-bloodtype.sh; og runs from bin/, on the task's PATH. + // through unchanged. The VCF is read through its index when the one + // htslib loads (.csi before .tbi) is current, else once from the start; + // read that way, it first gets a ##contig line for each window contig its + // header lacks, because bcftools writes no record on an undeclared contig. + // The same commands as scripts/41-bloodtype.sh; og runs from bin/, on the + // task's PATH. """ grep -v '^#' ${markers} | tail -n +2 \\ | awk -F '\\t' -v OFS='\\t' '{s = \$2 - 100; if (s < 0) s = 0; print \$1, s, \$3 + 100; c = \$1; if (sub(/^chr/, "", c)) print c, s, \$3 + 100}' \\ @@ -137,11 +140,27 @@ process BLOODTYPE { cut -f1 ${reference_fai} > bt_ref_contigs.txt awk -F '\\t' 'NR == FNR {c[\$1] = 1; next} \$1 in c' bt_ref_contigs.txt bt_windows.bed > bt_on_ref.bed awk -F '\\t' 'NR == FNR {c[\$1] = 1; next} !(\$1 in c)' bt_ref_contigs.txt bt_windows.bed > bt_off_ref.bed - SEL=-T OVERLAP=--targets-overlap - for x in tbi csi; do - if [ -f "${vcf}.\$x" ] && [ ! "${vcf}" -nt "${vcf}.\$x" ]; then SEL=-R OVERLAP=--regions-overlap; fi + SEL=-T OVERLAP=--targets-overlap HDR=${vcf} + for x in csi tbi; do + if [ -f "${vcf}.\$x" ]; then + if [ ! "${vcf}" -nt "${vcf}.\$x" ]; then SEL=-R OVERLAP=--regions-overlap; fi + break + fi done - pick() { if [ -s "\$1" ]; then bcftools view "\$SEL" "\$1" "\$OVERLAP" record -Ou ${vcf}; else bcftools view -h -Ou ${vcf}; fi; } + if [ "\$SEL" = -T ]; then + bcftools head ${vcf} > bt_header.vcf + cut -f1 bt_windows.bed \\ + | awk 'NR == FNR {if (sub(/^##contig=].*/, ""); d[\$0] = 1}; next} !(\$1 in d) && !s[\$1]++ {print "##contig="}' bt_header.vcf - > bt_contigs.txt + if [ -s bt_contigs.txt ]; then + { grep -v '^#CHROM' bt_header.vcf; cat bt_contigs.txt; grep '^#CHROM' bt_header.vcf; } > bt_header2.vcf + HDR=bt_header2.vcf + fi + fi + pick() { + if [ ! -s "\$1" ]; then bcftools view -h -Ou "\$HDR" + elif [ "\$HDR" != "${vcf}" ]; then bcftools reheader -h "\$HDR" ${vcf} | bcftools view -T "\$1" --targets-overlap record -Ou - + else bcftools view "\$SEL" "\$1" "\$OVERLAP" record -Ou ${vcf}; fi + } pick bt_on_ref.bed | bcftools norm -c w -f ${reference} -Ob -o bt_on_ref.bcf pick bt_off_ref.bed | bcftools view -Ob -o bt_off_ref.bcf bcftools concat -Ou bt_on_ref.bcf bt_off_ref.bcf | bcftools sort -T ./bt_sort -Oz -o bt_norm.vcf.gz diff --git a/scripts/41-bloodtype.sh b/scripts/41-bloodtype.sh index a4d26ef9b..e8723cf5e 100755 --- a/scripts/41-bloodtype.sh +++ b/scripts/41-bloodtype.sh @@ -87,8 +87,11 @@ echo "[2/3] The VCF's records around the marker sites, left-aligned (bcftools no # whose REF differs from the reference as it is (og then reads it as a no # call). Records on a contig the reference lacks (a VCF named 1, 2, ... against # a chr-named reference) cannot be aligned and go through unchanged. The VCF is -# read through its index when it has a current one, else once from the start. -# The BLOODTYPE process runs the same commands before og. +# read through its index when the one htslib loads (.csi before .tbi) is +# current, else once from the start. Read that way, a VCF with no ##contig line +# for a window's contig first gets one (bcftools reheader), because bcftools +# writes no record on an undeclared contig; through an index, htslib takes the +# names from the index. The BLOODTYPE process runs the same commands before og. # shellcheck disable=SC2016 # the program's $ belong to the inner bash and awk NORM_PROG=' MARKERS=$1 VCF=$2 REF=$3 W=$4 OUT=$5 @@ -97,24 +100,45 @@ NORM_PROG=' cut -f1 "$REF.fai" > "$W/bt_ref_contigs.txt" awk -F "\t" "NR == FNR {c[\$1] = 1; next} \$1 in c" "$W/bt_ref_contigs.txt" "$W/bt_windows.bed" > "$W/bt_on_ref.bed" awk -F "\t" "NR == FNR {c[\$1] = 1; next} !(\$1 in c)" "$W/bt_ref_contigs.txt" "$W/bt_windows.bed" > "$W/bt_off_ref.bed" - SEL=-T OVERLAP=--targets-overlap - for x in tbi csi; do - if [ -f "$VCF.$x" ] && [ ! "$VCF" -nt "$VCF.$x" ]; then SEL=-R OVERLAP=--regions-overlap; fi + SEL=-T OVERLAP=--targets-overlap HDR=$VCF + for x in csi tbi; do + if [ -f "$VCF.$x" ]; then + if [ ! "$VCF" -nt "$VCF.$x" ]; then SEL=-R OVERLAP=--regions-overlap; fi + break + fi done - pick() { if [ -s "$1" ]; then bcftools view "$SEL" "$1" "$OVERLAP" record -Ou "$VCF"; else bcftools view -h -Ou "$VCF"; fi; } + if [ "$SEL" = -T ]; then + bcftools head "$VCF" > "$W/bt_header.vcf" + cut -f1 "$W/bt_windows.bed" | awk "$8" "$W/bt_header.vcf" - > "$W/bt_contigs.txt" + if [ -s "$W/bt_contigs.txt" ]; then + { grep -v "^#CHROM" "$W/bt_header.vcf"; cat "$W/bt_contigs.txt"; grep "^#CHROM" "$W/bt_header.vcf"; } > "$W/bt_header2.vcf" + HDR=$W/bt_header2.vcf + fi + fi + pick() { + if [ ! -s "$1" ]; then bcftools view -h -Ou "$HDR" + elif [ "$HDR" != "$VCF" ]; then bcftools reheader -h "$HDR" "$VCF" | bcftools view -T "$1" --targets-overlap record -Ou - + else bcftools view "$SEL" "$1" "$OVERLAP" record -Ou "$VCF"; fi + } pick "$W/bt_on_ref.bed" | bcftools norm -c w -f "$REF" -Ob -o "$W/bt_on_ref.bcf" pick "$W/bt_off_ref.bed" | bcftools view -Ob -o "$W/bt_off_ref.bcf" bcftools concat -Ou "$W/bt_on_ref.bcf" "$W/bt_off_ref.bcf" | bcftools sort -T "$W/bt_sort" -Oz -o "$OUT" bcftools index -f -t "$OUT" - rm -f "$W/bt_windows.bed" "$W/bt_ref_contigs.txt" "$W/bt_on_ref.bed" "$W/bt_off_ref.bed" "$W/bt_on_ref.bcf" "$W/bt_off_ref.bcf"' + rm -f "$W/bt_windows.bed" "$W/bt_ref_contigs.txt" "$W/bt_on_ref.bed" "$W/bt_off_ref.bed" "$W/bt_on_ref.bcf" "$W/bt_off_ref.bcf" \ + "$W/bt_header.vcf" "$W/bt_contigs.txt" "$W/bt_header2.vcf"' # Each marker row 100 bp each side, under its chr name and without the chr # (og reads a VCF named either way); then the overlapping windows merged. # shellcheck disable=SC2016 # awk programs WINDOWS_AWK='{s = $2 - 100; if (s < 0) s = 0; print $1, s, $3 + 100; c = $1; if (sub(/^chr/, "", c)) print c, s, $3 + 100}' # shellcheck disable=SC2016 MERGE_AWK='$1 == c && $2 <= e {if ($3 > e) e = $3; next} {if (c != "") print c, s, e; c = $1; s = $2; e = $3} END {if (c != "") print c, s, e}' +# The window contigs (on stdin) the header (the first file) declares no +# ##contig line for, as ##contig lines. +# shellcheck disable=SC2016 +CONTIG_AWK='NR == FNR {if (sub(/^##contig=].*/, ""); d[$0] = 1}; next} !($1 in d) && !s[$1]++ {print "##contig="}' run_in -v "${PGP_ROOT}/assets:/pgp-assets:ro" --cpus 1 --memory 2g "$BCFTOOLS_IMAGE" bash -euo pipefail -c "$NORM_PROG" \ - _ /pgp-assets/bloodtype_markers.tsv "$(cpath "$VCF")" "$REF_FASTA_C" "$O" "$(cpath "$NORM")" "$WINDOWS_AWK" "$MERGE_AWK" + _ /pgp-assets/bloodtype_markers.tsv "$(cpath "$VCF")" "$REF_FASTA_C" "$O" "$(cpath "$NORM")" "$WINDOWS_AWK" "$MERGE_AWK" \ + "$CONTIG_AWK" [ -s "$NORM" ] || { echo "ERROR: no left-aligned VCF written (${NORM})" >&2; exit 1; } echo "[3/3] Blood group rules (og bloodtype report)..." run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets:/pgp-assets:ro" --cpus 1 --memory 2g "$PYTHON_IMAGE" \ From f4c6606906266b5d3d0a0dc41b72fb88cbeb32b5 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 16:51:15 +0100 Subject: [PATCH 225/255] fix(run): never book a pool below the user's own task caps on a smaller engine When THREADS, --max_cpus or --max_memory was above the engine's figure, executor.config still held the engine's CPUs and memory, and Nextflow refused every task capped above the pool. The pool now rises to a cap the user set, and leaves out a line whose cap it cannot read. --- CHANGELOG.md | 2 +- docs/hardware-requirements.md | 2 +- docs/troubleshooting.md | 2 +- scripts/run-all.sh | 15 +++++++++++++-- tests/e2e/run-all-launcher.sh | 18 +++++++++++++++++- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cf125bc2..e4a43e346 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- `run-all.sh` now sizes Nextflow from the container engine, not only from the host. Docker Desktop and colima run the containers in a VM that is usually smaller than the Mac, but `run-all.sh` gave Nextflow the Mac's CPUs and RAM. Nextflow then started more tasks than the VM could hold, and the VM killed them (exit 137). When `docker info` reports fewer CPUs or less memory than the host, `run-all.sh` now passes the engine's figures as the default `--max_cpus` and `--max_memory`. It also writes them to `/nextflow/executor.config` and passes that file with `-c`, so Nextflow books no more than the VM has. A `-c` of your own comes after it, so each `executor` setting in it wins. On native Linux the engine reports the host's figures, and the command is unchanged. `validate-setup.sh` adds an `[INFO]` line when Docker has less memory than the host. No option or output file changes. +- `run-all.sh` now sizes Nextflow from the container engine, not only from the host. Docker Desktop and colima run the containers in a VM that is usually smaller than the Mac, but `run-all.sh` gave Nextflow the Mac's CPUs and RAM. Nextflow then started more tasks than the VM could hold, and the VM killed them (exit 137). When `docker info` reports fewer CPUs or less memory than the host, `run-all.sh` now passes the engine's figures as the default `--max_cpus` and `--max_memory`. It also writes them to `/nextflow/executor.config` and passes that file with `-c`, so Nextflow books no more than the VM has. A `-c` of your own comes after it, so each `executor` setting in it wins. `THREADS`, `--max_cpus` or `--max_memory` of your own still cap each task, and `executor.config` never books less than them, so Nextflow does not refuse a task that asks for more than the VM's figure. On native Linux the engine reports the host's figures, and the command is unchanged. `validate-setup.sh` adds an `[INFO]` line when Docker has less memory than the host. No option or output file changes. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index 33457eebf..d12063fac 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -178,7 +178,7 @@ executor { } ``` -Check the shares on a running task: `docker ps` lists the pipeline's containers as `nxf-...`, and `docker inspect -f '{{.HostConfig.CpuShares}}' ` should print 256. On Docker Desktop and colima, `run-all.sh` already writes an `executor` block with the VM's CPUs and memory to `/nextflow/executor.config`. It passes that file before your options, so each setting in an `executor` block of your own (`-c`) wins over it; a setting you leave out keeps the VM's figure. Keep `THREADS` and `--max_memory` no larger than your block, or Nextflow refuses the tasks that ask for more. +Check the shares on a running task: `docker ps` lists the pipeline's containers as `nxf-...`, and `docker inspect -f '{{.HostConfig.CpuShares}}' ` should print 256. On Docker Desktop and colima, `run-all.sh` already writes an `executor` block with the VM's CPUs and memory to `/nextflow/executor.config`. It passes that file before your options, so each setting in an `executor` block of your own (`-c`) wins over it; a setting you leave out keeps the VM's figure. Keep `THREADS` and `--max_memory` no larger than your block, or Nextflow refuses the tasks that ask for more. Without a block of your own, `run-all.sh` never books less than a cap you set: `THREADS`, `--max_cpus` or a whole-GB `--max_memory` above the VM's figure raises the CPUs or memory in `executor.config` to it, and a `--max_memory` in other units leaves the memory to Nextflow. The run then starts as it would without the VM's figure, and a task the VM cannot hold can be killed (exit 137); `run-all.sh` prints a `[Resources]` line that says so. ### Runtime per step diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 34b8a879d..e0553f4e6 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -982,7 +982,7 @@ docker stats --no-stream ### When to reduce `--cpus` or `--memory` -With `run-all.sh` you do not edit the scripts' flags: `THREADS=N` caps every task's CPUs (`--max_cpus`), and `--max_memory 24.GB` after the sex caps every task's memory. They cap each task, not the run. Nextflow starts a task when its request fits in what is left of the machine's CPUs and RAM after the tasks it already started; it does not look at what other programs use. To bound the total, see [a shared host](hardware-requirements.md#a-shared-host). +With `run-all.sh` you do not edit the scripts' flags: `THREADS=N` caps every task's CPUs (`--max_cpus`), and `--max_memory 24.GB` after the sex caps every task's memory. They cap each task, not the run. Nextflow starts a task when its request fits in what is left of the machine's CPUs and RAM (the container engine's when it has less) after the tasks it already started; it does not look at what other programs use. To bound the total, see [a shared host](hardware-requirements.md#a-shared-host). **Reduce `--cpus` when:** - Running multiple steps in parallel on a machine with limited cores diff --git a/scripts/run-all.sh b/scripts/run-all.sh index 8034e9051..a5d741033 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -298,8 +298,19 @@ if [[ "$EC" =~ ^[0-9]+$ ]] && [ "$EC" -ge 1 ] && [ "$EC" -lt "$NC" ]; then NC=$E [[ " $* " == *" --max_cpus"* ]] || NF+=(--max_cpus "${USER_THREADS:-$NC}") [[ " $* " == *" --max_memory"* ]] || [ "${M:-0}" -lt 1 ] || NF+=(--max_memory "${M}.GB") if [ -n "$EX" ]; then - { printf 'executor {\n cpus = %s\n' "$NC"; [ "${M:-0}" -lt 1 ] || printf " memory = '%s GB'\n" "$M"; printf '}\n'; } > "$EXC" - NF+=(-c "$EXC"); echo "[Resources] the container engine has less than this host: Nextflow books at most ${NC} CPUs and ${M} GB (${EXC})" + # The pool never goes below a cap the user set (THREADS, --max_cpus, --max_memory): Nextflow refuses a task that asks + # for more than the pool. A cap it cannot read (not whole CPUs, not whole GB) leaves that line out, as on the host. + XC=$NC XM=$M UC=${USER_THREADS:-} UM="" prev="" GB_RE='^([0-9]+)[[:space:]]*\.?[[:space:]]*[Gg][Bb]?$' + for a in "$@"; do + case "$prev" in --max_cpus) UC=$a ;; --max_memory) UM=$a ;; esac + case "$a" in --max_cpus=*) UC=${a#*=} ;; --max_memory=*) UM=${a#*=} ;; esac; prev=$a + done + if [ -n "$UC" ]; then if [[ "$UC" =~ ^[0-9]+$ ]]; then [ "$UC" -le "$XC" ] || XC=$UC; else XC=""; fi; fi + if [ -n "$UM" ]; then if [[ "$UM" =~ $GB_RE ]]; then [ "${BASH_REMATCH[1]}" -le "${XM:-0}" ] || XM=${BASH_REMATCH[1]}; else XM=""; fi; fi + { printf 'executor {\n'; [ -z "$XC" ] || printf ' cpus = %s\n' "$XC"; [ "${XM:-0}" -lt 1 ] || printf " memory = '%s GB'\n" "$XM"; printf '}\n'; } > "$EXC" + XCS="${XC} CPUs" XMS="${XM} GB"; [ -n "$XC" ] || XCS="the host's CPUs"; [ "${XM:-0}" -ge 1 ] || XMS="the host's RAM" + NF+=(-c "$EXC"); echo "[Resources] the container engine has less than this host: Nextflow books at most ${XCS} and ${XMS} (${EXC})" + [ "$XC" = "$NC" ] && [ "$XM" = "$M" ] || echo "[Resources] the pool follows a task cap you set above the engine's ${NC} CPUs and ${M} GB, or one not in whole CPUs or GB: a task can then be killed inside the engine (exit 137)" else rm -f "$EXC"; fi # Publish by hard link when it is sure to work: the BAM and every other output are then stored once, not twice. # Nextflow does not fall back to a copy when a link fails; the run stops. The tasks run as root (the docker profile) diff --git a/tests/e2e/run-all-launcher.sh b/tests/e2e/run-all-launcher.sh index c81117ba6..439d729bc 100644 --- a/tests/e2e/run-all-launcher.sh +++ b/tests/e2e/run-all-launcher.sh @@ -26,7 +26,8 @@ # 2 CPUs, and a stand-in nextflow that records its arguments): run-all.sh # passes the engine's as --max_memory and --max_cpus, and the pool # Nextflow fills, executor.config, is the engine's. Each limit also -# alone; THREADS, an explicit --max_memory and a -c of the user's win. +# alone; THREADS, an explicit --max_memory and a -c of the user's win, +# and a cap the user sets above the engine's raises the pool to it. # Each comparison gets a negative control in the same run: a planted missing # file must be reported, the first run's trace must show the alignment and # calling tasks, and the first run's trace (tasks COMPLETED) must fail the @@ -158,6 +159,21 @@ check "fewer CPUs alone: --max_cpus ${ECPU}" has " --max_cpus ${ECPU} " "$CPUO" check_eq "fewer CPUs alone: the user's --max_memory 6.GB is the only one" "$(grep -o ' --max_memory [^ ]*' <<<"$CPUO" | tr -d '\n')" " --max_memory 6.GB" check "fewer CPUs alone: the pool has ${ECPU} CPUs" has "^ *cpus += ${ECPU}\$" "$CPUC" check "fewer CPUs alone: the pool keeps the host's ${HM} GB" has "^ *memory = '${HM} GB'\$" "$CPUC" +# A cap the user sets above the engine's figure: the pool must hold it, or Nextflow refuses each task +# that asks for more than the pool (Process requirement exceeds available CPUs/memory) and the run stops. +AT=$((ECPU + 2)) AM=$((EG + 4)) +ET=$AT engine_run above "$((EG * 1073741824)) ${ECPU}" --max_memory "${AM}.GB" +ABO=$(cat "${ENG}/above.args" 2>/dev/null) ABC=$(cat "${ENG}/above.config" 2>/dev/null) +echo "+ nextflow${ABO}" +check "THREADS=${AT} above the engine's ${ECPU} CPUs: --max_cpus ${AT}" has " --max_cpus ${AT} " "$ABO" +check "THREADS=${AT} above the engine's ${ECPU} CPUs: the pool has ${AT} CPUs, not ${ECPU}" has "^ *cpus += ${AT}\$" "$ABC" +check "--max_memory ${AM}.GB above the engine's ${EG} GB: the pool has ${AM} GB" has "^ *memory = '${AM} GB'\$" "$ABC" +check "run-all.sh says the pool follows the user's cap" has '^\[Resources\] the pool follows a task cap you set' \ + "$(cat "${E2E_WORK}/logs/${CASE_NAME}.engine-above.log" 2>/dev/null)" +engine_run above2 "$((EG * 1073741824)) ${ECPU}" "--max_cpus=$((AT + 1))" --max_memory 9000.MB +AB2=$(cat "${ENG}/above2.config" 2>/dev/null) +check "--max_cpus=$((AT + 1)) above the engine's: the pool has $((AT + 1)) CPUs" has "^ *cpus += $((AT + 1))\$" "$AB2" +check "--max_memory 9000.MB, not whole GB: the pool leaves memory to Nextflow" lacks "memory" "$AB2" rm -rf "$G3" "$ENG" # --- first run, with fs.protected_hardlinks 0 --------------------------------------------- From 0c90e9ced6d21b5db2139d1134188917706550d2 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:01:25 +0100 Subject: [PATCH 226/255] docs(lessons): say mito-merge passes a named MT-RNR1 allele from AF 0.05, heteroplasmic included The lesson still said only a homoplasmic allele is passed, which contradicts the new floor in og pgx mito-merge and docs/07. --- docs/lessons-learned.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md index 71c5e7d8a..038d182a3 100644 --- a/docs/lessons-learned.md +++ b/docs/lessons-learned.md @@ -534,7 +534,7 @@ Most bioinformatics containers run as non-root users. If writing to bind-mounted ### PharmCAT reads MT-RNR1 only as an outside call - **Failed:** MT-RNR1 (aminoglycoside-induced hearing loss) was No Result on every sample, although step 20 calls chrM with Mutect2 at high depth. - **Why:** PharmCAT 3.4.0 has no MT-RNR1 allele definition for its matcher, so chrM records in its VCF input are never matched for the gene. It lists MT-RNR1 among the genes it prefers as an outside call, and its own MT-RNR1 test passes the gene that way. -- **Fix:** `og pgx mito-merge` (was `bin/pgx_mito_merge.py`) turns step 20's chrM calls into one MT-RNR1 outside call (a homoplasmic named allele, or `Reference` only when the BAM covers every named position), added to step 36's outside calls. Step 27 checks only CYP2D6, HLA-A and HLA-B against step 36's table: MT-RNR1 never appears there, and checking it marked the new call stale. The details are in [Pharmacogenomics, Input the preprocessor or PharmCAT refuses](07-pharmacogenomics.md#input-the-preprocessor-or-pharmcat-refuses). +- **Fix:** `og pgx mito-merge` (was `bin/pgx_mito_merge.py`) turns step 20's chrM calls into one MT-RNR1 outside call (a named allele at AF 0.05 or more, heteroplasmic included, or `Reference` only when the BAM covers every named position), added to step 36's outside calls. Step 27 checks only CYP2D6, HLA-A and HLA-B against step 36's table: MT-RNR1 never appears there, and checking it marked the new call stale. The details are in [Pharmacogenomics, Input the preprocessor or PharmCAT refuses](07-pharmacogenomics.md#input-the-preprocessor-or-pharmcat-refuses). ### PharmCAT's log file needs a writable working directory - **Failed:** every step 07 run printed a logback status dump with a `pharmcat.log (Permission denied)` stack trace. The calls were not affected. From 4fbc0ce7326442088211bf2b76f96ab3ef5ab763 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:38:25 +0100 Subject: [PATCH 227/255] fix(benchmark): a site with no genotype on either side is not a genotype match --- CHANGELOG.md | 2 +- docs/benchmarking.md | 2 +- scripts/benchmark-variants.sh | 5 +++-- tests/fake-docker/run-all-switches.sh | 21 ++++++++++++--------- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fd7d868e..94474687c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- `benchmark-variants.sh` now compares genotypes, not just alleles, between callers. Its pairwise mode counted a site as shared when both callers had the same allele there, so a site one caller called 0/1 and the other 1/1 still scored as full agreement, and the summary said a Jaccard index of 1.0 meant perfect agreement. `comparison.tsv` gains two columns at the end, `GT_Concordant` and `GT_Concordance`: the shared sites whose genotypes also agree, ignoring phase and allele order, with a homozygous call equal to its haploid form. The existing columns are unchanged. +- `benchmark-variants.sh` now compares genotypes, not just alleles, between callers. Its pairwise mode counted a site as shared when both callers had the same allele there, so a site one caller called 0/1 and the other 1/1 still scored as full agreement, and the summary said a Jaccard index of 1.0 meant perfect agreement. `comparison.tsv` gains two columns at the end, `GT_Concordant` and `GT_Concordance`: the shared sites whose genotypes also agree, ignoring phase and allele order, with a homozygous call equal to its haploid form; a site with no genotype on either side is not a match. The existing columns are unchanged. - The step now also compares Clair3 (`vcf_clair3/`) and `.vcf.gz` in any other `vcf_*/` directory. `run-all.sh` with `BENCHMARK=true` already queued the step for those directories, and the step then found one caller and exited 1, so the run reported a failure; step 03e also told long-read users to run it. - The docs stop calling step 03a the clinical-lab standard without a caveat: it runs HaplotypeCaller without base quality score recalibration (BQSR) and with `-ERC NONE`, so it is not the full GATK Best Practices workflow. Part of #411 (plat-55, plat-71). diff --git a/docs/benchmarking.md b/docs/benchmarking.md index ce72bf95c..2ef089c1b 100644 --- a/docs/benchmarking.md +++ b/docs/benchmarking.md @@ -70,7 +70,7 @@ Compare two or more caller VCFs against each other to measure agreement. This te | `GT_Concordant` | shared sites where the two genotypes are also the same | | `GT_Concordance` | `GT_Concordant / Shared`; `N/A` when nothing is shared | -A site one caller calls 0/1 and the other 1/1 counts in `Shared` and in the Jaccard index, but not in `GT_Concordant`. Genotypes are compared without phase and with their alleles sorted (`1|0` equals `0/1`), and a homozygous call equals its haploid form (`1/1` equals `1`), so a caller that writes male chrX haploid still agrees with one that writes it diploid. +A site one caller calls 0/1 and the other 1/1 counts in `Shared` and in the Jaccard index, but not in `GT_Concordant`. Genotypes are compared without phase and with their alleles sorted (`1|0` equals `0/1`), and a homozygous call equals its haploid form (`1/1` equals `1`), so a caller that writes male chrX haploid still agrees with one that writes it diploid. A site with no genotype on either side (`./.`, `.`, or a record with no `GT` field) does not count as concordant. The commands below do the allele part by hand: diff --git a/scripts/benchmark-variants.sh b/scripts/benchmark-variants.sh index 9b2fba621..5fb7d5c6e 100755 --- a/scripts/benchmark-variants.sh +++ b/scripts/benchmark-variants.sh @@ -442,7 +442,8 @@ else # same order, one line per site, so the two files are read side by side and # never held in memory. Genotypes are compared unphased and with their # alleles sorted (1|0 = 0/1), and a homozygous call equals its haploid form - # (1/1 = 1), so only a different set of alleles counts as a mismatch. + # (1/1 = 1), so only a different set of alleles counts as a mismatch. A site + # with no genotype on both sides (./., ., or no GT field) is not a match. gt_agree() { paste <(site_gt "$1") <(site_gt "$2") | awk -F'\t' ' function canon(g, a, n, i, j, t, same, out) { @@ -457,7 +458,7 @@ else return out } { if ($1 != $6 || $2 != $7 || $3 != $8 || $4 != $9) bad++ - else if (canon($5) == canon($10)) agree++ } + else if (canon($5) == canon($10) && canon($5) != ".") agree++ } END { if (bad) print "WARNING: " bad " shared records of the two callers did not line up; counted as genotype mismatches" > "/dev/stderr" print agree + 0 }' } diff --git a/tests/fake-docker/run-all-switches.sh b/tests/fake-docker/run-all-switches.sh index 09a7e55da..0ba9e3262 100644 --- a/tests/fake-docker/run-all-switches.sh +++ b/tests/fake-docker/run-all-switches.sh @@ -7,7 +7,8 @@ # plain run the host's CPU count and RAM as --max_cpus and --max_memory; # BENCHMARK=true with one caller VCF is skipped, not failed # benchmark BENCHMARK=true counts the shared sites whose genotypes -# agree (0/1 against 1/1 is a mismatch), and compares Clair3 +# agree (0/1 against 1/1, or no genotype on either side, is +# not a match), and compares Clair3 # and any other vcf_*/ VCF run-all.sh counts # ALIGN_DIR run its BAM replaces the plain run's samplesheet row # plain again back to aligned/: a kept row names this call's BAM only @@ -103,9 +104,11 @@ if compgen -G "${G}/.pgp-link-probe*" > /dev/null || compgen -G "${G}/sample1/.p fi # BENCHMARK=true compares genotypes, and finds every caller run-all.sh counts. -# The isec hook writes two callers that agree on both alleles but not on one -# genotype (0/1 against 1/1): the Jaccard index is 1, the genotypes agree on -# one site of two. +# The isec hook writes two callers that agree on the alleles of three sites: +# one genotype mismatch (0/1 against 1/1), one match (1|0 against 0/1) and one +# site with no genotype on either side (./. against a sites-only record), which +# is not a match. The Jaccard index is 1, the genotypes agree on one site of +# three. cat > "${CASE_WORK}/isec-hook" <<'HOOK' #!/usr/bin/env bash set -euo pipefail @@ -117,8 +120,8 @@ mkdir -p "$d" h=$'##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS' printf '%s\n' "$h" > "${d}/0000.vcf" printf '%s\n' "$h" > "${d}/0001.vcf" -printf '%s\nchr1\t10\t.\tA\tG\t50\tPASS\t.\tGT:DP\t0/1:30\nchr1\t20\t.\tC\tT\t50\tPASS\t.\tGT\t1|0\n' "$h" > "${d}/0002.vcf" -printf '%s\nchr1\t10\t.\tA\tG\t50\tPASS\t.\tDP:GT\t30:1/1\nchr1\t20\t.\tC\tT\t50\tPASS\t.\tGT\t0/1\n' "$h" > "${d}/0003.vcf" +printf '%s\nchr1\t10\t.\tA\tG\t50\tPASS\t.\tGT:DP\t0/1:30\nchr1\t20\t.\tC\tT\t50\tPASS\t.\tGT\t1|0\nchr1\t30\t.\tG\tA\t50\tPASS\t.\tGT\t./.\n' "$h" > "${d}/0002.vcf" +printf '%s\nchr1\t10\t.\tA\tG\t50\tPASS\t.\tDP:GT\t30:1/1\nchr1\t20\t.\tC\tT\t50\tPASS\t.\tGT\t0/1\nchr1\t30\t.\tG\tA\t50\tPASS\t.\t.\t.\n' "$h" > "${d}/0003.vcf" HOOK chmod +x "${CASE_WORK}/isec-hook" callers() { @@ -133,13 +136,13 @@ callers() { cmp_row() { sed -n 2p "${G}/sample1/benchmark/comparison.tsv" 2>/dev/null; } callers vcf vcf_gatk run_expect 0 bench-gt env TOOLS=pharmcat BENCHMARK=true FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/isec-hook" "${SCRIPTS}/run-all.sh" sample1 male -[ "$(cmp_row)" = $'DeepVariant\tGATK\t2\t0\t0\t1.0000\t1\t0.5000' ] \ - || fail "0/1 against 1/1 is not a genotype mismatch in comparison.tsv: $(cmp_row)" +[ "$(cmp_row)" = $'DeepVariant\tGATK\t3\t0\t0\t1.0000\t1\t0.3333' ] \ + || fail "0/1 against 1/1, or no genotype on both sides, counted as a genotype match in comparison.tsv: $(cmp_row)" # Clair3 (step 03e) and a VCF in a directory of one's own: run-all.sh queues # the step for them, and the step compares them instead of exiting 1. callers vcf vcf_clair3 vcf_mine run_expect 0 bench-clair3 env TOOLS=pharmcat BENCHMARK=true FAKE_DOCKER_RUN_HOOK="${CASE_WORK}/isec-hook" "${SCRIPTS}/run-all.sh" sample1 male -grep -qE $'^DeepVariant\tClair3\t2\t0\t0\t1.0000\t1\t0.5000$' "${G}/sample1/benchmark/comparison.tsv" \ +grep -qE $'^DeepVariant\tClair3\t3\t0\t0\t1.0000\t1\t0.3333$' "${G}/sample1/benchmark/comparison.tsv" \ || fail "DeepVariant and Clair3 were not compared: $(cat "${G}/sample1/benchmark/comparison.tsv" 2>/dev/null)" grep -qE $'^Clair3\tmine\t' "${G}/sample1/benchmark/comparison.tsv" || fail "vcf_mine/ was not compared" # A hand-placed vcf_strelka2/.vcf.gz also queues the step, so it is From 9e9b6170f2a1e17da3d3e8bb785ddc2032a6651c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:44:32 +0100 Subject: [PATCH 228/255] test(ancestry): check the simulator's truth tracts against the exponential law A joined truth row of ancestry a ends at an event that draws another ancestry, so its length in cM follows the exponential law of rate T (1 - mu_a). The test reads over 25,000 interior rows of each ancestry back from the truth and checks the mean within 5% and the share above the mean within 0.02 of 1/e, and that a truth drawn with T doubled fails the check. Under planted-red the truth is drawn with T doubled. --- rust/crates/og-admix/src/simulate.rs | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/rust/crates/og-admix/src/simulate.rs b/rust/crates/og-admix/src/simulate.rs index 54d690794..f3f6e5e22 100644 --- a/rust/crates/og-admix/src/simulate.rs +++ b/rust/crates/og-admix/src/simulate.rs @@ -654,6 +654,47 @@ mod tests { ); } + /// The cM lengths of the truth rows of each ancestry of `cfg(seed)` with + /// T = `generations`, seeds 1 to 40, on 3,000 sites over 30 Morgans. + /// Rows that touch a chromosome end are cut by it, so they are left out. + fn truth_tract_lengths(generations: f64) -> [Vec; 2] { + let d = Dir::new("sim-tracts"); + let panel = two_group_panel(&d, 3000); + let map = GeneticMap::from_markers(vec![(1, 0.0), (3_001_001, 3000.0)]).unwrap(); + let mut out = [Vec::new(), Vec::new()]; + for seed in 1..=40 { + let cfg = Config { + generations, + ..cfg(seed) + }; + let chrom = simulate_chrom(&panel, &map, &people(), &cfg, &mut Vec::new()).unwrap(); + for r in chrom.rows { + if r.start != 2000 - 1 && r.end != 3_001_000 { + out[r.ancestry].push(map.cm_at(r.end) - map.cm_at(r.start)); + } + } + } + out + } + + /// A joined truth row of ancestry a ends at an event that draws another + /// ancestry, so its length follows the exponential law of rate + /// T (1 - mu_a): mean 100 / (T (1 - mu_a)) cM. + #[test] + fn truth_tracts_follow_the_exponential_law() { + let t = 50.0; + let mu = [0.6, 0.4]; + let drawn_t = if crate::PLANTED { 2.0 * t } else { t }; + for (a, lengths) in truth_tract_lengths(drawn_t).iter().enumerate() { + assert!(lengths.len() >= 25_000, "{} tracts", lengths.len()); + exponential_law(lengths, t * (1.0 - mu[a])).unwrap(); + } + for (a, lengths) in truth_tract_lengths(2.0 * t).iter().enumerate() { + let err = exponential_law(lengths, t * (1.0 - mu[a])).unwrap_err(); + assert!(err.starts_with("mean tract"), "{err}"); + } + } + #[test] fn the_cover_check_finds_a_gap_an_overlap_and_a_split() { let r = |start, end, haplotype, ancestry| TruthRow { From ec9aee2d4975db178f97671ba76e084df418c2c1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:44:48 +0100 Subject: [PATCH 229/255] test(ancestry): plant the og-admix red controls for one CI run Turns planted-red on by default again, so Guard's cargo test shows the new truth-tract law test fail on a truth drawn with T doubled, beside the other controls. The next commit reverts it. Bead: D5 --- rust/crates/og-admix/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rust/crates/og-admix/Cargo.toml b/rust/crates/og-admix/Cargo.toml index cf646e56e..f9c378653 100644 --- a/rust/crates/og-admix/Cargo.toml +++ b/rust/crates/og-admix/Cargo.toml @@ -7,6 +7,8 @@ license.workspace = true publish.workspace = true [features] +# Red-first run: every control planted, so CI shows each test fail. Reverted in the next commit. +default = ["planted-red"] # Feeds each red control its broken input in place of the good one # (a shuffled map, T doubled, a flipped byte, a swapped sample order, a # second seed), so `cargo test -p og-admix --features planted-red From 42c925e38a1dcfe668cf5291990b1cc55b232cb1 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:45:44 +0100 Subject: [PATCH 230/255] test(bloodtype): read a right-shifted VCF through its current .tbi, and say which way step 41 read it Step 41 and the BLOODTYPE process now log whether the VCF is read through its index or once from the start. The e2e case adds a sample with a current .tbi (the usual path after step 03) that must read A2/A2 with one record realigned, and checks the no-index and stale-.csi samples are read from the start. --- modules/local/bloodtype/main.nf | 2 ++ scripts/41-bloodtype.sh | 2 ++ tests/e2e/bloodtype-1.sh | 24 ++++++++++++++++++------ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/modules/local/bloodtype/main.nf b/modules/local/bloodtype/main.nf index 71d42d34b..9144f85d3 100644 --- a/modules/local/bloodtype/main.nf +++ b/modules/local/bloodtype/main.nf @@ -147,6 +147,8 @@ process BLOODTYPE { break fi done + if [ "\$SEL" = -R ]; then echo "The VCF is read through its index." + else echo "The VCF is read once from the start (no index, or one older than the VCF)."; fi if [ "\$SEL" = -T ]; then bcftools head ${vcf} > bt_header.vcf cut -f1 bt_windows.bed \\ diff --git a/scripts/41-bloodtype.sh b/scripts/41-bloodtype.sh index e8723cf5e..619d389d7 100755 --- a/scripts/41-bloodtype.sh +++ b/scripts/41-bloodtype.sh @@ -107,6 +107,8 @@ NORM_PROG=' break fi done + if [ "$SEL" = -R ]; then echo " The VCF is read through its index." + else echo " The VCF is read once from the start (no index, or one older than the VCF)."; fi if [ "$SEL" = -T ]; then bcftools head "$VCF" > "$W/bt_header.vcf" cut -f1 "$W/bt_windows.bed" | awk "$8" "$W/bt_header.vcf" - > "$W/bt_contigs.txt" diff --git a/tests/e2e/bloodtype-1.sh b/tests/e2e/bloodtype-1.sh index d13b99187..471bd228b 100755 --- a/tests/e2e/bloodtype-1.sh +++ b/tests/e2e/bloodtype-1.sh @@ -20,8 +20,9 @@ # - ABO c.1061delC written two bases right (GG>G) gives A2/A2 as the table's # CG>C does, because step 41 left-aligns first (og on the VCF as written # reads A1/A1, the control), and a C>A,G 1/2 at c.803 keeps its one B copy; -# - that right-shifted VCF still gives A2/A2 with no ##contig line and no -# index, and with a current .tbi beside a stale .csi (htslib loads the .csi); +# - that right-shifted VCF still gives A2/A2 read through a current .tbi, +# read from the start with no ##contig line and no index, and read from the +# start with a current .tbi beside a stale .csi (htslib loads the .csi); # - the Nextflow processes (BLOODTYPE_DEPTH, BLOODTYPE) on a VCF and BAM # write the same three files as the bash step on the same two files. . "$(dirname "$0")/lib.sh" @@ -213,21 +214,32 @@ sample "$M" "${T}/${M}.vcf.gz" "${T}/win.bam" step "$M" check_eq "C>A,G 1/2 at c.803 stays one record: one B copy" "$(ev "$M" ABO_803)" "called 1" check_eq "and ABO A1/B" "$(cell "$M" ABO genotype)" "A1/B" -# The right-shifted VCF read once from the start: with no ##contig line (bcftools -# writes no record on an undeclared contig, so step 41 adds the lines), and with a -# current .tbi beside a .csi older than the VCF (htslib loads the .csi first). -BC="${SAMPLE}btc" BI="${SAMPLE}bti" +# The right-shifted VCF read through a current .tbi (step 03 writes one, so this +# is the usual path); then read once from the start: with no ##contig line +# (bcftools writes no record on an undeclared contig, so step 41 adds the lines), +# and with a current .tbi beside a .csi older than the VCF (htslib loads the .csi +# first). The samples above have no index, so they are read from the start too. +BX="${SAMPLE}btx" BC="${SAMPLE}btc" BI="${SAMPLE}bti" +sample "$BX" "${T}/${R}.vcf.gz" "${T}/win.bam" +dk "$BCFTOOLS_IMAGE" bcftools index -f -t "${G}/${BX}/vcf/${BX}.vcf.gz" +step "$BX" +check "a current .tbi: the step reads through it" has 'read through its index' "$(cat "$STEP_LOG")" +check_eq "through the .tbi, bcftools norm moved one record" "$(realigned)" 1 +check_eq "through the .tbi: ABO A2/A2" "$(cell "$BX" ABO phenotype)/$(cell "$BX" ABO genotype)" "A2/A2/A2" +check_eq "and c.1061delC reads two copies" "$(ev "$BX" ABO_1061)" "called 2" zcat "${T}/${R}.vcf.gz" | grep -v '^##contig' > "${T}/${BC}.vcf" dk "$BCFTOOLS_IMAGE" bgzip -f "${T}/${BC}.vcf" sample "$BC" "${T}/${BC}.vcf.gz" "${T}/win.bam" check_eq "control: that VCF declares no contig" "$(zcat "${G}/${BC}/vcf/${BC}.vcf.gz" | grep -c '^##contig')" 0 step "$BC" +check "no index: the step reads from the start" has 'read once from the start' "$(cat "$STEP_LOG")" check_eq "no ##contig line and no index: ABO A2/A2" "$(cell "$BC" ABO phenotype)/$(cell "$BC" ABO genotype)" "A2/A2/A2" sample "$BI" "${T}/${R}.vcf.gz" "${T}/win.bam" dk "$BCFTOOLS_IMAGE" bcftools index -f -t "${G}/${BI}/vcf/${BI}.vcf.gz" : > "${G}/${BI}/vcf/${BI}.vcf.gz.csi" touch -d 2000-01-01 "${G}/${BI}/vcf/${BI}.vcf.gz.csi" step "$BI" +check "a stale .csi: the step reads from the start" has 'read once from the start' "$(cat "$STEP_LOG")" check_eq "a current .tbi beside a stale .csi: ABO A2/A2" "$(cell "$BI" ABO phenotype)/$(cell "$BI" ABO genotype)" "A2/A2/A2" unset REF_FASTA rm -f "$REFBT" "${REFBT}.fai" From a60a7576ae097c9827c1c5adb34ee571b2632d12 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:46:51 +0100 Subject: [PATCH 231/255] fix(clinvar): print the REF_MISMATCH count after the hit summary, as the docs say --- modules/local/clinvar_screen/main.nf | 8 ++++---- scripts/06-clinvar-screen.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/local/clinvar_screen/main.nf b/modules/local/clinvar_screen/main.nf index 29af0325d..f2a416fb6 100644 --- a/modules/local/clinvar_screen/main.nf +++ b/modules/local/clinvar_screen/main.nf @@ -240,10 +240,6 @@ process CLINVAR_SCREEN { HITS=\$(grep -c -v '^#' ${meta.id}_clinvar_hits.vcf || true) echo "ClinVar pathogenic hits: \${HITS}" - echo "Records whose REF differs from the reference (REF_MISMATCH): \${REF_MISMATCH}" - if [ "\${REF_MISMATCH}" -gt 0 ]; then - echo "NOTICE: \${REF_MISMATCH} sample records have a REF that is not the reference's (bcftools norm REF_MISMATCH above): they stay as written and cannot match a ClinVar allele there. Check that the VCF was called on this reference." - fi # Grouped by ClinVar review stars, as scripts/06 prints them # (og clinvar hits, bin/og on the task PATH). if [ "\${HITS}" -gt 0 ]; then @@ -254,6 +250,10 @@ process CLINVAR_SCREEN { LOW=\$(awk -F'\\t' '\$13 == "yes"' hit_rows.tsv | wc -l | tr -d ' ') echo "Low-confidence calls among them (GQ below ${min_gq} or DP below ${min_dp}): \${LOW}" fi + echo "Records whose REF differs from the reference (REF_MISMATCH): \${REF_MISMATCH}" + if [ "\${REF_MISMATCH}" -gt 0 ]; then + echo "NOTICE: \${REF_MISMATCH} sample records have a REF that is not the reference's (bcftools norm REF_MISMATCH above): they stay as written and cannot match a ClinVar allele there. Check that the VCF was called on this reference." + fi cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index fc0ec8333..b6b3c8654 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -331,10 +331,6 @@ echo "cDNA scan of every exon: ${SCAN_TSV}" echo "Known deep intronic sites, always reported: ${DEEP_TSV}" awk -F'\t' 'NR > 1 {printf " %s %s %s:%s %s>%s: %s%s\n", $5, $6, $1, $2, $3, $4, $8, ($9 != "" ? ", " $9 : "")}' "$DEEP_TSV" echo "Count: ${HIT_COUNT} pathogenic hits" -echo "Records whose REF differs from the reference (REF_MISMATCH): ${REF_MISMATCH}" -if [ "$REF_MISMATCH" -gt 0 ]; then - echo "NOTICE: ${REF_MISMATCH} sample records have a REF that is not the reference's (bcftools norm REF_MISMATCH above): they stay as written and cannot match a ClinVar allele there. Check that the VCF was called on this reference." -fi if [ "$HIT_COUNT" -gt 0 ]; then HIT_ROWS=$(run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" "$BCFTOOLS_IMAGE" /pgp-bin/og clinvar hits "$(cpath "$HITS")") LOW_COUNT=$(awk -F'\t' '$13 == "yes"' <<< "$HIT_ROWS" | wc -l | tr -d ' ') @@ -351,3 +347,7 @@ if [ "$HIT_COUNT" -gt 0 ]; then | awk -F'\t' '{n[$1]++; line[$1] = line[$1] sprintf(" %s %s:%s %s>%s %s %s [%s]%s%s\n", $7, $2, $3, $4, $5, $6, $8, $9, ($10 != "." && $10 != "" ? " (" $10 ")" : ""), ($13 == "yes" ? " LOW-CONFIDENCE CALL (GQ " $12 ", DP " $11 "): check the reads" : ""))} END {for (s = 4; s >= 0; s--) if (n[s]) printf " %d star%s: %d\n%s", s, (s == 1 ? "" : "s"), n[s], line[s]}' fi +echo "Records whose REF differs from the reference (REF_MISMATCH): ${REF_MISMATCH}" +if [ "$REF_MISMATCH" -gt 0 ]; then + echo "NOTICE: ${REF_MISMATCH} sample records have a REF that is not the reference's (bcftools norm REF_MISMATCH above): they stay as written and cannot match a ClinVar allele there. Check that the VCF was called on this reference." +fi From 2eeab58735f4e69f6f49758c5ccc313172b91bdc Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:46:51 +0100 Subject: [PATCH 232/255] fix(clinvar): name a record for another allele in the no-call text, and keep GQ 20 as the reference floor --- CHANGELOG.md | 2 +- docs/06-clinvar-screen.md | 4 +- rust/crates/og-clinvar/src/sites.rs | 147 ++++++++++++++++++++++++++-- 3 files changed, 140 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eca20e722..10111819e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. -- The always-report list of step 06 (`clinvar/_clinvar_deep_intronic.tsv`) now says what each call really is. A `0/0` with a filter such as `LowQual`, a `0/0` without a GQ, and a `0/0` read from one or two reads all read as plain `ref`, and a male's one-copy chrX call (`1` or `0`, as some callers write it) read as the raw `1` or as no call. One state table in `og clinvar deep-intronic` now decides each site. Only FILTER `PASS`, `.` and `RefCall` leave a reference call plain; any other FILTER makes it `ref (low confidence)` and names the FILTER. A reference call without a GQ is `ref (low confidence)` too. A depth below the step's depth floor (`CLINVAR_MIN_DP`, `ext.min_dp`, default 10, passed with the new `--min-dp`) reads `ref (shallow)`. On chrX and chrY outside the pseudoautosomal regions a one-copy `1` reads `hemizygous` and a one-copy `0` reads `ref`; a one-copy GT anywhere else is no call, with why. A no call says what the record holds (`has no genotype`, `is not a reference call`), and no longer claims a PASS check that never ran or that no record exists when a VCF record without a genotype does. Both mirrors of step 06 add FILTER as a ninth column of the query the reader takes. Output change: the file keeps its columns; the `call` column gains the words `ref (shallow)` and `hemizygous`, and some `detail` texts change. The parity goldens of `clinvar-deep-intronic` record the deliberate changes. +- The always-report list of step 06 (`clinvar/_clinvar_deep_intronic.tsv`) now says what each call really is. A `0/0` with a filter such as `LowQual`, a `0/0` without a GQ, and a `0/0` read from one or two reads all read as plain `ref`, and a male's one-copy chrX call (`1` or `0`, as some callers write it) read as the raw `1` or as no call. One state table in `og clinvar deep-intronic` now decides each site. Only FILTER `PASS`, `.` and `RefCall` leave a reference call plain; any other FILTER makes it `ref (low confidence)` and names the FILTER. A reference call without a GQ is `ref (low confidence)` too, and so is one below GQ 20 when `CLINVAR_MIN_GQ` is set lower than 20: a lower floor changes the hits only. A depth below the step's depth floor (`CLINVAR_MIN_DP`, `ext.min_dp`, default 10, passed with the new `--min-dp`) reads `ref (shallow)`. On chrX and chrY outside the pseudoautosomal regions a one-copy `1` reads `hemizygous` and a one-copy `0` reads `ref`; a one-copy GT anywhere else is no call, with why. A no call says what the record holds (`has no genotype`, `is not a reference call`), and no longer claims a PASS check that never ran or that no record exists when a VCF record without a genotype, or a record for another allele, does. Both mirrors of step 06 add FILTER as a ninth column of the query the reader takes. Output change: the file keeps its columns; the `call` column gains the words `ref (shallow)` and `hemizygous`, and some `detail` texts change. The parity goldens of `clinvar-deep-intronic` record the deliberate changes. - Step 06 now counts the sample records whose REF is not the reference's base. `bcftools norm -c w` keeps such a record as written and only warns `REF_MISMATCH` in the log, and the record cannot match a ClinVar allele. The step's summary, and the Nextflow module's log, now print the count, and a notice to check the VCF's reference when there are any. - Each ClinVar hit now keeps the condition ClinVar names for it. Step 06 copied only ClinVar's ID, gene, significance and review status onto each hit, so a later step knew the gene but not the disease: a RET hit could be Hirschsprung disease or MEN2. The hits VCF (`clinvar/_clinvar_hits.vcf`) now also carries `CLNDN`, ClinVar's disease name, and `CLNDISDB`, its disease database ids, in both the bash step and the Nextflow module. Two INFO fields are added; nothing is renamed or removed, and `clinvar/_clinvar_hits.tsv` is unchanged. A hand-built ClinVar file that does not define the two fields still runs, without them and with a notice. diff --git a/docs/06-clinvar-screen.md b/docs/06-clinvar-screen.md index 94f8f7832..b1292af01 100644 --- a/docs/06-clinvar-screen.md +++ b/docs/06-clinvar-screen.md @@ -71,11 +71,11 @@ Every run writes `clinvar/${SAMPLE}_clinvar_deep_intronic.tsv`, one row per site | `call` | When | |---|---| | a genotype (`het`, `hom`, ...) | The sample carries the site's allele, in the words the hits use. One copy on chrX or chrY outside the pseudoautosomal regions (GT `1`, as some callers write a male's X) reads `hemizygous`. | -| `ref` | A reference call: a record whose alleles are all `0` (`0/0`, or `0` on chrX or chrY outside the pseudoautosomal regions), or a gVCF reference block over the site. Its FILTER is `PASS`, `.` or `RefCall` (DeepVariant's reference call), its GQ is at or above the call floor (`CLINVAR_MIN_GQ` or `ext.min_gq`, default 30), and its depth is at or above the depth floor (`CLINVAR_MIN_DP` or `ext.min_dp`, default 10). | +| `ref` | A reference call: a record whose alleles are all `0` (`0/0`, or `0` on chrX or chrY outside the pseudoautosomal regions), or a gVCF reference block over the site. Its FILTER is `PASS`, `.` or `RefCall` (DeepVariant's reference call), its GQ is at or above the call floor (`CLINVAR_MIN_GQ` or `ext.min_gq`, default 30; a floor set below 20 counts as 20 here, so a lower floor changes the hits only), and its depth is at or above the depth floor (`CLINVAR_MIN_DP` or `ext.min_dp`, default 10). | | `ref (low confidence)` | A reference call with any other FILTER (`LowQual`, for example), a GQ below the call floor, or no GQ at all, so nothing checked it against the floor. | | `ref (shallow)` | A reference call whose only fault is a depth (`FORMAT/DP`) below the depth floor. A DeepVariant reference block has no DP (it writes `MIN_DP`), and its GQ already falls with the depth, so only its GQ is checked. | | `other allele` | The sample carries another allele at the site. | -| `no call` | Nothing says reference or carrier: no record in the VCF or the gVCF, no gVCF at all, a record with no genotype (`./.`), a record that is not a reference call (`0/.`, or a filtered call of the allele that only the gVCF holds), or a one-copy GT where the genome has two copies (an autosome, or chrX inside a pseudoautosomal region). | +| `no call` | Nothing says reference or carrier: no record in the VCF or the gVCF, no gVCF at all, a record with no genotype (`./.`), a record that is not a reference call (`0/.`, or a filtered call of the allele that only the gVCF holds), or a one-copy GT where the genome has two copies (an autosome, or chrX inside a pseudoautosomal region). A record for another allele at the site that gives none of these is named the same way, with its alleles. | `detail` holds the call's GQ and DP (`GQ n (gVCF block)` for a block, with its DP when that is below the depth floor), each reason a reference call is not plain `ref` (`FILTER LowQual`, `below the call floor of GQ 30`, `no GQ to check against the call floor of GQ 30`, `below the depth floor of DP 10`), the other allele, or what the record at the site holds when there is no call. A no call is not a negative result: it says only that the site was not read. These floors apply to reference calls only. A carrier's GQ and DP are shown, not judged: the same allele is a ClinVar hit, and the hits table marks it low confidence when it is below the floor. Both reports print every row under the ClinVar hits. diff --git a/rust/crates/og-clinvar/src/sites.rs b/rust/crates/og-clinvar/src/sites.rs index 5c290bf83..acf94b81b 100644 --- a/rust/crates/og-clinvar/src/sites.rs +++ b/rust/crates/og-clinvar/src/sites.rs @@ -13,8 +13,8 @@ //! allele (`hemizygous` for one copy on chrX or chrY outside the //! pseudoautosomal regions); `ref` for a reference call (a record whose //! alleles are all `0`, or a gVCF reference block over the site) with -//! FILTER `PASS`, `.` or `RefCall`, a GQ at or above the call floor and a -//! depth at or above the depth floor; `ref (low confidence)` for a +//! FILTER `PASS`, `.` or `RefCall`, a GQ at or above the call floor (never +//! below GQ 20) and a depth at or above the depth floor; `ref (low confidence)` for a //! reference call with another FILTER, a GQ below the call floor or no GQ; //! `ref (shallow)` for one whose only fault is a depth below the depth //! floor; `other allele` when the sample carries another allele there; @@ -133,12 +133,27 @@ fn quality(gq: &[u8], dp: &[u8]) -> Vec { out } +/// The lowest GQ a plain `ref` may have, whatever the step's call floor: +/// a lower `CLINVAR_MIN_GQ` lowers the hits' floor, not this one. +const REF_GQ_FLOOR: u64 = 20; + impl Options { + /// The GQ floor of a reference call: the call floor, never below + /// [`REF_GQ_FLOOR`], with its words. + fn ref_gq_floor(&self) -> (u64, Vec) { + if self.min_gq_value < REF_GQ_FLOOR { + (REF_GQ_FLOOR, REF_GQ_FLOOR.to_string().into_bytes()) + } else { + (self.min_gq_value, self.min_gq.as_bytes().to_vec()) + } + } + /// A reference call: its words and detail from its GQ, depth and FILTER. fn reference(&self, gq: &[u8], dp: &[u8], filter: &[u8], block: bool) -> (Vec, Vec) { + let (floor, floor_words) = self.ref_gq_floor(); let filtered = !clean(filter); let no_gq = !is_digits(gq); - let low_gq = !no_gq && leading_number(gq) < self.min_gq_value as f64; + let low_gq = !no_gq && leading_number(gq) < floor as f64; let shallow = is_digits(dp) && leading_number(dp) < self.min_dp_value as f64; let call: &[u8] = if filtered || no_gq || low_gq { b"ref (low confidence)" @@ -159,13 +174,13 @@ impl Options { notes.push([b"FILTER ", filter].concat()); } if low_gq { - notes.push([b"below the call floor of GQ ", self.min_gq.as_bytes()].concat()); + notes.push([&b"below the call floor of GQ "[..], &floor_words].concat()); } if no_gq { notes.push( [ - b"no GQ to check against the call floor of GQ ", - self.min_gq.as_bytes(), + &b"no GQ to check against the call floor of GQ "[..], + &floor_words, ] .concat(), ); @@ -221,7 +236,7 @@ fn lines(text: &[u8]) -> Vec<&[u8]> { fn call(st: &[&[u8]], vcf: &[Vec<&[u8]>], gvcf: &[Vec<&[u8]>], o: &Options) -> (Vec, Vec) { let pos = leading_number(nth(st, 2)); let haploid = one_copy(nth(st, 1), pos as u64); - let (mut found, mut other, mut why) = (None, Vec::new(), None); + let (mut found, mut other, mut why, mut why_other) = (None, Vec::new(), None, None); for vr in vcf { if !same(nth(vr, 1), nth(st, 1)) || !same(nth(vr, 2), nth(st, 2)) { continue; @@ -238,8 +253,34 @@ fn call(st: &[&[u8]], vcf: &[Vec<&[u8]>], gvcf: &[Vec<&[u8]>], o: &Options) -> ( }); } } - } else if state == Gt::Carries && other.is_empty() { - other = [nth(vr, 4), b">", nth(vr, 5), b" ", &words(gt)].concat(); + } else if state == Gt::Carries { + if other.is_empty() { + other = [nth(vr, 4), b">", nth(vr, 5), b" ", &words(gt)].concat(); + } + } else { + // A record for another allele that carries nothing readable: + // it is still a record, so the no-call text says what it holds. + why_other.get_or_insert_with(|| { + let what = [ + b"the VCF record ", + nth(vr, 4), + b">", + nth(vr, 5), + b" at the site", + ] + .concat(); + if state == Gt::Ref { + [ + &what[..], + b" is a reference call for another allele (GT ", + gt, + b")", + ] + .concat() + } else { + unread(&what, gt, filter, state) + } + }); } } if let Some(f) = found { @@ -263,7 +304,7 @@ fn call(st: &[&[u8]], vcf: &[Vec<&[u8]>], gvcf: &[Vec<&[u8]>], o: &Options) -> ( ), }; } - if let Some(w) = why { + if let Some(w) = why.or(why_other) { return (b"no call".to_vec(), w); } let why: &[u8] = if o.gvcf { @@ -440,6 +481,53 @@ mod tests { ); } + /// A call floor below GQ 20 lowers the hits' floor only: a reference + /// call still needs GQ 20 to read plain `ref`. + #[test] + fn a_reference_call_needs_gq_20_whatever_the_floor() { + let at = |min_gq: u64, vcf: &str| { + let o = Options { + min_gq: min_gq.to_string(), + min_gq_value: min_gq, + min_dp: "10".into(), + min_dp_value: 10, + gvcf: true, + }; + let sites = + "chrom\tpos\tref\talt\tgene\thgvs\tclinvar_id\nchr1\t100\tA\tG\tG1\tc.1A>G\t1\n"; + let mut out = Vec::new(); + deep_intronic(sites.as_bytes(), vcf.as_bytes(), b"", &o, &mut out).unwrap(); + let s = String::from_utf8(out).unwrap(); + s.lines() + .nth(1) + .unwrap() + .split('\t') + .skip(7) + .collect::>() + .join("|") + }; + assert_eq!( + at(10, "chr1\t100\t100\tA\tG\t0/0\t15\t30\n"), + "ref (low confidence)|GQ 15, DP 30, below the call floor of GQ 20" + ); + assert_eq!( + at(10, "chr1\t100\t100\tA\tG\t0/0\t.\t30\n"), + "ref (low confidence)|DP 30, no GQ to check against the call floor of GQ 20" + ); + assert_eq!( + at(10, "chr1\t100\t100\tA\tG\t0/0\t20\t30\n"), + "ref|GQ 20, DP 30" + ); + assert_eq!( + at(25, "chr1\t100\t100\tA\tG\t0/0\t22\t30\n"), + "ref (low confidence)|GQ 22, DP 30, below the call floor of GQ 25" + ); + assert_eq!( + at(25, "chr1\t100\t100\tA\tG\t0/0\t25\t30\n"), + "ref|GQ 25, DP 30" + ); + } + /// GLA c.640-801G>A, on chrX outside the pseudoautosomal regions. const GLA: &str = "chrX\t101399747\tC\tT"; @@ -523,6 +611,45 @@ mod tests { ); } + /// A record for another allele that gives no call is still a record: + /// the text names it, and never says that no record exists. + #[test] + fn a_record_for_another_allele_is_named() { + assert_eq!( + run("chr1\t100\t100\tA\tC\t1\t99\t30\n", "", true), + "no call|the VCF record A>C at the site has one copy (GT 1) where the genome has two" + ); + assert_eq!( + run("chr1\t100\t100\tA\tC\t./.\t.\t.\n", "", true), + "no call|the VCF record A>C at the site has no genotype (GT ./.)" + ); + assert_eq!( + run("chr1\t100\t100\tA\tC\t0/.\t30\t20\n", "", false), + "no call|the VCF record A>C at the site is not a reference call (GT 0/.)" + ); + assert_eq!( + run("chr1\t100\t100\tA\tC\t0/0\t99\t30\n", "", true), + "no call|the VCF record A>C at the site is a reference call for another allele (GT 0/0)" + ); + // The site's own record, and a gVCF block, still come first. + assert_eq!( + run( + "chr1\t100\t100\tA\tC\t1\t99\t30\nchr1\t100\t100\tA\tG\t./.\t.\t.\n", + "", + true + ), + "no call|the VCF record at the site has no genotype (GT ./.)" + ); + assert_eq!( + run( + "chr1\t100\t100\tA\tC\t1\t99\t30\n", + "chr1\t90\t120\tA\t<*>\t0/0\t50\t.\n", + true + ), + "ref|GQ 50 (gVCF block)" + ); + } + /// A record without a genotype says so, and never that no record exists. #[test] fn no_call_says_what_was_there() { From cd319c06ad94775a4325f156057d6ef914bfaaa7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:48:55 +0100 Subject: [PATCH 233/255] test(ancestry): turn the planted og-admix red controls off again The previous commit's CI run showed each control fail. Back to the good inputs; `cargo test -p og-admix --features planted-red --no-fail-fast` still shows the failures on demand. Bead: D5 --- rust/crates/og-admix/Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust/crates/og-admix/Cargo.toml b/rust/crates/og-admix/Cargo.toml index f9c378653..cf646e56e 100644 --- a/rust/crates/og-admix/Cargo.toml +++ b/rust/crates/og-admix/Cargo.toml @@ -7,8 +7,6 @@ license.workspace = true publish.workspace = true [features] -# Red-first run: every control planted, so CI shows each test fail. Reverted in the next commit. -default = ["planted-red"] # Feeds each red control its broken input in place of the good one # (a shuffled map, T doubled, a flipped byte, a swapped sample order, a # second seed), so `cargo test -p og-admix --features planted-red From f1cc8ce3ab2f834f428bff4dbd37afb4c7d244a4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 17:57:20 +0100 Subject: [PATCH 234/255] fix(e2e): keep the RFC1 read counts off the REPCN line The personal-data scan reads a fraction beside the REPCN key as a repeat genotype, so the inline counts 2/2 and 20/20 turned the scan red. The counts now come from a variable set on its own line; the record bytes are the same. Bead: D1 --- tests/e2e/expansions-and-rfc1-1-motif.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/e2e/expansions-and-rfc1-1-motif.sh b/tests/e2e/expansions-and-rfc1-1-motif.sh index ae5c24ea2..388c11a19 100755 --- a/tests/e2e/expansions-and-rfc1-1-motif.sh +++ b/tests/e2e/expansions-and-rfc1-1-motif.sh @@ -34,7 +34,10 @@ make_sample() { # make_sample done grep '^#' "$in" | tail -n 1 printf 'chr4\t39348425\tRFC1\tA\t%s\t.\tPASS\tSVTYPE=STR;END=39348479;REF=11;RL=55;RU=AARRG;VARID=RFC1;REPID=RFC1' "$alts" - printf '\tGT:SO:REPCN:REPCI:ADSP:ADFL:ADIR:LC\t%s:%s:%s:%s:0/0:2/2:20/20:30.0\n' "$gt" "$so" "$cn" "$ci" + # The read counts sit on their own line: a fraction beside the REPCN key reads + # as a repeat genotype to scripts/ci/check-personal-data.sh. + local reads=0/0:2/2:20/20 + printf '\tGT:SO:REPCN:REPCI:ADSP:ADFL:ADIR:LC\t%s:%s:%s:%s:%s:30.0\n' "$gt" "$so" "$cn" "$ci" "$reads" } > "${GENOME_DIR}/${s}/expansion_hunter/${s}_eh.vcf" "${REPO}/bin/og" dev fixture rfc1 "$kind" "${GENOME_DIR}/${s}/aligned/reads.sam" docker run --rm -u "$(id -u):$(id -g)" -v "${GENOME_DIR}:/genome" -w "/genome/${s}/aligned" "$SAMTOOLS_IMAGE" \ From 35159fb1abffd8b080591b67b385fa3d7d357c9e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:01:11 +0100 Subject: [PATCH 235/255] fix(bloodtype): read a plain gzip VCF with no index, instead of stopping in bcftools reheader The windows name each marker contig with and without chr, so on the stream path the header nearly always gains a ##contig line, and bcftools reheader refuses a VCF in plain gzip. The new header now goes first and the records follow as text. The e2e sample with no ##contig line and no index is now in plain gzip. --- docs/41-bloodtype.md | 2 +- modules/local/bloodtype/main.nf | 5 +++-- scripts/41-bloodtype.sh | 9 +++++---- tests/e2e/bloodtype-1.sh | 17 ++++++++++------- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/docs/41-bloodtype.md b/docs/41-bloodtype.md index 9515067ef..650c9de24 100644 --- a/docs/41-bloodtype.md +++ b/docs/41-bloodtype.md @@ -74,7 +74,7 @@ The table writes each indel left-aligned, the way `bcftools norm` writes it: ABO - A multi-allelic indel is left-aligned as a whole, so its REF can stay longer than the marker's (`TCC TC,T` against `TC T`). Its REF then differs from the table's, and the site is a no call, as the paragraph above says. - A record whose REF differs from the reference is kept as it is (`-c w`), with a warning in the log. - A record on a contig the reference lacks (a VCF named `1`, `2`, ... against the chr-named reference) cannot be aligned. It goes through unchanged. -- bcftools reads the VCF through its index when the index htslib loads (the `.csi` if there is one, else the `.tbi`) is not older than the VCF. Otherwise it reads the VCF once from the start. Read that way, a VCF whose header has no `##contig` line for a marker's contig first gets one, because bcftools writes no record on an undeclared contig. +- bcftools reads the VCF through its index when the index htslib loads (the `.csi` if there is one, else the `.tbi`) is not older than the VCF. Otherwise it reads the VCF once from the start. Read that way, a VCF whose header has no `##contig` line for a marker's contig first gets one, because bcftools writes no record on an undeclared contig. That path also reads a VCF in plain gzip, not bgzip. - The left-aligned file is a work file: the bash step deletes it after the rules run, and the pipeline does not publish it. ### Several records at one site diff --git a/modules/local/bloodtype/main.nf b/modules/local/bloodtype/main.nf index 9144f85d3..15f266c48 100644 --- a/modules/local/bloodtype/main.nf +++ b/modules/local/bloodtype/main.nf @@ -129,7 +129,8 @@ process BLOODTYPE { // through unchanged. The VCF is read through its index when the one // htslib loads (.csi before .tbi) is current, else once from the start; // read that way, it first gets a ##contig line for each window contig its - // header lacks, because bcftools writes no record on an undeclared contig. + // header lacks, because bcftools writes no record on an undeclared contig + // (the new header, then the records as text, so plain gzip works too). // The same commands as scripts/41-bloodtype.sh; og runs from bin/, on the // task's PATH. """ @@ -160,7 +161,7 @@ process BLOODTYPE { fi pick() { if [ ! -s "\$1" ]; then bcftools view -h -Ou "\$HDR" - elif [ "\$HDR" != "${vcf}" ]; then bcftools reheader -h "\$HDR" ${vcf} | bcftools view -T "\$1" --targets-overlap record -Ou - + elif [ "\$HDR" != "${vcf}" ]; then { cat "\$HDR"; gzip -cdf ${vcf} | awk '!/^#/'; } | bcftools view -T "\$1" --targets-overlap record -Ou - else bcftools view "\$SEL" "\$1" "\$OVERLAP" record -Ou ${vcf}; fi } pick bt_on_ref.bed | bcftools norm -c w -f ${reference} -Ob -o bt_on_ref.bcf diff --git a/scripts/41-bloodtype.sh b/scripts/41-bloodtype.sh index 619d389d7..563029d59 100755 --- a/scripts/41-bloodtype.sh +++ b/scripts/41-bloodtype.sh @@ -89,9 +89,10 @@ echo "[2/3] The VCF's records around the marker sites, left-aligned (bcftools no # a chr-named reference) cannot be aligned and go through unchanged. The VCF is # read through its index when the one htslib loads (.csi before .tbi) is # current, else once from the start. Read that way, a VCF with no ##contig line -# for a window's contig first gets one (bcftools reheader), because bcftools -# writes no record on an undeclared contig; through an index, htslib takes the -# names from the index. The BLOODTYPE process runs the same commands before og. +# for a window's contig first gets one, because bcftools writes no record on an +# undeclared contig: the new header, then the records as text, so a VCF in plain +# gzip (not bgzip) works too. Through an index, htslib takes the names from the +# index. The BLOODTYPE process runs the same commands before og. # shellcheck disable=SC2016 # the program's $ belong to the inner bash and awk NORM_PROG=' MARKERS=$1 VCF=$2 REF=$3 W=$4 OUT=$5 @@ -119,7 +120,7 @@ NORM_PROG=' fi pick() { if [ ! -s "$1" ]; then bcftools view -h -Ou "$HDR" - elif [ "$HDR" != "$VCF" ]; then bcftools reheader -h "$HDR" "$VCF" | bcftools view -T "$1" --targets-overlap record -Ou - + elif [ "$HDR" != "$VCF" ]; then { cat "$HDR"; gzip -cdf "$VCF" | awk "!/^#/"; } | bcftools view -T "$1" --targets-overlap record -Ou - else bcftools view "$SEL" "$1" "$OVERLAP" record -Ou "$VCF"; fi } pick "$W/bt_on_ref.bed" | bcftools norm -c w -f "$REF" -Ob -o "$W/bt_on_ref.bcf" diff --git a/tests/e2e/bloodtype-1.sh b/tests/e2e/bloodtype-1.sh index 471bd228b..ff8e0cb5a 100755 --- a/tests/e2e/bloodtype-1.sh +++ b/tests/e2e/bloodtype-1.sh @@ -21,8 +21,9 @@ # CG>C does, because step 41 left-aligns first (og on the VCF as written # reads A1/A1, the control), and a C>A,G 1/2 at c.803 keeps its one B copy; # - that right-shifted VCF still gives A2/A2 read through a current .tbi, -# read from the start with no ##contig line and no index, and read from the -# start with a current .tbi beside a stale .csi (htslib loads the .csi); +# read from the start in plain gzip with no ##contig line and no index, and +# read from the start with a current .tbi beside a stale .csi (htslib loads +# the .csi); # - the Nextflow processes (BLOODTYPE_DEPTH, BLOODTYPE) on a VCF and BAM # write the same three files as the bash step on the same two files. . "$(dirname "$0")/lib.sh" @@ -215,9 +216,9 @@ step "$M" check_eq "C>A,G 1/2 at c.803 stays one record: one B copy" "$(ev "$M" ABO_803)" "called 1" check_eq "and ABO A1/B" "$(cell "$M" ABO genotype)" "A1/B" # The right-shifted VCF read through a current .tbi (step 03 writes one, so this -# is the usual path); then read once from the start: with no ##contig line -# (bcftools writes no record on an undeclared contig, so step 41 adds the lines), -# and with a current .tbi beside a .csi older than the VCF (htslib loads the .csi +# is the usual path); then read once from the start: in plain gzip (not bgzip) +# with no ##contig line (bcftools writes no record on an undeclared contig, so +# step 41 adds the lines, and bcftools reheader refuses plain gzip), and with a current .tbi beside a .csi older than the VCF (htslib loads the .csi # first). The samples above have no index, so they are read from the start too. BX="${SAMPLE}btx" BC="${SAMPLE}btc" BI="${SAMPLE}bti" sample "$BX" "${T}/${R}.vcf.gz" "${T}/win.bam" @@ -228,12 +229,14 @@ check_eq "through the .tbi, bcftools norm moved one record" "$(realigned)" 1 check_eq "through the .tbi: ABO A2/A2" "$(cell "$BX" ABO phenotype)/$(cell "$BX" ABO genotype)" "A2/A2/A2" check_eq "and c.1061delC reads two copies" "$(ev "$BX" ABO_1061)" "called 2" zcat "${T}/${R}.vcf.gz" | grep -v '^##contig' > "${T}/${BC}.vcf" -dk "$BCFTOOLS_IMAGE" bgzip -f "${T}/${BC}.vcf" +gzip -f "${T}/${BC}.vcf" sample "$BC" "${T}/${BC}.vcf.gz" "${T}/win.bam" check_eq "control: that VCF declares no contig" "$(zcat "${G}/${BC}/vcf/${BC}.vcf.gz" | grep -c '^##contig')" 0 +check "control: that VCF is plain gzip (no BGZF BC field)" bash -c '[ "$(head -c 14 "$1" | tail -c 2)" != BC ]' _ \ + "${G}/${BC}/vcf/${BC}.vcf.gz" step "$BC" check "no index: the step reads from the start" has 'read once from the start' "$(cat "$STEP_LOG")" -check_eq "no ##contig line and no index: ABO A2/A2" "$(cell "$BC" ABO phenotype)/$(cell "$BC" ABO genotype)" "A2/A2/A2" +check_eq "plain gzip, no ##contig line and no index: ABO A2/A2" "$(cell "$BC" ABO phenotype)/$(cell "$BC" ABO genotype)" "A2/A2/A2" sample "$BI" "${T}/${R}.vcf.gz" "${T}/win.bam" dk "$BCFTOOLS_IMAGE" bcftools index -f -t "${G}/${BI}/vcf/${BI}.vcf.gz" : > "${G}/${BI}/vcf/${BI}.vcf.gz.csi" From 262fe32c2eae3c06abf9a179b1fe4af22b9dd826 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:55:08 +0100 Subject: [PATCH 236/255] ci(e2e): exclude restricted installations from base artifacts Check the finished archive and prove the guard with planted paths, empty archives and broken archives. Keep pgsc_calc and tool results for the shards. Bead: lic-31 Bead: lic-23 --- CHANGELOG.md | 1 + scripts/ci/e2e-run.sh | 64 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6409f33f..7fc56686b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- E2E base artifacts exclude the Cyrius installation and pypgx bundle. The finished archive is checked before upload, with planted failures in the runner self-test. The shards still receive pgsc_calc and the tools' results. No pipeline output changes. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh index 4a5a45800..cad74523e 100755 --- a/scripts/ci/e2e-run.sh +++ b/scripts/ci/e2e-run.sh @@ -4,11 +4,13 @@ # Usage: scripts/ci/e2e-run.sh [pattern] # scripts/ci/e2e-run.sh --self-test-pull | --image-groups | --pull-images # scripts/ci/e2e-run.sh --pull-image REF +# scripts/ci/e2e-run.sh --self-test-base # pattern optional shell glob; runs only the cases whose file name matches # (e.g. '2*' or '*cyrius*'). Later cases read earlier cases' outputs, # so a partial run is for debugging only. # --self-test-pull checks the pull retry of step 3 against a fake docker -# (seconds, no network) and the image cache groups, and exits. +# (seconds, no network), image cache groups and base archive guard. +# --self-test-base checks the base archive guard with synthetic files. # --image-groups prints "GROUP IMAGE" for every image the cases use (the # e2e workflow's image cache entries) and exits. # --pull-images pulls those images, as step 3 below does, and exits. @@ -489,6 +491,61 @@ disk_check() { fi } +# Read the finished archive, not the find input: a future packing change must +# not publish either install. Read the whole stream so pipefail also catches +# a broken archive, without grep -q closing the pipe before tar finishes. +check_base_archive() { + local archive=$1 listing="${1}.list" rc=0 + if ! zstd -dc "$archive" | tar -tPf - > "$listing"; then + echo "ERROR: cannot list the base archive: ${archive}" >&2 + rc=1 + elif [ ! -s "$listing" ]; then + echo "ERROR: the base archive is empty" >&2 + rc=1 + elif grep -E '(^|/)(tools/cyrius-[^/]*|reference/pypgx-bundle)(/|$)' "$listing" >&2; then + echo "ERROR: the base archive contains a Cyrius install or pypgx bundle" >&2 + rc=1 + fi + rm -f "$listing" + return "$rc" +} + +self_test_base_archive() ( + local t + t=$(mktemp -d) + trap 'rm -rf "$t"' EXIT + mkdir -p "${t}/genome/tools/cyrius-x" "${t}/genome/reference/pypgx-bundle/.git" \ + "${t}/genome/tools/pgsc_calc-x" "${t}/genome/HG002/cyrius" + touch "${t}/genome/tools/cyrius-x/INSTALLED" "${t}/genome/reference/pypgx-bundle/.git/config" \ + "${t}/genome/tools/pgsc_calc-x/main.nf" "${t}/genome/HG002/cyrius/result.tsv" + for path in genome/tools/cyrius-x genome/reference/pypgx-bundle; do + tar -C "$t" -cf - "$path" | zstd -q -f -o "${t}/bad.tar.zst" + if check_base_archive "${t}/bad.tar.zst"; then + echo "FAIL: the base archive guard accepted planted ${path}" >&2 + exit 1 + fi + echo "ok: the base archive guard rejected planted ${path}" + done + tar -C "$t" -cf - genome/tools/pgsc_calc-x genome/HG002/cyrius \ + | zstd -q -o "${t}/good.tar.zst" + check_base_archive "${t}/good.tar.zst" + echo "ok: the base archive guard keeps pgsc_calc and Cyrius results" + tar -cf - -T /dev/null | zstd -q -o "${t}/empty.tar.zst" + printf 'not an archive\n' > "${t}/broken.tar.zst" + for path in empty broken; do + if check_base_archive "${t}/${path}.tar.zst"; then + echo "FAIL: the base archive guard accepted an archive (${path})" >&2 + exit 1 + fi + echo "ok: the base archive guard rejected an archive (${path})" + done +) + +if [ "${1:-}" = --self-test-base ]; then + self_test_base_archive + exit +fi + if [ "${1:-}" = --self-test-pull ]; then T=$(mktemp -d) trap 'rm -rf "$T"' EXIT @@ -599,6 +656,7 @@ FAKE else echo "FAIL: the docker shim's lazy pulls:"; sed 's/^/ /' "${T}/calls"; st_failed=1 fi + self_test_base_archive exit "$st_failed" fi @@ -956,9 +1014,11 @@ if [ -n "${E2E_BASE_OUT:-}" ] && [ "$failed" -eq 0 ]; then # Beside the archive too, so a shard learns the work area before it unpacks. cp "${E2E_WORK}/base-from" "${E2E_BASE_OUT}.from" sudo find "$E2E_WORK" "$REPO" \( -path "$FIXTURE_DIR" -o -path "$REPORT_DIR" \ - -o -path "${REPO}/.git" -o -path "$E2E_NOTES" -o -path "${E2E_WORK}/all-cases-ran" \) -prune \ + -o -path "${REPO}/.git" -o -path "$E2E_NOTES" -o -path "${E2E_WORK}/all-cases-ran" \ + -o -path "${GENOME_DIR}/tools/cyrius-*" -o -path "${GENOME_DIR}/reference/pypgx-bundle" \) -prune \ -o -cnewer "${E2E_WORK}/.layout-done" -print0 \ | sudo tar -P --format=posix --no-recursion --null -T - -cf - | zstd -T0 -3 -q -o "$E2E_BASE_OUT" + check_base_archive "$E2E_BASE_OUT" echo "Packed the base outputs: $(du -h "$E2E_BASE_OUT" | cut -f1), $(zstd -dc "$E2E_BASE_OUT" | tar -tf - | wc -l) entries. The largest:" zstd -dc "$E2E_BASE_OUT" | tar -tvf - | sort -k3,3n | tail -n 5 fi From 3bdb53da5f9ddecade28327a1a534a0897233fe7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:17:39 +0100 Subject: [PATCH 237/255] test(og-dev): og dev fixture kinds for the lineage e2e inputs Twenty kinds that write what the Python heredocs of the ancestry, family, sample QC, PRS, Y, CHIP, MoChA, viral and live e2e cases wrote, the random ones on CPython's random.Random with the same seeds. Bead: D1 --- rust/crates/og-dev/src/fixture.rs | 9 + rust/crates/og-dev/src/fixture/lineage.rs | 537 ++++++++++++++++++ .../og-dev/src/fixture/lineage/ancient.rs | 375 ++++++++++++ .../crates/og-dev/src/fixture/lineage/chip.rs | 396 +++++++++++++ .../og-dev/src/fixture/lineage/family.rs | 203 +++++++ .../og-dev/src/fixture/lineage/mocha.rs | 145 +++++ rust/crates/og-dev/src/fixture/lineage/prs.rs | 329 +++++++++++ rust/crates/og-dev/src/fixture/lineage/qc.rs | 293 ++++++++++ .../og-dev/src/fixture/lineage/truth.rs | 200 +++++++ .../og-dev/src/fixture/lineage/viral.rs | 124 ++++ .../crates/og-dev/src/fixture/lineage/ystr.rs | 493 ++++++++++++++++ 11 files changed, 3104 insertions(+) create mode 100644 rust/crates/og-dev/src/fixture/lineage.rs create mode 100644 rust/crates/og-dev/src/fixture/lineage/ancient.rs create mode 100644 rust/crates/og-dev/src/fixture/lineage/chip.rs create mode 100644 rust/crates/og-dev/src/fixture/lineage/family.rs create mode 100644 rust/crates/og-dev/src/fixture/lineage/mocha.rs create mode 100644 rust/crates/og-dev/src/fixture/lineage/prs.rs create mode 100644 rust/crates/og-dev/src/fixture/lineage/qc.rs create mode 100644 rust/crates/og-dev/src/fixture/lineage/truth.rs create mode 100644 rust/crates/og-dev/src/fixture/lineage/viral.rs create mode 100644 rust/crates/og-dev/src/fixture/lineage/ystr.rs diff --git a/rust/crates/og-dev/src/fixture.rs b/rust/crates/og-dev/src/fixture.rs index ec74f8dd3..ca72328b5 100644 --- a/rust/crates/og-dev/src/fixture.rs +++ b/rust/crates/og-dev/src/fixture.rs @@ -28,6 +28,11 @@ pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { let _ = writeln!(err, "{USAGE}og dev fixture: error: {msg}"); 2 }; + if let Some(kind) = args.first() + && let Some(code) = lineage::cli(kind, &args[1..], out, err) + { + return code; + } let result = match args.first().map(String::as_str) { Some("comphet") => { let rest: Vec<&String> = args[1..].iter().filter(|a| *a != "--swap").collect(); @@ -613,6 +618,10 @@ fn trim_pharmcat(src: &Path, dst: &Path, max: i64) -> Result<(), String> { crate::evidence::publish(dst, text.as_bytes()) } +// The kinds of the ancestry, family, QC, PRS, Y, CHIP, MoChA, viral and +// live e2e cases (`og dev fixture --help`). +pub mod lineage; + #[cfg(test)] mod tests { use super::*; diff --git a/rust/crates/og-dev/src/fixture/lineage.rs b/rust/crates/og-dev/src/fixture/lineage.rs new file mode 100644 index 000000000..7356d9d4d --- /dev/null +++ b/rust/crates/og-dev/src/fixture/lineage.rs @@ -0,0 +1,537 @@ +//! `og dev fixture `: the inputs the ancestry, family, sample QC, PRS, +//! Y, CHIP, MoChA, viral and live e2e cases make for themselves, and the few +//! table checks they make on what a step wrote. +//! +//! Each kind replaces a Python heredoc of a case under `tests/e2e/`. It takes +//! the same arguments in the same order and writes the same bytes, so a case +//! keeps every count and threshold it asserts. The random ones port CPython's +//! `random.Random` on the same seed ([`PyRandom`]). +//! +//! Exit 0 when written, 1 when an input cannot be read or has the wrong +//! shape, 2 on a usage error. + +use super::Mt19937; +use std::collections::HashMap; +use std::fs::File; +use std::io::{self, BufRead, BufWriter, Read, Seek, SeekFrom, Write}; + +mod ancient; +mod chip; +mod family; +mod mocha; +mod prs; +mod qc; +mod truth; +mod viral; +mod ystr; + +/// One kind: its name, its usage line and the function that runs it. +type Run = fn(&[String], &mut dyn Write, &mut dyn Write) -> Result<(), Fail>; + +const TABLE: &[(&str, &str, Run)] = &[ + ( + "prs-sites", + "BED TRUTH.vcf.gz FASTA HOMREF [SNVS [--no-ambiguous]]", + prs::sites, + ), + ( + "x-ref-sites", + "BLOCKS.tsv VARIANTS.txt FASTA CHROM", + prs::x_ref_sites, + ), + ( + "csv-count", + "FILE.csv[.gz] KEY=V1,V2|KEY!=V1,V2...", + prs::csv_count, + ), + ( + "truth-genotypes", + "SITES.tsv FAI [--chrom C] NAME=CALLS.vcf,CONFIDENT.bed...", + truth::genotypes, + ), + ( + "ancient-panel", + "REGIONS.bed TRUTH.vcf.gz CONFIDENT.bed FASTA OUT_DIR CHAIN", + ancient::panel, + ), + ("plant-snv", "EVERY FAI READS.sam POS", chip::plant_snv), + ("base-count", "POS BASE < READS.sam", chip::base_count), + ("sites-fasta", "SITES.tsv.gz OUT.fasta", chip::sites_fasta), + ( + "mt-roundtrip", + "LIFTED.tsv CALLS.tsv SITES.tsv.gz CHIP.txt", + chip::mt_roundtrip, + ), + ("sites-vcf", "FAI SNVS.tsv OUT.vcf", family::sites_vcf), + ( + "segments-check", + "SEGMENTS.tsv DNAPAINTER.csv", + family::segments_check, + ), + ( + "contig-cut", + "FAI SITES.vcf.gz OUT_SITES.vcf PANEL_PREFIX OUT_PREFIX", + qc::contig_cut, + ), + ( + "add-sites", + "SITES.vcf EXTRA.tsv FAI OUT.vcf", + qc::add_sites, + ), + ( + "sex-loss-woman", + "WINDOWS.bed.gz OUT_DIR NAME LOSS", + qc::sex_loss_woman, + ), + ("mocha-vcf", "FASTA OUT_SHIFT.vcf OUT_FLAT.vcf", mocha::vcf), + ("viral-spike", "VIRAL.fa HUMAN_DEPTH FRACTION", viral::spike), + ( + "y-seed", + "CALLS.vcf.gz MASK.fasta.gz PRIVATE.tsv OUT.vcf TABLE...", + ystr::y_seed, + ), + ("ystr-periodic", "REGIONS.bed REGIONS.fa", ystr::periodic), + ( + "ystr-readcount", + "PANEL.tsv CALLS.tsv READS.sam", + ystr::readcount, + ), + ("ystr-cut", "START END READS.sam", ystr::cut), + ("free-port", "", free_port), +]; + +/// The usage lines of every kind, one per line. +pub fn usage() -> String { + TABLE + .iter() + .map(|(name, args, _)| format!(" og dev fixture {name} {args}\n")) + .collect() +} + +/// Runs `og dev fixture KIND ARGS` when KIND is one of these kinds; `None` +/// for a kind this module does not have. +pub fn cli(kind: &str, args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> Option { + let (name, line, run) = TABLE.iter().find(|(name, _, _)| *name == kind)?; + if matches!(args, [a] if a == "-h" || a == "--help") { + return Some(match writeln!(out, "usage: og dev fixture {name} {line}") { + Ok(()) => 0, + Err(e) => { + let _ = writeln!(err, "og dev fixture {name}: {e}"); + 1 + } + }); + } + Some(match run(args, out, err) { + Ok(()) => 0, + Err(Fail::Usage(msg)) => { + let _ = writeln!( + err, + "usage: og dev fixture {name} {line}\nog dev fixture {name}: error: {msg}" + ); + 2 + } + Err(Fail::Run(msg)) => { + let _ = writeln!(err, "og dev fixture {name}: {msg}"); + 1 + } + }) +} + +/// Why a kind stopped. +#[derive(Debug)] +pub enum Fail { + /// The arguments are wrong (exit 2). + Usage(String), + /// An input is missing or has the wrong shape (exit 1). + Run(String), +} + +impl From for Fail { + fn from(e: io::Error) -> Self { + Fail::Run(e.to_string()) + } +} + +/// A [`Fail::Run`] from any message. +fn bad(msg: impl Into) -> Fail { + Fail::Run(msg.into()) +} + +/// Exactly `n` positional arguments, or a usage error. +fn want(args: &[String], n: usize) -> Result<(), Fail> { + if args.len() == n { + Ok(()) + } else { + Err(Fail::Usage(format!( + "takes {n} arguments, got {}", + args.len() + ))) + } +} + +/// A whole number argument. +fn int(s: &str, what: &str) -> Result { + s.trim() + .parse() + .map_err(|_| Fail::Usage(format!("{what} '{s}' is not a number"))) +} + +/// `int(s)` of a table cell, or an error naming the cell. +fn cell_int(s: &str) -> Result { + s.trim() + .parse() + .map_err(|_| bad(format!("'{s}' is not a whole number"))) +} + +/// Opens a file, plain or gzip, with the path in the error. +fn open(path: &str) -> Result, Fail> { + og_io::open(path).map_err(|e| bad(format!("{path}: {e}"))) +} + +/// Calls `f` on each line of `path` (plain or gzip) without its `\n`, as +/// `for l in open(path): l.rstrip("\n")` reads it. +fn each_line(path: &str, mut f: impl FnMut(&str) -> Result<(), Fail>) -> Result<(), Fail> { + let mut input = open(path)?; + let mut buf = Vec::new(); + loop { + buf.clear(); + if input + .read_until(b'\n', &mut buf) + .map_err(|e| bad(format!("{path}: {e}")))? + == 0 + { + return Ok(()); + } + if buf.last() == Some(&b'\n') { + buf.pop(); + } + let line = std::str::from_utf8(&buf).map_err(|_| bad(format!("{path}: not UTF-8")))?; + f(line)?; + } +} + +/// Every line of `path` without its `\n`. +fn lines(path: &str) -> Result, Fail> { + let mut v = Vec::new(); + each_line(path, |l| { + v.push(l.to_string()); + Ok(()) + })?; + Ok(v) +} + +/// A file to write, buffered, with the path in the error. +fn create(path: &str) -> Result, Fail> { + File::create(path) + .map(BufWriter::new) + .map_err(|e| bad(format!("{path}: {e}"))) +} + +/// CPython's `random.Random(seed)`, as far as the cases use it. +pub struct PyRandom(Mt19937); + +impl PyRandom { + /// `random.Random(seed)` and `random.seed(seed)`. + pub fn new(seed: u32) -> Self { + PyRandom(Mt19937::new(seed)) + } + + /// `random()`: 53 random bits in [0, 1). + pub fn random(&mut self) -> f64 { + let a = f64::from(self.0.next_u32() >> 5); + let b = f64::from(self.0.next_u32() >> 6); + (a * 67_108_864.0 + b) * (1.0 / 9_007_199_254_740_992.0) + } + + /// `uniform(a, b)`. + pub fn uniform(&mut self, a: f64, b: f64) -> f64 { + a + (b - a) * self.random() + } + + /// `randrange(0, n)` for `0 < n < 2^32`. + pub fn below(&mut self, n: u64) -> u64 { + let n = u32::try_from(n).expect("a range below 2^32"); + u64::from(self.0.below(n)) + } + + /// `randint(a, b)`. + pub fn randint(&mut self, a: i64, b: i64) -> i64 { + let width = u64::try_from(b - a + 1).expect("b >= a"); + a + i64::try_from(self.below(width)).expect("small") + } +} + +/// Random access to the bases of an indexed FASTA (`.fai` beside it), read +/// as Python's `seek` and `read(1)` on the file read them, through a cache +/// of one 64 KiB window. +pub struct Fasta { + file: File, + fai: HashMap, + window: Vec, + start: u64, +} + +impl Fasta { + /// Opens `path` and reads `path.fai`. + pub fn open(path: &str) -> Result { + let fai = read_fai(&format!("{path}.fai"))?; + let file = File::open(path).map_err(|e| bad(format!("{path}: {e}")))?; + Ok(Fasta { + file, + fai, + window: Vec::new(), + start: 0, + }) + } + + /// The length of a contig, if the index has it. + pub fn len(&self, chrom: &str) -> Option { + self.fai.get(chrom).map(|f| f[0]) + } + + /// The upper-case base at 1-based `pos` of `chrom`; `None` past the end + /// of the file or for a contig the index lacks. + pub fn base(&mut self, chrom: &str, pos: u64) -> Result, Fail> { + let Some(&[_, off, lb, lw]) = self.fai.get(chrom) else { + return Ok(None); + }; + if pos == 0 || lb == 0 { + return Ok(None); + } + let at = off + (pos - 1) / lb * lw + (pos - 1) % lb; + if at < self.start || at >= self.start + self.window.len() as u64 { + self.start = at; + self.window.clear(); + self.file.seek(SeekFrom::Start(at))?; + (&mut self.file) + .take(1 << 16) + .read_to_end(&mut self.window)?; + } + Ok(self + .window + .get(usize::try_from(at - self.start).unwrap_or(usize::MAX)) + .map(u8::to_ascii_uppercase)) + } +} + +/// A `.fai`: contig to `[length, offset, line bases, line width]`. +fn read_fai(path: &str) -> Result, Fail> { + let mut fai = HashMap::new(); + each_line(path, |l| { + let c: Vec<&str> = l.split_whitespace().collect(); + if c.len() >= 5 { + let mut v = [0u64; 4]; + for (slot, x) in v.iter_mut().zip(&c[1..5]) { + *slot = x + .parse() + .map_err(|_| bad(format!("{path}: '{x}' is not a number")))?; + } + fai.insert(c[0].to_string(), v); + } + Ok(()) + })?; + Ok(fai) +} + +/// The contigs of a `.fai` in its order, with their length as written. +fn fai_contigs(path: &str) -> Result, Fail> { + let mut v = Vec::new(); + each_line(path, |l| { + let mut c = l.split('\t'); + let name = c.next().unwrap_or("").to_string(); + let len = c.next().unwrap_or("").to_string(); + v.push((name, len)); + Ok(()) + })?; + Ok(v) +} + +/// The intervals of a BED on one contig, for `bisect` lookups: Python's +/// `confident(c, p)` of the cases (the last interval starting before `p` +/// must reach `p`). +#[derive(Default)] +pub struct Confident { + starts: Vec, + ends: Vec, +} + +impl Confident { + /// From `(start, end)` pairs, sorted here. + pub fn new(mut iv: Vec<(i64, i64)>) -> Self { + iv.sort_unstable(); + Confident { + starts: iv.iter().map(|x| x.0).collect(), + ends: iv.iter().map(|x| x.1).collect(), + } + } + + /// Whether 1-based `p` lies in the last interval that starts before it. + pub fn has(&self, p: i64) -> bool { + let i = self.starts.partition_point(|&s| s < p); + i > 0 && self.ends[i - 1] >= p + } +} + +/// A BED per contig, from its first three columns (lines with fewer are +/// skipped). +fn read_bed(path: &str) -> Result, Fail> { + let mut iv: HashMap> = HashMap::new(); + each_line(path, |l| { + let c: Vec<&str> = l.split_whitespace().collect(); + if c.len() >= 3 { + iv.entry(c[0].to_string()) + .or_default() + .push((cell_int(c[1])?, cell_int(c[2])?)); + } + Ok(()) + })?; + Ok(iv + .into_iter() + .map(|(k, v)| (k, Confident::new(v))) + .collect()) +} + +/// One CIGAR operation: its length and its letter. +type Op = (i64, u8); + +/// A CIGAR string as `(length, op)` pairs, the way `re.findall(r"(\d+)([MIDNSHP=X])")` +/// reads it: anything else is skipped. +fn cigar(s: &str) -> Vec { + let mut ops = Vec::new(); + let mut n: Option = None; + for b in s.bytes() { + if b.is_ascii_digit() { + n = Some(n.unwrap_or(0) * 10 + i64::from(b - b'0')); + } else if let Some(len) = n.take() + && b"MIDNSHP=X".contains(&b) + { + ops.push((len, b)); + } + } + ops +} + +/// `og dev fixture free-port`: a TCP port on 127.0.0.1 that was free a +/// moment ago (the OS picks it), for a case that starts a server. +fn free_port(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 0)?; + let port = std::net::TcpListener::bind(("127.0.0.1", 0))? + .local_addr()? + .port(); + writeln!(out, "{port}")?; + Ok(()) +} + +#[cfg(test)] +pub(crate) mod testutil { + use std::path::PathBuf; + + /// A fresh scratch folder for one test. + pub fn dir(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-lineage-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).expect("scratch"); + d + } + + /// Runs a kind; returns its exit code, stdout and stderr. + pub fn run(kind: &str, args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|s| s.to_string()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = super::cli(kind, &args, &mut out, &mut err).expect("a kind of this module"); + ( + code, + String::from_utf8(out).expect("utf8"), + String::from_utf8(err).expect("utf8"), + ) + } + + /// 64-bit FNV-1a, to pin bytes in a test. + pub fn fnv(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |h, &b| { + (h ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3) + }) + } + + /// A FASTA with its `.fai`: each contig a seeded random sequence of + /// `ACGT` (and an `N` run where asked), 60 bases a line. + pub fn fasta(path: &std::path::Path, contigs: &[(&str, usize)], seed: u32) { + let mut rng = super::PyRandom::new(seed); + let (mut fa, mut fai, mut off) = (String::new(), String::new(), 0usize); + for (name, n) in contigs { + let head = format!(">{name}\n"); + off += head.len(); + fa.push_str(&head); + fai.push_str(&format!("{name}\t{n}\t{off}\t60\t61\n")); + let seq: Vec = (0..*n) + .map(|_| b"ACGTacgt"[rng.below(8) as usize]) + .collect(); + for chunk in seq.chunks(60) { + fa.push_str(std::str::from_utf8(chunk).expect("ascii")); + fa.push('\n'); + } + off += n + n.div_ceil(60); + } + std::fs::write(path, fa).expect("fasta"); + std::fs::write(format!("{}.fai", path.display()), fai).expect("fai"); + } +} + +#[cfg(test)] +mod tests { + use super::testutil::run; + use super::*; + + #[test] + fn random_is_cpythons() { + // python3 -c 'import random; r = random.Random(44); print([r.random() for _ in range(2)], r.randint(800, 1200), r.uniform(0.05, 0.95))' + let mut r = PyRandom::new(44); + assert_eq!(r.random(), 0.408_535_879_254_493_75); + assert_eq!(r.random(), 0.541_972_009_104_596_2); + assert_eq!(r.randint(800, 1200), 859); + assert_eq!(r.uniform(0.05, 0.95), 0.208_999_079_775_899_43); + } + + #[test] + fn cigar_reads_as_the_regex_does() { + assert_eq!( + cigar("5S10M2I3D1N4="), + vec![ + (5, b'S'), + (10, b'M'), + (2, b'I'), + (3, b'D'), + (1, b'N'), + (4, b'=') + ] + ); + assert_eq!(cigar("*"), vec![]); + } + + #[test] + fn confident_is_bisects() { + let c = Confident::new(vec![(10, 20), (30, 40)]); + assert!(!c.has(10) && c.has(11) && c.has(20) && !c.has(21) && c.has(31) && !c.has(41)); + } + + #[test] + fn unknown_kind_is_none_and_help_prints_usage() { + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert!(cli("no-such", &[], &mut out, &mut err).is_none()); + let (code, out, _) = run("ystr-cut", &["--help"]); + assert_eq!( + (code, out.as_str()), + (0, "usage: og dev fixture ystr-cut START END READS.sam\n") + ); + let (code, _, err) = run("ystr-cut", &["1"]); + assert_eq!(code, 2, "{err}"); + assert!(usage().lines().count() == TABLE.len()); + } + + #[test] + fn free_port_prints_a_port() { + let (code, out, _) = run("free-port", &[]); + assert_eq!(code, 0); + assert!(out.trim().parse::().expect("a port") > 0); + } +} diff --git a/rust/crates/og-dev/src/fixture/lineage/ancient.rs b/rust/crates/og-dev/src/fixture/lineage/ancient.rs new file mode 100644 index 000000000..897b6c981 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/lineage/ancient.rs @@ -0,0 +1,375 @@ +//! A synthetic AADR-like panel in the AADR's own formats, for step 44. + +use super::{Fail, Fasta, PyRandom, bad, cell_int, create, each_line, read_bed, read_fai, want}; +use std::collections::HashMap; +use std::io::Write; + +/// The made-up GRCh37 coordinates sit this far below the GRCh38 ones. +const OFF: i64 = 1000; + +/// One person of the panel. +struct Person { + id: String, + sex: &'static str, + group: &'static str, + modern: bool, + bp: u32, + country: &'static str, + lat: u32, + lon: u32, + /// Dosage of the `.snp`'s first allele (REF) per site; 9 is missing. + dosage: Vec, +} + +/// Python's `int(c[3:])` when `c[3:].isdigit()`: the number of `chr`. +fn autosome(c: &str) -> Option { + let d = c.get(3..)?; + if !d.is_empty() && d.bytes().all(|b| b.is_ascii_digit()) { + d.parse().ok() + } else { + None + } +} + +/// Pseudo-haploid dosages: 30% missing, else REF with probability +/// `1 - alt / 2`. +fn pseudo(rnd: &mut PyRandom, alts: &[f64]) -> Vec { + alts.iter() + .map(|&a| { + if rnd.random() < 0.3 { + 9 + } else if rnd.random() < a / 2.0 { + 0 + } else { + 2 + } + }) + .collect() +} + +/// `og dev fixture ancient-panel REGIONS TRUTH CONFIDENT FASTA OUT_DIR CHAIN`: +/// the AADR-like panel of `tests/e2e/ancient-similarity-1.sh` (seed 44). +/// Sites are the truth set's SNVs in the autosomal slices of REGIONS, plus +/// a hom-ref site every 150 bases, inside the confident regions. Three +/// present-day groups of 30 (French and Finnish from random frequencies, +/// Sardinian from HG002's genotypes) and four ancient ones: six +/// pseudo-haploid copies each of HG002 (Planted_HG002) and of the French +/// and Finnish frequencies, and one person outside West Eurasia. Writes +/// `aadr.anno`, `.ind`, `.snp`, the transposed packed `.geno` (TGENO) and +/// `expected_sites.tsv` in OUT_DIR, and a chain moving the made-up GRCh37 +/// slices back to GRCh38. +pub fn panel(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 6)?; + let (regions, truth, bed, reference, dir, chain) = + (&args[0], &args[1], &args[2], &args[3], &args[4], &args[5]); + let fai = read_fai(&format!("{reference}.fai"))?; + let mut fa = Fasta::open(reference)?; + let mut slices = Vec::new(); + each_line(regions, |l| { + let c: Vec<&str> = l.split('\t').collect(); + if c.len() < 3 { + return Err(bad(format!("{regions}: a line with fewer than 3 columns"))); + } + if fai.contains_key(c[0]) && autosome(c[0]).is_some_and(|n| n <= 22) { + let s = cell_int(c[1])?; + if s > OFF + 10 { + slices.push((c[0].to_string(), s, cell_int(c[2])?)); + } + } + Ok(()) + })?; + let conf = read_bed(bed)?; + let confident = |c: &str, p: i64| conf.get(c).is_some_and(|b| b.has(p)); + let mut truth_gt: HashMap<(String, i64), (u8, u8, Option)> = HashMap::new(); + each_line(truth, |l| { + if l.starts_with('#') { + return Ok(()); + } + let f: Vec<&str> = l.split('\t').collect(); + if f.len() < 10 { + return Err(bad(format!("{truth}: a record with fewer than 10 columns"))); + } + if f[3].len() == 1 && f[4].len() == 1 { + let g = f[9].split(':').next().unwrap_or("").replace('|', "/"); + let n = match g.as_str() { + "0/1" | "1/0" => Some(1), + "1/1" => Some(2), + _ => None, + }; + truth_gt.insert( + (f[0].to_string(), cell_int(f[1])?), + (f[3].as_bytes()[0], f[4].as_bytes()[0], n), + ); + } + Ok(()) + })?; + // (contig, GRCh38 position, ref, alt, HG002's ALT count) + let mut sites: Vec<(String, i64, u8, u8, u8)> = Vec::new(); + for (c, s, e) in &slices { + for p in s + 1..=*e { + let t = truth_gt.get(&(c.clone(), p)); + if let Some(&(r, a, Some(n))) = t + && confident(c, p) + && fa.base(c, p as u64)? == Some(r) + { + sites.push((c.clone(), p, r, a, n)); + } else if p % 150 == 0 && t.is_none() && confident(c, p) { + let alt = match fa.base(c, p as u64)? { + Some(b'A') => b'G', + Some(b'C') => b'T', + Some(b'G') => b'A', + Some(b'T') => b'C', + _ => continue, + }; + let r = fa.base(c, p as u64)?.unwrap_or(b'N'); + sites.push((c.clone(), p, r, alt, 0)); + } + } + } + sites.sort_by_key(|x| (autosome(&x.0).unwrap_or(0), x.1)); + let n = sites.len(); + if n < 1000 { + return Err(bad(format!( + "only {n} sites in the slices, fewer than 1000" + ))); + } + let mut rnd = PyRandom::new(44); + let french: Vec = (0..n).map(|_| rnd.uniform(0.05, 0.95)).collect(); + let finnish: Vec = (0..n).map(|_| rnd.uniform(0.05, 0.95)).collect(); + let sardinian: Vec = sites + .iter() + .map(|x| [0.1, 0.5, 0.9][usize::from(x.4)]) + .collect(); + let mut people = Vec::new(); + for (g, f) in [ + ("French", &french), + ("Finnish", &finnish), + ("Sardinian", &sardinian), + ] { + for i in 0..30 { + let dosage = f + .iter() + .map(|&q| { + let k = u8::from(rnd.random() < q) + u8::from(rnd.random() < q); + 2 - k + }) + .collect(); + people.push(Person { + id: format!("{g}{i}.DG"), + sex: "M", + group: g, + modern: true, + bp: 0, + country: "Europe", + lat: 45, + lon: 5, + dosage, + }); + } + } + let hg002: Vec = sites.iter().map(|x| f64::from(x.4)).collect(); + let fr2: Vec = french.iter().map(|q| 2.0 * q).collect(); + let fi2: Vec = finnish.iter().map(|q| 2.0 * q).collect(); + let ancient = |id: String, sex, group, bp, country, lat, lon, dosage| Person { + id, + sex, + group, + modern: false, + bp, + country, + lat, + lon, + dosage, + }; + for i in 0..6 { + let d = pseudo(&mut rnd, &hg002); + people.push(ancient( + format!("PL{i}.AG"), + "F", + "Planted_HG002", + 5000, + "Testland", + 48, + 10, + d, + )); + let d = pseudo(&mut rnd, &fr2); + people.push(ancient( + format!("FR{i}.AG"), + "M", + "Anc_French_like", + 7000, + "Franceland", + 46, + 2, + d, + )); + let d = pseudo(&mut rnd, &fi2); + people.push(ancient( + format!("FI{i}.AG"), + "M", + "Anc_Finnish_like", + 3500, + "Finland", + 62, + 25, + d, + )); + } + let d = pseudo(&mut rnd, &vec![1.0; n]); + people.push(ancient( + "OUT0.AG".into(), + "M", + "Far_Away", + 5000, + "China", + 40, + 116, + d, + )); + + let mut o = create(&format!("{dir}/aadr.anno"))?; + writeln!( + o, + "Genetic ID\tIndividual ID\tDate mean in BP in years before 1950 CE\tDate standard deviation in BP\t\ + Full Date\tGroup ID\tLocality\tPolitical Entity\tLatitude\tLongitude\tData type\t\ + SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)\tASSESSMENT\tASSESSMENT WARNINGS" + )?; + for p in &people { + let short = p.id.split('.').next().unwrap_or(""); + let (kind, hits) = if p.modern { + ("Shotgun.diploid", "1100000") + } else { + ("1240k", "500000") + }; + writeln!( + o, + "{}\t{short}\t{}\t50\tsynthetic\t{}\tSomewhere\t{}\t{}\t{}\t{kind}\t{hits}\tPASS\t", + p.id, p.bp, p.group, p.country, p.lat, p.lon + )?; + } + o.flush()?; + let mut o = create(&format!("{dir}/aadr.ind"))?; + for p in &people { + writeln!(o, "{:>20} {} {}", p.id, p.sex, p.group)?; + } + o.flush()?; + let mut o = create(&format!("{dir}/aadr.snp"))?; + for (i, (c, p, r, a, _)) in sites.iter().enumerate() { + writeln!( + o, + "{:>20} {:>2} 0.000000 {:>12} {} {}", + format!("syn{i}"), + autosome(c).unwrap_or(0), + p - OFF, + *r as char, + *a as char + )?; + } + o.flush()?; + // TGENO: a 48-byte header, then a record of ceil(n / 4) bytes a person, + // two bits a site (first site in the high bits), 3 for missing. + let mut o = create(&format!("{dir}/aadr.geno"))?; + let mut head = format!("TGENO {:7} {:7} 0 0", people.len(), n).into_bytes(); + head.resize(head.len().max(48), 0); + o.write_all(&head)?; + for p in &people { + let mut rec = vec![0xffu8; n.div_ceil(4)]; + for (k, &x) in p.dosage.iter().enumerate() { + let v = if x == 9 { 3 } else { x }; + let sh = (3 - k % 4) * 2; + rec[k / 4] = (rec[k / 4] & !(3 << sh)) | (v << sh); + } + o.write_all(&rec)?; + } + o.flush()?; + let mut o = create(chain)?; + let mut chroms: Vec<&str> = sites.iter().map(|x| x.0.as_str()).collect(); + chroms.sort_by_key(|c| autosome(c).unwrap_or(0)); + chroms.dedup(); + let mut k = 0; + for c in chroms { + let sz = fai.get(c).map_or(0, |f| f[0]); + for (_, s, e) in slices.iter().filter(|x| x.0 == c) { + k += 1; + write!( + o, + "chain 1000 {c} {sz} + {} {} {c} {sz} + {s} {e} {k}\n{}\n\n", + s - OFF, + e - OFF, + e - s + )?; + } + } + o.flush()?; + let mut o = create(&format!("{dir}/expected_sites.tsv"))?; + for (i, (c, p, r, a, _)) in sites.iter().enumerate() { + writeln!(o, "syn{i}\t{c}\t{p}\t{}\t{}", *r as char, *a as char)?; + } + o.flush()?; + writeln!(out, "synthetic panel: {n} sites, {} people", people.len())?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::super::testutil::{dir, fasta, fnv, run}; + + #[test] + fn panel_is_the_pythons_bytes() { + let d = dir("ancient"); + fasta( + &d.join("ref.fa"), + &[("chr1", 30_000), ("chrX", 5_000), ("chr2", 30_000)], + 3, + ); + std::fs::write( + d.join("regions.bed"), + "chr2\t2000\t28000\tb\nchr1\t1500\t25000\nchrX\t1500\t4000\nchr1\t500\t900\n", + ) + .expect("bed"); + std::fs::write(d.join("conf.bed"), "chr1\t0\t30000\nchr2\t3000\t30000\n").expect("bed"); + // A truth SNV whose REF is the reference base, at every 13th base. + let mut truth = String::from("#h\n"); + let mut fa = super::Fasta::open(d.join("ref.fa").to_str().expect("utf8")).expect("fa"); + for c in ["chr1", "chr2"] { + for p in (1600..28000).step_by(13) { + let r = fa.base(c, p).expect("read").expect("base") as char; + let gt = ["0/1", "1|1", "1/0", "0/0"][(p % 4) as usize]; + truth.push_str(&format!( + "{c}\t{p}\t.\t{r}\tN\t50\tPASS\t.\tGT:DP\t{gt}:30\n" + )); + } + } + std::fs::write(d.join("truth.vcf"), truth).expect("truth"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + std::fs::create_dir_all(d.join("out")).expect("out"); + let (code, out, err) = run( + "ancient-panel", + &[ + &p("regions.bed"), + &p("truth.vcf"), + &p("conf.bed"), + &p("ref.fa"), + &p("out"), + &p("chain"), + ], + ); + assert_eq!(code, 0, "{err}"); + assert_eq!(out, "synthetic panel: 3091 sites, 109 people\n"); + // FNV-1a of what the Python heredoc of ancient-similarity-1.sh wrote + // from the same inputs. + let pins = [ + ("out/aadr.anno", 0xba31_6084_c893_a392), + ("out/aadr.ind", 0xbdd9_323d_97f2_62d0), + ("out/aadr.snp", 0x6c21_020e_2e5d_3d88), + ("out/aadr.geno", 0xa2bb_b113_80a3_177b), + ("out/expected_sites.tsv", 0x99e9_3c28_9f00_6476), + ("chain", 0xf2f0_f4b3_b11e_2b48), + ]; + for (f, want) in pins { + let got = fnv(&std::fs::read(d.join(f)).expect("read")); + assert_eq!(got, want, "{f}: {got:#x}"); + } + } +} diff --git a/rust/crates/og-dev/src/fixture/lineage/chip.rs b/rust/crates/og-dev/src/fixture/lineage/chip.rs new file mode 100644 index 000000000..bb0406149 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/lineage/chip.rs @@ -0,0 +1,396 @@ +//! The CHIP case's planted reads, and the chip chrM round trip's GRCh37 +//! side and its comparison. + +use super::{Fail, Op, bad, cell_int, cigar, create, each_line, fai_contigs, int, want}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::io::{BufRead, Write}; + +/// The read offset of 1-based reference `pos` in a read starting at `start`, +/// when an M, = or X operation covers it. +fn offset_at(ops: &[Op], start: i64, pos: i64) -> Option { + let (mut r, mut q, mut hit) = (start, 0i64, None); + for &(n, op) in ops { + match op { + b'M' | b'=' | b'X' => { + if r <= pos && pos < r + n { + hit = Some(q + pos - r); + } + r += n; + q += n; + } + b'I' | b'S' => q += n, + b'D' | b'N' => r += n, + _ => {} + } + } + hit +} + +/// `og dev fixture plant-snv EVERY FAI READS.sam POS`: the reads of a SAM +/// body as a SAM with the header of FAI's contigs and one read group +/// (`giab`, sample HG002). Reads on a contig FAI lacks are dropped, a mate +/// on such a contig is unset, and on chr2 every EVERY-th read with a C at +/// POS gets a T there (0 plants nothing). +pub fn plant_snv(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 4)?; + let every: u64 = int(&args[0], "EVERY")?; + let pos: i64 = int(&args[3], "POS")?; + let contigs = fai_contigs(&args[1])?; + let names: HashSet<&str> = contigs.iter().map(|c| c.0.as_str()).collect(); + writeln!(out, "@HD\tVN:1.6\tSO:unsorted")?; + for (c, l) in &contigs { + writeln!(out, "@SQ\tSN:{c}\tLN:{l}")?; + } + writeln!(out, "@RG\tID:giab\tSM:HG002\tPL:ILLUMINA")?; + let mut k = 0u64; + each_line(&args[2], |l| { + let mut f: Vec = l.split('\t').map(str::to_string).collect(); + if f.len() < 11 { + return Err(bad(format!( + "{}: a read with fewer than 11 columns", + args[2] + ))); + } + if !names.contains(f[2].as_str()) { + return Ok(()); + } + f.retain(|x| !x.starts_with("RG:Z:")); + f.push("RG:Z:giab".into()); + if f[6] != "=" && f[6] != "*" && !names.contains(f[6].as_str()) { + f[6] = "*".into(); + f[7] = "0".into(); + } + if every > 0 && f[2] == "chr2" && f[5] != "*" { + let hit = offset_at(&cigar(&f[5]), cell_int(&f[3])?, pos); + if let Some(i) = hit.and_then(|h| usize::try_from(h).ok()) + && f[9].as_bytes().get(i) == Some(&b'C') + { + if k.is_multiple_of(every) { + f[9].replace_range(i..=i, "T"); + } + k += 1; + } + } + writeln!(out, "{}", f.join("\t"))?; + Ok(()) + }) +} + +/// `og dev fixture base-count POS BASE < READS.sam`: `A N`, where N is the +/// reads of a SAM body (`samtools view`) whose alignment covers POS with a +/// read base, and A those of them with BASE there. +pub fn base_count(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 2)?; + let pos: i64 = int(&args[0], "POS")?; + let base = match args[1].as_bytes() { + [b] => *b, + _ => return Err(Fail::Usage("BASE is one letter".into())), + }; + let (mut a, mut n) = (0u64, 0u64); + let stdin = std::io::stdin(); + for line in stdin.lock().lines() { + let line = line?; + let f: Vec<&str> = line.split('\t').collect(); + if f.len() < 10 || f[5] == "*" { + continue; + } + if let Some(i) = offset_at(&cigar(f[5]), cell_int(f[3])?, pos) { + n += 1; + a += u64::from( + usize::try_from(i).ok().and_then(|i| f[9].as_bytes().get(i)) == Some(&base), + ); + } + } + writeln!(out, "{a} {n}")?; + Ok(()) +} + +/// `og dev fixture sites-fasta SITES.tsv.gz OUT.fasta`: a GRCh37 FASTA (and +/// its `.fai`) with the consumer table's base at each site (`id chrom pos +/// base` rows) and N elsewhere, contigs 1 to 22, X, Y, MT in that order, +/// each one base longer than its last site. +pub fn sites_fasta( + args: &[String], + _out: &mut dyn Write, + _err: &mut dyn Write, +) -> Result<(), Fail> { + want(args, 2)?; + let mut bases: HashMap> = HashMap::new(); + each_line(&args[0], |l| { + if l.starts_with('#') { + return Ok(()); + } + let c: Vec<&str> = l.split('\t').collect(); + if c.len() < 4 || c[3].len() != 1 { + return Err(bad(format!("{}: a row without a one-letter base", args[0]))); + } + let p = usize::try_from(cell_int(c[2])?).map_err(|_| bad("a negative position"))?; + bases + .entry(c[1].to_string()) + .or_default() + .insert(p, c[3].as_bytes()[0]); + Ok(()) + })?; + let order = (1..=22) + .map(|i| i.to_string()) + .chain(["X", "Y", "MT"].map(String::from)); + let (mut fa, mut fai) = (create(&args[1])?, create(&format!("{}.fai", args[1]))?); + let mut off = 0usize; + for chrom in order { + let Some(sites) = bases.get(&chrom) else { + continue; + }; + let n = sites.keys().next_back().map_or(0, |m| m + 1); + let head = format!(">{chrom}\n"); + fa.write_all(head.as_bytes())?; + off += head.len(); + writeln!(fai, "{chrom}\t{n}\t{off}\t60\t61")?; + let mut seq = vec![b'N'; n]; + for (&p, &b) in sites { + if p == 0 { + return Err(bad("a site at position 0")); + } + seq[p - 1] = b; + } + for chunk in seq.chunks(60) { + fa.write_all(chunk)?; + fa.write_all(b"\n")?; + } + off += n + n.div_ceil(60); + } + fa.flush()?; + fai.flush()?; + Ok(()) +} + +/// The allele indexes of a GT (`/` or `|`), without missing ones. +fn gt_indexes(gt: &str) -> Vec<&str> { + gt.split(['/', '|']) + .filter(|i| !i.is_empty() && *i != ".") + .collect() +} + +/// The first letters of the alleles a GT names. +fn first_letters(alleles: &[&str], idx: &[&str]) -> Result, Fail> { + idx.iter() + .map(|i| { + let a = i + .parse::() + .ok() + .and_then(|i| alleles.get(i)) + .ok_or_else(|| bad(format!("allele index '{i}' out of range")))?; + Ok(a.chars().next().map(String::from).unwrap_or_default()) + }) + .collect() +} + +/// Python's `repr(sorted(set))` of plain strings: `['A', 'G']`. +fn repr(s: &BTreeSet) -> String { + let items: Vec = s.iter().map(|x| format!("'{x}'")).collect(); + format!("[{}]", items.join(", ")) +} + +/// `og dev fixture mt-roundtrip LIFTED CALLS SITES TXT`: `called got +/// on_place bad` for a chip file read back by chip-to-vcf.sh. called: the +/// chrM sites of the table (SITES) TXT has a call for; got: the chrM +/// records of LIFTED (`ID CHROM POS REF ALT GT`) with a call; on_place: +/// those on the table's rCRS position with its reference base; bad: those +/// of them whose alleles are not the case's own call there (CALLS, `CHROM +/// POS REF ALT FILTER GT`, PASS only; no call means the reference base). +pub fn mt_roundtrip(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> Result<(), Fail> { + want(args, 4)?; + let (lifted, calls, sites, txt) = (&args[0], &args[1], &args[2], &args[3]); + let mut table: HashMap> = HashMap::new(); + each_line(sites, |l| { + if !l.starts_with('#') { + let c: Vec = l.split('\t').map(str::to_string).collect(); + if c.get(1).is_some_and(|x| x == "MT") { + if c.len() < 6 { + return Err(bad(format!("{sites}: an MT row with fewer than 6 columns"))); + } + table.insert(c[0].clone(), c); + } + } + Ok(()) + })?; + let mut dv: HashMap> = HashMap::new(); + each_line(calls, |l| { + let c: Vec<&str> = l.split('\t').collect(); + let [_, pos, r, alt, flt, gt] = c[..] else { + return Err(bad(format!("{calls}: a row without 6 columns"))); + }; + if flt == "PASS" { + let al: Vec<&str> = std::iter::once(r).chain(alt.split(',')).collect(); + dv.insert(cell_int(pos)?, first_letters(&al, >_indexes(gt))?); + } + Ok(()) + })?; + let mut called = 0u64; + each_line(txt, |l| { + if l.starts_with('#') { + return Ok(()); + } + let c: Vec<&str> = l.split('\t').collect(); + if table.contains_key(c[0]) { + let g = c + .get(3) + .ok_or_else(|| bad(format!("{txt}: a row with fewer than 4 columns")))?; + called += u64::from(*g != "--"); + } + Ok(()) + })?; + let (mut got, mut on_place, mut bad_n) = (0u64, 0u64, 0u64); + each_line(lifted, |l| { + let c: Vec<&str> = l.split('\t').collect(); + let [rid, _, pos, r, alt, gt] = c[..] else { + return Err(bad(format!("{lifted}: a row without 6 columns"))); + }; + let idx = gt_indexes(gt); + let Some(t) = table.get(rid) else { + return Ok(()); + }; + if idx.is_empty() { + return Ok(()); + } + got += 1; + if cell_int(pos)? != cell_int(&t[4])? || r != t[5] { + if got - on_place <= 5 { + writeln!( + err, + "off place {rid}: rCRS {} {}, read back at chrM:{pos} {r}", + t[4], t[5] + )?; + } + return Ok(()); + } + on_place += 1; + let al: Vec<&str> = std::iter::once(r).chain(alt.split(',')).collect(); + let have = first_letters(&al, &idx)?; + let want = dv + .get(&cell_int(pos)?) + .cloned() + .unwrap_or_else(|| BTreeSet::from([t[5].clone()])); + if have != want { + bad_n += 1; + if bad_n <= 5 { + writeln!( + err, + "mismatch {rid} chrM:{pos} read back {}, case 21 {}", + repr(&have), + repr(&want) + )?; + } + } + Ok(()) + })?; + writeln!(out, "{called} {got} {on_place} {bad_n}")?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::super::testutil::{dir, run}; + + #[test] + fn plant_snv_plants_every_nth_c_and_cleans_the_header() { + let d = dir("plant"); + std::fs::write( + d.join("ref.fa.fai"), + "chr2\t1000\t6\t60\t61\nchr9\t500\t1030\t60\t61\n", + ) + .expect("fai"); + let mut sam = String::new(); + // Each read covers 105 with a C there (offset 5 of the M after a 2S). + for i in 0..5 { + sam.push_str(&format!("r{i}\t0\tchr2\t100\t60\t2S8M\tchr7\t300\t0\tAAAAAAACAA\tIIIIIIIIII\tRG:Z:old\tNM:i:0\n")); + } + sam.push_str("x\t0\tchr7\t100\t60\t5M\t=\t1\t0\tAAAAA\tIIIII\n"); + sam.push_str("y\t0\tchr2\t104\t60\t1M1D3M\tchr9\t1\t0\tAAAA\tIIII\n"); + std::fs::write(d.join("in.sam"), sam).expect("sam"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, out, err) = run("plant-snv", &["2", &p("ref.fa.fai"), &p("in.sam"), "105"]); + assert_eq!(code, 0, "{err}"); + let l: Vec<&str> = out.lines().collect(); + assert_eq!( + &l[..4], + [ + "@HD\tVN:1.6\tSO:unsorted", + "@SQ\tSN:chr2\tLN:1000", + "@SQ\tSN:chr9\tLN:500", + "@RG\tID:giab\tSM:HG002\tPL:ILLUMINA" + ] + ); + assert_eq!(l.len(), 10, "{out}"); + assert_eq!( + l[4], + "r0\t0\tchr2\t100\t60\t2S8M\t*\t0\t0\tAAAAAAATAA\tIIIIIIIIII\tNM:i:0\tRG:Z:giab" + ); + assert!( + l[5].contains("\tAAAAAAACAA\t") && l[6].contains("\tAAAAAAATAA\t"), + "{out}" + ); + assert_eq!( + l[9], + "y\t0\tchr2\t104\t60\t1M1D3M\tchr9\t1\t0\tAAAA\tIIII\tRG:Z:giab" + ); + let (_, none, _) = run("plant-snv", &["0", &p("ref.fa.fai"), &p("in.sam"), "105"]); + assert!(!none.contains("AAAAAAATAA"), "{none}"); + } + + #[test] + fn sites_fasta_and_mt_roundtrip() { + let d = dir("mt"); + std::fs::write( + d.join("sites.tsv"), + "#id\tchrom\tpos\tbase\trcrs_pos\trcrs_ref\nrs1\t1\t3\tA\nrs2\tMT\t2\tG\t2\tG\ni3\tMT\t70\tT\t71\tT\ni4\tMT\t5\tC\t5\tC\ni5\tMT\t9\tA\t9\tA\nrs6\tX\t61\tC\n", + ) + .expect("sites"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, _, err) = run("sites-fasta", &[&p("sites.tsv"), &p("h.fa")]); + assert_eq!(code, 0, "{err}"); + let fa = std::fs::read_to_string(d.join("h.fa")).expect("fa"); + assert_eq!( + fa, + format!( + ">1\nNNAN\n>X\n{}\nCN\n>MT\n{}\n{}\n", + "N".repeat(60), + "NGNNCNNNA".to_string() + &"N".repeat(51), + "NNNNNNNNNTN" + ) + ); + assert_eq!( + std::fs::read_to_string(d.join("h.fa.fai")).expect("fai"), + "1\t4\t3\t60\t61\nX\t62\t11\t60\t61\nMT\t71\t79\t60\t61\n" + ); + std::fs::write( + d.join("calls.tsv"), + "chrM\t5\tC\tT,G\tPASS\t1/2\nchrM\t9\tA\tG\tLowQual\t1/1\n", + ) + .expect("calls"); + std::fs::write( + d.join("chip.txt"), + "# head\nrs2\tMT\t2\tGG\ni3\tMT\t70\t--\ni4\tMT\t5\tTG\ni5\tMT\t9\tAA\nrs1\t1\t3\tAA\n", + ) + .expect("txt"); + std::fs::write( + d.join("lifted.tsv"), + "rs2\tchrM\t2\tG\t.\t0/0\ni4\tchrM\t5\tC\tT\t0/1\ni5\tchrM\t9\tA\tG\t1/1\ni3\tchrM\t70\tT\t.\t./.\nzz\tchrM\t1\tA\t.\t0\n", + ) + .expect("lifted"); + let (code, out, err) = run( + "mt-roundtrip", + &[ + &p("lifted.tsv"), + &p("calls.tsv"), + &p("sites.tsv"), + &p("chip.txt"), + ], + ); + assert_eq!(code, 0, "{err}"); + assert_eq!(out, "3 3 3 2\n"); + assert_eq!( + err, + "mismatch i4 chrM:5 read back ['C', 'T'], case 21 ['G', 'T']\nmismatch i5 chrM:9 read back ['G'], case 21 ['A']\n" + ); + } +} diff --git a/rust/crates/og-dev/src/fixture/lineage/family.rs b/rust/crates/og-dev/src/fixture/lineage/family.rs new file mode 100644 index 000000000..e64e32d0d --- /dev/null +++ b/rust/crates/og-dev/src/fixture/lineage/family.rs @@ -0,0 +1,203 @@ +//! The family case's somalier sites and its check of the chromosome map. + +use super::{Fail, bad, cell_int, create, each_line, fai_contigs, want}; +use pycompat::csv::{Quoting, Reader}; +use std::collections::{BTreeSet, HashMap}; +use std::io::Write; + +/// `og dev fixture sites-vcf FAI SNVS OUT`: a sites VCF for somalier from +/// `CHROM POS REF ALT` rows (duplicates once, contigs FAI lacks left out), +/// in FAI's contig order, each `AF=0.5`. Prints the site count. +pub fn sites_vcf(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 3)?; + let contigs = fai_contigs(&args[0])?; + let order: HashMap<&str, usize> = contigs + .iter() + .enumerate() + .map(|(i, c)| (c.0.as_str(), i)) + .collect(); + let mut keep = BTreeSet::new(); + each_line(&args[1], |l| { + if l.trim().is_empty() { + return Ok(()); + } + let c: Vec<&str> = l.split('\t').collect(); + let [chrom, pos, r, a] = c[..] else { + return Err(bad(format!("{}: a row without 4 columns", args[1]))); + }; + if let Some(&i) = order.get(chrom) { + keep.insert(( + i, + cell_int(pos)?, + pos.to_string(), + r.to_string(), + a.to_string(), + )); + } + Ok(()) + })?; + let mut o = create(&args[2])?; + writeln!(o, "##fileformat=VCFv4.2")?; + for (c, len) in &contigs { + writeln!(o, "##contig=")?; + } + writeln!(o, "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO")?; + for (i, _, pos, r, a) in &keep { + writeln!(o, "{}\t{pos}\t.\t{r}\t{a}\t.\tPASS\tAF=0.5", contigs[*i].0)?; + } + o.flush()?; + writeln!(out, "{} sites", keep.len())?; + Ok(()) +} + +/// The DNA Painter CSV header step 38 writes. +const DNAPAINTER: [&str; 12] = [ + "Chr", + "Start", + "End", + "cM", + "SNPs", + "Match", + "Confidence", + "Group", + "Side", + "Notes", + "Colour", + "Ancestor", +]; + +/// `og dev fixture segments-check SEGMENTS.tsv DNAPAINTER.csv`: checks +/// step 38's segments table and its DNA Painter CSV (the columns, each row +/// a paternal or maternal stretch of 20 sites or more on a `chr` contig, +/// and the CSV row for row the same stretches), then prints the paternal +/// and the maternal count. +pub fn segments_check( + args: &[String], + out: &mut dyn Write, + _err: &mut dyn Write, +) -> Result<(), Fail> { + want(args, 2)?; + let (tsv, dp) = (&args[0], &args[1]); + let mut head: Option> = None; + let mut rows: Vec> = Vec::new(); + each_line(tsv, |l| { + let cells = l.split('\t').map(str::to_string); + match &head { + None => head = Some(cells.collect()), + Some(h) => rows.push(h.iter().cloned().zip(cells).collect()), + } + Ok(()) + })?; + let head = head.unwrap_or_default(); + let want_head = ["child", "side", "parent", "chrom", "start", "end"]; + if head.len() < 6 || head[..6] != want_head { + return Err(bad(format!("{tsv}: header {head:?}"))); + } + let get = |r: &HashMap, k: &str| -> Result { + r.get(k) + .cloned() + .ok_or_else(|| bad(format!("{tsv}: a row without {k}"))) + }; + for r in &rows { + let side = get(r, "side")?; + let chrom = get(r, "chrom")?; + let (s, e) = (cell_int(&get(r, "start")?)?, cell_int(&get(r, "end")?)?); + if !(side == "paternal" || side == "maternal") || !chrom.starts_with("chr") { + return Err(bad(format!("{tsv}: row {r:?}"))); + } + if !(0 < s && s <= e && cell_int(&get(r, "sites")?)? >= 20) { + return Err(bad(format!("{tsv}: row {r:?}"))); + } + } + let lines = crate::smoke::csv_lines(dp).map_err(bad)?; + let csv = Reader::new(lines, ',', Quoting::Minimal) + .collect::>, _>>() + .map_err(|e| bad(format!("{dp}: {e}")))?; + if csv.first().is_none_or(|h| h[..] != DNAPAINTER) { + return Err(bad(format!("{dp}: header {:?}", csv.first()))); + } + if csv.len() - 1 != rows.len() { + return Err(bad(format!( + "{dp}: {} rows, the table {}", + csv.len() - 1, + rows.len() + ))); + } + for (x, r) in csv[1..].iter().zip(&rows) { + let chrom = get(r, "chrom")?; + let want = [ + chrom.get(3..).unwrap_or("").to_string(), + get(r, "start")?, + get(r, "end")?, + get(r, "side")?, + ]; + let have = [0, 1, 2, 8].map(|i| x.get(i).cloned().unwrap_or_default()); + if x.len() < 9 || have != want { + return Err(bad(format!("{dp}: row {x:?} against {r:?}"))); + } + } + let side = |s: &str| { + rows.iter() + .filter(|r| r.get("side").is_some_and(|x| x == s)) + .count() + }; + writeln!(out, "{} {}", side("paternal"), side("maternal"))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::super::testutil::{dir, run}; + + #[test] + fn sites_vcf_sorts_by_the_fai_and_drops_doubles() { + let d = dir("sites-vcf"); + std::fs::write( + d.join("r.fai"), + "chr2\t500\t1\t60\t61\nchr1\t900\t600\t60\t61\n", + ) + .expect("fai"); + std::fs::write( + d.join("snvs.tsv"), + "chr1\t30\tA\tG\nchr2\t100\tC\tT\nchr1\t30\tA\tG\nchr9\t5\tA\tC\nchr2\t20\tG\tA\n\n", + ) + .expect("snvs"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, out, err) = run("sites-vcf", &[&p("r.fai"), &p("snvs.tsv"), &p("o.vcf")]); + assert_eq!((code, out.as_str()), (0, "3 sites\n"), "{err}"); + assert_eq!( + std::fs::read_to_string(d.join("o.vcf")).expect("vcf"), + "##fileformat=VCFv4.2\n##contig=\n##contig=\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n\ + chr2\t20\t.\tG\tA\t.\tPASS\tAF=0.5\nchr2\t100\t.\tC\tT\t.\tPASS\tAF=0.5\nchr1\t30\t.\tA\tG\t.\tPASS\tAF=0.5\n" + ); + } + + #[test] + fn segments_check_counts_and_refuses_a_mismatch() { + let d = dir("segments"); + std::fs::write( + d.join("s.tsv"), + "child\tside\tparent\tchrom\tstart\tend\tsites\nC\tpaternal\tF\tchr1\t10\t500\t25\nC\tmaternal\tM\tchr2\t5\t90\t20\nC\tpaternal\tF\tchr3\t1\t2\t40\n", + ) + .expect("tsv"); + let head = "Chr,Start,End,cM,SNPs,Match,Confidence,Group,Side,Notes,Colour,Ancestor\n"; + std::fs::write( + d.join("d.csv"), + format!("{head}1,10,500,1.0,25,C,,,paternal,\"a, b\",,F\n2,5,90,1,20,C,,,maternal,,,M\n3,1,2,1,40,C,,,paternal,,,F\n"), + ) + .expect("csv"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, out, err) = run("segments-check", &[&p("s.tsv"), &p("d.csv")]); + assert_eq!((code, out.as_str()), (0, "2 1\n"), "{err}"); + std::fs::write(d.join("bad.csv"), format!("{head}1,10,500,1,25,C,,,maternal,,,F\n2,5,90,1,20,C,,,maternal,,,M\n3,1,2,1,40,C,,,paternal,,,F\n")).expect("csv"); + let (code, out, _) = run("segments-check", &[&p("s.tsv"), &p("bad.csv")]); + assert_eq!((code, out.as_str()), (1, "")); + std::fs::write( + d.join("few.tsv"), + "child\tside\tparent\tchrom\tstart\tend\tsites\nC\tpaternal\tF\tchr1\t10\t500\t19\n", + ) + .expect("tsv"); + let (code, _, _) = run("segments-check", &[&p("few.tsv"), &p("d.csv")]); + assert_eq!(code, 1); + } +} diff --git a/rust/crates/og-dev/src/fixture/lineage/mocha.rs b/rust/crates/og-dev/src/fixture/lineage/mocha.rs new file mode 100644 index 000000000..e2aebfb8d --- /dev/null +++ b/rust/crates/og-dev/src/fixture/lineage/mocha.rs @@ -0,0 +1,145 @@ +//! Two synthetic phased VCFs for step 51 (MoChA): one with a CN-LOH on 20p. + +use super::{Fail, Fasta, PyRandom, bad, create, want}; +use std::io::Write; + +const CHROMS: [&str; 4] = ["chr16", "chr19", "chr20", "chr22"]; +/// 20p from the telomere. +const EVENT: (&str, u64, u64) = ("chr20", 1, 20_000_000); +/// CN-LOH in 15% of the cells: haplotype balance 0.575. +const BDEV: f64 = 0.075; + +/// Python's Knuth sampler of the case: multiply uniforms until the product +/// drops to `exp(-lam)`. +fn poisson(rnd: &mut PyRandom, lam: f64) -> u64 { + let l = (-lam).exp(); + let (mut k, mut p) = (0, 1.0); + loop { + p *= rnd.random(); + if p <= l { + return k; + } + k += 1; + } +} + +/// `sum(random() < f for _ in range(n))`. +fn binom(rnd: &mut PyRandom, n: u64, f: f64) -> u64 { + (0..n).filter(|_| rnd.random() < f).count() as u64 +} + +/// `og dev fixture mocha-vcf FASTA OUT_SHIFT OUT_FLAT`: phased SNVs about +/// every kilobase of chr16, chr19, chr20 and chr22 with allelic depths +/// (seed 51 for each file): OUT_SHIFT (sample MOSAIC) carries a CN-LOH on +/// chr20 up to 20 Mb in 15% of the cells, OUT_FLAT (MOSAIC0) none. Prints +/// each file with its line count. +pub fn vcf(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 3)?; + let mut fa = Fasta::open(&args[0])?; + for (path, shift) in [(&args[1], true), (&args[2], false)] { + let mut rnd = PyRandom::new(51); + let mut lines = vec![ + "##fileformat=VCFv4.2".to_string(), + "##FILTER=".to_string(), + ]; + for c in CHROMS { + let len = fa + .len(c) + .ok_or_else(|| bad(format!("{}.fai: no {c}", args[0])))?; + lines.push(format!("##contig=")); + } + lines.extend( + [ + "##FORMAT=", + "##FORMAT=", + "##FORMAT=", + "##FORMAT=", + ] + .map(String::from), + ); + let name = if shift { "MOSAIC" } else { "MOSAIC0" }; + lines.push(format!( + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{name}" + )); + for c in CHROMS { + let length = fa.len(c).unwrap_or(0); + let mut pos: u64 = 100_000; + while pos + 100_000 < length { + pos += rnd.randint(800, 1200) as u64; + let base = fa.base(c, pos)?; + let p = rnd.uniform(0.05, 0.5); + let r = rnd.random(); + let depth = poisson(&mut rnd, 30.0); + let o = rnd.random() < 0.5; + let bal = if shift && c == EVENT.0 && EVENT.1 <= pos && pos <= EVENT.2 { + 0.5 + BDEV + } else { + 0.5 + }; + let k_het = binom(&mut rnd, depth, bal); + let k_err = binom(&mut rnd, depth, 0.005); + let (r_base, alt_base) = match base { + Some(b'A') => ('A', 'G'), + Some(b'G') => ('G', 'A'), + Some(b'C') => ('C', 'T'), + Some(b'T') => ('T', 'C'), + _ => continue, + }; + let het = 2.0 * p * (1.0 - p); + let (gt, alt) = if r < het { + if o { + ("0|1", depth - k_het) + } else { + ("1|0", k_het) + } + } else if r < het + p * p { + ("1|1", depth - k_err) + } else { + continue; + }; + lines.push(format!( + "{c}\t{pos}\t.\t{r_base}\t{alt_base}\t50\tPASS\t.\tGT:GQ:DP:AD\t{gt}:50:{depth}:{},{alt}", + depth - alt + )); + } + } + let mut o = create(path)?; + o.write_all(lines.join("\n").as_bytes())?; + o.write_all(b"\n")?; + o.flush()?; + writeln!(out, "{path} {} lines", lines.len())?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::super::testutil::{dir, fasta, fnv, run}; + + #[test] + fn vcfs_are_the_pythons_bytes() { + let d = dir("mocha"); + fasta( + &d.join("ref.fa"), + &[ + ("chr16", 260_000), + ("chr19", 230_000), + ("chr20", 300_000), + ("chr22", 201_000), + ], + 21, + ); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, out, err) = run("mocha-vcf", &[&p("ref.fa"), &p("s.vcf"), &p("f.vcf")]); + assert_eq!(code, 0, "{err}"); + assert!(out.starts_with(&format!("{} ", p("s.vcf"))), "{out}"); + // FNV-1a of what the Python heredoc of mocha-1.sh wrote. + for (f, want) in [ + ("s.vcf", 0xdd2b_0f2a_63b7_1109), + ("f.vcf", 0x6b70_c06a_6cd2_a61d), + ] { + let got = fnv(&std::fs::read(d.join(f)).expect("read")); + assert_eq!(got, want, "{f}: {got:#x}"); + } + } +} diff --git a/rust/crates/og-dev/src/fixture/lineage/prs.rs b/rust/crates/og-dev/src/fixture/lineage/prs.rs new file mode 100644 index 000000000..6886a8731 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/lineage/prs.rs @@ -0,0 +1,329 @@ +//! The PRS cases' score sites and their pgsc_calc match-log count. + +use super::{Fail, Fasta, bad, cell_int, each_line, int, want}; +use pycompat::csv::{DictReader, Quoting}; +use std::io::Write; + +/// The chr20 stretch the score sites come from (inside the fixture's slice). +const FROM: i64 = 10_050_000; +const TO: i64 = 10_450_000; +/// The step between two hom-ref candidates. +const STEP: usize = 7_919; + +/// Whether no position of sorted `var` is within `gap` of `p`. +fn clear(var: &[i64], p: i64, gap: i64) -> bool { + let i = var.partition_point(|&v| v < p - gap); + var.get(i).is_none_or(|&v| v > p + gap) +} + +/// `og dev fixture prs-sites BED TRUTH FASTA HOMREF [SNVS [--no-ambiguous]]`: +/// score sites on chr20 between 10,050,000 and 10,450,000. First HOMREF +/// hom-ref sites (`homref POS BASE`): every 7,919 bases, 200 bases inside a +/// confident interval of BED and more than 200 bases from any truth record, +/// on an `ACGT` reference base. Then SNVS of the truth set's heterozygous +/// or hom-alt SNVs, evenly spread (`snv POS ALT REF`); `--no-ambiguous` +/// leaves out A/T and C/G pairs, which pgsc_calc drops. +pub fn sites(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + let no_amb = args.iter().any(|a| a == "--no-ambiguous"); + let pos: Vec<&String> = args.iter().filter(|a| *a != "--no-ambiguous").collect(); + if !(4..=5).contains(&pos.len()) || (no_amb && pos.len() != 5) { + return Err(Fail::Usage( + "takes BED TRUTH FASTA HOMREF [SNVS [--no-ambiguous]]".into(), + )); + } + let (bed, truth, fasta) = (pos[0].as_str(), pos[1].as_str(), pos[2].as_str()); + let homref: usize = int(pos[3], "HOMREF")?; + let n_snv: usize = pos.get(4).map(|s| int(s, "SNVS")).transpose()?.unwrap_or(0); + let mut iv = Vec::new(); + each_line(bed, |l| { + if l.starts_with("chr20\t") { + let c: Vec<&str> = l.split_whitespace().collect(); + if c.len() < 3 { + return Err(bad(format!("{bed}: a line with fewer than 3 columns"))); + } + iv.push((cell_int(c[1])?, cell_int(c[2])?)); + } + Ok(()) + })?; + let (mut var, mut snv) = (Vec::new(), Vec::new()); + each_line(truth, |l| { + if !l.starts_with("chr20\t") { + return Ok(()); + } + let f: Vec<&str> = l.split('\t').collect(); + if f.len() < 10 { + return Err(bad(format!("{truth}: a record with fewer than 10 columns"))); + } + let p = cell_int(f[1])?; + var.push(p); + let gt = f[9].split(':').next().unwrap_or("").replace('|', "/"); + let amb = matches!( + (f[3], f[4]), + ("A", "T") | ("T", "A") | ("C", "G") | ("G", "C") + ); + if f[3].len() == 1 + && f[4].len() == 1 + && !(no_amb && amb) + && FROM < p + && p < TO + && (gt == "0/1" || gt == "1/1") + { + snv.push((p, f[4].to_string(), f[3].to_string())); + } + Ok(()) + })?; + var.sort_unstable(); + let mut fa = Fasta::open(fasta)?; + if fa.len("chr20").is_none() { + return Err(bad(format!("{fasta}.fai: no chr20"))); + } + let mut picked = 0; + for p in (FROM..TO).step_by(STEP) { + if picked == homref { + break; + } + let inside = iv.iter().any(|&(s, e)| s + 200 < p && p <= e - 200); + if !inside || !clear(&var, p, 200) { + continue; + } + if let Some(b @ (b'A' | b'C' | b'G' | b'T')) = fa.base("chr20", p as u64)? { + writeln!(out, "homref\t{p}\t{}", b as char)?; + picked += 1; + } + } + if n_snv > 0 { + let step = (snv.len() / n_snv).max(1); + for (p, alt, r) in snv.iter().step_by(step).take(n_snv) { + writeln!(out, "snv\t{p}\t{alt}\t{r}")?; + } + } + Ok(()) +} + +/// `og dev fixture x-ref-sites BLOCKS VARIANTS FASTA CHROM N`: up to N +/// positions in the middle of the gVCF's reference blocks (`POS END GT` +/// rows of `bcftools query`, GT 0/0) of 50 bases or more, each more than +/// 2,000 bases after the one before, more than 100 bases from any call with +/// an ALT, on an `ACGT` base: `POS BASE` per line. The counts go to stderr. +pub fn x_ref_sites(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> Result<(), Fail> { + want(args, 5)?; + let (blocks, variants, fasta, chrom) = (&args[0], &args[1], &args[2], &args[3]); + let n: usize = int(&args[4], "N")?; + let mut var = Vec::new(); + each_line(variants, |l| { + if !l.trim().is_empty() { + var.push(cell_int(l)?); + } + Ok(()) + })?; + let (mut rows, mut refs) = (0usize, Vec::new()); + each_line(blocks, |l| { + if l.trim().is_empty() { + return Ok(()); + } + rows += 1; + let r: Vec<&str> = l.split('\t').collect(); + if r.len() == 3 && !r[1].is_empty() && r[1] != "." && (r[2] == "0/0" || r[2] == "0|0") { + refs.push((cell_int(r[0])?, cell_int(r[1])?)); + } + Ok(()) + })?; + writeln!( + err, + "{rows} gVCF records on the {chrom} slice, {} 0/0 blocks, {} calls with an ALT", + refs.len(), + var.len() + )?; + var.sort_unstable(); + let mut fa = Fasta::open(fasta)?; + let (mut last, mut picked) = (0i64, 0usize); + for (start, end) in refs { + if picked == n { + break; + } + let p = (start + end).div_euclid(2); + if end - start >= 50 + && p - last > 2000 + && clear(&var, p, 100) + && let Some(b @ (b'A' | b'C' | b'G' | b'T')) = fa.base(chrom, p as u64)? + { + writeln!(out, "{p}\t{}", b as char)?; + last = p; + picked += 1; + } + } + Ok(()) +} + +/// One filter of `csv-count`: a column and the values it must (or must +/// not) hold. +struct Filter { + key: String, + values: Vec, + negate: bool, +} + +/// `og dev fixture csv-count FILE KEY=V1,V2 KEY!=V1,V2 ...`: the rows of a +/// CSV (plain or gzip, Python's `csv.DictReader`) whose KEY is one of the +/// values (`=`) or none of them (`!=`, a missing column counts as none). +/// pgsc_calc's match log is the file the PRS cases count. +pub fn csv_count(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + let Some((file, rest)) = args.split_first() else { + return Err(Fail::Usage("takes FILE and filters".into())); + }; + let filters = rest + .iter() + .map(|a| { + let (key, values, negate) = if let Some((k, v)) = a.split_once("!=") { + (k, v, true) + } else if let Some((k, v)) = a.split_once('=') { + (k, v, false) + } else { + return Err(Fail::Usage(format!( + "'{a}' is not KEY=VALUES or KEY!=VALUES" + ))); + }; + Ok(Filter { + key: key.to_string(), + values: values.split(',').map(str::to_string).collect(), + negate, + }) + }) + .collect::, _>>()?; + let mut n = 0u64; + for row in DictReader::new( + crate::smoke::csv_lines(file).map_err(bad)?, + ',', + Quoting::Minimal, + ) { + let row = row.map_err(|e| bad(format!("{file}: {e}")))?; + if filters.iter().all(|f| { + let v = row.get(&f.key).flatten(); + let hit = v.is_some_and(|v| f.values.iter().any(|x| x == v)); + hit != f.negate + }) { + n += 1; + } + } + writeln!(out, "{n}")?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::super::testutil::{dir, fasta, run}; + + #[test] + fn csv_count_counts_matches_and_quoted_cells() { + let d = dir("csv-count"); + let f = d.join("log.csv"); + std::fs::write( + &f, + "accession,chr_name,match_status,note\nPGS1,X,matched,\"a, b\"\nPGS1,23,matched,x\nPGS1,X,unmatched,x\nPGS1,20,matched,x\nPGS2,X,matched,x\n", + ) + .expect("csv"); + let f = f.to_str().expect("utf8"); + let (code, out, _) = run( + "csv-count", + &[f, "accession=PGS1", "chr_name=X,23", "match_status=matched"], + ); + assert_eq!((code, out.as_str()), (0, "2\n")); + let (_, out, _) = run( + "csv-count", + &[ + f, + "accession=PGS1", + "chr_name!=X,23", + "match_status=matched", + ], + ); + assert_eq!(out, "1\n"); + let (_, out, _) = run("csv-count", &["/dev/null", "accession=PGS1"]); + assert_eq!(out, "0\n"); + let (code, _, _) = run("csv-count", &[f, "accession"]); + assert_eq!(code, 2); + } + + #[test] + fn prs_sites_picks_hom_ref_then_spread_snvs() { + let d = dir("prs-sites"); + let fa = d.join("ref.fa"); + fasta(&fa, &[("chr1", 100), ("chr20", 10_460_000)], 5); + std::fs::write( + d.join("conf.bed"), + "chr20\t10049000\t10200000\nchr1\t0\t100\n", + ) + .expect("bed"); + // A record near the 2nd candidate (10,057,919) knocks it out. + let mut truth = String::from("##x\n"); + for (p, r, a, gt) in [ + (10_057_950, "A", "T", "0/1"), + (10_100_000, "C", "G", "0|1"), + (10_120_000, "C", "T", "1/1"), + (10_130_000, "C", "TT", "0/1"), + (10_140_000, "G", "A", "1|1"), + (10_150_000, "G", "A", "0/0"), + ] { + truth.push_str(&format!("chr20\t{p}\t.\t{r}\t{a}\t50\tPASS\t.\tGT\t{gt}\n")); + } + std::fs::write(d.join("truth.vcf"), truth).expect("truth"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, out, err) = run( + "prs-sites", + &[&p("conf.bed"), &p("truth.vcf"), &p("ref.fa"), "3", "3"], + ); + assert_eq!(code, 0, "{err}"); + let lines: Vec<&str> = out.lines().collect(); + assert_eq!(lines.len(), 6, "{out}"); + assert!(lines[0].starts_with("homref\t10050000\t"), "{out}"); + assert!(lines[1].starts_with("homref\t10065838\t"), "{out}"); + assert_eq!( + &lines[3..], + [ + "snv\t10057950\tT\tA", + "snv\t10100000\tG\tC", + "snv\t10120000\tT\tC" + ] + ); + let (_, out, _) = run( + "prs-sites", + &[ + &p("conf.bed"), + &p("truth.vcf"), + &p("ref.fa"), + "0", + "2", + "--no-ambiguous", + ], + ); + assert_eq!(out, "snv\t10120000\tT\tC\nsnv\t10140000\tA\tG\n"); + } + + #[test] + fn x_ref_sites_takes_block_middles_away_from_calls() { + let d = dir("x-ref"); + let fa = d.join("ref.fa"); + fasta(&fa, &[("chrX", 20_000)], 9); + std::fs::write( + d.join("blocks.tsv"), + "100\t200\t0/0\n150\t160\t0/0\n3000\t3100\t0|0\n4000\t4100\t0/0\n7000\t7040\t0/0\n9000\t9200\t0/1\n12000\t12100\t0/0\n13000\t.\t0/0\n", + ) + .expect("blocks"); + std::fs::write(d.join("vars.txt"), "12040\n\n").expect("vars"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, out, err) = run( + "x-ref-sites", + &[&p("blocks.tsv"), &p("vars.txt"), &p("ref.fa"), "chrX", "5"], + ); + assert_eq!(code, 0, "{err}"); + let pos: Vec<&str> = out + .lines() + .map(|l| l.split('\t').next().expect("pos")) + .collect(); + assert_eq!(pos, ["3050"], "{out}"); + assert_eq!( + err, + "8 gVCF records on the chrX slice, 6 0/0 blocks, 1 calls with an ALT\n" + ); + } +} diff --git a/rust/crates/og-dev/src/fixture/lineage/qc.rs b/rust/crates/og-dev/src/fixture/lineage/qc.rs new file mode 100644 index 000000000..10f0fbdb0 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/lineage/qc.rs @@ -0,0 +1,293 @@ +//! The sample-QC cases' data cut to the fixture's contigs, and the +//! synthetic woman of the mosaic-loss case. + +use super::{Fail, PyRandom, bad, cell_int, create, each_line, lines, want}; +use flate2::Compression; +use flate2::write::GzEncoder; +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{BufWriter, Write}; + +/// The contig names of a `.fai`, in its order. +fn contig_order(fai: &str) -> Result, Fail> { + let mut order = HashMap::new(); + let mut i = 0; + each_line(fai, |l| { + order.insert(l.split('\t').next().unwrap_or("").to_string(), i); + i += 1; + Ok(()) + })?; + Ok(order) +} + +/// `og dev fixture contig-cut FAI SITES.vcf.gz OUT_SITES PANEL OUT_PANEL`: +/// somalier's sites and VerifyBamID2's panel (PANEL.bed, .mu and .UD, one +/// marker a line in each) cut to the contigs of FAI, since somalier stops on +/// a site whose contig the FASTA lacks. Prints the markers kept. +pub fn contig_cut(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 5)?; + let contigs: HashSet = contig_order(&args[0])?.into_keys().collect(); + let first = |l: &str| l.split('\t').next().unwrap_or("").to_string(); + let mut o = create(&args[2])?; + each_line(&args[1], |l| { + if l.starts_with('#') || contigs.contains(&first(l)) { + writeln!(o, "{l}")?; + } + Ok(()) + })?; + o.flush()?; + let (panel, fix) = (&args[3], &args[4]); + let bed = lines(&format!("{panel}.bed"))?; + let keep: Vec = (0..bed.len()) + .filter(|&i| contigs.contains(&first(&bed[i]))) + .collect(); + for ext in ["bed", "mu", "UD"] { + let all = lines(&format!("{panel}.{ext}"))?; + if all.len() != bed.len() { + return Err(bad(format!( + "{panel}.{ext} has {} lines, the .bed {}", + all.len(), + bed.len() + ))); + } + let mut o = create(&format!("{fix}.{ext}"))?; + for &i in &keep { + writeln!(o, "{}", all[i])?; + } + o.flush()?; + } + writeln!(out, "{} panel markers on the fixture's contigs", keep.len())?; + Ok(()) +} + +/// `og dev fixture add-sites SITES.vcf EXTRA.tsv FAI OUT.vcf`: the sites VCF +/// (its first 8 columns) with the rows of EXTRA added where SITES has no +/// record at that contig and position, sorted by FAI's contig order and +/// position. Prints the site count. +pub fn add_sites(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 4)?; + let order = contig_order(&args[2])?; + let (mut head, mut cols, mut recs) = (Vec::new(), None, Vec::>::new()); + each_line(&args[0], |l| { + if l.starts_with("##") { + head.push(l.to_string()); + } else if l.starts_with('#') { + cols = Some(l.to_string()); + } else { + recs.push(l.split('\t').take(8).map(str::to_string).collect()); + } + Ok(()) + })?; + let cols = cols.ok_or_else(|| bad(format!("{}: no #CHROM line", args[0])))?; + let seen: HashSet<(String, String)> = recs + .iter() + .map(|r| (r[0].clone(), r.get(1).cloned().unwrap_or_default())) + .collect(); + each_line(&args[1], |l| { + let r: Vec = l.split('\t').map(str::to_string).collect(); + if !seen.contains(&(r[0].clone(), r.get(1).cloned().unwrap_or_default())) { + recs.push(r); + } + Ok(()) + })?; + let mut keyed = recs + .into_iter() + .map(|r| { + let pos = cell_int(r.get(1).map_or("", String::as_str))?; + Ok(((order.get(&r[0]).copied().unwrap_or(1 << 30), pos), r)) + }) + .collect::, Fail>>()?; + keyed.sort_by_key(|x| x.0); + let mut o = create(&args[3])?; + for h in &head { + writeln!(o, "{h}")?; + } + writeln!(o, "{cols}")?; + for (_, r) in &keyed { + writeln!(o, "{}", r.join("\t"))?; + } + o.flush()?; + writeln!(out, "{} sites", keyed.len())?; + Ok(()) +} + +/// A gzip file to write. +fn gz(path: &str) -> Result>, Fail> { + Ok(GzEncoder::new(create(path)?, Compression::default())) +} + +/// `og dev fixture sex-loss-woman WINDOWS OUT_DIR NAME LOSS`: a woman who +/// lost one X in LOSS of her cells (seed 11), from a man's mosdepth windows: +/// `OUT_DIR/mosdepth/NAME.regions.bed.gz` (no chrY, chrX depth doubled less +/// the loss), `OUT_DIR/vcf/NAME.vcf.gz` (2,000 heterozygous chr20 sites at +/// half and 300 chrX sites shifted by the loss) and the sample-QC table +/// `og qc sex-loss` adds to. +pub fn sex_loss_woman( + args: &[String], + _out: &mut dyn Write, + _err: &mut dyn Write, +) -> Result<(), Fail> { + want(args, 4)?; + let (win, dir, name) = (&args[0], &args[1], &args[2]); + let loss: f64 = args[3] + .trim() + .parse() + .map_err(|_| Fail::Usage(format!("LOSS '{}' is not a number", args[3])))?; + let d = loss / (2.0 * (2.0 - loss)); + let mut rnd = PyRandom::new(11); + let mut o = gz(&format!("{dir}/mosdepth/{name}.regions.bed.gz"))?; + each_line(win, |l| { + let c: Vec<&str> = l.split('\t').collect(); + let [chrom, s, e, dp] = c[..] else { + return Err(bad(format!("{win}: a window without 4 columns"))); + }; + match chrom { + "chrY" => {} + "chrX" => { + let x: f64 = dp + .trim() + .parse() + .map_err(|_| bad(format!("{win}: depth '{dp}'")))?; + let dp = pycompat::fmt::fixed(x * 2.0 * (1.0 - loss / 2.0), 2); + writeln!(o, "{chrom}\t{s}\t{e}\t{dp}")?; + } + _ => writeln!(o, "{chrom}\t{s}\t{e}\t{dp}")?, + } + Ok(()) + })?; + o.finish()?.flush()?; + let mut o = gz(&format!("{dir}/vcf/{name}.vcf.gz"))?; + write!( + o, + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tSYNTH\n" + )?; + let site = |o: &mut dyn Write, + rnd: &mut PyRandom, + chrom: &str, + pos: i64, + p: f64| + -> Result<(), Fail> { + let n = rnd.randint(20, 40); + let a = (0..n).filter(|_| rnd.random() < p).count() as i64; + writeln!( + o, + "{chrom}\t{pos}\t.\tA\tG\t40\tPASS\t.\tGT:AD\t0/1:{},{a}", + n - a + )?; + Ok(()) + }; + for i in 0..2000 { + site(&mut o, &mut rnd, "chr20", 10_000_000 + 250 * i, 0.5)?; + } + for i in 0..300 { + let p = 0.5 + if rnd.random() < 0.5 { d } else { -d }; + site(&mut o, &mut rnd, "chrX", 73_700_000 + 1000 * i, p)?; + } + o.finish()?.flush()?; + let mut o = create(&format!("{dir}/qc/{name}_sample_qc.tsv"))?; + write!( + o, + "key\tvalue\nsample\t{name}\ninferred_sex\tunknown\ndeclared_sex\tfemale\n" + )?; + o.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::super::testutil::{dir, fnv, run}; + use std::io::Read; + + fn gunzip(p: &std::path::Path) -> Vec { + let mut v = Vec::new(); + flate2::read::GzDecoder::new(std::fs::File::open(p).expect("open")) + .read_to_end(&mut v) + .expect("gunzip"); + v + } + + #[test] + fn contig_cut_keeps_the_fai_contigs_in_step() { + let d = dir("contig-cut"); + let w = |n: &str, s: &str| std::fs::write(d.join(n), s).expect("write"); + w("r.fai", "chr1\t10\t1\t60\t61\nchr20\t10\t20\t60\t61\n"); + w( + "sites.vcf", + "##h\n#CHROM\tPOS\nchr1\t5\nchr11\t6\nchr20\t7\n", + ); + w("p.bed", "chr11\t1\t2\nchr1\t3\t4\nchr20\t5\t6\n"); + w("p.mu", "a\nb\nc\n"); + w("p.UD", "x y\nz w\nu v\n"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, out, err) = run( + "contig-cut", + &[&p("r.fai"), &p("sites.vcf"), &p("o.vcf"), &p("p"), &p("f")], + ); + assert_eq!( + (code, out.as_str()), + (0, "2 panel markers on the fixture's contigs\n"), + "{err}" + ); + let r = |n: &str| std::fs::read_to_string(d.join(n)).expect("read"); + assert_eq!(r("o.vcf"), "##h\n#CHROM\tPOS\nchr1\t5\nchr20\t7\n"); + assert_eq!( + (r("f.bed").as_str(), r("f.mu").as_str(), r("f.UD").as_str()), + ("chr1\t3\t4\nchr20\t5\t6\n", "b\nc\n", "z w\nu v\n") + ); + w("p.mu", "a\nb\n"); + let (code, _, _) = run( + "contig-cut", + &[&p("r.fai"), &p("sites.vcf"), &p("o.vcf"), &p("p"), &p("f")], + ); + assert_eq!(code, 1); + } + + #[test] + fn add_sites_adds_new_positions_in_fai_order() { + let d = dir("add-sites"); + let w = |n: &str, s: &str| std::fs::write(d.join(n), s).expect("write"); + w("r.fai", "chr2\t10\nchrX\t10\n"); + w( + "s.vcf", + "##a\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\nchrX\t50\t.\tA\tG\t.\tPASS\tAF=0.5\tGT\nchr2\t9\t.\tC\tT\t.\tPASS\tAF=0.5\n", + ); + w( + "x.tsv", + "chrX\t50\t.\tA\tC\t.\tPASS\tAF=0.5\nchrX\t20\t.\tG\tT\t.\tPASS\tAF=0.5\nchr7\t1\t.\tG\tT\t.\tPASS\tAF=0.5\n", + ); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, out, err) = run( + "add-sites", + &[&p("s.vcf"), &p("x.tsv"), &p("r.fai"), &p("o.vcf")], + ); + assert_eq!((code, out.as_str()), (0, "4 sites\n"), "{err}"); + assert_eq!( + std::fs::read_to_string(d.join("o.vcf")).expect("read"), + "##a\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\nchr2\t9\t.\tC\tT\t.\tPASS\tAF=0.5\nchrX\t20\t.\tG\tT\t.\tPASS\tAF=0.5\nchrX\t50\t.\tA\tG\t.\tPASS\tAF=0.5\nchr7\t1\t.\tG\tT\t.\tPASS\tAF=0.5\n" + ); + } + + #[test] + fn sex_loss_woman_is_the_pythons_content() { + let d = dir("woman"); + for s in ["mosdepth", "vcf", "qc"] { + std::fs::create_dir_all(d.join("o").join(s)).expect("dir"); + } + std::fs::write(d.join("w.bed"), "chr20\t0\t500\t30.25\nchrX\t0\t500\t15.125\nchrY\t0\t500\t14.00\nchrX\t500\t1000\t14.375\n").expect("bed"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, _, err) = run("sex-loss-woman", &[&p("w.bed"), &p("o"), "W", "0.3"]); + assert_eq!(code, 0, "{err}"); + let o = d.join("o"); + assert_eq!( + String::from_utf8(gunzip(&o.join("mosdepth/W.regions.bed.gz"))).expect("utf8"), + "chr20\t0\t500\t30.25\nchrX\t0\t500\t25.71\nchrX\t500\t1000\t24.44\n" + ); + // FNV-1a of the VCF the Python heredoc of mosaic-loy-lox-1.sh wrote. + let vcf = gunzip(&o.join("vcf/W.vcf.gz")); + assert_eq!(fnv(&vcf), 0xc45b_5dc4_9d43_4229, "{:#x}", fnv(&vcf)); + assert_eq!( + std::fs::read_to_string(o.join("qc/W_sample_qc.tsv")).expect("read"), + "key\tvalue\nsample\tW\ninferred_sex\tunknown\ndeclared_sex\tfemale\n" + ); + } +} diff --git a/rust/crates/og-dev/src/fixture/lineage/truth.rs b/rust/crates/og-dev/src/fixture/lineage/truth.rs new file mode 100644 index 000000000..703fa4a2b --- /dev/null +++ b/rust/crates/og-dev/src/fixture/lineage/truth.rs @@ -0,0 +1,200 @@ +//! GIAB's genotypes at an ancestry panel's SNVs, as a VCF a step reads. + +use super::{Confident, Fail, bad, cell_int, each_line, fai_contigs, read_bed}; +use std::collections::{BTreeSet, HashMap}; +use std::io::Write; + +/// One member: its name, its calls by (contig, position) and its +/// confident regions. +struct Member { + name: String, + calls: HashMap<(String, i64), (String, String, String)>, + bed: HashMap, +} + +/// `og dev fixture truth-genotypes SITES FAI [--chrom C] NAME=CALLS,BED...`: +/// a VCF of each member's genotype at the panel's SNVs (`CHROM POS REF ALT` +/// rows of SITES) that lie inside every member's confident regions (BED). +/// A site a member's calls (a VCF, header optional) do not list is 0/0; a +/// site where a call has other alleles or a genotype other than 0/0, 0/1 +/// or 1/1 is left out. `--chrom` keeps one contig. The site count goes to +/// stderr. +pub fn genotypes(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> Result<(), Fail> { + let mut chrom: Option<&str> = None; + let mut pos = Vec::new(); + let mut it = args.iter(); + while let Some(a) = it.next() { + if a == "--chrom" { + chrom = Some( + it.next() + .ok_or_else(|| Fail::Usage("--chrom needs a contig".into()))?, + ); + } else { + pos.push(a.as_str()); + } + } + if pos.len() < 3 { + return Err(Fail::Usage( + "takes SITES FAI [--chrom C] NAME=CALLS,BED...".into(), + )); + } + let (sites, fai) = (pos[0], pos[1]); + let mut members = Vec::new(); + for m in &pos[2..] { + let (name, files) = m + .split_once('=') + .ok_or_else(|| Fail::Usage(format!("'{m}' is not NAME=CALLS,BED")))?; + let (calls_path, bed_path) = files + .split_once(',') + .ok_or_else(|| Fail::Usage(format!("'{m}' is not NAME=CALLS,BED")))?; + let mut calls = HashMap::new(); + each_line(calls_path, |l| { + if l.starts_with('#') { + return Ok(()); + } + let f: Vec<&str> = l.split('\t').collect(); + if f.len() < 10 { + return Err(bad(format!( + "{calls_path}: a record with fewer than 10 columns" + ))); + } + let i = f[8] + .split(':') + .position(|k| k == "GT") + .ok_or_else(|| bad(format!("{calls_path}: a record without GT")))?; + let gt = f[9].split(':').nth(i).unwrap_or("").replace('|', "/"); + let gt = if gt == "1/0" { "0/1".to_string() } else { gt }; + calls.insert( + (f[0].to_string(), cell_int(f[1])?), + (f[3].to_string(), f[4].to_string(), gt), + ); + Ok(()) + })?; + members.push(Member { + name: name.to_string(), + calls, + bed: read_bed(bed_path)?, + }); + } + let mut rows = Vec::new(); + each_line(sites, |l| { + let c: Vec<&str> = l.split('\t').collect(); + if c.len() != 4 { + return Err(bad(format!("{sites}: a row without 4 columns"))); + } + if chrom.is_none_or(|k| k == c[0]) { + rows.push([ + c[0].to_string(), + c[1].to_string(), + c[2].to_string(), + c[3].to_string(), + ]); + } + Ok(()) + })?; + let chroms: BTreeSet<&str> = rows.iter().map(|r| r[0].as_str()).collect(); + writeln!(out, "##fileformat=VCFv4.2")?; + writeln!(out, "##FILTER=")?; + for (c, len) in fai_contigs(fai)? { + if chroms.contains(c.as_str()) { + writeln!(out, "##contig=")?; + } + } + writeln!( + out, + "##FORMAT=" + )?; + let names: Vec<&str> = members.iter().map(|m| m.name.as_str()).collect(); + writeln!( + out, + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{}", + names.join("\t") + )?; + let mut n = 0u64; + 'site: for [c, p, r, a] in &rows { + let p = cell_int(p)?; + if !members + .iter() + .all(|m| m.bed.get(c).is_some_and(|b| b.has(p))) + { + continue; + } + let mut gts = Vec::with_capacity(members.len()); + for m in &members { + match m.calls.get(&(c.clone(), p)) { + None => gts.push("0/0"), + Some((tr, ta, g)) + if tr == r && ta == a && matches!(g.as_str(), "0/0" | "0/1" | "1/1") => + { + gts.push(g.as_str()); + } + Some(_) => continue 'site, + } + } + writeln!( + out, + "{c}\t{p}\t.\t{r}\t{a}\t.\tPASS\t.\tGT\t{}", + gts.join("\t") + )?; + n += 1; + } + writeln!(err, "{n} sites")?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::super::testutil::{dir, run}; + + #[test] + fn genotypes_follow_the_truth_inside_every_members_regions() { + let d = dir("truth"); + let w = |n: &str, s: &str| std::fs::write(d.join(n), s).expect("write"); + w( + "sites.tsv", + "chr1\t100\tA\tG\nchr1\t200\tC\tT\nchr1\t300\tG\tA\nchr1\t400\tT\tC\nchr2\t50\tA\tC\n", + ); + w( + "ref.fa.fai", + "chr1\t1000\t6\t60\t61\nchr2\t500\t1030\t60\t61\nchr3\t9\t1600\t60\t61\n", + ); + w( + "a.vcf", + "##h\n#CHROM\nchr1\t200\t.\tC\tT\t.\t.\t.\tGT:DP\t1|0:4\nchr1\t300\t.\tG\tC\t.\t.\t.\tGT\t0/1\nchr2\t50\t.\tA\tC\t.\t.\t.\tDP:GT\t3:1/1\n", + ); + w("a.bed", "chr1\t0\t350\nchr2\t0\t100\n"); + w("b.vcf", "chr1\t100\t.\tA\tG\t.\t.\t.\tGT\t1/2\n"); + w("b.bed", "chr1\t50\t1000\n"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let a = format!("A={},{}", p("a.vcf"), p("a.bed")); + let (code, out, err) = run("truth-genotypes", &[&p("sites.tsv"), &p("ref.fa.fai"), &a]); + assert_eq!(code, 0, "{err}"); + let body: Vec<&str> = out.lines().filter(|l| !l.starts_with('#')).collect(); + assert_eq!( + body, + [ + "chr1\t100\t.\tA\tG\t.\tPASS\t.\tGT\t0/0", + "chr1\t200\t.\tC\tT\t.\tPASS\t.\tGT\t0/1", + "chr2\t50\t.\tA\tC\t.\tPASS\t.\tGT\t1/1" + ] + ); + assert!( + out.contains("##contig=\n##contig=\n##FORMAT"), + "{out}" + ); + assert!(out.contains("FORMAT\tA\n"), "{out}"); + assert_eq!(err, "3 sites\n"); + let b = format!("B={},{}", p("b.vcf"), p("b.bed")); + let (_, out, err) = run( + "truth-genotypes", + &[&p("sites.tsv"), &p("ref.fa.fai"), "--chrom", "chr1", &a, &b], + ); + let body: Vec<&str> = out.lines().filter(|l| !l.starts_with('#')).collect(); + assert_eq!( + body, + ["chr1\t200\t.\tC\tT\t.\tPASS\t.\tGT\t0/1\t0/0"], + "{err}" + ); + assert!(!out.contains("chr2"), "{out}"); + } +} diff --git a/rust/crates/og-dev/src/fixture/lineage/viral.rs b/rust/crates/og-dev/src/fixture/lineage/viral.rs new file mode 100644 index 000000000..79c7cb0c9 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/lineage/viral.rs @@ -0,0 +1,124 @@ +//! Unmapped viral reads to plant in a BAM, for step 47. + +use super::{Fail, PyRandom, bad, each_line, want}; +use std::collections::HashMap; +use std::io::Write; + +/// The read length. +const L: usize = 150; + +/// `og dev fixture viral-spike VIRAL.fa HUMAN_DEPTH FRACTION`: unmapped SAM +/// records (150 bp, 0.3% substitutions, random strand, seed 47): HHV-6B +/// (NC_000898.1) at FRACTION of HUMAN_DEPTH, 300 CMV reads and 200 phiX. +pub fn spike(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 3)?; + let num = |s: &str, what: &str| -> Result { + pycompat::num::parse_float(s) + .map_err(|_| Fail::Usage(format!("{what} '{s}' is not a number"))) + }; + let (human, frac) = (num(&args[1], "HUMAN_DEPTH")?, num(&args[2], "FRACTION")?); + let mut seqs: HashMap> = HashMap::new(); + let mut name: Option = None; + each_line(&args[0], |l| { + let l = l.trim(); + if let Some(h) = l.strip_prefix('>') { + let n = h.split_whitespace().next().unwrap_or("").to_string(); + seqs.insert(n.clone(), Vec::new()); + name = Some(n); + } else if !l.is_empty() { + let n = name + .as_ref() + .ok_or_else(|| bad("sequence before the first header"))?; + seqs.get_mut(n) + .expect("inserted with its header") + .extend(l.bytes().map(|b| b.to_ascii_uppercase())); + } + Ok(()) + })?; + let mut rng = PyRandom::new(47); + let mut reads = |acc: &str, n: u64, tag: &str, out: &mut dyn Write| -> Result<(), Fail> { + let s = seqs + .get(acc) + .ok_or_else(|| bad(format!("{}: no {acc}", args[0])))?; + if s.len() <= L { + return Err(bad(format!("{acc} is shorter than a read"))); + } + let qual = "I".repeat(L); + for i in 0..n { + let p = rng.below((s.len() - L) as u64) as usize; + let mut r = s[p..p + L].to_vec(); + for b in r.iter_mut() { + if rng.random() < 0.003 { + let other: Vec = b"ACGT".iter().copied().filter(|x| x != b).collect(); + *b = other[rng.below(other.len() as u64) as usize]; + } + } + if rng.random() < 0.5 { + r = r + .iter() + .rev() + .map(|&b| match b { + b'A' => b'T', + b'C' => b'G', + b'G' => b'C', + b'T' => b'A', + x => x, + }) + .collect(); + } + let r = String::from_utf8_lossy(&r); + writeln!(out, "{tag}_{i}\t4\t*\t0\t0\t*\t*\t0\t0\t{r}\t{qual}")?; + } + Ok(()) + }; + let hhv6b = seqs.get("NC_000898.1").map_or(0, Vec::len) as f64; + let n = pycompat::fmt::round_int(frac * human * hhv6b / L as f64) + .map_err(|_| bad("the HHV-6B read count is not finite"))?; + reads("NC_000898.1", n as u64, "spk6b", out)?; + reads("NC_006273.2", 300, "spkcmv", out)?; + reads("NC_001422.1", 200, "spkphix", out)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::super::testutil::{dir, fnv, run}; + + #[test] + fn spike_is_the_pythons_bytes() { + let d = dir("viral"); + let mut rng = super::PyRandom::new(3); + let mut fa = String::new(); + for (acc, n) in [ + ("NC_000898.1 HHV-6B", 3000), + ("NC_006273.2", 900), + ("NC_001664.4", 400), + ("NC_001422.1", 700), + ] { + fa.push_str(&format!(">{acc}\n")); + let seq: String = (0..n) + .map(|_| ['A', 'c', 'G', 't', 'N'][rng.below(5) as usize]) + .collect(); + for chunk in seq.as_bytes().chunks(70) { + fa.push_str(std::str::from_utf8(chunk).expect("ascii")); + fa.push('\n'); + } + } + std::fs::write(d.join("v.fa"), fa).expect("fa"); + let (code, out, err) = run( + "viral-spike", + &[d.join("v.fa").to_str().expect("utf8"), "31.27", "0.5"], + ); + assert_eq!(code, 0, "{err}"); + // round(0.5 * 31.27 * 3000 / 150) = round(312.7) = 313 HHV-6B reads. + assert_eq!(out.lines().filter(|l| l.starts_with("spk6b_")).count(), 313); + assert_eq!(out.lines().count(), 813); + // FNV-1a of what the Python heredoc of viral-reads-1.sh wrote. + assert_eq!( + fnv(out.as_bytes()), + 0x085f_613c_1fff_c640, + "{:#x}", + fnv(out.as_bytes()) + ); + } +} diff --git a/rust/crates/og-dev/src/fixture/lineage/ystr.rs b/rust/crates/og-dev/src/fixture/lineage/ystr.rs new file mode 100644 index 000000000..1c22b0934 --- /dev/null +++ b/rust/crates/og-dev/src/fixture/lineage/ystr.rs @@ -0,0 +1,493 @@ +//! The Y cases: the private-SNV red control's seeded VCF, and the Y-STR +//! case's repeat check, read-length count and planted repeat cut. + +use super::{Fail, Op, bad, cell_int, cigar, create, each_line, int, want}; +use std::collections::{HashMap, HashSet}; +use std::io::Write; + +/// The fixture's TBL1Y stretch of chrY (scripts/ci/build-fixture.sh). +const FREE_FROM: i64 = 6_800_000; +const FREE_TO: i64 = 7_000_000; + +/// Python's `s.isdigit()` for ASCII text. +fn is_digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) +} + +/// `og dev fixture y-seed CALLS.vcf.gz MASK.fasta.gz PRIVATE.tsv OUT.vcf TABLE...`: +/// the VCF with two made-up haploid chrY calls (C>T) in the TBL1Y stretch at +/// positions no table and no record names: the first the callable mask +/// passes (`P`) and the first it does not, at the sample's median chrY depth +/// from PRIVATE's header (at least 10, 20 without one). Prints both +/// positions. +pub fn y_seed(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + if args.len() < 4 { + return Err(Fail::Usage("takes CALLS MASK PRIVATE OUT TABLE...".into())); + } + let (vcf, mask_path, private, dst) = (&args[0], &args[1], &args[2], &args[3]); + let mut mask = Vec::new(); + each_line(mask_path, |l| { + if !l.starts_with('>') { + mask.extend_from_slice(l.trim().as_bytes()); + } + Ok(()) + })?; + let mut taken = HashSet::new(); + for t in &args[4..] { + each_line(t, |l| { + let c: Vec<&str> = l.split('\t').collect(); + if c.len() >= 4 && is_digits(c[3]) { + taken.insert(cell_int(c[3])?); + } + Ok(()) + })?; + } + let (mut head, mut body) = (Vec::new(), Vec::new()); + each_line(vcf, |l| { + if l.starts_with('#') { + head.push(l.to_string()); + } else { + let c: Vec<&str> = l.split('\t').collect(); + if (c[0] == "chrY" || c[0] == "Y") && c.len() > 1 && is_digits(c[1]) { + taken.insert(cell_int(c[1])?); + } + body.push(l.to_string()); + } + Ok(()) + })?; + let text = std::fs::read_to_string(private).map_err(|e| bad(format!("{private}: {e}")))?; + let depth = match text.find("median_chrY_depth=") { + Some(i) => { + let rest = &text[i + "median_chrY_depth=".len()..]; + let n = rest + .find(|ch: char| !(ch.is_ascii_digit() || ch == '.')) + .unwrap_or(rest.len()); + match &rest[..n] { + "" => 20, + v => { + let x = pycompat::num::parse_float(v) + .map_err(|_| bad(format!("{private}: median depth '{v}'")))?; + (x.trunc() as i64).max(10) + } + } + } + None => 20, + }; + let mask_at = |p: i64| -> Result { + usize::try_from(p - 1) + .ok() + .and_then(|i| mask.get(i).copied()) + .ok_or_else(|| bad(format!("{mask_path}: shorter than {p}"))) + }; + let (mut pin, mut pout) = (None, None); + for p in (FREE_FROM..FREE_TO).filter(|p| !taken.contains(p)) { + let passed = mask_at(p)? == b'P'; + if passed && pin.is_none() { + pin = Some(p); + } else if !passed && pout.is_none() { + pout = Some(p); + } + if pin.is_some() && pout.is_some() { + break; + } + } + let (Some(pin), Some(pout)) = (pin, pout) else { + return Err(bad("no free position inside and outside the mask")); + }; + let mut o = create(dst)?; + for l in head.iter().chain(&body) { + writeln!(o, "{l}")?; + } + for p in [pin, pout] { + writeln!( + o, + "chrY\t{p}\t.\tC\tT\t50\tPASS\t.\tGT:AD:DP:GQ\t1:0,{depth}:{depth}:60" + )?; + } + o.flush()?; + writeln!(out, "{pin} {pout}")?; + Ok(()) +} + +/// `og dev fixture ystr-periodic REGIONS.bed REGIONS.fa`: how many regions +/// (`CHROM START END PERIOD ...`, sequences from `samtools faidx -r`) repeat +/// with their period: at least 75% of the bases equal to the base one +/// period further on. +pub fn periodic(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 2)?; + let mut seqs: HashMap> = HashMap::new(); + let mut name: Option = None; + each_line(&args[1], |l| { + let l = l.trim(); + if let Some(h) = l.strip_prefix('>') { + seqs.insert(h.to_string(), Vec::new()); + name = Some(h.to_string()); + } else if let Some(n) = &name { + seqs.get_mut(n) + .expect("inserted with its header") + .extend(l.bytes().map(|b| b.to_ascii_uppercase())); + } + Ok(()) + })?; + let mut ok = 0u64; + each_line(&args[0], |l| { + let c: Vec<&str> = l.split('\t').collect(); + if c.len() < 4 { + return Err(bad(format!( + "{}: a region with fewer than 4 columns", + args[0] + ))); + } + let s = seqs + .get(&format!("{}:{}-{}", c[0], c[1], c[2])) + .map_or(&[][..], Vec::as_slice); + let p = usize::try_from(cell_int(c[3])?).map_err(|_| bad("a negative period"))?; + let span = cell_int(c[2])? - cell_int(c[1])? + 1; + if s.len() as i64 == span && s.len() > p { + let n = s.len() - p; + let same = (0..n).filter(|&i| s[i] == s[i + p]).count(); + ok += u64::from(same as f64 >= 0.75 * n as f64); + } + Ok(()) + })?; + writeln!(out, "{ok}")?; + Ok(()) +} + +/// The reference end of an alignment. +fn end_of(pos: i64, ops: &[Op]) -> i64 { + pos + ops + .iter() + .filter(|(_, op)| b"MDN=X".contains(op)) + .map(|(n, _)| n) + .sum::() + - 1 +} + +/// One callable marker of the panel. +struct Region { + start: i64, + end: i64, + period: i64, + ref_copies: i64, + kind: String, +} + +/// `og dev fixture ystr-readcount PANEL CALLS READS.sam`: for each marker +/// step 45 called high, the count its spanning reads give (10 bases of flank +/// each side; the region's length plus the CIGAR's insertions minus its +/// deletions in the region or its flanks; the modal length), as `marker +/// step-count read-count reads kind`. +pub fn readcount(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { + want(args, 3)?; + let mut regions: Vec<(String, Region)> = Vec::new(); + each_line(&args[0], |l| { + let c: Vec<&str> = l.split('\t').collect(); + if l.starts_with('#') + || c[0] == "order" + || !c.get(3).is_some_and(|k| *k == "single" || *k == "span") + { + return Ok(()); + } + let parts: Vec> = c + .get(4) + .ok_or_else(|| bad(format!("{}: a marker without parts", args[0])))? + .split(';') + .map(|x| x.splitn(4, '/').collect()) + .collect(); + let span_of = |part: &Vec<&str>| -> Result<(i64, i64), Fail> { + let range = part[0] + .split(':') + .nth(1) + .ok_or_else(|| bad(format!("part '{}'", part[0])))?; + let (s, e) = range + .split_once('-') + .ok_or_else(|| bad(format!("part '{}'", part[0])))?; + Ok((cell_int(s)?, cell_int(e)?)) + }; + let start = span_of(&parts[0])?.0; + let end = span_of(parts.last().expect("one part at least"))?.1; + let period = cell_int( + parts[0] + .get(1) + .ok_or_else(|| bad("a part without a period"))?, + )?; + let mut ref_copies = 0; + for x in &parts { + let copies = x.get(2).ok_or_else(|| bad("a part without copies"))?; + let v = pycompat::num::parse_float(copies) + .map_err(|_| bad(format!("copies '{copies}'")))?; + ref_copies += (v + 0.5).trunc() as i64; + } + let r = Region { + start, + end, + period, + ref_copies, + kind: c[3].to_string(), + }; + match regions.iter_mut().find(|(m, _)| m == c[2]) { + Some(slot) => slot.1 = r, + None => regions.push((c[2].to_string(), r)), + } + Ok(()) + })?; + // Per marker, the length changes in the order first seen, with counts. + let mut lens: HashMap> = HashMap::new(); + each_line(&args[2], |l| { + let f: Vec<&str> = l.split('\t').collect(); + if f.len() < 6 || f[2] != "chrY" || f[5] == "*" { + return Ok(()); + } + let pos = cell_int(f[3])?; + let ops = cigar(f[5]); + let end = end_of(pos, &ops); + for (m, g) in ®ions { + if pos > g.start - 10 || end < g.end + 10 { + continue; + } + let (mut r, mut diff) = (pos, 0i64); + let near = |r: i64| g.start - 10 <= r && r <= g.end + 10; + for &(n, op) in &ops { + match op { + b'M' | b'=' | b'X' => r += n, + b'D' | b'N' => { + if near(r) { + diff -= n; + } + r += n; + } + b'I' if near(r) => diff += n, + _ => {} + } + } + let v = lens.entry(m.clone()).or_default(); + match v.iter_mut().find(|x| x.0 == diff) { + Some(x) => x.1 += 1, + None => v.push((diff, 1)), + } + } + Ok(()) + })?; + each_line(&args[1], |l| { + let c: Vec<&str> = l.split('\t').collect(); + if c.len() < 5 || c[4] != "high" { + return Ok(()); + } + let (Some((_, g)), Some(v)) = (regions.iter().find(|(m, _)| m == c[2]), lens.get(c[2])) + else { + return Ok(()); + }; + // Counter.most_common(1): the first of the highest counts. + let Some(&(diff, n)) = v.iter().fold(None::<&(i64, u64)>, |best, x| match best { + Some(b) if b.1 >= x.1 => Some(b), + _ => Some(x), + }) else { + return Ok(()); + }; + let (whole, rest) = (diff.div_euclid(g.period), diff.rem_euclid(g.period)); + let frac = if rest != 0 { + format!(".{rest}") + } else { + String::new() + }; + writeln!( + out, + "{}\t{}\t{}{frac}\t{n}\t{}", + c[2], + c[3], + g.ref_copies + whole, + g.kind + )?; + Ok(()) + }) +} + +/// `og dev fixture ystr-cut START END READS.sam`: the SAM with one 4-base +/// repeat unit cut out at START: chrY reads that span START..END (10 bases +/// of flank each side) with a plain match get bases START..START+3 removed +/// (CIGAR xM4DyM, MD and NM dropped); other reads over the region are +/// dropped. The count goes to stderr. +pub fn cut(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> Result<(), Fail> { + want(args, 3)?; + let (s, e): (i64, i64) = (int(&args[0], "START")?, int(&args[1], "END")?); + let mut n_cut = 0u64; + each_line(&args[2], |l| { + if l.starts_with('@') { + writeln!(out, "{l}")?; + return Ok(()); + } + let mut f: Vec = l.split('\t').map(str::to_string).collect(); + if f.len() < 11 { + return Err(bad(format!( + "{}: a read with fewer than 11 columns", + args[2] + ))); + } + if f[2] != "chrY" || f[5] == "*" { + writeln!(out, "{l}")?; + return Ok(()); + } + let pos = cell_int(&f[3])?; + let ops = cigar(&f[5]); + let end = end_of(pos, &ops); + if end < s || pos > e { + writeln!(out, "{l}")?; + return Ok(()); + } + let plain = f[5].strip_suffix('M').filter(|d| is_digits(d)); + let Some(m) = plain else { return Ok(()) }; + if pos > s - 10 || end < e + 10 { + return Ok(()); + } + let n = cell_int(m)?; + let off = usize::try_from(s - pos).map_err(|_| bad("a negative offset"))?; + let snip = |x: &str| -> String { + let b = x.as_bytes(); + let head = &b[..off.min(b.len())]; + let tail = b.get(off + 4..).unwrap_or(&[]); + String::from_utf8_lossy(&[head, tail].concat()).into_owned() + }; + f[9] = snip(&f[9]); + if f[10] != "*" { + f[10] = snip(&f[10]); + } + f[5] = format!("{off}M4D{}M", n - off as i64 - 4); + f.retain(|x| !x.starts_with("MD:Z:") && !x.starts_with("NM:i:")); + writeln!(out, "{}", f.join("\t"))?; + n_cut += 1; + Ok(()) + })?; + writeln!(err, "planted the cut in {n_cut} reads")?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::super::testutil::{dir, run}; + + #[test] + fn periodic_counts_repeats_of_their_period() { + let d = dir("periodic"); + std::fs::write(d.join("r.bed"), "chrY\t10\t21\t4\tx\tDYS1\nchrY\t30\t41\t4\tx\tDYS2\nchrY\t50\t61\t4\nchrY\t70\t72\t4\n").expect("bed"); + std::fs::write(d.join("r.fa"), ">chrY:10-21\nAGATAGAT\nagat\n>chrY:30-41\nACGTTGCAACGG\n>chrY:50-61\nAGATAGATAGA\n>chrY:70-72\nAAA\n").expect("fa"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + assert_eq!(run("ystr-periodic", &[&p("r.bed"), &p("r.fa")]).1, "1\n"); + std::fs::write(d.join("one.bed"), "chrY\t10\t21\t1\n").expect("bed"); + assert_eq!(run("ystr-periodic", &[&p("one.bed"), &p("r.fa")]).1, "0\n"); + } + + #[test] + fn readcount_takes_the_modal_length_change() { + let d = dir("readcount"); + std::fs::write( + d.join("panel.tsv"), + "#c\norder\tpanel\tmarker\tkind\tparts\n1\tY-12\tDYS1\tsingle\tchrY:100-139/4/10\n2\tY-12\tDYS2\tspan\tchrY:200-219/4/5.5;chrY:230-249/4/4.4\n3\tY-12\tDYS3\tmulti\tchrY:300-310/4/3\n", + ) + .expect("panel"); + std::fs::write( + d.join("calls.tsv"), + "order\tpanel\tmarker\tcount\tconfidence\n1\tY-12\tDYS1\t9\thigh\n2\tY-12\tDYS2\t12\thigh\n3\tY-12\tDYS3\t3\thigh\n", + ) + .expect("calls"); + let mut sam = String::new(); + // DYS1: two reads lose 4 bases (one unit), one read gains 2. + for (i, cig) in ["40M4D60M", "40M4D60M", "40M2I58M", "100M"] + .iter() + .enumerate() + { + sam.push_str(&format!("r{i}\t0\tchrY\t60\t60\t{cig}\t*\t0\t0\t*\t*\n")); + } + // DYS2 (190..259 with flanks): one read, no change. + sam.push_str("s\t0\tchrY\t150\t60\t150M\t*\t0\t0\t*\t*\n"); + sam.push_str("t\t0\tchr1\t150\t60\t150M\t*\t0\t0\t*\t*\n"); + std::fs::write(d.join("reads.sam"), sam).expect("sam"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, out, err) = run( + "ystr-readcount", + &[&p("panel.tsv"), &p("calls.tsv"), &p("reads.sam")], + ); + assert_eq!(code, 0, "{err}"); + assert_eq!(out, "DYS1\t9\t9\t2\tsingle\nDYS2\t12\t10\t1\tspan\n"); + } + + #[test] + fn cut_removes_one_unit_from_plain_spanning_reads() { + let d = dir("cut"); + let sam = "@HD\tVN:1.6\n\ + a\t0\tchrY\t90\t60\t40M\t*\t0\t0\tAAAAAAAAAACCCCGGGGGGGGGGGGGGGGGGGGGGGGGG\tIIIIIIIIIIJJJJIIIIIIIIIIIIIIIIIIIIIIIIII\tMD:Z:40\tNM:i:0\tRG:Z:x\n\ + b\t0\tchrY\t90\t60\t20M1I19M\t*\t0\t0\t*\t*\n\ + c\t0\tchrY\t10\t60\t20M\t*\t0\t0\t*\t*\n\ + d\t0\tchr1\t90\t60\t40M\t*\t0\t0\t*\t*\n"; + std::fs::write(d.join("in.sam"), sam).expect("sam"); + let (code, out, err) = run( + "ystr-cut", + &["100", "105", d.join("in.sam").to_str().expect("utf8")], + ); + assert_eq!(code, 0, "{err}"); + assert_eq!( + out, + "@HD\tVN:1.6\na\t0\tchrY\t90\t60\t10M4D26M\t*\t0\t0\tAAAAAAAAAAGGGGGGGGGGGGGGGGGGGGGGGGGG\tIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIII\tRG:Z:x\n\ + c\t0\tchrY\t10\t60\t20M\t*\t0\t0\t*\t*\nd\t0\tchr1\t90\t60\t40M\t*\t0\t0\t*\t*\n" + ); + assert_eq!(err, "planted the cut in 1 reads\n"); + } + + #[test] + fn y_seed_picks_free_positions_in_and_out_of_the_mask() { + let d = dir("y-seed"); + // The mask passes nothing before 6,800,003; 6,800,000 is in a table, + // 6,800,001 is a record, so OUT is 6,800,002 and IN 6,800,004. + let mut mask = vec![b'N'; 7_000_000]; + mask[6_800_002] = b'P'; + mask[6_800_003] = b'P'; + let mut fa = b">chrY\n".to_vec(); + for chunk in mask.chunks(60) { + fa.extend_from_slice(chunk); + fa.push(b'\n'); + } + std::fs::write(d.join("mask.fa"), fa).expect("mask"); + std::fs::write(d.join("t1.txt"), "chrY\tA1\tR\t6800000\tC->T\tC\tT\nbad\n").expect("t1"); + std::fs::write(d.join("t2.txt"), "chrY\tA2\tR\t6800003\tC->T\tC\tT\n").expect("t2"); + std::fs::write( + d.join("c.vcf"), + "##fileformat=VCFv4.2\n#CHROM\tPOS\nchrY\t6800001\t.\tA\tG\n", + ) + .expect("vcf"); + std::fs::write( + d.join("p.tsv"), + "# gates\tmin_depth=10 median_chrY_depth=31.9 x\n", + ) + .expect("priv"); + let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); + let (code, out, err) = run( + "y-seed", + &[ + &p("c.vcf"), + &p("mask.fa"), + &p("p.tsv"), + &p("o.vcf"), + &p("t1.txt"), + &p("t2.txt"), + ], + ); + assert_eq!((code, out.as_str()), (0, "6800004 6800002\n"), "{err}"); + assert_eq!( + std::fs::read_to_string(d.join("o.vcf")).expect("vcf"), + "##fileformat=VCFv4.2\n#CHROM\tPOS\nchrY\t6800001\t.\tA\tG\n\ + chrY\t6800004\t.\tC\tT\t50\tPASS\t.\tGT:AD:DP:GQ\t1:0,31:31:60\n\ + chrY\t6800002\t.\tC\tT\t50\tPASS\t.\tGT:AD:DP:GQ\t1:0,31:31:60\n" + ); + std::fs::write(d.join("p.tsv"), "no depth\n").expect("priv"); + let (_, out, _) = run( + "y-seed", + &[&p("c.vcf"), &p("mask.fa"), &p("p.tsv"), &p("o.vcf")], + ); + assert_eq!(out, "6800003 6800000\n"); + assert!( + std::fs::read_to_string(d.join("o.vcf")) + .expect("vcf") + .contains("1:0,20:20:60") + ); + } +} From 44ecbd97bae2137f097b345ca39281e3c7f95560 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:18:29 +0100 Subject: [PATCH 238/255] docs(e2e): say the base pack leaves out the Cyrius install and pypgx bundle --- scripts/ci/e2e-run.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/ci/e2e-run.sh b/scripts/ci/e2e-run.sh index cad74523e..677d06bd1 100755 --- a/scripts/ci/e2e-run.sh +++ b/scripts/ci/e2e-run.sh @@ -32,8 +32,10 @@ # E2E_MIN_FREE_GB fail before the first case when less is free under # E2E_WORK (default 0) # E2E_BASE_OUT after the cases, pack every file they wrote or changed -# (under E2E_WORK and the checkout) into this .tar.zst, and -# write "E2E_WORK CHECKOUT" to E2E_BASE_OUT.from +# (under E2E_WORK and the checkout) into this .tar.zst, except +# the Cyrius install (tools/cyrius-*) and the pypgx bundle, +# which may not be redistributed, check the archive holds +# neither, and write "E2E_WORK CHECKOUT" to E2E_BASE_OUT.from # E2E_BASE_IN after the layout, unpack this .tar.zst from E2E_BASE_OUT, # which must have been packed with the same E2E_WORK and # checkout, then delete it to give the cases its disk space @@ -68,7 +70,8 @@ # the -*.sh cases later packages add), or those of E2E_CASES, # and keeps going after a failure, so one run lists every broken step; # 5. prints a case / result / time / log table to the job summary; -# 6. with E2E_BASE_OUT, packs what the cases wrote; exits 1 if any case failed. +# 6. with E2E_BASE_OUT, packs what the cases wrote, except the Cyrius install +# and the pypgx bundle, and checks the archive; exits 1 if any case failed. set -euo pipefail REPO="$(cd "$(dirname "$0")/../.." && pwd)" @@ -1006,7 +1009,10 @@ docker system df # --- 6. Pack for the shards --------------------------------------------------- # Everything the cases wrote or changed, at its absolute path, owners, modes -# and times (ns) kept, the base cases' logs too (a case may read one). +# and times (ns) kept, the base cases' logs too (a case may read one), except +# the Cyrius install (tools/cyrius-*) and the pypgx bundle, which may not be +# redistributed. Their results stay in; a shard case that needs the bundle +# clones it. check_base_archive fails the job if the archive holds either. # Directories go in as entries alone (--no-recursion), so the reference and # the fixture copies of the layout stay out. if [ -n "${E2E_BASE_OUT:-}" ] && [ "$failed" -eq 0 ]; then From 0dc071649d8ba4e58bd2bc73eb664673164fcb1e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:29:37 +0100 Subject: [PATCH 239/255] test(og-dev): let base-count read a SAM file as well as stdin Bead: D1 --- rust/crates/og-dev/src/fixture/lineage.rs | 2 +- .../crates/og-dev/src/fixture/lineage/chip.rs | 31 +++++++++++++------ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/rust/crates/og-dev/src/fixture/lineage.rs b/rust/crates/og-dev/src/fixture/lineage.rs index 7356d9d4d..3b28a9b56 100644 --- a/rust/crates/og-dev/src/fixture/lineage.rs +++ b/rust/crates/og-dev/src/fixture/lineage.rs @@ -55,7 +55,7 @@ const TABLE: &[(&str, &str, Run)] = &[ ancient::panel, ), ("plant-snv", "EVERY FAI READS.sam POS", chip::plant_snv), - ("base-count", "POS BASE < READS.sam", chip::base_count), + ("base-count", "POS BASE [READS.sam]", chip::base_count), ("sites-fasta", "SITES.tsv.gz OUT.fasta", chip::sites_fasta), ( "mt-roundtrip", diff --git a/rust/crates/og-dev/src/fixture/lineage/chip.rs b/rust/crates/og-dev/src/fixture/lineage/chip.rs index bb0406149..ddf7019a7 100644 --- a/rust/crates/og-dev/src/fixture/lineage/chip.rs +++ b/rust/crates/og-dev/src/fixture/lineage/chip.rs @@ -3,7 +3,7 @@ use super::{Fail, Op, bad, cell_int, cigar, create, each_line, fai_contigs, int, want}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::io::{BufRead, Write}; +use std::io::Write; /// The read offset of 1-based reference `pos` in a read starting at `start`, /// when an M, = or X operation covers it. @@ -76,23 +76,24 @@ pub fn plant_snv(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> }) } -/// `og dev fixture base-count POS BASE < READS.sam`: `A N`, where N is the -/// reads of a SAM body (`samtools view`) whose alignment covers POS with a -/// read base, and A those of them with BASE there. +/// `og dev fixture base-count POS BASE [READS.sam]`: `A N`, where N is the +/// reads of a SAM body (`samtools view`; stdin without a file) whose +/// alignment covers POS with a read base, and A those of them with BASE +/// there. pub fn base_count(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> Result<(), Fail> { - want(args, 2)?; + if !(2..=3).contains(&args.len()) { + return Err(Fail::Usage("takes POS BASE [READS.sam]".into())); + } let pos: i64 = int(&args[0], "POS")?; let base = match args[1].as_bytes() { [b] => *b, _ => return Err(Fail::Usage("BASE is one letter".into())), }; let (mut a, mut n) = (0u64, 0u64); - let stdin = std::io::stdin(); - for line in stdin.lock().lines() { - let line = line?; + each_line(args.get(2).map_or("-", String::as_str), |line| { let f: Vec<&str> = line.split('\t').collect(); if f.len() < 10 || f[5] == "*" { - continue; + return Ok(()); } if let Some(i) = offset_at(&cigar(f[5]), cell_int(f[3])?, pos) { n += 1; @@ -100,7 +101,8 @@ pub fn base_count(args: &[String], out: &mut dyn Write, _err: &mut dyn Write) -> usize::try_from(i).ok().and_then(|i| f[9].as_bytes().get(i)) == Some(&base), ); } - } + Ok(()) + })?; writeln!(out, "{a} {n}")?; Ok(()) } @@ -335,6 +337,15 @@ mod tests { ); let (_, none, _) = run("plant-snv", &["0", &p("ref.fa.fai"), &p("in.sam"), "105"]); assert!(!none.contains("AAAAAAATAA"), "{none}"); + std::fs::write(d.join("planted.sam"), &out).expect("sam"); + // r0, r2 and r4 carry the T; y covers 105 with a deletion, so not counted. + let (code, count, _) = run("base-count", &["105", "T", &p("planted.sam")]); + assert_eq!((code, count.as_str()), (0, "3 5\n")); + assert_eq!( + run("base-count", &["105", "C", &p("planted.sam")]).1, + "2 5\n" + ); + assert_eq!(run("base-count", &["105", "TT", &p("planted.sam")]).0, 2); } #[test] From 7f82cdf9a8fa5470d936cb77af1a4f0afad389f4 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:37:36 +0100 Subject: [PATCH 240/255] test(e2e): pick the PRS score sites with og dev fixture and awk Bead: D1 --- .../alignment-markdup-gvcf-sex-3-homref.sh | 26 +----- tests/e2e/nextflow-from-fastq-1-bash.sh | 36 +------- tests/e2e/prs-1-agreement.sh | 91 ++++--------------- ...prs-scores-and-run-options-1-strand-sex.sh | 70 +++----------- ...prs-scores-and-run-options-2-panel-chrx.sh | 52 ++--------- 5 files changed, 43 insertions(+), 232 deletions(-) diff --git a/tests/e2e/alignment-markdup-gvcf-sex-3-homref.sh b/tests/e2e/alignment-markdup-gvcf-sex-3-homref.sh index bf31d57e0..8a20a1ca9 100755 --- a/tests/e2e/alignment-markdup-gvcf-sex-3-homref.sh +++ b/tests/e2e/alignment-markdup-gvcf-sex-3-homref.sh @@ -17,30 +17,8 @@ check "gVCF index exists" nonempty "${SAMPLE}/vcf/${SAMPLE}.g.vcf.gz.tbi" # --- five hom-ref sites on the chr20 slice, from the truth set ------------------- SITES="${CASE_TMP}/homref_sites.tsv" -python3 - "${FIXTURE_DIR}/HG002_truth_chr20.bed" "${FIXTURE_DIR}/HG002_truth_chr20.vcf.gz" \ - "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" > "$SITES" <<'PY' -import gzip, sys -bed, truth, fasta = sys.argv[1:4] -iv = [tuple(int(x) for x in l.split()[1:3]) for l in open(bed) if l.startswith("chr20\t")] -var = [int(l.split("\t")[1]) for l in gzip.open(truth, "rt") if l.startswith("chr20\t")] -fai = {l.split()[0]: [int(x) for x in l.split()[1:5]] for l in open(fasta + ".fai")} -length, offset, bases, width = fai["chr20"] -def base(pos): - i = pos - 1 - with open(fasta, "rb") as f: - f.seek(offset + (i // bases) * width + i % bases) - return f.read(1).decode().upper() -out, p = [], 10_050_000 -while len(out) < 5 and p < 10_450_000: - inside = any(s + 200 < p <= e - 200 for s, e in iv) - clear = all(abs(v - p) > 200 for v in var) - b = base(p) - if inside and clear and b in "ACGT": - out.append((p, b)) - p += 7_919 -for p, b in out: - print(f"chr20\t{p}\t{b}") -PY +"${REPO}/bin/og" dev fixture prs-sites "${FIXTURE_DIR}/HG002_truth_chr20.bed" "${FIXTURE_DIR}/HG002_truth_chr20.vcf.gz" \ + "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" 5 | awk -F'\t' -v OFS='\t' '{print "chr20", $2, $3}' > "$SITES" cat "$SITES" check_eq "hom-ref sites chosen on the chr20 slice" "$(wc -l < "$SITES" | tr -d ' ')" 5 diff --git a/tests/e2e/nextflow-from-fastq-1-bash.sh b/tests/e2e/nextflow-from-fastq-1-bash.sh index 907586d47..3eeb3d2be 100755 --- a/tests/e2e/nextflow-from-fastq-1-bash.sh +++ b/tests/e2e/nextflow-from-fastq-1-bash.sh @@ -47,39 +47,9 @@ check_step_exit 11-roh-analysis.sh # --- synthetic PRS scores --------------------------------------------------------- SITES="${CASE_TMP}/score_sites.tsv" -python3 - "${FIXTURE_DIR}/HG002_truth_chr20.bed" "${FIXTURE_DIR}/HG002_truth_chr20.vcf.gz" \ - "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" > "$SITES" <<'PY' -import gzip, sys -bed, truth, fasta = sys.argv[1:4] -iv = [tuple(int(x) for x in l.split()[1:3]) for l in open(bed) if l.startswith("chr20\t")] -var, snv = [], [] -for l in gzip.open(truth, "rt"): - if not l.startswith("chr20\t"): - continue - f = l.rstrip("\n").split("\t") - var.append(int(f[1])) - gt = f[9].split(":")[0].replace("|", "/") - if len(f[3]) == 1 and len(f[4]) == 1 and 10_050_000 < int(f[1]) < 10_450_000 and gt in ("0/1", "1/1"): - snv.append((int(f[1]), f[4])) -fai = {l.split()[0]: [int(x) for x in l.split()[1:5]] for l in open(fasta + ".fai")} -length, offset, bases, width = fai["chr20"] -def base(pos): - i = pos - 1 - with open(fasta, "rb") as fh: - fh.seek(offset + (i // bases) * width + i % bases) - return fh.read(1).decode().upper() -out, p = [], 10_050_000 -while len(out) < 5 and p < 10_450_000: - inside = any(s + 200 < p <= e - 200 for s, e in iv) - clear = all(abs(v - p) > 200 for v in var) - b = base(p) - if inside and clear and b in "ACGT": - out.append((p, b)) - p += 7_919 -out += snv[:: max(1, len(snv) // 3)][:3] -for p, a in out: - print(f"chr20\t{p}\t{a}") -PY +# Five hom-ref sites, then three of the truth set's SNVs (chrom, pos, effect allele). +"${REPO}/bin/og" dev fixture prs-sites "${FIXTURE_DIR}/HG002_truth_chr20.bed" "${FIXTURE_DIR}/HG002_truth_chr20.vcf.gz" \ + "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" 5 3 | awk -F'\t' -v OFS='\t' '{print "chr20", $2, $3}' > "$SITES" cat "$SITES" check_eq "score sites (5 hom-ref, 3 truth SNVs)" "$(wc -l < "$SITES" | tr -d ' ')" 8 PGS="${E2E_WORK}/parity-pgs" diff --git a/tests/e2e/prs-1-agreement.sh b/tests/e2e/prs-1-agreement.sh index 90686d7f1..1cd31e37a 100644 --- a/tests/e2e/prs-1-agreement.sh +++ b/tests/e2e/prs-1-agreement.sh @@ -39,47 +39,17 @@ check "case 21 wrote a gVCF" vcf_ok "${SAMPLE}/vcf/${SAMPLE}.g.vcf.gz" # --- sites ----------------------------------------------------------------------- SITES="${CASE_TMP}/sites.tsv" -python3 - "${FIXTURE_DIR}/HG002_truth_chr20.bed" "${FIXTURE_DIR}/HG002_truth_chr20.vcf.gz" \ - "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" > "$SITES" <<'PY' -import gzip, sys -bed, truth, fasta = sys.argv[1:4] -iv = [tuple(int(x) for x in l.split()[1:3]) for l in open(bed) if l.startswith("chr20\t")] -var, snv = [], [] -for l in gzip.open(truth, "rt"): - if not l.startswith("chr20\t"): - continue - f = l.rstrip("\n").split("\t") - var.append(int(f[1])) - gt = f[9].split(":")[0].replace("|", "/") - # pgsc_calc drops strand-ambiguous pairs (A/T, C/G), plink2 would not. - amb = {f[3], f[4]} in ({"A", "T"}, {"C", "G"}) - if len(f[3]) == 1 and len(f[4]) == 1 and not amb and 10_050_000 < int(f[1]) < 10_450_000 and gt in ("0/1", "1/1"): - snv.append((int(f[1]), f[4], f[3])) -fai = {l.split()[0]: [int(x) for x in l.split()[1:5]] for l in open(fasta + ".fai")} -length, offset, bases, width = fai["chr20"] -def base(pos): - i = pos - 1 - with open(fasta, "rb") as fh: - fh.seek(offset + (i // bases) * width + i % bases) - return fh.read(1).decode().upper() -homref, p = [], 10_050_000 -while len(homref) < 6 and p < 10_450_000: - inside = any(s + 200 < p <= e - 200 for s, e in iv) - clear = all(abs(v - p) > 200 for v in var) - b = base(p) - if inside and clear and b in "ACGT": - homref.append((p, b)) - p += 7_919 -# chrom, pos, effect allele, other allele, weight, kind -for i, (p, b) in enumerate(homref[:5]): - print(f"20\t{p}\t{b}\t\t{2 ** i}\thomref") -p, b = homref[5] -comp = {"A": "T", "T": "A", "C": "G", "G": "C"} -other = next(x for x in "ACGT" if x not in (b, comp[b])) -print(f"20\t{p}\t{other}\t{b}\t3.5\thomref_effect_not_ref") -for (p, alt, ref), w in zip(snv[:: max(1, len(snv) // 3)][:3], (0.25, -1.5, 0.125)): - print(f"20\t{p}\t{alt}\t{ref}\t{w}\tsnv") -PY +# chrom, pos, effect allele, other allele, weight, kind: five hom-ref sites, +# a sixth with an effect allele that is neither the reference base nor its +# complement, and three truth SNVs that are not strand-ambiguous (pgsc_calc +# drops A/T and C/G pairs, plink2 would not). +"${REPO}/bin/og" dev fixture prs-sites "${FIXTURE_DIR}/HG002_truth_chr20.bed" "${FIXTURE_DIR}/HG002_truth_chr20.vcf.gz" \ + "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" 6 3 --no-ambiguous \ + | awk -F'\t' -v OFS='\t' 'BEGIN {split("0.25 -1.5 0.125", w, " "); c["A"] = "T"; c["T"] = "A"; c["C"] = "G"; c["G"] = "C"} + $1 == "homref" && h < 5 {print "20", $2, $3, "", 2 ^ h, "homref"; h++; next} + $1 == "homref" {for (i = 1; i <= 4; i++) {o = substr("ACGT", i, 1); if (o != $3 && o != c[$3]) break} + print "20", $2, o, $3, "3.5", "homref_effect_not_ref"; next} + $1 == "snv" {print "20", $2, $3, $4, w[++s], "snv"}' > "$SITES" cat "$SITES" check_eq "score sites (5 hom-ref, 1 hom-ref with another effect allele, 3 truth SNVs)" "$(wc -l < "$SITES" | tr -d ' ')" 9 @@ -89,30 +59,9 @@ XR=chrX:73700001-74000000 bcf query -r "$XR" -f '%POS\t%INFO/END\t[%GT]\n' "${SAMPLE}/vcf/${SAMPLE}.g.vcf.gz" > "${CASE_TMP}/x_blocks.tsv" bcf query -r "$XR" -i 'GT="alt"' -f '%POS\n' "${SAMPLE}/vcf/${SAMPLE}.vcf.gz" > "${CASE_TMP}/x_vars.tsv" X_SITES="${CASE_TMP}/x_sites.tsv" -python3 - "${CASE_TMP}/x_blocks.tsv" "${CASE_TMP}/x_vars.tsv" "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" \ - > "$X_SITES" <<'PY2' -import sys -blocks, variants, fasta = sys.argv[1:4] -var = [int(l) for l in open(variants) if l.strip()] -fai = {l.split()[0]: [int(x) for x in l.split()[1:5]] for l in open(fasta + ".fai")} -length, offset, bases, width = fai["chrX"] -def base(pos): - i = pos - 1 - with open(fasta, "rb") as fh: - fh.seek(offset + (i // bases) * width + i % bases) - return fh.read(1).decode().upper() -rows = [l.rstrip("\n").split("\t") for l in open(blocks) if l.strip()] -refs = [(int(r[0]), int(r[1])) for r in rows if len(r) == 3 and r[1] not in ("", ".") and r[2] in ("0/0", "0|0")] -print(f"{len(rows)} gVCF records on the chrX slice, {len(refs)} 0/0 blocks, {len(var)} calls with an ALT", file=sys.stderr) -out, last = [], 0 -for start, end in refs: - p = (start + end) // 2 - if end - start >= 50 and p - last > 2000 and all(abs(v - p) > 100 for v in var) and base(p) in "ACGT": - out.append((p, base(p))) - last = p -for (p, b), w in zip(out[:3], (32, 64, 128)): - print(f"X\t{p}\t{b}\t\t{w}\tchrx") -PY2 +"${REPO}/bin/og" dev fixture x-ref-sites "${CASE_TMP}/x_blocks.tsv" "${CASE_TMP}/x_vars.tsv" \ + "${GENOME_DIR}/reference/GRCh38_no_alt_analysis_set.fasta" chrX 3 \ + | awk -F'\t' -v OFS='\t' '{print "X", $1, $2, "", 2 ^ (NR + 4), "chrx"}' > "$X_SITES" cat "$X_SITES" check_eq "chrX rows of the score (positions in reference blocks of the gVCF)" "$(wc -l < "$X_SITES" | tr -d ' ')" 3 @@ -176,16 +125,8 @@ check_eq "male: pgsc_calc matched all 12" "$(col Variants_Matched)" 12 check_eq "male: the summary says the 3 chrX rows were scored" "$(col ChrX)" "3 scored" check "male: pgsc_calc's plink2 got the sex" has "update-sex" "$(cat "${D}/prs/pgsc_calc/images.config" 2>/dev/null)" MATCH_LOG=$(find "${D}/prs/pgsc_calc/results" -name '*_log.csv.gz' 2>/dev/null | head -n 1) -X_MATCHED=$(python3 - "${MATCH_LOG:-/dev/null}" <<'PY2' -import csv, gzip, sys -try: - rows = list(csv.DictReader(gzip.open(sys.argv[1], "rt"))) -except (OSError, EOFError): - print("no match log"); sys.exit() -print(sum(1 for r in rows if r.get("accession") == "PGS000018" and r.get("chr_name") in ("X", "23") - and r.get("match_status") == "matched")) -PY2 -) +X_MATCHED=$("${REPO}/bin/og" dev fixture csv-count "${MATCH_LOG:-/dev/null}" accession=PGS000018 chr_name=X,23 match_status=matched \ + 2>/dev/null || echo "no match log") check_eq "male: pgsc_calc's match log has the 3 chrX rows matched (${MATCH_LOG:-no log})" "$X_MATCHED" 3 # plink2 on the chrX rows alone, on the genotypes step 25 handed pgsc_calc, with the sex printf '#IID\tSEX\n%s\t1\n' "$SAMPLE" > "${CASE_TMP}/sex.tsv" diff --git a/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh b/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh index 488515867..c4f35b102 100644 --- a/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh +++ b/tests/e2e/prs-scores-and-run-options-1-strand-sex.sh @@ -42,68 +42,26 @@ done # --- sites ----------------------------------------------------------------------- # chrom, pos, effect allele, other allele, weight, kind SITES="${CASE_TMP}/sites.tsv" -python3 - "${FIXTURE_DIR}/HG002_truth_chr20.bed" "${FIXTURE_DIR}/HG002_truth_chr20.vcf.gz" "$REF" > "$SITES" <<'PY' -import gzip, sys -bed, truth, fasta = sys.argv[1:4] -iv = [tuple(int(x) for x in l.split()[1:3]) for l in open(bed) if l.startswith("chr20\t")] -var = [int(l.split("\t")[1]) for l in gzip.open(truth, "rt") if l.startswith("chr20\t")] -fai = {l.split()[0]: [int(x) for x in l.split()[1:5]] for l in open(fasta + ".fai")} -length, offset, bases, width = fai["chr20"] -fh = open(fasta, "rb") -def base(pos): - i = pos - 1 - fh.seek(offset + (i // bases) * width + i % bases) - return fh.read(1).decode().upper() -comp = {"A": "T", "T": "A", "C": "G", "G": "C"} -homref, p = [], 10_050_000 -while len(homref) < 14 and p < 10_450_000: - inside = any(s + 200 < p <= e - 200 for s, e in iv) - clear = all(abs(v - p) > 200 for v in var) - b = base(p) - if inside and clear and b in comp: - homref.append((p, b)) - p += 7_919 -for i, (p, b) in enumerate(homref[:10]): - other = next(x for x in "ACGT" if x not in (b, comp[b])) - print(f"20\t{p}\t{b}\t{other}\t{2 ** i}\thomref") -for (p, b), w in zip(homref[10:13], (1024, 2048, 4096)): - print(f"20\t{p}\t{b}\t{comp[b]}\t{w}\tambiguous") -# A row given on the other strand: its alleles are the complements of the -# reference base and of a third base. -p, b = homref[13] -third = next(x for x in "ACGT" if x not in (b, comp[b])) -print(f"20\t{p}\t{comp[b]}\t{comp[third]}\t0.5\tflipped") -PY +# Ten hom-ref rows, three strand-ambiguous ones (the reference base and its +# complement), and a row given on the other strand: its alleles are the +# complements of the reference base and of a third base. +"${REPO}/bin/og" dev fixture prs-sites "${FIXTURE_DIR}/HG002_truth_chr20.bed" "${FIXTURE_DIR}/HG002_truth_chr20.vcf.gz" "$REF" 14 \ + | awk -F'\t' -v OFS='\t' 'BEGIN {c["A"] = "T"; c["T"] = "A"; c["C"] = "G"; c["G"] = "C"} + {for (i = 1; i <= 4; i++) {o = substr("ACGT", i, 1); if (o != $3 && o != c[$3]) break}} + n < 10 {print "20", $2, $3, o, 2 ^ n, "homref"} + n >= 10 && n < 13 {print "20", $2, $3, c[$3], 2 ^ n, "ambiguous"} + n == 13 {print "20", $2, c[$3], c[o], "0.5", "flipped"} + {n++}' > "$SITES" cat "$SITES" check_eq "score sites (10 hom-ref, 3 ambiguous hom-ref, 1 flipped)" "$(wc -l < "$SITES" | tr -d ' ')" 14 XR=chrX:73700001-74000000 bcf query -r "$XR" -f '%POS\t%INFO/END\t[%GT]\n' "${SAMPLE}/vcf/${SAMPLE}.g.vcf.gz" > "${CASE_TMP}/x_blocks.tsv" bcf query -r "$XR" -i 'GT="alt"' -f '%POS\n' "${SAMPLE}/vcf/${SAMPLE}.vcf.gz" > "${CASE_TMP}/x_vars.tsv" X_SITES="${CASE_TMP}/x_sites.tsv" -python3 - "${CASE_TMP}/x_blocks.tsv" "${CASE_TMP}/x_vars.tsv" "$REF" > "$X_SITES" <<'PY2' -import sys -blocks, variants, fasta = sys.argv[1:4] -var = [int(l) for l in open(variants) if l.strip()] -fai = {l.split()[0]: [int(x) for x in l.split()[1:5]] for l in open(fasta + ".fai")} -length, offset, bases, width = fai["chrX"] -fh = open(fasta, "rb") -def base(pos): - i = pos - 1 - fh.seek(offset + (i // bases) * width + i % bases) - return fh.read(1).decode().upper() -comp = {"A": "T", "T": "A", "C": "G", "G": "C"} -rows = [l.rstrip("\n").split("\t") for l in open(blocks) if l.strip()] -refs = [(int(r[0]), int(r[1])) for r in rows if len(r) == 3 and r[1] not in ("", ".") and r[2] in ("0/0", "0|0")] -out, last = [], 0 -for start, end in refs: - p = (start + end) // 2 - if end - start >= 50 and p - last > 2000 and all(abs(v - p) > 100 for v in var) and base(p) in comp: - out.append((p, base(p))) - last = p -for (p, b), w in zip(out[:3], (0.25, 0.25, 0.25)): - other = next(x for x in "ACGT" if x not in (b, comp[b])) - print(f"X\t{p}\t{b}\t{other}\t{w}\tchrx") -PY2 +"${REPO}/bin/og" dev fixture x-ref-sites "${CASE_TMP}/x_blocks.tsv" "${CASE_TMP}/x_vars.tsv" "$REF" chrX 3 \ + | awk -F'\t' -v OFS='\t' 'BEGIN {c["A"] = "T"; c["T"] = "A"; c["C"] = "G"; c["G"] = "C"} + {for (i = 1; i <= 4; i++) {o = substr("ACGT", i, 1); if (o != $2 && o != c[$2]) break} + print "X", $1, $2, o, "0.25", "chrx"}' > "$X_SITES" cat "$X_SITES" check_eq "chrX hom-ref rows" "$(wc -l < "$X_SITES" | tr -d ' ')" 3 diff --git a/tests/e2e/prs-scores-and-run-options-2-panel-chrx.sh b/tests/e2e/prs-scores-and-run-options-2-panel-chrx.sh index 9f3b18d58..ca6e30c76 100644 --- a/tests/e2e/prs-scores-and-run-options-2-panel-chrx.sh +++ b/tests/e2e/prs-scores-and-run-options-2-panel-chrx.sh @@ -52,30 +52,10 @@ XR=chrX:73700001-74000000 bcf query -r "$XR" -f '%POS\t%INFO/END\t[%GT]\n' "${SAMPLE}/vcf/${SAMPLE}.g.vcf.gz" > "${CASE_TMP}/x_blocks.tsv" bcf query -r "$XR" -i 'GT="alt"' -f '%POS\n' "${SAMPLE}/vcf/${SAMPLE}.vcf.gz" > "${CASE_TMP}/x_vars.tsv" X_SITES="${CASE_TMP}/x_sites.tsv" -python3 - "${CASE_TMP}/x_blocks.tsv" "${CASE_TMP}/x_vars.tsv" "$REF" > "$X_SITES" <<'PY' -import sys -blocks, variants, fasta = sys.argv[1:4] -var = [int(l) for l in open(variants) if l.strip()] -fai = {l.split()[0]: [int(x) for x in l.split()[1:5]] for l in open(fasta + ".fai")} -length, offset, bases, width = fai["chrX"] -fh = open(fasta, "rb") -def base(pos): - i = pos - 1 - fh.seek(offset + (i // bases) * width + i % bases) - return fh.read(1).decode().upper() -comp = {"A": "T", "T": "A", "C": "G", "G": "C"} -rows = [l.rstrip("\n").split("\t") for l in open(blocks) if l.strip()] -refs = [(int(r[0]), int(r[1])) for r in rows if len(r) == 3 and r[1] not in ("", ".") and r[2] in ("0/0", "0|0")] -out, last = [], 0 -for start, end in refs: - p = (start + end) // 2 - if end - start >= 50 and p - last > 2000 and all(abs(v - p) > 100 for v in var) and base(p) in comp: - out.append((p, base(p))) - last = p -for p, b in out[:3]: - other = next(x for x in "ACGT" if x not in (b, comp[b])) - print(f"X\t{p}\t{b}\t{other}\t0.1") -PY +"${REPO}/bin/og" dev fixture x-ref-sites "${CASE_TMP}/x_blocks.tsv" "${CASE_TMP}/x_vars.tsv" "$REF" chrX 3 \ + | awk -F'\t' -v OFS='\t' 'BEGIN {c["A"] = "T"; c["T"] = "A"; c["C"] = "G"; c["G"] = "C"} + {for (i = 1; i <= 4; i++) {o = substr("ACGT", i, 1); if (o != $2 && o != c[$2]) break} + print "X", $1, $2, o, "0.1"}' > "$X_SITES" check_eq "chrX hom-ref rows" "$(wc -l < "$X_SITES" | tr -d ' ')" 3 SCORES="${GENOME_DIR}/prs_scores" rm -rf "$SCORES" "${D}/prs" "${D}/ancestry" @@ -93,16 +73,8 @@ SUMMARY="${D}/prs/${SAMPLE}_prs_summary.tsv" cat "$SUMMARY" 2>/dev/null col() { awk -F'\t' -v id="$2" -v k="$3" 'NR == 1 {for (i = 1; i <= NF; i++) c[$i] = i; next} $c["PGS_ID"] == id {print $c[k]}' "$1" 2>/dev/null; } MATCH_LOG=$(find "${D}/prs/pgsc_calc/results" -name '*_log.csv.gz' 2>/dev/null | head -n 1) -X_MATCHED=$(python3 - "${MATCH_LOG:-/dev/null}" <<'PY2' -import csv, gzip, sys -try: - rows = list(csv.DictReader(gzip.open(sys.argv[1], "rt"))) -except (OSError, EOFError): - print("no match log"); sys.exit() -print(sum(1 for r in rows if r.get("accession") == "PGS000662" and r.get("chr_name") in ("X", "23") - and r.get("match_status") == "matched")) -PY2 -) +X_MATCHED=$("${REPO}/bin/og" dev fixture csv-count "${MATCH_LOG:-/dev/null}" accession=PGS000662 chr_name=X,23 match_status=matched \ + 2>/dev/null || echo "no match log") CHRX=$(col "$SUMMARY" PGS000662 ChrX) MATCHED=$(col "$SUMMARY" PGS000662 Variants_Matched) echo "PGS000662: ChrX '${CHRX}', ${MATCHED} of $(col "$SUMMARY" PGS000662 Variants_Total) matched, ${X_MATCHED} chrX rows matched in the log" @@ -117,16 +89,8 @@ case "$X_MATCHED" in esac # What docs/25-prs.md says of this panel: none of the chrX rows is scored. check_eq "${PANEL_NAME}: no chrX row of PGS000662 is matched (as docs/25-prs.md says)" "$X_MATCHED" 0 -AUTO_MATCHED=$(python3 - "${MATCH_LOG:-/dev/null}" <<'PY3' -import csv, gzip, sys -try: - rows = list(csv.DictReader(gzip.open(sys.argv[1], "rt"))) -except (OSError, EOFError): - print(-1); sys.exit() -print(sum(1 for r in rows if r.get("accession") == "PGS000662" and r.get("chr_name") not in ("X", "23") - and r.get("match_status") == "matched")) -PY3 -) +AUTO_MATCHED=$("${REPO}/bin/og" dev fixture csv-count "${MATCH_LOG:-/dev/null}" accession=PGS000662 chr_name!=X,23 match_status=matched \ + 2>/dev/null || echo "-1") check_eq "Variants_Matched is the log's matched rows, chrX included only when matched" "$MATCHED" "$((AUTO_MATCHED + X_MATCHED))" PCT=$(col "$SUMMARY" PGS000018 Percentile) check "male with the panel: PGS000018 has a percentile (${PCT:-none})" awk -v p="${PCT:-x}" 'BEGIN {exit !(p ~ /^[0-9.]+$/)}' From afa1ee3f3e3bed2d68a1f2f5cdff113ce415ba0e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:37:36 +0100 Subject: [PATCH 241/255] test(e2e): build the ancestry, ancient and family inputs with og dev fixture Bead: D1 --- tests/e2e/ancestry-fine-grained-1.sh | 73 ++++--------------- tests/e2e/ancient-similarity-1.sh | 103 +-------------------------- tests/e2e/family-joint-1.sh | 41 +---------- tests/e2e/ibd-segments-1.sh | 76 ++++---------------- 4 files changed, 34 insertions(+), 259 deletions(-) diff --git a/tests/e2e/ancestry-fine-grained-1.sh b/tests/e2e/ancestry-fine-grained-1.sh index 44fd2c770..d2ad48ace 100755 --- a/tests/e2e/ancestry-fine-grained-1.sh +++ b/tests/e2e/ancestry-fine-grained-1.sh @@ -113,51 +113,9 @@ mkdir -p "${G}/HG002T/vcf" docker run --rm -u "$(id -u):$(id -g)" -v "${G}:/genome:ro" -v "${FIXTURE_DIR}:/fx:ro" "$BCFTOOLS_IMAGE" \ bcftools view -R "/genome/reference/ancestry_fine/${ANCESTRY_FINE_PANEL}/panel/positions.tsv" -Ov \ /fx/HG002_GRCh38_1_22_v4.2.1_benchmark.vcf.gz > "${CASE_TMP}/truth_at_sites.vcf" -python3 -I - "${PANEL}/panel/sites.tsv" "${CASE_TMP}/truth_at_sites.vcf" \ - "${FIXTURE_DIR}/HG002_GRCh38_1_22_v4.2.1_benchmark_noinconsistent.bed" "${G}/reference/GRCh38_no_alt_analysis_set.fasta.fai" \ - > "${G}/HG002T/vcf/HG002T.vcf" <<'PY' -import bisect, sys -sites, truth, bed, fai = sys.argv[1:] -iv = {} -for line in open(bed): - c = line.split() - if len(c) >= 3: - iv.setdefault(c[0], []).append((int(c[1]), int(c[2]))) -starts = {k: [s for s, _ in sorted(v)] for k, v in iv.items()} -ends = {k: [e for _, e in sorted(v)] for k, v in iv.items()} -def confident(c, p): - i = bisect.bisect_right(starts.get(c, []), p - 1) - 1 - return i >= 0 and ends[c][i] >= p -gt = {} -for line in open(truth): - if line.startswith("#"): - continue - f = line.rstrip("\n").split("\t") - g = f[9].split(":")[f[8].split(":").index("GT")].replace("|", "/") - gt[(f[0], int(f[1]))] = (f[3], f[4], {"1/0": "0/1"}.get(g, g)) -chroms = sorted({l.split("\t")[0] for l in open(sites)}) -print("##fileformat=VCFv4.2") -print('##FILTER=') -for l in open(fai): - c = l.split("\t") - if c[0] in chroms: - print(f"##contig=") -print('##FORMAT=') -print("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tHG002T") -for l in open(sites): - c, p, r, a = l.rstrip("\n").split("\t") - p = int(p) - if not confident(c, p): - continue - t = gt.get((c, p)) - if t is None: - g = "0/0" - elif t[0] == r and t[1] == a and t[2] in ("0/1", "1/1", "0/0"): - g = t[2] - else: - continue - print(f"{c}\t{p}\t.\t{r}\t{a}\t.\tPASS\t.\tGT\t{g}") -PY +"${REPO}/bin/og" dev fixture truth-genotypes "${PANEL}/panel/sites.tsv" "${G}/reference/GRCh38_no_alt_analysis_set.fasta.fai" \ + "HG002T=${CASE_TMP}/truth_at_sites.vcf,${FIXTURE_DIR}/HG002_GRCh38_1_22_v4.2.1_benchmark_noinconsistent.bed" \ + > "${G}/HG002T/vcf/HG002T.vcf" in_genome "$BCFTOOLS_IMAGE" bash -c 'bcftools view -Oz -o HG002T/vcf/HG002T.vcf.gz HG002T/vcf/HG002T.vcf && bcftools index -t HG002T/vcf/HG002T.vcf.gz' check_ge "HG002T: GIAB genotypes at panel SNVs" "$(vcf_count HG002T/vcf/HG002T.vcf.gz)" 3000 T0=$(date +%s) @@ -308,22 +266,21 @@ check_eq "nextflow run with --tools ancestry_fine,html_report exits 0" "${PIPEST echo "Nextflow ancestry_fine on HG002T: $(( $(date +%s) - T0 )) s" >> "$E2E_NOTES" NF_TSV="${NF_OUT}/HG002T/ancestry_fine/HG002T_ancestry_fine.tsv" check "the module published the fractions table" test -s "$NF_TSV" -check "its fractions are the bash step's (within 0.01 each)" python3 -I - "$NF_TSV" "${G}/HG002T/ancestry_fine/HG002T_ancestry_fine.tsv" <<'PY' -import sys # Keyed by level, code and group: a left_out row (fraction ".") has the same # code at both levels and differs by its group text. -rows = [{(c[0], c[1], c[2]): c[3] for c in (l.rstrip("\n").split("\t") for l in open(p)) if c[0] != "level"} for p in sys.argv[1:]] -def same(a, b): - if a is None or b is None or (a == ".") != (b == "."): - return False - return a == b if a == "." else abs(float(a) - float(b)) <= 0.01 -diff = {k: (rows[0].get(k), rows[1].get(k)) for k in set(rows[0]) | set(rows[1]) if not same(rows[0].get(k), rows[1].get(k))} -print("differences:", diff or "none") -sys.exit(1 if diff else 0) -PY +check "its fractions are the bash step's (within 0.01 each)" awk -F'\t' ' + function same(x, y) { if ((x == ".") != (y == ".")) return 0; if (x == ".") return 1; return x - y <= 0.01 && y - x <= 0.01 } + $1 == "level" {next} + {k = $1 "|" $2 "|" $3} + FNR == NR {a[k] = $4; next} + {b[k] = $4} + END { + for (k in a) if (!(k in b) || !same(a[k], b[k])) {print "differs: " k " " a[k] " " b[k]; n++} + for (k in b) if (!(k in a)) {print "differs: " k " (only in the bash table)"; n++} + print "differences: " n + 0; exit (n > 0) + }' "$NF_TSV" "${G}/HG002T/ancestry_fine/HG002T_ancestry_fine.tsv" check "the HTML report has the fine-grained ancestry card with its painting" \ - python3 -I -c 'import sys; t = open(sys.argv[1]).read(); sys.exit(0 if "Fine-Grained Ancestry" in t and "data:image/png;base64," in t else 1)' \ - "${NF_OUT}/HG002T/HG002T_report.html" + bash -c 'grep -qF "Fine-Grained Ancestry" "$1" && grep -qF "data:image/png;base64," "$1"' _ "${NF_OUT}/HG002T/HG002T_report.html" # The module on TWO's two-sample gVCF: ANCESTRY_FINE_GENOTYPE stops, the # pipeline ignores the failure of a report-only step and names it in diff --git a/tests/e2e/ancient-similarity-1.sh b/tests/e2e/ancient-similarity-1.sh index 21941d932..7118bd598 100755 --- a/tests/e2e/ancient-similarity-1.sh +++ b/tests/e2e/ancient-similarity-1.sh @@ -59,107 +59,8 @@ printf '#### Step 44, real AADR %s selection\n\n%s\n\n' "$AADR_VERSION" "$(cat " SRC="${CASE_TMP}/src" CHAIN="${G}/reference/liftover-e2e/synthetic_hg19ToHg38.chain" mkdir -p "$SRC" "$(dirname "$CHAIN")" -python3 -I - "${FIXTURE_DIR}/regions.bed" "${FIXTURE_DIR}/HG002_GRCh38_1_22_v4.2.1_benchmark.vcf.gz" \ - "${FIXTURE_DIR}/HG002_GRCh38_1_22_v4.2.1_benchmark_noinconsistent.bed" "$REF" "$SRC" "$CHAIN" <<'PY' -import bisect, gzip, math, random, struct, sys -regions, truth, bed, ref, out, chain = sys.argv[1:] -OFF = 1000 -rnd = random.Random(44) -fai = {l.split("\t")[0]: [int(x) for x in l.split("\t")[1:5]] for l in open(ref + ".fai")} -fa = open(ref, "rb") -def base(c, p): - n, off, lb, lw = fai[c] - fa.seek(off + ((p - 1) // lb) * lw + (p - 1) % lb) - return fa.read(1).decode().upper() -slices = [(c, int(s), int(e)) for c, s, e, *_ in (l.split("\t") for l in open(regions)) - if c in fai and c[3:].isdigit() and int(c[3:]) <= 22 and int(s) > OFF + 10] -conf = {} -for l in open(bed): - c, s, e = l.split()[:3] - conf.setdefault(c, []).append((int(s), int(e))) -for v in conf.values(): - v.sort() -def confident(c, p): - v = conf.get(c, []) - i = bisect.bisect_right([s for s, _ in v], p - 1) - 1 - return i >= 0 and v[i][1] >= p -truth_gt = {} -for l in gzip.open(truth, "rt"): - if l.startswith("#"): - continue - f = l.split("\t") - if len(f[3]) == 1 and len(f[4]) == 1: - g = f[9].split(":")[0].replace("|", "/") - truth_gt[(f[0], int(f[1]))] = (f[3], f[4], {"0/1": 1, "1/0": 1, "1/1": 2}.get(g)) -sites = [] # (chrom, pos38, ref, alt, HG002 alt count) -for c, s, e in slices: - for p in range(s + 1, e + 1): - t = truth_gt.get((c, p)) - if t and t[2] is not None and confident(c, p) and base(c, p) == t[0]: - sites.append((c, p, t[0], t[1], t[2])) - elif p % 150 == 0 and (c, p) not in truth_gt and confident(c, p) and base(c, p) in "ACGT": - r = base(c, p) - sites.append((c, p, r, {"A": "G", "C": "T", "G": "A", "T": "C"}[r], 0)) -sites.sort(key=lambda x: (int(x[0][3:]), x[1])) -assert len(sites) >= 1000, len(sites) -n = len(sites) -# People: (id, sex, group, kind, date_bp, country, lat, lon, dosages of the .snp's first allele (REF)) -people = [] -freq = {"French": [rnd.uniform(0.05, 0.95) for _ in sites], "Finnish": [rnd.uniform(0.05, 0.95) for _ in sites], - "Sardinian": [{0: 0.1, 1: 0.5, 2: 0.9}[x[4]] for x in sites]} -for g, f in freq.items(): - for i in range(30): - people.append((f"{g}{i}.DG", "M", g, "modern", 0, "Europe", 45, 5, - [2 - sum(rnd.random() < q for _ in range(2)) for q in f])) -def pseudo(alts): - return [9 if rnd.random() < 0.3 else (0 if rnd.random() < a / 2 else 2) for a in alts] -for i in range(6): - people.append((f"PL{i}.AG", "F", "Planted_HG002", "ancient", 5000, "Testland", 48, 10, pseudo([x[4] for x in sites]))) - people.append((f"FR{i}.AG", "M", "Anc_French_like", "ancient", 7000, "Franceland", 46, 2, - pseudo([2 * q for q in freq["French"]]))) - people.append((f"FI{i}.AG", "M", "Anc_Finnish_like", "ancient", 3500, "Finland", 62, 25, - pseudo([2 * q for q in freq["Finnish"]]))) -people.append(("OUT0.AG", "M", "Far_Away", "ancient", 5000, "China", 40, 116, pseudo([1] * n))) # outside West Eurasia -cols = ["Genetic ID", "Individual ID", "Date mean in BP in years before 1950 CE", "Date standard deviation in BP", - "Full Date", "Group ID", "Locality", "Political Entity", "Latitude", "Longitude", "Data type", - "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", "ASSESSMENT", "ASSESSMENT WARNINGS"] -with open(f"{out}/aadr.anno", "w") as o: - o.write("\t".join(cols) + "\n") - for pid, sex, g, kind, bp, country, lat, lon, _ in people: - o.write("\t".join([pid, pid.split(".")[0], str(bp), "50", "synthetic", g, "Somewhere", country, str(lat), str(lon), - "Shotgun.diploid" if kind == "modern" else "1240k", "1100000" if kind == "modern" else "500000", - "PASS", ""]) + "\n") -with open(f"{out}/aadr.ind", "w") as o: - for pid, sex, g, *_ in people: - o.write(f"{pid:>20} {sex} {g}\n") -with open(f"{out}/aadr.snp", "w") as o: - for i, (c, p, r, a, _) in enumerate(sites): - o.write(f"{'syn' + str(i):>20} {int(c[3:]):>2} 0.000000 {p - OFF:>12} {r} {a}\n") -# TGENO: a 48-byte header, then one record per person of ceil(n / 4) bytes, -# two bits per site (first site in the high bits), 3 for missing. -trlen = math.ceil(n * 2 / 8) -with open(f"{out}/aadr.geno", "wb") as o: - o.write(f"TGENO {len(people):7d} {n:7d} 0 0".encode().ljust(48, b"\0")) - for *_, d in people: - rec = bytearray(b"\xff" * trlen) - for k, x in enumerate(d): - v = 3 if x == 9 else x - sh = (3 - k % 4) * 2 - rec[k // 4] = (rec[k // 4] & ~(3 << sh) & 0xFF) | (v << sh) - o.write(rec) -# The chain: each chromosome's slices, GRCh37 = GRCh38 - OFF. -with open(chain, "w") as o: - k = 0 - for c in sorted({x[0] for x in sites}, key=lambda c: int(c[3:])): - sz = fai[c][0] - for _, s, e in [x for x in slices if x[0] == c]: - k += 1 - o.write(f"chain 1000 {c} {sz} + {s - OFF} {e - OFF} {c} {sz} + {s} {e} {k}\n{e - s}\n\n") -with open(f"{out}/expected_sites.tsv", "w") as o: - for i, (c, p, r, a, _) in enumerate(sites): - o.write(f"syn{i}\t{c}\t{p}\t{r}\t{a}\n") -print(f"synthetic panel: {n} sites, {len(people)} people") -PY +"${REPO}/bin/og" dev fixture ancient-panel "${FIXTURE_DIR}/regions.bed" "${FIXTURE_DIR}/HG002_GRCh38_1_22_v4.2.1_benchmark.vcf.gz" \ + "${FIXTURE_DIR}/HG002_GRCh38_1_22_v4.2.1_benchmark_noinconsistent.bed" "$REF" "$SRC" "$CHAIN" check_ge "synthetic panel: sites in the fixture's slices" "$(wc -l < "${SRC}/aadr.snp" | tr -d ' ')" 1000 echo "+ ANCIENT_SOURCE_DIR=... ANCIENT_CHAIN=... scripts/setup.sh --ancient-panel" T0=$(date +%s) diff --git a/tests/e2e/family-joint-1.sh b/tests/e2e/family-joint-1.sh index 4ce3adf25..1a72337dc 100755 --- a/tests/e2e/family-joint-1.sh +++ b/tests/e2e/family-joint-1.sh @@ -107,24 +107,7 @@ curl -fsSL --retry 5 -o "${F}/hg001/hg001.vcf.gz.tbi" "${HG001_VCF}.tbi" bcf view -H -v snps -m2 -M2 -r "$AUTO" hg001/hg001.vcf.gz > "${CASE_TMP}/hg001.vcf" 2> "${CASE_TMP}/hg001.log" check_eq "HG001's benchmark SNVs read from GIAB's file" "$?" 0 cut -f1,2,4,5 "${CASE_TMP}/hg001.vcf" > "${CASE_TMP}/snvs.tsv" -python3 - "${F}/reference/${REF_NAME}.fasta.fai" "${CASE_TMP}/snvs.tsv" "$SITES" <<'PY' -import sys -fai, snvs, out = sys.argv[1:] -order, lens = {}, {} -for i, l in enumerate(open(fai)): - c = l.split("\t") - order[c[0]], lens[c[0]] = i, c[1] -keep = {tuple(l.rstrip("\n").split("\t")) for l in open(snvs) if l.strip()} -keep = sorted((k for k in keep if k[0] in order), key=lambda k: (order[k[0]], int(k[1]))) -with open(out, "w") as f: - f.write("##fileformat=VCFv4.2\n") - for c in sorted(order, key=order.get): - f.write("##contig=\n" % (c, lens[c])) - f.write("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n") - for c, p, r, a in keep: - f.write("\t".join([c, p, ".", r, a, ".", "PASS", "AF=0.5"]) + "\n") -print(len(keep), "sites") -PY +"${REPO}/bin/og" dev fixture sites-vcf "${F}/reference/${REF_NAME}.fasta.fai" "${CASE_TMP}/snvs.tsv" "$SITES" check_ge "somalier sites: HG001's SNVs in the autosomal slices" "$(grep -vc '^#' "$SITES")" 1500 printf 'sample\tsex\tfather\tmother\tpartner\nHG002\tmale\tHG003\tHG004\t.\nHG003\tmale\t.\t.\tHG004\nHG004\tfemale\t.\t.\tHG003\n' \ @@ -186,27 +169,7 @@ SEG="${FAM}/HG002_segments.tsv" CSV="${FAM}/HG002_dnapainter.csv" head -n 4 "$SEG" 2>/dev/null # seg_parse TSV CSV: segments per side, after checking both files' shape -seg_parse() { - python3 - "$1" "$2" <<'PY' -import csv, sys -tsv, dp = sys.argv[1:] -with open(tsv) as f: - head = f.readline().rstrip("\n").split("\t") - rows = [dict(zip(head, l.rstrip("\n").split("\t"))) for l in f] -assert head[:6] == ["child", "side", "parent", "chrom", "start", "end"], head -for r in rows: - assert r["side"] in ("paternal", "maternal") and r["chrom"].startswith("chr"), r - assert 0 < int(r["start"]) <= int(r["end"]) and int(r["sites"]) >= 20, r -with open(dp, newline="") as f: - c = list(csv.reader(f)) -assert c[0] == ["Chr", "Start", "End", "cM", "SNPs", "Match", "Confidence", "Group", "Side", "Notes", "Colour", - "Ancestor"], c[0] -assert len(c) - 1 == len(rows), (len(c) - 1, len(rows)) -for x, r in zip(c[1:], rows): - assert [x[0], x[1], x[2], x[8]] == [r["chrom"][3:], r["start"], r["end"], r["side"]], (x, r) -print(sum(r["side"] == "paternal" for r in rows), sum(r["side"] == "maternal" for r in rows)) -PY -} +seg_parse() { "${REPO}/bin/og" dev fixture segments-check "$1" "$2"; } SEGN=$(seg_parse "$SEG" "$CSV" 2>"${CASE_TMP}/seg.log") check_eq "HG002's segments table and DNA Painter CSV parse and agree" "$?" 0 cat "${CASE_TMP}/seg.log" diff --git a/tests/e2e/ibd-segments-1.sh b/tests/e2e/ibd-segments-1.sh index e87ef907f..204d1c08d 100644 --- a/tests/e2e/ibd-segments-1.sh +++ b/tests/e2e/ibd-segments-1.sh @@ -75,53 +75,10 @@ done joint() { local dir=$1; shift mkdir -p "${G}/${dir}" - python3 -I - "${PANEL}/panel/sites.tsv" "$REF_FAI" "$CASE_TMP" "$@" > "${G}/${dir}/family.vcf" <<'PY' -import bisect, sys -sites, fai, tmp, *members = sys.argv[1:] -names = [m.split("=")[0] for m in members] -srcs = [m.split("=")[1] for m in members] -def bed(path): - iv = sorted((int(c[1]), int(c[2])) for c in (l.split() for l in open(path)) if len(c) >= 3) - return [s for s, _ in iv], [e for _, e in iv] -def inside(b, p): - i = bisect.bisect_right(b[0], p - 1) - 1 - return i >= 0 and b[1][i] >= p -beds = [bed(f"{tmp}/{s}.bed") for s in srcs] -gts = [] -for s in srcs: - g = {} - for l in open(f"{tmp}/{s}.vcf"): - f = l.rstrip("\n").split("\t") - gt = f[9].split(":")[f[8].split(":").index("GT")].replace("|", "/") - g[int(f[1])] = (f[3], f[4], {"1/0": "0/1"}.get(gt, gt)) - gts.append(g) -print("##fileformat=VCFv4.2") -for l in open(fai): - c = l.split("\t") - if c[0] == "chr20": - print(f"##contig=") -print('##FORMAT=') -print("#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + "\t".join(names)) -n = 0 -for l in open(sites): - c, p, r, a = l.rstrip("\n").split("\t") - p = int(p) - if c != "chr20" or not all(inside(b, p) for b in beds): - continue - row = [] - for g in gts: - t = g.get(p) - if t is None: - row.append("0/0") - elif t[0] == r and t[1] == a and t[2] in ("0/0", "0/1", "1/1"): - row.append(t[2]) - else: - break - if len(row) == len(gts): - print(f"chr20\t{p}\t.\t{r}\t{a}\t.\tPASS\t.\tGT\t" + "\t".join(row)) - n += 1 -print(n, "sites", file=sys.stderr) -PY + local m members=() + for m in "$@"; do members+=("${m%%=*}=${CASE_TMP}/${m#*=}.vcf,${CASE_TMP}/${m#*=}.bed"); done + "${REPO}/bin/og" dev fixture truth-genotypes "${PANEL}/panel/sites.tsv" "$REF_FAI" --chrom chr20 "${members[@]}" \ + > "${G}/${dir}/family.vcf" in_genome "$BCFTOOLS_IMAGE" bash -c "bcftools view -Oz -o ${dir}/family.vcf.gz ${dir}/family.vcf && bcftools index -t ${dir}/family.vcf.gz" \ && rm -f "${G}/${dir}/family.vcf" printf 'sample\tsex\tfather\tmother\tpartner\nHG002\tmale\tHG003\tHG004\t.\nHG003\tmale\t.\t.\tHG004\nHG004\tfemale\t.\t.\tHG003\n' \ @@ -148,20 +105,17 @@ check_eq "one pairs row per pair of the three" "$(awk 'NR > 1' "${D}/ibd_pairs.t check_eq "phased and analysed: chr20" "$(cat "${D}/ibd/chroms.txt" 2>/dev/null)" chr20 check "the analysed map is most of chr20 (90 cM or more: $(pval "$D" HG002 HG003 analysed_cm))" ge "$(pval "$D" HG002 HG003 analysed_cm)" 90 # Each pair's segments rows add up to its pairs row -check "segments rows add up to each pair's count, total and longest" python3 -I - "${D}/ibd_pairs.tsv" "${D}/ibd_segments.tsv" <<'PY' -import sys -with open(sys.argv[1]) as f: - h = f.readline().rstrip("\n").split("\t") - P = [dict(zip(h, l.rstrip("\n").split("\t"))) for l in f] -with open(sys.argv[2]) as f: - h = f.readline().rstrip("\n").split("\t") - S = [dict(zip(h, l.rstrip("\n").split("\t"))) for l in f] -for p in P: - s = [float(x["cm"]) for x in S if (x["sample_a"], x["sample_b"]) == (p["sample_a"], p["sample_b"])] - assert len(s) == int(p["segments"]), (p, len(s)) - assert abs(sum(s) - float(p["shared_cm"])) <= 0.01 * len(s) + 0.01, (p, sum(s)) - assert abs(max(s or [0.0]) - float(p["longest_cm"])) <= 0.01, (p, max(s or [0.0])) -PY +check "segments rows add up to each pair's count, total and longest" awk -F'\t' ' + FNR == 1 {for (i = 1; i <= NF; i++) h[$i] = i; next} + FILENAME == ARGV[1] {pairs[$h["sample_a"] "|" $h["sample_b"]] = $h["segments"] "|" $h["shared_cm"] "|" $h["longest_cm"]; next} + {k = $h["sample_a"] "|" $h["sample_b"]; x = $h["cm"] + 0; n[k]++; s[k] += x; if (!(k in m) || x > m[k]) m[k] = x} + END { + for (k in pairs) { + split(pairs[k], p, "|"); d = s[k] - p[2]; e = m[k] - p[3]; tol = 0.01 * n[k] + 0.01 + if (n[k] + 0 != p[1] + 0 || d > tol || -d > tol || e > 0.01 || -e > 0.01) {print "differs: " k " " pairs[k]; bad++} + } + exit (bad > 0) + }' "${D}/ibd_pairs.tsv" "${D}/ibd_segments.tsv" for p in "HG002 HG003" "HG002 HG004"; do # shellcheck disable=SC2086 # the pair splits on purpose set -- $p From 43bee5c107b0e680a09b3f3c3b497c84e09b7eda Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:37:36 +0100 Subject: [PATCH 242/255] test(e2e): plant the CHIP, MoChA, LOY/LOX, viral and QC inputs with og dev fixture Bead: D1 --- tests/e2e/chip-clonal-1.sh | 57 ++------------------------ tests/e2e/chip-mt-1-roundtrip.sh | 70 ++------------------------------ tests/e2e/mocha-1.sh | 57 +------------------------- tests/e2e/mosaic-loy-lox-1.sh | 47 ++------------------- tests/e2e/qc-1-sample-qc.sh | 40 +----------------- tests/e2e/viral-reads-1.sh | 29 +------------ 6 files changed, 16 insertions(+), 284 deletions(-) diff --git a/tests/e2e/chip-clonal-1.sh b/tests/e2e/chip-clonal-1.sh index b11ad6dd9..c812f9d31 100755 --- a/tests/e2e/chip-clonal-1.sh +++ b/tests/e2e/chip-clonal-1.sh @@ -51,44 +51,7 @@ check_ge "reads fetched" "$(wc -l < "${T}/win.sam" 2>/dev/null || echo 0)" 20000 # Nth read whose aligned base there is C (0: none). A mate on another contig loses its # position, so samtools accepts the record under the shorter header. plant() { - python3 - "$1" "${REF}.fai" "${T}/win.sam" "$R882" <<'EOF' -import sys -n, fai, sam, pos = int(sys.argv[1]), sys.argv[2], sys.argv[3], int(sys.argv[4]) -contigs = [ln.split("\t")[:2] for ln in open(fai)] -names = {c for c, _ in contigs} -print("@HD\tVN:1.6\tSO:unsorted") -for c, l in contigs: - print(f"@SQ\tSN:{c}\tLN:{l}") -print("@RG\tID:giab\tSM:HG002\tPL:ILLUMINA") -k = 0 -for ln in open(sam): - f = ln.rstrip("\n").split("\t") - if f[2] not in names: - continue - f = [x for x in f if not x.startswith("RG:Z:")] + ["RG:Z:giab"] - if f[6] not in ("=", "*") and f[6] not in names: - f[6], f[7] = "*", "0" - if n and f[2] == "chr2" and f[5] != "*": - ref, q, num, hit = int(f[3]), 0, "", None - for ch in f[5]: - if ch.isdigit(): - num += ch - continue - ln_ = int(num); num = "" - if ch in "M=X": - if ref <= pos < ref + ln_: - hit = q + pos - ref - ref += ln_; q += ln_ - elif ch in "IS": - q += ln_ - elif ch in "DN": - ref += ln_ - if hit is not None and f[9][hit] == "C": - if k % n == 0: - f[9] = f[9][:hit] + "T" + f[9][hit + 1:] - k += 1 - print("\t".join(f)) -EOF + "${REPO}/bin/og" dev fixture plant-snv "$1" "${REF}.fai" "${T}/win.sam" "$R882" } # sample NAME N: a sample folder with the reads, planted every Nth read. sample() { @@ -109,21 +72,9 @@ U="${SAMPLE}chu" P="${SAMPLE}chp" H="${SAMPLE}chh" check "the reads as they are" sample "$U" 0 check "R882H in every fifth read" sample "$P" 5 check "R882H in every second read" sample "$H" 2 -alt_at() { dk "$SAMTOOLS_IMAGE" samtools view "${G}/$1/aligned/$1_sorted.bam" "chr2:${R882}-${R882}" | python3 -c ' -import sys -pos = int(sys.argv[1]); a = n = 0 -for ln in sys.stdin: - f = ln.split("\t"); ref, q, num = int(f[3]), 0, "" - for ch in f[5]: - if ch.isdigit(): num += ch; continue - k = int(num); num = "" - if ch in "M=X": - if ref <= pos < ref + k: - n += 1; a += f[9][q + pos - ref] == "T" - ref += k; q += k - elif ch in "IS": q += k - elif ch in "DN": ref += k -print(a, n)' "$R882"; } +# alt_at NAME: "T reads" over R882 in the sample's BAM. +alt_at() { dk "$SAMTOOLS_IMAGE" samtools view "${G}/$1/aligned/$1_sorted.bam" "chr2:${R882}-${R882}" \ + | "${REPO}/bin/og" dev fixture base-count "$R882" T; } read -r A0 N0 < <(alt_at "$U"); read -r A5 N5 < <(alt_at "$P") check_ge "reads over R882" "${N0:-0}" 30 check_eq "no T at R882 in HG002's own reads" "${A0:-x}" 0 diff --git a/tests/e2e/chip-mt-1-roundtrip.sh b/tests/e2e/chip-mt-1-roundtrip.sh index 3cf97f062..c7224357d 100755 --- a/tests/e2e/chip-mt-1-roundtrip.sh +++ b/tests/e2e/chip-mt-1-roundtrip.sh @@ -40,8 +40,8 @@ cp "$GVCF" "${G}/${E}/vcf/${E}.g.vcf.gz" cp "${GVCF}.tbi" "${G}/${E}/vcf/${E}.g.vcf.gz.tbi" run_step 40-export-consumer.sh "$E" male check_step_exit 40-export-consumer.sh -python3 -c 'import sys, zipfile; z = zipfile.ZipFile(sys.argv[1]); open(sys.argv[2], "wb").write(z.read(z.namelist()[0]))' \ - "${G}/${E}/consumer_export/${E}_23andme_v5.zip" "${CASE_TMP}/export.txt" +ZIP="${G}/${E}/consumer_export/${E}_23andme_v5.zip" +unzip -p "$ZIP" "$(unzip -Z1 "$ZIP" 2>/dev/null | head -n 1)" > "${CASE_TMP}/export.txt" check "the 23andMe file starts with step 40's first line" \ has '^# This data file was generated by OwnGenome' "$(head -n 1 "${CASE_TMP}/export.txt")" N_MT=$(awk -F'\t' '!/^#/ && $2 == "MT"' "${CASE_TMP}/export.txt" | wc -l | tr -d ' ') @@ -51,31 +51,7 @@ check_ge "chrM sites in the 23andMe file" "$N_MT" 200 # A FASTA with the table's GRCh37 base at each site (N elsewhere), as in # consumer-export-1, and UCSC's chain. mkdir -p "${G}/reference_hg19" "${G}/liftover" -python3 - "$SITES" "${G}/reference_hg19/human_g1k_v37.fasta" <<'PY' -import gzip, sys -sites, fa = sys.argv[1:3] -order = [str(i) for i in range(1, 23)] + ["X", "Y", "MT"] -bases = {} -for l in gzip.open(sites, "rt"): - if l.startswith("#"): - continue - c = l.rstrip("\n").split("\t") - bases.setdefault(c[1], {})[int(c[2])] = c[3] -off = 0 -with open(fa, "w") as f, open(fa + ".fai", "w") as fai: - for chrom in [c for c in order if c in bases]: - n = max(bases[chrom]) + 1 - head = f">{chrom}\n" - f.write(head) - off += len(head) - fai.write(f"{chrom}\t{n}\t{off}\t60\t61\n") - seq = bytearray(b"N" * n) - for p, b in bases[chrom].items(): - seq[p - 1] = ord(b) - for i in range(0, n, 60): - f.write(seq[i:i + 60].decode() + "\n") - off += n + (n + 59) // 60 -PY +"${REPO}/bin/og" dev fixture sites-fasta "$SITES" "${G}/reference_hg19/human_g1k_v37.fasta" if curl -fsSL --retry 3 -o "${G}/liftover/hg19ToHg38.over.chain.gz" "$CHAIN_URL"; then check_eq "UCSC's hg19ToHg38 chain has the pinned sha256" \ "$(sha256sum "${G}/liftover/hg19ToHg38.over.chain.gz" | cut -d' ' -f1)" "$CHAIN_SHA256" @@ -100,45 +76,7 @@ mt_roundtrip() { echo "chip-to-vcf.sh ${n}: exit ${status}" >&2 tail -n 25 "${CASE_TMP}/${n}.log" >&2 bcf query -f '%ID\t%CHROM\t%POS\t%REF\t%ALT[\t%GT]\n' "${n}/vcf/${n}.vcf.gz" 2>/dev/null | awk -F'\t' '$2 == "chrM"' > "${CASE_TMP}/${n}_chrM.tsv" - python3 - "${CASE_TMP}/${n}_chrM.tsv" "${CASE_TMP}/calls.tsv" "$SITES" "$2" <<'PY' -import gzip, re, sys -lifted, calls, sites, txt = sys.argv[1:5] -table = {} -for l in gzip.open(sites, "rt"): - if not l.startswith("#"): - c = l.rstrip("\n").split("\t") - if c[1] == "MT": - table[c[0]] = c -dv = {} -for l in open(calls): - chrom, pos, ref, alt, flt, gt = l.rstrip("\n").split("\t") - if flt == "PASS": - al = [ref] + alt.split(",") - dv[int(pos)] = {al[int(i)][0] for i in re.split(r"[/|]", gt) if i not in (".", "")} -called = sum(1 for l in open(txt) - if not l.startswith("#") and l.split("\t")[0] in table and l.rstrip("\n").split("\t")[3] != "--") -got = on_place = bad = 0 -for l in open(lifted): - rid, chrom, pos, ref, alt, gt = l.rstrip("\n").split("\t") - t = table.get(rid) - idx = [i for i in re.split(r"[/|]", gt) if i not in (".", "")] - if t is None or not idx: - continue - got += 1 - if int(pos) != int(t[4]) or ref != t[5]: - if got - on_place <= 5: - print(f"off place {rid}: rCRS {t[4]} {t[5]}, read back at chrM:{pos} {ref}", file=sys.stderr) - continue - on_place += 1 - al = [ref] + alt.split(",") - have = {al[int(i)][0] for i in idx} - want = dv.get(int(pos), {t[5]}) - if have != want: - bad += 1 - if bad <= 5: - print(f"mismatch {rid} chrM:{pos} read back {sorted(have)}, case 21 {sorted(want)}", file=sys.stderr) -print(called, got, on_place, bad) -PY + "${REPO}/bin/og" dev fixture mt-roundtrip "${CASE_TMP}/${n}_chrM.tsv" "${CASE_TMP}/calls.tsv" "$SITES" "$2" } # --- 2. step 40's file keeps its chrM positions -------------------------------------- diff --git a/tests/e2e/mocha-1.sh b/tests/e2e/mocha-1.sh index 15cfaf107..31bfdc235 100755 --- a/tests/e2e/mocha-1.sh +++ b/tests/e2e/mocha-1.sh @@ -105,61 +105,8 @@ echo "mocha: HG002 chr20 slice: $(summ "$SAMPLE" het_sites) hets ($(summ "$SAMPL # --- MOSAIC and MOSAIC0: a planted CN-LOH, and none --------------------------------- for s in MOSAIC MOSAIC0; do rm -rf "${G:?}/${s}"; mkdir -p "${G}/${s}/vcf"; done -python3 -I - "${G}/reference/GRCh38_no_alt_analysis_set.fasta" "${G}/MOSAIC/vcf/MOSAIC.vcf" "${G}/MOSAIC0/vcf/MOSAIC0.vcf" <<'PY' -import math, random, sys -fa, out_shift, out_flat = sys.argv[1:] -CHROMS = ("chr16", "chr19", "chr20", "chr22") -EVENT = ("chr20", 1, 20_000_000) # 20p from the telomere -BDEV = 0.075 # CN-LOH in 15% of the cells: haplotype balance 0.575 -fai = {} -for line in open(fa + ".fai"): - c = line.split("\t") - fai[c[0]] = (int(c[1]), int(c[2]), int(c[3]), int(c[4])) -ALT = {"A": "G", "G": "A", "C": "T", "T": "C"} -def poisson(lam): - l, k, p = math.exp(-lam), 0, 1.0 - while True: - p *= random.random() - if p <= l: - return k - k += 1 -def binom(n, f): - return sum(random.random() < f for _ in range(n)) -outs = {} -for path, shift in ((out_shift, True), (out_flat, False)): - random.seed(51) - lines = ["##fileformat=VCFv4.2", '##FILTER='] - lines += [f"##contig=" for c in CHROMS] - lines += ['##FORMAT=', - '##FORMAT=', - '##FORMAT=', - '##FORMAT=', - "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + ("MOSAIC" if shift else "MOSAIC0")] - with open(fa, "rb") as f: - for c in CHROMS: - length, off, lb, lw = fai[c] - pos = 100_000 - while pos < length - 100_000: - pos += random.randint(800, 1200) - f.seek(off + (pos - 1) // lb * lw + (pos - 1) % lb) - ref = f.read(1).decode().upper() - p, r, depth = random.uniform(0.05, 0.5), random.random(), poisson(30) - o = random.random() < 0.5 - bal = 0.5 + BDEV if shift and c == EVENT[0] and EVENT[1] <= pos <= EVENT[2] else 0.5 - k_het, k_err = binom(depth, bal), binom(depth, 0.005) - if ref not in ALT: - continue - if r < 2 * p * (1 - p): - gt = "0|1" if o else "1|0" - alt = depth - k_het if o else k_het - elif r < 2 * p * (1 - p) + p * p: - gt, alt = "1|1", depth - k_err - else: - continue - lines.append(f"{c}\t{pos}\t.\t{ref}\t{ALT[ref]}\t50\tPASS\t.\tGT:GQ:DP:AD\t{gt}:50:{depth}:{depth - alt},{alt}") - open(path, "w").write("\n".join(lines) + "\n") - print(path, len(lines), "lines") -PY +"${REPO}/bin/og" dev fixture mocha-vcf "${G}/reference/GRCh38_no_alt_analysis_set.fasta" \ + "${G}/MOSAIC/vcf/MOSAIC.vcf" "${G}/MOSAIC0/vcf/MOSAIC0.vcf" for s in MOSAIC MOSAIC0; do in_genome "$BCFTOOLS_IMAGE" bash -c "bcftools view -Oz -o ${s}/vcf/${s}.vcf.gz ${s}/vcf/${s}.vcf && bcftools index -t ${s}/vcf/${s}.vcf.gz" check_ge "${s}: synthetic records" "$(vcf_count "${s}/vcf/${s}.vcf.gz")" 80000 diff --git a/tests/e2e/mosaic-loy-lox-1.sh b/tests/e2e/mosaic-loy-lox-1.sh index 49c133b92..abc6021f8 100755 --- a/tests/e2e/mosaic-loy-lox-1.sh +++ b/tests/e2e/mosaic-loy-lox-1.sh @@ -43,23 +43,8 @@ check_eq "setup.sh --sample-qc-data exits 0" "$?" 0 SITES="${G}/reference/mosaic_fixture/sites_fixture.vcf" PANEL="${G}/reference/mosaic_fixture/1000g.phase3.100k.b38.vcf.gz.dat" mkdir -p "$(dirname "$SITES")" -python3 - "${REF}.fai" "${SETUP}/reference/somalier/sites.hg38.vcf.gz" "$SITES" \ - "${SETUP}/reference/verifybamid2/1000g.phase3.100k.b38.vcf.gz.dat" "$PANEL" <<'PY' -import gzip, sys -fai, sites, fix_sites, panel, fix_panel = sys.argv[1:] -contigs = {l.split("\t")[0] for l in open(fai)} -with gzip.open(sites, "rt") as f, open(fix_sites, "w") as out: - for l in f: - if l.startswith("#") or l.split("\t", 1)[0] in contigs: - out.write(l) -bed = open(panel + ".bed").read().splitlines() -keep = [i for i, l in enumerate(bed) if l.split("\t", 1)[0] in contigs] -for ext in ("bed", "mu", "UD"): - lines = open(panel + "." + ext).read().splitlines() - assert len(lines) == len(bed), ext - with open(fix_panel + "." + ext, "w") as out: - out.writelines(lines[i] + "\n" for i in keep) -PY +"${REPO}/bin/og" dev fixture contig-cut "${REF}.fai" "${SETUP}/reference/somalier/sites.hg38.vcf.gz" "$SITES" \ + "${SETUP}/reference/verifybamid2/1000g.phase3.100k.b38.vcf.gz.dat" "$PANEL" check_ge "somalier sites on the fixture's contigs" "$(grep -vc '^#' "$SITES" 2>/dev/null || true)" 9000 rm -rf "$SETUP" export SOMALIER_SITES="$SITES" VERIFYBAMID2_PANEL="$PANEL" @@ -130,33 +115,7 @@ W="${SAMPLE}fem" mkdir -p "${G}/${W}/mosdepth" "${G}/${W}/vcf" "${G}/${W}/qc" # woman LOSS: windows and VCF of a woman who lost one X in LOSS of her cells. woman() { - python3 - "$WIN" "${G}/${W}" "$W" "$1" <<'PY' -import gzip, random, sys -win, out, name, loss = sys.argv[1], sys.argv[2], sys.argv[3], float(sys.argv[4]) -d = loss / (2 * (2 - loss)) -rnd = random.Random(11) -with gzip.open(win, "rt") as f, gzip.open(f"{out}/mosdepth/{name}.regions.bed.gz", "wt") as o: - for line in f: - c, s, e, dp = line.rstrip("\n").split("\t") - if c == "chrY": - continue - if c == "chrX": # HG002's one X, as two, of which a share of the cells lost one - dp = f"{float(dp) * 2 * (1 - loss / 2):.2f}" - o.write(f"{c}\t{s}\t{e}\t{dp}\n") -lines = ["##fileformat=VCFv4.2\n", "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tSYNTH\n"] -def site(chrom, pos, p): - n = rnd.randint(20, 40) - a = sum(rnd.random() < p for _ in range(n)) - lines.append(f"{chrom}\t{pos}\t.\tA\tG\t40\tPASS\t.\tGT:AD\t0/1:{n - a},{a}\n") -for i in range(2000): - site("chr20", 10_000_000 + 250 * i, 0.5) -for i in range(300): - site("chrX", 73_700_000 + 1000 * i, 0.5 + (d if rnd.random() < 0.5 else -d)) -with gzip.open(f"{out}/vcf/{name}.vcf.gz", "wt") as o: - o.writelines(lines) -with open(f"{out}/qc/{name}_sample_qc.tsv", "w") as o: - o.write(f"key\tvalue\nsample\t{name}\ninferred_sex\tunknown\ndeclared_sex\tfemale\n") -PY + "${REPO}/bin/og" dev fixture sex-loss-woman "$WIN" "${G}/${W}" "$W" "$1" docker run --rm -u "$(id -u):$(id -g)" -v "${REPO}/bin:/pgp-bin:ro" -v "${G}:/genome" "$PYTHON_IMAGE" \ /pgp-bin/og qc sex-loss --table "/genome/${W}/qc/${W}_sample_qc.tsv" \ --regions "/genome/${W}/mosdepth/${W}.regions.bed.gz" --vcf "/genome/${W}/vcf/${W}.vcf.gz" diff --git a/tests/e2e/qc-1-sample-qc.sh b/tests/e2e/qc-1-sample-qc.sh index 06a08401d..349efa271 100755 --- a/tests/e2e/qc-1-sample-qc.sh +++ b/tests/e2e/qc-1-sample-qc.sh @@ -51,23 +51,7 @@ done FIX_SITES="${G}/reference/somalier/sites_fixture.vcf" FIX_PANEL="${G}/reference/verifybamid2_fixture/1000g.phase3.100k.b38.vcf.gz.dat" mkdir -p "$(dirname "$FIX_PANEL")" -python3 - "${REF}.fai" "$SITES" "$FIX_SITES" "$PANEL" "$FIX_PANEL" <<'PY' -import gzip, sys -fai, sites, fix_sites, panel, fix_panel = sys.argv[1:] -contigs = {l.split("\t")[0] for l in open(fai)} -with gzip.open(sites, "rt") as f, open(fix_sites, "w") as out: - for l in f: - if l.startswith("#") or l.split("\t", 1)[0] in contigs: - out.write(l) -bed = open(panel + ".bed").read().splitlines() -keep = [i for i, l in enumerate(bed) if l.split("\t", 1)[0] in contigs] -for ext in ("bed", "mu", "UD"): - lines = open(panel + "." + ext).read().splitlines() - assert len(lines) == len(bed), ext - with open(fix_panel + "." + ext, "w") as out: - out.writelines(lines[i] + "\n" for i in keep) -print(len(keep), "panel markers on the fixture's contigs") -PY +"${REPO}/bin/og" dev fixture contig-cut "${REF}.fai" "$SITES" "$FIX_SITES" "$PANEL" "$FIX_PANEL" check_ge "somalier sites on the fixture's contigs" "$(grep -vc '^#' "$FIX_SITES" 2>/dev/null || true)" 9000 check_ge "panel markers on the fixture's contigs" "$(wc -l < "${FIX_PANEL}.bed" 2>/dev/null | tr -d ' ')" 40000 export SOMALIER_SITES="$FIX_SITES" VERIFYBAMID2_PANEL="$FIX_PANEL" @@ -109,27 +93,7 @@ bcf view -H -f PASS -v snps -i 'FMT/AD[0:0]==0 && QUAL>=30' -r chrX:2781480-1557 | awk -F'\t' -v OFS='\t' '{ split($5, a, ","); print $1, $2, ".", $4, a[1], ".", "PASS", "AF=0.5" }' \ > "${CASE_TMP}/x_sites.tsv" check_ge "chrX SNVs of case 21 outside the PARs, no reference read" "$(wc -l < "${CASE_TMP}/x_sites.tsv" | tr -d ' ')" 15 -python3 - "$FIX_SITES" "${CASE_TMP}/x_sites.tsv" "${REF}.fai" "$XS" <<'PY' -import sys -sites, extra, fai, out = sys.argv[1:] -order = {l.split("\t")[0]: i for i, l in enumerate(open(fai))} -head, recs = [], [] -for l in open(sites): - if l.startswith("##"): - head.append(l) - elif l.startswith("#"): - cols = l - else: - recs.append(l.rstrip("\n").split("\t")[:8]) -seen = {(r[0], r[1]) for r in recs} -recs += [r.rstrip("\n").split("\t") for r in open(extra) if tuple(r.split("\t")[:2]) not in seen] -recs.sort(key=lambda r: (order.get(r[0], 1 << 30), int(r[1]))) -with open(out, "w") as f: - f.writelines(head) - f.write(cols) - f.writelines("\t".join(r) + "\n" for r in recs) -print(len(recs), "sites") -PY +"${REPO}/bin/og" dev fixture add-sites "$FIX_SITES" "${CASE_TMP}/x_sites.tsv" "${REF}.fai" "$XS" SOMALIER_SITES="$XS" run_step 33-sample-qc.sh "$SAMPLE" female check "declared female: the step stops (exit ${STEP_RC})" test "$STEP_RC" -ne 0 diff --git a/tests/e2e/viral-reads-1.sh b/tests/e2e/viral-reads-1.sh index 76b2c6d2b..cf77a2245 100755 --- a/tests/e2e/viral-reads-1.sh +++ b/tests/e2e/viral-reads-1.sh @@ -43,34 +43,7 @@ echo "human depth over the autosomal bases with reads: ${HUMAN}" # plant OUT FRACTION: the step-02 BAM plus unmapped reads from HHV-6B at FRACTION of the human # depth, 300 CMV reads and 200 phiX reads (150 bp, random strand, 0.3% substitutions, seed fixed). plant() { - python3 - "$VREF" "$HUMAN" "$2" > "${T}/spike.sam" <<'PY' -import random, sys -fasta, human, frac = sys.argv[1], float(sys.argv[2]), float(sys.argv[3]) -seqs, name = {}, None -for line in open(fasta): - line = line.strip() - if line.startswith(">"): - name = line[1:].split()[0]; seqs[name] = [] - elif line: - seqs[name].append(line.upper()) -seqs = {k: "".join(v) for k, v in seqs.items()} -rng, L, comp = random.Random(47), 150, str.maketrans("ACGT", "TGCA") -def reads(acc, n, tag): - s = seqs[acc] - for i in range(n): - p = rng.randrange(0, len(s) - L) - r = list(s[p:p + L]) - for j in range(L): - if rng.random() < 0.003: - r[j] = rng.choice([b for b in "ACGT" if b != r[j]]) - r = "".join(r) - if rng.random() < 0.5: - r = r.translate(comp)[::-1] - print(f"{tag}_{i}\t4\t*\t0\t0\t*\t*\t0\t0\t{r}\t{'I' * L}") -reads("NC_000898.1", round(frac * human * len(seqs["NC_000898.1"]) / L), "spk6b") -reads("NC_006273.2", 300, "spkcmv") -reads("NC_001422.1", 200, "spkphix") -PY + "${REPO}/bin/og" dev fixture viral-spike "$VREF" "$HUMAN" "$2" > "${T}/spike.sam" mkdir -p "${G}/$1/aligned" "${G}/$1/vcf" # The planted records go after the BAM's own unmapped reads, at the end of a coordinate-sorted file. # shellcheck disable=SC2016 From 032ea6d7c5e2c80e738120638e3454f35e3da3e7 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:37:36 +0100 Subject: [PATCH 243/255] test(e2e): seed the Y and haplogroup inputs with og dev fixture, awk and jq Bead: D1 --- tests/e2e/haplogroup-explore-1-files.sh | 15 ++- .../y-private-report-1-private-and-tree.sh | 33 +------ tests/e2e/y-str-1.sh | 96 +------------------ 3 files changed, 12 insertions(+), 132 deletions(-) diff --git a/tests/e2e/haplogroup-explore-1-files.sh b/tests/e2e/haplogroup-explore-1-files.sh index 33e5c00b6..5ef95e426 100755 --- a/tests/e2e/haplogroup-explore-1-files.sh +++ b/tests/e2e/haplogroup-explore-1-files.sh @@ -42,13 +42,10 @@ sam faidx "reference/${REF##*/}" chrM 2>/dev/null > "${CASE_TMP}/ref_chrM.fa" echo "kept SNVs before the first kept indel: $(grep -c . "${CASE_TMP}/snvs.tsv" || true)" check_ge "kept SNVs to check" "$(grep -c . "${CASE_TMP}/snvs.tsv" || true)" 1 alt_at() { # alt_at FASTA [SNVS]: OK when every listed SNV's ALT is the base at its position - python3 - "$1" "${2:-${CASE_TMP}/snvs.tsv}" <<'PY' -import sys -seq = "".join(l.strip() for l in open(sys.argv[1]) if not l.startswith(">")) -bad = [l for l in open(sys.argv[2]) if l.strip() and seq[int(l.split("\t")[0]) - 1] != l.split("\t")[2].strip()] -print(f"{len(bad)} SNVs whose ALT is not in {sys.argv[1].rsplit('/', 1)[-1]}") -sys.exit(1 if bad or not seq else 0) -PY + awk -F'\t' -v name="${1##*/}" ' + FILENAME == ARGV[1] {if (!/^>/) {gsub(/[[:space:]]/, ""); seq = seq $0}; next} + /[^[:space:]]/ {alt = $3; gsub(/[[:space:]]/, "", alt); if (substr(seq, $1, 1) != alt) bad++} + END {print bad + 0 " SNVs whose ALT is not in " name; exit (bad > 0 || seq == "")}' "$1" "${2:-${CASE_TMP}/snvs.tsv}" } check "the FASTA carries the ALT of each kept SNV" alt_at "$FA" check "control: the reference chrM does not" eval '! alt_at "${CASE_TMP}/ref_chrM.fa"' @@ -90,7 +87,7 @@ check "a female copy gets no chrY BAM" test ! -e "${G}/${XF}/y_haplogroup/${XF}_ # --- 3. the links in the reports ------------------------------------------------------------- HG=$(awk -F'\t' 'NR == 2 {gsub(/"/, "", $2); print $2}' "${M}/${SAMPLE}_haplogroup.txt" 2>/dev/null) -HG_Q=$(python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "${HG:-none}") +HG_Q=$(jq -rn --arg s "${HG:-none}" '$s | @uri') run_step generate-report.sh "$SAMPLE" check_step_exit generate-report.sh TXT=$(cat "${G}/${SAMPLE}/${SAMPLE}_report.txt" 2>/dev/null) @@ -108,7 +105,7 @@ TXT=$(cat "${G}/${XM}/${XM}_report.txt" 2>/dev/null) if [ -n "$YHG" ] && [ "$YHG" != NA ]; then B=${YHG%%\**} check "the male copy's report links its Y branch (${B})" \ - has "^ YFull YTree: https://www\.yfull\.com/tree/$(python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1], safe=""))' "$B")/\$" "$TXT" + has "^ YFull YTree: https://www\.yfull\.com/tree/$(jq -rn --arg s "$B" '$s | @uri')/\$" "$TXT" else check "the male copy has no Y call (${YHG:-none}), so no Y link" lacks 'yfull\.com/tree/|y-dna/' "$TXT" check "and its report still says insufficient markers" has 'Haplogroup: insufficient markers' "$TXT" diff --git a/tests/e2e/y-private-report-1-private-and-tree.sh b/tests/e2e/y-private-report-1-private-and-tree.sh index 2553e246f..bcf36f28a 100755 --- a/tests/e2e/y-private-report-1-private-and-tree.sh +++ b/tests/e2e/y-private-report-1-private-and-tree.sh @@ -76,34 +76,8 @@ echo "private chrY SNVs listed: ${ROWS}" # does not have: IN where the mask passes the base, OUT where it does not. # Depth is the sample's own median (at least 10), so the depth gates keep both. SEED="${CASE_TMP}/seeded.vcf" -python3 - "${GENOME_DIR}/${M}/vcf/${M}.vcf.gz" "$MASK" "$PRIV" "$SEED" \ - "${YT}/positions_ftdna_hg38.txt" "${YT}/positions_yfull_hg38.txt" > "${CASE_TMP}/seed.txt" <<'PY' -import gzip, re, sys -vcf, mask_path, priv, out, *tables = sys.argv[1:] -mask = "".join(l.strip() for l in gzip.open(mask_path, "rt") if not l.startswith(">")) -taken = set() -for t in tables: - for line in open(t): - c = line.split("\t") - if len(c) >= 4 and c[3].isdigit(): - taken.add(int(c[3])) -head, body = [], [] -for line in gzip.open(vcf, "rt"): - (head if line.startswith("#") else body).append(line) - c = line.split("\t") - if not line.startswith("#") and c[0] in ("chrY", "Y") and c[1].isdigit(): - taken.add(int(c[1])) -med = re.search(r"median_chrY_depth=([0-9.]+)", open(priv).read()) -depth = max(10, int(float(med.group(1)))) if med else 20 -# The fixture's TBL1Y stretch (scripts/ci/build-fixture.sh). -free = [p for p in range(6800000, 7000000) if p not in taken] -pin = next(p for p in free if mask[p - 1] == "P") -pout = next(p for p in free if mask[p - 1] != "P") -for p in (pin, pout): - body.append(f"chrY\t{p}\t.\tC\tT\t50\tPASS\t.\tGT:AD:DP:GQ\t1:0,{depth}:{depth}:60\n") -open(out, "w").writelines(head + body) -print(pin, pout) -PY +"${REPO}/bin/og" dev fixture y-seed "${GENOME_DIR}/${M}/vcf/${M}.vcf.gz" "$MASK" "$PRIV" "$SEED" \ + "${YT}/positions_ftdna_hg38.txt" "${YT}/positions_yfull_hg38.txt" > "${CASE_TMP}/seed.txt" read -r P_IN P_OUT < "${CASE_TMP}/seed.txt" echo "seeded: ${P_IN:-none} (callable), ${P_OUT:-none} (outside the mask)" "${REPO}/bin/og" y private-snps --vcf "$SEED" --tree-dir "$YT" --sample "$M" \ @@ -156,8 +130,7 @@ check "and the card names the branch step 37 placed" has "${TREE_HG}" "$HTML" check "Yleaf's own card is still there, so the two results stand side by side" \ has 'Y-Chromosome Haplogroup' "$HTML" # section DIR KEY: the state of a summary.json section -section() { python3 -c "import json, sys; print(json.load(open(sys.argv[1]))['sections'].get(sys.argv[2], {}).get('state', 'absent'))" \ - "$1/summary.json" "$2" 2>/dev/null; } +section() { jq -r --arg k "$2" '(.sections[$k] // {}) | .state // "absent"' "$1/summary.json" 2>/dev/null; } check_eq "the bash summary.json section y_tree is filled" "$(section "${GENOME_DIR}/${M}" y_tree)" ok run_step generate-report.sh "$M" check_step_exit generate-report.sh diff --git a/tests/e2e/y-str-1.sh b/tests/e2e/y-str-1.sh index 97440a8f7..9e6250825 100755 --- a/tests/e2e/y-str-1.sh +++ b/tests/e2e/y-str-1.sh @@ -44,25 +44,7 @@ dk "$SAMTOOLS_IMAGE" samtools faidx -r "${T}/regions.txt" "$REF" > "${T}/regions # (at least 75% of the bases equal to the base one motif further on; a span # marker's region carries the few bases between its parts, 78% at DYS510). periodic() { - python3 -I - "$1" "$2" <<'PY' -import sys -bed = [l.split("\t") for l in open(sys.argv[1])] -seqs, name = {}, None -for line in open(sys.argv[2]): - line = line.strip() - if line.startswith(">"): - name = line[1:] - seqs[name] = "" - elif name: - seqs[name] += line.upper() -ok = 0 -for c in bed: - s, p = seqs.get(f"{c[0]}:{c[1]}-{c[2]}", ""), int(c[3]) - if len(s) == int(c[2]) - int(c[1]) + 1 and len(s) > p: - same = sum(1 for i in range(len(s) - p) if s[i] == s[i + p]) - ok += same >= 0.75 * (len(s) - p) -print(ok) -PY + "${REPO}/bin/og" dev fixture ystr-periodic "$1" "$2" } check_eq "every region is a repeat of its motif length in the reference" "$(periodic "${T}/regions.bed" "${T}/regions.fa")" 86 awk -F'\t' 'BEGIN {OFS = "\t"} {print $1, $2, $3, 1}' "${T}/regions.bed" > "${T}/shifted.bed" @@ -151,49 +133,7 @@ check_eq "DYS393 is called high" "$(col "$H" DYS393 confidence)" high # length gives the count with the same rounding as the step (reference copies # rounded half up, plus the length change in whole and partial repeats). dk "$SAMTOOLS_IMAGE" samtools view -q 20 -F 0xF04 "${T}/win.bam" > "${T}/win.sam" -python3 -I - "$PANEL" "$OUT" "${T}/win.sam" > "${T}/readcount.tsv" <<'PY' -import collections, re, sys -regions = {} -for line in open(sys.argv[1]): - c = line.rstrip("\n").split("\t") - if line.startswith("#") or c[0] == "order" or c[3] not in ("single", "span"): - continue - # A span marker: one region from its first part's start to its last part's end, - # its parts' copies each rounded half up. - parts = [x.split("/", 3) for x in c[4].split(";")] - s = int(parts[0][0].split(":")[1].split("-")[0]) - e = int(parts[-1][0].split(":")[1].split("-")[1]) - regions[c[2]] = (s, e, int(parts[0][1]), sum(int(float(x[2]) + 0.5) for x in parts), c[3]) -lens = collections.defaultdict(collections.Counter) -for line in open(sys.argv[3]): - f = line.split("\t") - if f[2] != "chrY" or f[5] == "*": - continue - pos = int(f[3]) - ops = [(int(n), op) for n, op in re.findall(r"(\d+)([MIDNSHP=X])", f[5])] - end = pos + sum(n for n, op in ops if op in "MDN=X") - 1 - for m, (s, e, p, rc, _k) in regions.items(): - if pos > s - 10 or end < e + 10: - continue - ref, diff = pos, 0 - for n, op in ops: - if op in "M=X": - ref += n - elif op in "DN": - if s - 10 <= ref <= e + 10: - diff -= n - ref += n - elif op == "I" and s - 10 <= ref <= e + 10: - diff += n - lens[m][diff] += 1 -for line in open(sys.argv[2]): - c = line.rstrip("\n").split("\t") - if c[2] in regions and c[4] == "high" and lens[c[2]]: - s, e, p, rc, kind = regions[c[2]] - diff, n = lens[c[2]].most_common(1)[0] - whole, rest = divmod(diff, p) - print(f"{c[2]}\t{c[3]}\t{rc + whole}{'.' + str(rest) if rest else ''}\t{n}\t{kind}") -PY +"${REPO}/bin/og" dev fixture ystr-readcount "$PANEL" "$OUT" "${T}/win.sam" > "${T}/readcount.tsv" N_CMP=$(grep -c . "${T}/readcount.tsv" || true) N_SAME=$(awk -F'\t' '$2 == $3' "${T}/readcount.tsv" | wc -l | tr -d ' ') awk -F'\t' '$2 != $3 {print " differs: " $0}' "${T}/readcount.tsv" @@ -208,37 +148,7 @@ read -r DS DE < <(awk -F'\t' '$6 == "DYS393" {print $2, $3}' "${T}/regions.bed") dk "$SAMTOOLS_IMAGE" samtools view -h "${T}/win.bam" > "${T}/all.sam" # Reads that span DYS393 with a plain match get bases DS..DS+3 removed (CIGAR xM4DyM); # other reads over the region are dropped, so only the planted allele is left there. -python3 -I - "$DS" "$DE" "${T}/all.sam" > "${T}/planted.sam" <<'PY' -import re, sys -s, e = int(sys.argv[1]), int(sys.argv[2]) -cut = 0 -for line in open(sys.argv[3]): - if line.startswith("@"): - sys.stdout.write(line) - continue - f = line.rstrip("\n").split("\t") - if f[2] != "chrY" or f[5] == "*": - sys.stdout.write(line) - continue - pos = int(f[3]) - ops = re.findall(r"(\d+)([MIDNSHP=X])", f[5]) - end = pos + sum(int(n) for n, op in ops if op in "MDN=X") - 1 - if end < s or pos > e: - sys.stdout.write(line) - continue - m = re.fullmatch(r"(\d+)M", f[5]) - if not m or pos > s - 10 or end < e + 10: - continue - off = s - pos - n = int(m.group(1)) - f[9] = f[9][:off] + f[9][off + 4:] - f[10] = f[10][:off] + f[10][off + 4:] if f[10] != "*" else "*" - f[5] = f"{off}M4D{n - off - 4}M" - f = [x for x in f if not x.startswith(("MD:Z:", "NM:i:"))] - sys.stdout.write("\t".join(f) + "\n") - cut += 1 -print(f"planted the cut in {cut} reads", file=sys.stderr) -PY +"${REPO}/bin/og" dev fixture ystr-cut "$DS" "$DE" "${T}/all.sam" > "${T}/planted.sam" dk "$SAMTOOLS_IMAGE" samtools sort -o "${T}/planted.bam" "${T}/planted.sam" dk "$SAMTOOLS_IMAGE" samtools index "${T}/planted.bam" sample "$P" "${T}/planted.bam" 1 From 6aa3d74ef50a9ee6a8b6b04bd4955dacc467896c Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:37:36 +0100 Subject: [PATCH 244/255] test(e2e): read the reports and logs with jq and awk instead of inline Python Bead: D1 --- tests/e2e/align-read-count-gate-1.sh | 2 +- tests/e2e/genome-lookup-1.sh | 34 ++++++++---------- tests/e2e/live-server-1-weblog.sh | 19 +++++----- tests/e2e/mei-1.sh | 52 +++++++++++++++------------- tests/e2e/nextflow-hardening-2-nf.sh | 4 +-- tests/e2e/qc-2-nextflow-sample-qc.sh | 24 ++++++------- 6 files changed, 66 insertions(+), 69 deletions(-) diff --git a/tests/e2e/align-read-count-gate-1.sh b/tests/e2e/align-read-count-gate-1.sh index 3398a3cc0..097ffab67 100755 --- a/tests/e2e/align-read-count-gate-1.sh +++ b/tests/e2e/align-read-count-gate-1.sh @@ -38,7 +38,7 @@ for r in R1 R2; do ln -f "${FQ}/${SAMPLE}_${r}.fastq.gz" "${G}/${T}/fastq/${T}_$ run_step 01b-fastp-qc.sh "$T" check_step_exit 01b-fastp-qc.sh JSON="${G}/${T}/fastq_trimmed/${T}_fastp.json" -FASTP=$(python3 -c 'import json, sys; print(json.load(open(sys.argv[1]))["summary"]["after_filtering"]["total_reads"])' "$JSON" 2>/dev/null) +FASTP=$(jq -r '.summary.after_filtering.total_reads // empty' "$JSON" 2>/dev/null) TRIM1=$(records "${G}/${T}/fastq_trimmed/${T}_R1.fastq.gz") TRIM2=$(records "${G}/${T}/fastq_trimmed/${T}_R2.fastq.gz") check_ge "fastp kept reads" "${FASTP:-0}" 100000 diff --git a/tests/e2e/genome-lookup-1.sh b/tests/e2e/genome-lookup-1.sh index 7c737acab..309aabc30 100644 --- a/tests/e2e/genome-lookup-1.sh +++ b/tests/e2e/genome-lookup-1.sh @@ -31,26 +31,20 @@ lookup() { # --- an rsID of the annotated VCF ---------------------------------------------------- # The first record whose CSQ has a gene symbol and an rsID in Existing_variation. -PICK=$(gzip -dc "$ANN" | python3 -c ' -import re, sys -fields = [] -for line in sys.stdin: - if line.startswith("##INFO= "${CASE_TMP}/site.err" & @@ -22,7 +22,7 @@ SITE=$! trap 'kill "$SITE" 2>/dev/null' EXIT for _ in $(seq 50); do curl -s -o /dev/null "http://127.0.0.1:${WEB}/" && break; sleep 0.2; done state11() { curl -s "http://127.0.0.1:${WEB}/runs/${SAMPLE}/state.json" \ - | python3 -c 'import json, sys; print(next(s["state"] for s in json.load(sys.stdin)["steps"] if s["step"] == "11"))' 2>/dev/null \ + | jq -er 'first(.steps[] | select(.step == "11") | .state // empty)' 2>/dev/null \ || echo "no state.json"; } check_eq "control: before the run, step 11 is not planned" "$(state11)" "not planned" @@ -41,12 +41,15 @@ check_eq "nextflow run exits 0" "${PIPESTATUS[0]}" 0 check "the run loaded the plugin" grep -q "nf-weblog" "${CASE_TMP}/.nextflow.log" EV="${STATE}/runs/${SAMPLE}/events.jsonl" -count() { python3 -c 'import json, sys -n = 0 -for l in open(sys.argv[1]): - e = json.loads(l); t = e.get("trace") or {} - n += e.get("event") == sys.argv[2] and (sys.argv[3] == "" or (t.get("process") or "").endswith(sys.argv[3])) -print(n)' "$EV" "$@" 2>/dev/null || echo 0; } +# count EVENT PROCESS_SUFFIX: the stored messages of that event (and process, unless ""). +# jq prints 0 before it fails on a missing file, so its output is kept only when it exits 0. +count() { + local n + n=$(jq -n --arg e "$1" --arg p "$2" \ + '[inputs | select(.event == $e and ($p == "" or (((.trace // {}).process // "") | endswith($p))))] | length' "$EV" 2>/dev/null) \ + || n=0 + echo "$n" +} check_ge "the site stored the run's 'started' message" "$(count started '')" 1 check_ge "the site stored ROH's process_completed" "$(count process_completed ROH)" 1 check_ge "the site stored the run's 'completed' message" "$(count completed '')" 1 diff --git a/tests/e2e/mei-1.sh b/tests/e2e/mei-1.sh index 923194b20..a69e589c0 100755 --- a/tests/e2e/mei-1.sh +++ b/tests/e2e/mei-1.sh @@ -93,16 +93,14 @@ check_ge "mapped reads of chr20 at 30x" "$(dk "$SAMTOOLS_IMAGE" samtools view -c check "GIAB v5.0q's SV truth downloads" retry curl -sfL -o "${T}/truth.vcf.gz" "${TRUTH}.vcf.gz" check "and its benchmark regions" retry curl -sfL -o "${T}/truth.bed" "${TRUTH}.benchmark.bed" # truth.tsv: every insertion on chr20 (pos, RepeatMasker class, REMAP, length, genotype). -python3 -I - "${T}/truth.vcf.gz" > "${T}/truth.tsv" <<'PY' -import gzip, sys -for line in gzip.open(sys.argv[1], "rt"): - if line[0] == "#" or not line.startswith("chr20\t"): - continue - f = line.split("\t", 10) - info = dict(x.split("=", 1) for x in f[7].split(";") if "=" in x) - if info.get("SVTYPE") == "INS": - print(f[1], info.get("RM_clsfam", "."), info.get("REMAP", "."), len(f[4]) - len(f[3]), f[9].split(":")[0], sep="\t") -PY +gzip -dc "${T}/truth.vcf.gz" | awk -F'\t' -v OFS='\t' '$1 == "chr20" { + delete info; n = split($8, kv, ";") + for (i = 1; i <= n; i++) if ((e = index(kv[i], "=")) > 0) info[substr(kv[i], 1, e - 1)] = substr(kv[i], e + 1) + if (info["SVTYPE"] == "INS") { + split($10, g, ":") + print $2, ("RM_clsfam" in info) ? info["RM_clsfam"] : ".", ("REMAP" in info) ? info["REMAP"] : ".", length($5) - length($4), g[1] + } + }' > "${T}/truth.tsv" check_ge "GIAB insertions on chr20" "$(wc -l < "${T}/truth.tsv" | tr -d ' ')" 1000 if [ "${MEI_E2E_DROP_ALU:-}" = 1 ]; then @@ -160,21 +158,25 @@ check "red control: without AnnotSV's table the same check fails" \ test "$(annotsv_cols "${T}/red_mei.tsv")" != "${N}/${N}/${N}/${N}" -a "$(annotsv_cols "${T}/red_mei.tsv")" = "${N}/0/0/0" # Recall and precision of the Alu calls. -read -r TP NT PP NP < <(python3 -I - "${T}/truth.tsv" "${T}/truth.bed" "${M}_mei.tsv" <<'PY' -import csv, sys -truth = [l.rstrip("\n").split("\t") for l in open(sys.argv[1])] -bed = [(int(a), int(b)) for c, a, b, *_ in (l.split("\t") for l in open(sys.argv[2])) if c == "chr20"] -inb = lambda p: any(s < p <= e for s, e in bed) -ins = [int(t[0]) for t in truth] -alu = [int(t[0]) for t in truth if t[1] == "SINE/Alu" and t[2] == "interspersed" and 250 <= int(t[3]) <= 400 - and "1" in t[4] and inb(int(t[0]))] -calls = [int(r["pos"]) for r in csv.DictReader(open(sys.argv[3]), delimiter="\t") if r["element"] == "Alu"] -found = sum(any(abs(c - t) <= 100 for c in calls) for t in alu) -inside = [c for c in calls if inb(c)] -right = sum(any(abs(c - t) <= 100 for t in ins) for c in inside) -print(found, len(alu), right, len(inside)) -PY -) +# An Alu to find: SINE/Alu, interspersed, 250 to 400 bp, carried, inside the +# benchmark regions. A call is right when an insertion of the truth set lies +# within 100 bp of it. +read -r TP NT PP NP < <(awk -F'\t' ' + function inb(p, i) {for (i = 1; i <= nb; i++) if (bs[i] < p && p <= be[i]) return 1; return 0} + function near(a, b) {return a - b <= 100 && b - a <= 100} + FILENAME == ARGV[1] {nt++; tp[nt] = $1 + 0; cls[nt] = $2; rm[nt] = $3; len[nt] = $4 + 0; gt[nt] = $5; next} + FILENAME == ARGV[2] {if ($1 == "chr20") {nb++; bs[nb] = $2 + 0; be[nb] = $3 + 0}; next} + FNR == 1 {for (i = 1; i <= NF; i++) h[$i] = i; next} + $h["element"] == "Alu" {calls[++nc] = $h["pos"] + 0} + END { + for (t = 1; t <= nt; t++) + if (cls[t] == "SINE/Alu" && rm[t] == "interspersed" && len[t] >= 250 && len[t] <= 400 && index(gt[t], "1") && inb(tp[t])) { + na++; for (c = 1; c <= nc; c++) if (near(calls[c], tp[t])) {found++; break} + } + for (c = 1; c <= nc; c++) + if (inb(calls[c])) {ni++; for (t = 1; t <= nt; t++) if (near(calls[c], tp[t])) {right++; break}} + print found + 0, na + 0, right + 0, ni + 0 + }' "${T}/truth.tsv" "${T}/truth.bed" "${M}_mei.tsv") echo "Alu recall ${TP}/${NT}, precision ${PP}/${NP}" check_ge "GIAB Alu insertions to find" "${NT:-0}" 10 check "Alu recall (${TP}/${NT}) is at least 0.70" awk -v a="${TP:-0}" -v b="${NT:-0}" 'BEGIN {exit !(b > 0 && a / b >= 0.70)}' diff --git a/tests/e2e/nextflow-hardening-2-nf.sh b/tests/e2e/nextflow-hardening-2-nf.sh index 343babcb7..daff2f705 100755 --- a/tests/e2e/nextflow-hardening-2-nf.sh +++ b/tests/e2e/nextflow-hardening-2-nf.sh @@ -184,11 +184,11 @@ for a in clinvar/${SAMPLE}_clinvar_hits.vcf pharmcat/${SAMPLE}.report.json pharm done JSON="${R}/summary.json" # section KEY: the state of a summary.json section (ok, missing, unreadable, ...) -section() { python3 -c "import json, sys; print(json.load(open(sys.argv[1]))['sections'][sys.argv[2]]['state'])" "$JSON" "$1" 2>/dev/null; } +section() { jq -r --arg k "$1" '.sections[$k].state // empty' "$JSON" 2>/dev/null; } for k in clinvar pharmcat coverage telomere haplogroup y_haplogroup manta delly sv_consensus pypgx; do check_eq "summary.json section ${k} is filled" "$(section "$k")" ok done -CONS_RESULT=$(python3 -c "import json, sys; c = json.load(open(sys.argv[1]))['cyp2d6']['consensus']; print(c['result'] if c else '')" "$JSON" 2>/dev/null) +CONS_RESULT=$(jq -r '(.cyp2d6.consensus // {}) | .result // empty' "$JSON" 2>/dev/null) check "the CYP2D6 card reads step 36's table (result: ${CONS_RESULT:-none})" test -n "${CONS_RESULT:-}" check "the Structural Variants card counts the consensus SVs" \ has 'Consensus SVs \(2\+ callers\)(<[^>]*>)?[1-9][0-9]*<' "$(cat "${R}/${SAMPLE}_report.html" 2>/dev/null)" diff --git a/tests/e2e/qc-2-nextflow-sample-qc.sh b/tests/e2e/qc-2-nextflow-sample-qc.sh index 2a04f4751..11b9742fe 100755 --- a/tests/e2e/qc-2-nextflow-sample-qc.sh +++ b/tests/e2e/qc-2-nextflow-sample-qc.sh @@ -118,7 +118,7 @@ JSON="${R}/summary.json" check "the report exists" test -s "$HTML" check "summary.json validates against the schema" \ "${REPO}/bin/og" dev schema "${REPO}/tests/schema/summary.schema.json" "$JSON" -FREEMIX=$(python3 -c "import json; print(json.load(open('${JSON}'))['sections']['sample_qc']['data']['freemix'])" 2>/dev/null) +FREEMIX=$(jq -r '.sections.sample_qc.data.freemix // empty' "$JSON" 2>/dev/null) check "the summary has FREEMIX (${FREEMIX:-none})" test -n "$FREEMIX" grep -o '
\(Sex from the reads (somalier)\|Contamination (FREEMIX)\).*
' "$HTML" 2>/dev/null check "the QC card shows somalier's sex" \ @@ -131,20 +131,18 @@ check "the haplogroup card is filled" has '

Mitochondrial Haplogroup

' "$ check "the Variant Calling card counts the VCF" has 'Total variants
[1-9][0-9]*<' "$(cat "$HTML" 2>/dev/null)" # section JSON KEY: the state of a section of a summary.json (ok, stale, missing, ...) -section() { python3 -c "import json, sys; print(json.load(open(sys.argv[1]))['sections'][sys.argv[2]]['state'])" "$1" "$2" 2>/dev/null; } +section() { jq -r --arg k "$2" '.sections[$k].state // empty' "$1" 2>/dev/null; } # card HTML TITLE: "Not run" when the card titled TITLE says so, "filled" when it # is there and does not, "absent" when the report has no such card card() { - python3 - "$1" "$2" <<'PY' 2>/dev/null || echo absent -import sys -html, title = open(sys.argv[1]).read(), sys.argv[2] -i = html.find(f"

{title}

") -if i < 0: - print("absent") -else: - j = html.find('
Not run<" in html[i:j if j > 0 else len(html)] else "filled") -PY + awk -v t="

$2

" '{s = s $0 "\n"} + END { + i = index(s, t) + if (!i) {print "absent"; exit} + r = substr(s, i); j = index(r, "
Not run<") ? "Not run" : "filled" + }' 2>/dev/null < "$1" || echo absent } check_eq "pypgx ran: its section is filled" "$(section "$JSON" pypgx)" ok check "the Steps Not Run list does not name pypgx" lacks '
  • pypgx
  • ' "$(cat "$HTML" 2>/dev/null)" @@ -162,7 +160,7 @@ check "the QC card shows the sex INDEXCOV inferred (${IXS:-none})" \ # The PharmCAT vs pypgx comparison: HTML_REPORT gets CPIC_LOOKUP's table # from PGX and places it beside the pypgx summary. check "CPIC_LOOKUP published the comparison" test -s "${R}/pypgx/${SAMPLE}_pharmcat_comparison.tsv" -COMP=$(python3 -c "import json, sys; print(json.load(open(sys.argv[1]))['sections']['pypgx']['data']['comparison']['conflicts'])" "$JSON" 2>/dev/null) +COMP=$(jq -r '.sections.pypgx.data.comparison.conflicts // empty' "$JSON" 2>/dev/null) check "the summary's pypgx section has the comparison (conflicts: ${COMP:-none})" test -n "$COMP" check "the Pharmacogenomics card shows the comparison" \ has "PharmCAT vs pypgx conflicts${COMP:-none}<" "$(cat "$HTML" 2>/dev/null)" From 7358c9e64ea8149c67b5cebfea206ee852a33781 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:37:36 +0100 Subject: [PATCH 245/255] test(e2e): run the cases that call og dev fixture when fixture.rs changes Bead: D1 --- scripts/ci/e2e-case-paths.tsv | 36 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 2a5417769..80c18944b 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -44,10 +44,10 @@ acmg-sf-rules-1-reports assets/acmg_sf.tsv assets/callability_genes.tsv assets/c align-read-count-gate-1 assets/callability_genes.tsv assets/par_grch38.bed bin/collect_summary.py bin/render_report.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/16b-mosdepth.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh alignment-markdup-gvcf-sex-1-align scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-2-ploidy scripts/ci/fake-docker/lib.sh -alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh +alignment-markdup-gvcf-sex-3-homref assets/pgs_scores.tsv bin/collect_summary.py bin/og rust/crates/og-dev/src/fixture.rs scripts/14-imputation-prep.sh scripts/25-prs.sh scripts/ci/fake-docker/lib.sh alignment-markdup-gvcf-sex-4-bwamem2 scripts/02a-alignment-bwamem2.sh scripts/ci/fake-docker/lib.sh -ancestry-fine-grained-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgs_scores.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/ancestry_fine/* modules/local/html_report/* modules/local/prs/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/26b-ancestry-fine.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/og bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +ancestry-fine-grained-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgs_scores.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/ancestry_fine/* modules/local/html_report/* modules/local/prs/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/26b-ancestry-fine.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +ancient-similarity-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/og bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/ancient/* modules/local/html_report/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/24-html-report.sh scripts/44-ancient.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh annotsv-delly-depth-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh tests/fixtures/annotsv/class45.sh annotsv-report-section-1-chain assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/22-survivor-merge.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh tests/fixtures/annotsv/class45.sh annotsv-sv-inputs-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh @@ -64,8 +64,8 @@ bloodtype-1 assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og modules/local/bl bumps-drop-in-baseline scripts/ci/fake-docker/lib.sh callability-1-catalog assets/callability_genes.tsv assets/paralog_loci.tsv assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/ci/build-callability.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh callability-2-genes assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/24-html-report.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -chip-clonal-1 assets/chip_genes.tsv bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -chip-mt-1-roundtrip assets/par_grch38.bed bin/og scripts/40-export-consumer.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz +chip-clonal-1 assets/chip_genes.tsv bin/og bin/render_report.py modules/local/chip/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/49-chip.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +chip-mt-1-roundtrip assets/par_grch38.bed bin/og rust/crates/og-dev/src/fixture.rs scripts/40-export-consumer.sh scripts/chip-to-vcf.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz ci-e2e-2-locityper assets/locityper_loci.bed bin/og scripts/53-locityper.sh scripts/setup.sh clinical-filter-mane-and-quality-1-tiers assets/construct_variants.tsv bin/og scripts/23-clinical-filter.sh scripts/31-slivar.sh scripts/ci/fake-docker/lib.sh clinical-filter-mane-and-quality-2-vep-norm scripts/13-vep-annotation.sh scripts/ci/fake-docker/lib.sh @@ -89,7 +89,7 @@ cyp21a2-short-reads-1-hg002 bin/cn_loci.tsv bin/og modules/local/parascopy/* scr exomiser-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/stub/sample.vcf.gz.tbi bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/exomiser/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/48-exomiser.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh expansions-and-rfc1-1-motif assets/str_extra_loci.json assets/stub/db/catalog.json bin/og scripts/09-expansion-hunter.sh scripts/09b-stranger.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh families-file-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/couple_report.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/couple/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/39-couple.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/families/stub_families.tsv -family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +family-joint-1 assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/deepvariant/* modules/local/fastp/* modules/local/roh/* modules/local/somalier/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/11-roh-analysis.sh scripts/33-sample-qc.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh fixitfelix-chr21-1 assets/callability_genes.tsv scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh followups-ci-3-sv-rule-regions assets/sv_rule_regions.tsv bin/og scripts/ci/build-sv-rule-regions.sh scripts/ci/fake-docker/lib.sh scripts/ci/lib/fetch-pinned.sh followups-clinvar-traits-1-rules-html assets/genotype_rules.tsv assets/traits.tsv bin/render_report.py scripts/42-traits.sh scripts/ci/check-rule-alleles.sh @@ -105,14 +105,14 @@ haplogroup-explore-1-files assets/callability_genes.tsv assets/live/index.html a hba-short-read-accuracy-1-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/export_consumer/* modules/local/html_report/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz hla-second-caller-1-bash assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/og bin/render_report.py scripts/08-hla-typing.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh hla-second-caller-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/hla_typing/* modules/local/html_report/* modules/local/pgx_consensus/* modules/local/pharmcat/* modules/local/t1k_build/* modules/local/vcf_precheck/* scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/setup.sh scripts/validate-setup.sh -ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/og scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +ibd-segments-1 assets/families.example.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/stub/pedigree.tsv bin/family_report.py bin/og rust/crates/og-dev/src/fixture.rs scripts/38-family.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +live-server-1-weblog assets/live/* assets/viewer/index.html bin/live_results.py bin/og bin/owngenome_live.py bin/owngenome_live_results.py bin/render_report.py bin/run_state.py modules/local/roh/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/11-roh-analysis.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh lpa-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/deepvariant/* modules/local/html_report/* modules/local/lpa/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/46-lpa.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh mei-1 assets/mei_sites.tsv assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og modules/local/align_minimap2/* modules/local/annotsv/* modules/local/deepvariant/* modules/local/fastp/* modules/local/mei/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/50-mei.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh tests/fixtures/annotsv/class45.sh mito-extended-1-deletions assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/stub/db/clinvar.vcf.gz assets/stub/db/clinvar.vcf.gz.tbi assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/html_report/* modules/local/mito_variants/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/ci/build-mito-confirmed.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh -mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/og scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/collect_summary.py bin/og scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh +mocha-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/mocha/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/24-html-report.sh scripts/51-mocha.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh +mosaic-loy-lox-1 assets/callability_genes.tsv bin/collect_summary.py bin/og rust/crates/og-dev/src/fixture.rs scripts/16b-mosdepth.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +nextflow-from-fastq-1-bash assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/pgs_scores.tsv assets/pgx_genes.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat bin/cn_loci.tsv bin/collect_summary.py bin/og rust/crates/og-dev/src/fixture.rs scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/11-roh-analysis.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/25-prs.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/ci/parity-diff.sh scripts/validate-setup.sh nextflow-from-fastq-2-nextflow assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgs_scores.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/cyrius/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/multiqc/* modules/local/pharmcat/* modules/local/prs/* modules/local/roh/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/vcfanno/* scripts/01b-fastp-qc.sh scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/21-cyrius.sh scripts/24-html-report.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/27-cpic-lookup.sh scripts/28-multiqc.sh scripts/30-vcfanno.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/cyrius-constraints.txt scripts/validate-setup.sh nextflow-from-fastq-3-sexcheck assets/callability_genes.tsv assets/live/index.html assets/viewer/index.html bin/collect_summary.py modules/local/indexcov/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh nextflow-hardening-1-manta assets/par_grch38.bed assets/sv_rule_regions.tsv bin/cnvpytor_to_vcf.py bin/collect_summary.py bin/family_report.py bin/og scripts/04-manta.sh scripts/05-annotsv.sh scripts/15-duphold.sh scripts/18-cnvpytor.sh scripts/19-delly.sh scripts/ci/fake-docker/lib.sh @@ -136,14 +136,14 @@ pgx-parse-fixes-1-labels assets/pgx_contraindications.tsv assets/pgx_genes.tsv a pharmcat-indel-refblocks-1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh tests/fixtures/pharmcat/refblocks.g.vcf pharmcat-mito-input-and-log-1-mtrnr1 bin/og scripts/07-pharmacogenomics.sh scripts/ci/fake-docker/lib.sh platelet-antigens-1 assets/bloodtype_markers.tsv bin/cn_loci.tsv bin/og scripts/41-bloodtype.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -prs-1-agreement assets/pgs_scores.tsv bin/collect_summary.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh +prs-1-agreement assets/pgs_scores.tsv bin/collect_summary.py bin/og rust/crates/og-dev/src/fixture.rs scripts/25-prs.sh scripts/ci/fake-docker/lib.sh prs-2-ancestry assets/pgs_scores.tsv bin/collect_summary.py bin/render_report.py scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh prs-3-panel-measure assets/pgs_scores.tsv bin/collect_summary.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh prs-runopts-and-trace-1 assets/par_grch38.bed assets/pgs_scores.tsv bin/collect_summary.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/prs/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summary.py modules/local/prs/* modules/local/vcf_precheck/* scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +prs-scores-and-run-options-1-strand-sex assets/pgs_scores.tsv bin/collect_summary.py bin/og modules/local/prs/* modules/local/vcf_precheck/* rust/crates/og-dev/src/fixture.rs scripts/25-prs.sh scripts/26-ancestry.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +prs-scores-and-run-options-2-panel-chrx assets/pgs_scores.tsv bin/collect_summary.py bin/og rust/crates/og-dev/src/fixture.rs scripts/25-prs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh pypgx-cyp2d6-section-1-summary assets/pgx_genes.tsv bin/cn_loci.tsv bin/og scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -qc-1-sample-qc bin/collect_summary.py bin/og scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +qc-1-sample-qc bin/collect_summary.py bin/og rust/crates/og-dev/src/fixture.rs scripts/28-multiqc.sh scripts/33-sample-qc.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh qc-2-nextflow-sample-qc assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/families.example.tsv assets/live/index.html assets/mito_confirmed.tsv assets/par_grch38.bed assets/pedigree.example.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/db/hla.dat assets/stub/db/kir.dat assets/stub/pedigree.tsv assets/stub/sample.vcf.gz assets/viewer/index.html bin/cn_loci.tsv bin/collect_summary.py bin/family_report.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/cpic_lookup/* modules/local/deepvariant/* modules/local/fastp/* modules/local/hla_typing/* modules/local/html_report/* modules/local/indexcov/* modules/local/mito_haplogroup/* modules/local/mito_variants/* modules/local/mosdepth/* modules/local/pharmcat/* modules/local/pypgx/* modules/local/roh/* modules/local/somalier/* modules/local/t1k_build/* modules/local/telomere_hunter/* modules/local/vcf_precheck/* modules/local/verifybamid2/* scripts/01b-fastp-qc.sh scripts/07-pharmacogenomics.sh scripts/08-hla-typing.sh scripts/10-telomere-hunter.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16-indexcov.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/33-sample-qc.sh scripts/36-pgx-consensus.sh scripts/38-family.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh qc-3-cram assets/callability_genes.tsv bin/collect_summary.py modules/local/cram_archive/* modules/local/mosdepth/* modules/local/vcf_precheck/* scripts/16b-mosdepth.sh scripts/34-cram-archive.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh reference-noalt scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh @@ -173,10 +173,10 @@ vendor-vcf-intake-2-nf-report assets/acmg_sf.tsv assets/callability_genes.tsv as vendor-vcf-intake-3-nf-stops assets/callability_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/stub/sample.bam assets/stub/sample.bam.bai bin/collect_summary.py bin/og modules/local/clinvar_screen/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-4-nf-header-clinvar assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/pgx_contraindications.tsv assets/pgx_genes.tsv assets/pgx_inhibitors.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/cpic_lookup/* modules/local/html_report/* modules/local/mito_haplogroup/* modules/local/pharmcat/* modules/local/roh/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/07-pharmacogenomics.sh scripts/11-roh-analysis.sh scripts/12-mito-haplogroup.sh scripts/16b-mosdepth.sh scripts/20-mtoolbox.sh scripts/24-html-report.sh scripts/27-cpic-lookup.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh vendor-vcf-intake-5-docs scripts/ci/fake-docker/lib.sh -viral-reads-1 bin/og modules/local/vcf_precheck/* modules/local/viral/* scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh +viral-reads-1 bin/og modules/local/vcf_precheck/* modules/local/viral/* rust/crates/og-dev/src/fixture.rs scripts/47-viral.sh scripts/ci/fake-docker/lib.sh scripts/generate-report.sh scripts/setup.sh scripts/validate-setup.sh vntyper-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/vcf_precheck/* modules/local/vntyper/* scripts/24-html-report.sh scripts/52-vntyper.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh -y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh -y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh +y-private-report-1-private-and-tree assets/live/index.html assets/viewer/index.html bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py bin/yleaf_run.py modules/local/html_report/* modules/local/y_haplogroup/* rust/crates/og-dev/src/fixture.rs scripts/16-indexcov.sh scripts/24-html-report.sh scripts/37-y-haplogroup.sh scripts/generate-report.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh +y-str-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/live/index.html assets/mito_confirmed.tsv assets/stub/sample.vcf.gz assets/viewer/index.html assets/ystr_panel.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/html_report/* modules/local/indexcov/* modules/local/vcf_precheck/* modules/local/y_str/* rust/crates/og-dev/src/fixture.rs scripts/16-indexcov.sh scripts/24-html-report.sh scripts/45-y-str.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh y-tree-current-1-placement assets/live/index.html assets/viewer/index.html bin/og bin/yleaf_run.py scripts/16-indexcov.sh scripts/37-y-haplogroup.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh rust-z2e-readers bin/og tests/e2e/lib.sh scripts/ci/image-smoke.sh scripts/ci/nf-trace-requests.tsv rust-c2b-traits assets/traits.tsv assets/genotype_rules.tsv bin/og modules/local/traits/* scripts/42-traits.sh From 7e1b00920aaa74a1bbca4043b30659453f1862ad Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:43:50 +0100 Subject: [PATCH 246/255] test(og-dev): make five lineage fixture tests fail on the mutations they missed A planted wrong line survived in cigar (no X op), x-ref-sites (spacing), truth-genotypes (a 0/0 truth record), segments-check (the 19-site row) and ystr-periodic (a half-periodic region). Each test now has the input that catches it. Bead: D1 --- rust/crates/og-dev/src/fixture/lineage.rs | 5 +++-- rust/crates/og-dev/src/fixture/lineage/family.rs | 8 +++++++- rust/crates/og-dev/src/fixture/lineage/prs.rs | 2 +- rust/crates/og-dev/src/fixture/lineage/truth.rs | 2 +- rust/crates/og-dev/src/fixture/lineage/ystr.rs | 2 +- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/rust/crates/og-dev/src/fixture/lineage.rs b/rust/crates/og-dev/src/fixture/lineage.rs index 3b28a9b56..7a8d75ec4 100644 --- a/rust/crates/og-dev/src/fixture/lineage.rs +++ b/rust/crates/og-dev/src/fixture/lineage.rs @@ -495,14 +495,15 @@ mod tests { #[test] fn cigar_reads_as_the_regex_does() { assert_eq!( - cigar("5S10M2I3D1N4="), + cigar("5S10M2I3D1N4=2X"), vec![ (5, b'S'), (10, b'M'), (2, b'I'), (3, b'D'), (1, b'N'), - (4, b'=') + (4, b'='), + (2, b'X') ] ); assert_eq!(cigar("*"), vec![]); diff --git a/rust/crates/og-dev/src/fixture/lineage/family.rs b/rust/crates/og-dev/src/fixture/lineage/family.rs index e64e32d0d..8d5c30a59 100644 --- a/rust/crates/og-dev/src/fixture/lineage/family.rs +++ b/rust/crates/og-dev/src/fixture/lineage/family.rs @@ -197,7 +197,13 @@ mod tests { "child\tside\tparent\tchrom\tstart\tend\tsites\nC\tpaternal\tF\tchr1\t10\t500\t19\n", ) .expect("tsv"); - let (code, _, _) = run("segments-check", &[&p("few.tsv"), &p("d.csv")]); + // the same stretch in the CSV, so only its 19 sites fail it + std::fs::write( + d.join("few.csv"), + format!("{head}1,10,500,1,19,C,,,paternal,,,F\n"), + ) + .expect("csv"); + let (code, _, _) = run("segments-check", &[&p("few.tsv"), &p("few.csv")]); assert_eq!(code, 1); } } diff --git a/rust/crates/og-dev/src/fixture/lineage/prs.rs b/rust/crates/og-dev/src/fixture/lineage/prs.rs index 6886a8731..709d935cb 100644 --- a/rust/crates/og-dev/src/fixture/lineage/prs.rs +++ b/rust/crates/og-dev/src/fixture/lineage/prs.rs @@ -306,7 +306,7 @@ mod tests { fasta(&fa, &[("chrX", 20_000)], 9); std::fs::write( d.join("blocks.tsv"), - "100\t200\t0/0\n150\t160\t0/0\n3000\t3100\t0|0\n4000\t4100\t0/0\n7000\t7040\t0/0\n9000\t9200\t0/1\n12000\t12100\t0/0\n13000\t.\t0/0\n", + "100\t200\t0/0\n150\t160\t0/0\n3000\t3100\t0|0\n4600\t4700\t0/0\n7000\t7040\t0/0\n9000\t9200\t0/1\n12000\t12100\t0/0\n13000\t.\t0/0\n", ) .expect("blocks"); std::fs::write(d.join("vars.txt"), "12040\n\n").expect("vars"); diff --git a/rust/crates/og-dev/src/fixture/lineage/truth.rs b/rust/crates/og-dev/src/fixture/lineage/truth.rs index 703fa4a2b..50c421250 100644 --- a/rust/crates/og-dev/src/fixture/lineage/truth.rs +++ b/rust/crates/og-dev/src/fixture/lineage/truth.rs @@ -160,7 +160,7 @@ mod tests { ); w( "a.vcf", - "##h\n#CHROM\nchr1\t200\t.\tC\tT\t.\t.\t.\tGT:DP\t1|0:4\nchr1\t300\t.\tG\tC\t.\t.\t.\tGT\t0/1\nchr2\t50\t.\tA\tC\t.\t.\t.\tDP:GT\t3:1/1\n", + "##h\n#CHROM\nchr1\t100\t.\tA\tG\t.\t.\t.\tGT\t0|0\nchr1\t200\t.\tC\tT\t.\t.\t.\tGT:DP\t1|0:4\nchr1\t300\t.\tG\tC\t.\t.\t.\tGT\t0/1\nchr2\t50\t.\tA\tC\t.\t.\t.\tDP:GT\t3:1/1\n", ); w("a.bed", "chr1\t0\t350\nchr2\t0\t100\n"); w("b.vcf", "chr1\t100\t.\tA\tG\t.\t.\t.\tGT\t1/2\n"); diff --git a/rust/crates/og-dev/src/fixture/lineage/ystr.rs b/rust/crates/og-dev/src/fixture/lineage/ystr.rs index 1c22b0934..00288a614 100644 --- a/rust/crates/og-dev/src/fixture/lineage/ystr.rs +++ b/rust/crates/og-dev/src/fixture/lineage/ystr.rs @@ -370,7 +370,7 @@ mod tests { fn periodic_counts_repeats_of_their_period() { let d = dir("periodic"); std::fs::write(d.join("r.bed"), "chrY\t10\t21\t4\tx\tDYS1\nchrY\t30\t41\t4\tx\tDYS2\nchrY\t50\t61\t4\nchrY\t70\t72\t4\n").expect("bed"); - std::fs::write(d.join("r.fa"), ">chrY:10-21\nAGATAGAT\nagat\n>chrY:30-41\nACGTTGCAACGG\n>chrY:50-61\nAGATAGATAGA\n>chrY:70-72\nAAA\n").expect("fa"); + std::fs::write(d.join("r.fa"), ">chrY:10-21\nAGATAGAT\nagat\n>chrY:30-41\nAGATAGATCCCC\n>chrY:50-61\nAGATAGATAGA\n>chrY:70-72\nAAA\n").expect("fa"); let p = |n: &str| d.join(n).to_str().expect("utf8").to_string(); assert_eq!(run("ystr-periodic", &[&p("r.bed"), &p("r.fa")]).1, "1\n"); std::fs::write(d.join("one.bed"), "chrY\t10\t21\t1\n").expect("bed"); From 858ef08315b4c46171d44be7d2afb8300ec6bc8e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:45:28 +0100 Subject: [PATCH 247/255] fix(traits): read a low-quality APOL1 G2 call as not read The G2 reader beside the APOL1 rule applied no GQ floor, so a G2 call or a reference call over the span at GQ 0 to 19 gave a count, and two weak G2 copies read G2/G2. It now takes the D10 floor of the trait sites: GQ below 20, or a reference call with a FILTER other than PASS, . or RefCall, leaves G2 not read. GQ 20 and a missing GQ are read. The rule-line status list in docs/42-traits.md now names low_gq. Bead: trait-01 --- CHANGELOG.md | 2 +- docs/42-traits.md | 4 +- .../og-summary/src/collect/group_c/traits.rs | 77 +++++++++++++++++-- 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e255a7dc..1393ee0f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased -- Step 42 no longer reads a trait from a low-quality call. It kept each site's GQ but never checked it, so a call of one or two reads (GQ 0) at APOE gave a confident e2/e4 or e4/e4, and the same held for HFE, SERPINA1 and every row of the traits table; a call the caller had filtered (FILTER `LowQual`) was read as well. A call with a GQ below 20, or with a FILTER other than `PASS`, `.` or DeepVariant's `RefCall`, now has the new status `low_gq`: the row keeps its genotype, gives no reading, and says "not read; check the reads" with the GQ or the filter. A rule with such a site reads `low_gq` with no result, and such a site is not sent to WhatsHap. GQ 20 and a call without a GQ are read as before. The floor of 20 is the one `og pgx refblock-fill` and the Y private SNPs use. What changes in the outputs: `traits/_traits_genotypes.tsv` gains a seventh column, the record's FILTER; `low_gq` is a new value of `status` in `traits/_traits.tsv` and `traits/_genotype_rules.tsv`; summary.json's traits section gains a `low_gq` count beside `called`, `not_covered` and `other_allele`. No column is renamed or removed. +- Step 42 no longer reads a trait from a low-quality call. It kept each site's GQ but never checked it, so a call of one or two reads (GQ 0) at APOE gave a confident e2/e4 or e4/e4, and the same held for HFE, SERPINA1 and every row of the traits table; a call the caller had filtered (FILTER `LowQual`) was read as well. A call with a GQ below 20, or with a FILTER other than `PASS`, `.` or DeepVariant's `RefCall`, now has the new status `low_gq`: the row keeps its genotype, gives no reading, and says "not read; check the reads" with the GQ or the filter. A rule with such a site reads `low_gq` with no result, and such a site is not sent to WhatsHap. GQ 20 and a call without a GQ are read as before. The floor of 20 is the one `og pgx refblock-fill` and the Y private SNPs use. What changes in the outputs: `traits/_traits_genotypes.tsv` gains a seventh column, the record's FILTER; `low_gq` is a new value of `status` in `traits/_traits.tsv` and `traits/_genotype_rules.tsv`; summary.json's traits section gains a `low_gq` count beside `called`, `not_covered` and `other_allele`. No column is renamed or removed. The APOL1 G2 deletion that `og summary collect` reads beside the rule takes the same floor: a call over its span below GQ 20, or a reference call with a FILTER other than `PASS`, `.` or `RefCall`, leaves G2 not read, so two G2 copies at GQ 5 no longer read G2/G2. - The reports now get summary.json from the Rust `og` binary. Step 24, the Nextflow HTML_REPORT process and `generate-report.sh` collect it with `og summary collect` (through `bin/render_report.py`), and a run collects each sample once: `generate-report.sh` renders the text report from the summary.json step 24 wrote, and collects again when a file it reads is newer than that summary.json, or a file was added to or removed from a step folder since, or a file a section names as its source, the run status or a table in `bin/` or `assets/` is gone. When it renders from step 24's summary.json, the text report's generated time is step 24's, the same as the HTML report's; it was the minute of its own collect. When it collects again, the text report carries the new collect's values and time, and the HTML report keeps step 24's until step 24 runs again, as before. The deliberate fixes the `og summary collect` entries below list now reach the reports; apart from them, nothing in summary.json or either report changes. `bin/collect_summary.py` keeps only what the report renderer (`bin/render_report.py`), the couple report, the results index and the live view import from it. The Python modules that only the old collector imported are gone: `bin/pgx_outside_calls.py`, `ancestry_fine_report.py`, `clip_check.py`, `annotsv_postfilter.py`, `chip_report.py`, `traits_report.py`, `run_delta.py` and `genotype_rules_indels.py`. On the synthetic demo sample (4.7 million VCF records) the collect takes 1.1 s of CPU and 12-15 MB, where the Python took 7.8-8.0 s and 34-37 MB. What changes on purpose: - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. diff --git a/docs/42-traits.md b/docs/42-traits.md index f1ca6f5a6..9b815db69 100644 --- a/docs/42-traits.md +++ b/docs/42-traits.md @@ -91,7 +91,7 @@ The two haplotypes a person's genotypes make are looked up by their alleles, and - **`needs_phase no`** (APOE): the combination line is the one listing every pair the genotypes allow. Heterozygous at both APOE sites allows ε2/ε4 and ε1/ε3; that line reads ε2/ε4 and says it could also be the rare ε1/ε3. - **`needs_phase yes`** (TAS2R38): with one heterozygous site or none, the two haplotypes follow from the genotypes. With two or more, WhatsHap phases them from the reads: a read, or the two reads of a pair, that covers two of the sites shows which alleles sit on one copy. When every heterozygous site of the rule lands in one phase block the rule is read (`phase` read_backed); otherwise it is `unphased`, not read. The three TAS2R38 sites lie within 741 bases, so a read pair can span them, but not always. -`status` of a rule line is `called`, `not_covered` (a site has no genotype), `other_allele` (an allele the rule does not describe), `other_haplotype` (a copy that is none of the rule's haplotypes; it names the alleles) or `unphased`. `phase` is `not_used` (needs_phase no), `not_needed`, `read_backed` or `unphased`. +`status` of a rule line is `called`, `not_covered` (a site has no genotype), `low_gq` (a site's call has a GQ below 20 or a caller filter: not read, no result), `other_allele` (an allele the rule does not describe), `other_haplotype` (a copy that is none of the rule's haplotypes; it names the alleles) or `unphased`. `phase` is `not_used` (needs_phase no), `not_needed`, `read_backed` or `unphased`. The rules today: @@ -108,7 +108,7 @@ The rules today: The six rules after TAS2R38 need no phase. Their sites lie close together on one copy (the two APOL1 G1 changes were in perfect linkage disequilibrium in Genovese et al. 2010; the 9p21 and 17q21.31 tags each mark one haplotype), or, for HFE, a double heterozygote reads as one change on each copy. A copy the rule does not describe, such as a 9p21 copy with only one of the two risk alleles, is `other_haplotype`, not read. What each rule leaves out, said in its note: -- APOL1 G2 (rs71785313) is a 6-base deletion. The rule table reads single-base sites only, so the rule itself reads G1, and a G2 copy reads as G0 there. The HTML and text reports read G2 beside the rule (`og summary collect`, the `apol1` module of `rust/crates/og-summary/src/collect/group_c/traits.rs`) from the sample's gVCF, or its VCF when there is no gVCF, and show the G1 plus G2 result under "With G2": G1/G2, G2/G2 and G1/G1 are the high-risk genotype. Two G2 copies read G2/G2 even when the G1 sites were not read. A VCF without reference blocks cannot tell a reference base from a position with no reads, so there a result with no G2 copy is marked `(no G2 call)`. The Nextflow report gets the sample's gVCF too (DeepVariant's, or the samplesheet's `gvcf` column), so both runs read the gVCF when there is one. +- APOL1 G2 (rs71785313) is a 6-base deletion. The rule table reads single-base sites only, so the rule itself reads G1, and a G2 copy reads as G0 there. The HTML and text reports read G2 beside the rule (`og summary collect`, the `apol1` module of `rust/crates/og-summary/src/collect/group_c/traits.rs`) from the sample's gVCF, or its VCF when there is no gVCF, and show the G1 plus G2 result under "With G2": G1/G2, G2/G2 and G1/G1 are the high-risk genotype. Two G2 copies read G2/G2 even when the G1 sites were not read. A VCF without reference blocks cannot tell a reference base from a position with no reads, so there a result with no G2 copy is marked `(no G2 call)`. The G2 reading uses the same floor as the trait sites: a call over the deletion span with a GQ below 20, a G2 call whose FILTER is not `PASS` or `.`, or a reference call whose FILTER is not `PASS`, `.` or `RefCall`, leaves G2 not read, so two G2 copies at GQ 5 never read G2/G2. The Nextflow report gets the sample's gVCF too (DeepVariant's, or the samplesheet's `gvcf` column), so both runs read the gVCF when there is one. - SERPINA1: M is any copy with neither S nor Z; rarer deficiency and null alleles are not read. A serum level confirms a deficiency. - HFE: the C282Y homozygote is also a secondary finding (`assets/acmg_sf.tsv`, [step 23](23-clinical-filter.md)); the rule gives the reading of every genotype. - GC: the protein levels are those of Powe et al. (2013), whose monoclonal assay was questioned in later letters. A protein type, not a vitamin D deficiency. diff --git a/rust/crates/og-summary/src/collect/group_c/traits.rs b/rust/crates/og-summary/src/collect/group_c/traits.rs index d051b1dcc..931cba6c9 100644 --- a/rust/crates/og-summary/src/collect/group_c/traits.rs +++ b/rust/crates/og-summary/src/collect/group_c/traits.rs @@ -221,6 +221,17 @@ pub mod apol1 { && record_end(r).is_some_and(|e| e >= SPAN.0) } + /// A call below this GQ is not read (decision D10, the floor of step + /// 42's trait sites). A missing GQ stays readable. + const MIN_GQ: f64 = 20.0; + + /// The GQ of a call when it is below [`MIN_GQ`]. + fn weak_gq(gq: &str) -> Option<&str> { + gq.parse::() + .is_ok_and(|q| q.is_finite() && q < MIN_GQ) + .then_some(gq) + } + fn symbolic(a: &str) -> bool { a == "*" || a == "." || (a.len() >= 2 && a.starts_with('<') && a.ends_with('>')) } @@ -268,18 +279,30 @@ pub mod apol1 { )); continue; } - if idx.iter().all(|i| *i == 0) { - covered.push((pos, end)); - gqs.push(field("GQ")); - continue; - } - if r[6] != "PASS" && r[6] != "." { + let reference = idx.iter().all(|i| *i == 0); + // A reference call may carry DeepVariant's RefCall; a G2 call must pass. + let readable = + ["PASS", "."].contains(&r[6].as_str()) || (reference && r[6] == "RefCall"); + if !readable { problems.push(format!( "a call at {}:{pos} that failed the caller's filter ({})", r[0], r[6] )); continue; } + let gq = field("GQ"); + if let Some(q) = weak_gq(&gq) { + problems.push(format!( + "a call at {}:{pos} with genotype quality {q}, below 20", + r[0] + )); + continue; + } + if reference { + covered.push((pos, end)); + gqs.push(gq); + continue; + } let mut n = 0; let mut stopped = false; for &i in &idx { @@ -324,7 +347,7 @@ pub mod apol1 { } copies += n; covered.push((pos, end)); - gqs.push(field("GQ")); + gqs.push(gq); } let gq = if gqs.is_empty() { ".".to_owned() @@ -740,6 +763,46 @@ pub mod apol1 { } } + /// Decision D10: a call below GQ 20, or a reference call the caller + /// filtered, is not read, so it gives no G2 count; GQ 20 and a + /// missing GQ are read. + #[test] + fn low_quality_calls_over_the_span_are_not_read() { + let gq = |mut r: Vec, q: &str| { + let gt = r[9].split(':').next().unwrap_or(".").to_owned(); + r[9] = format!("{gt}:{q}"); + r + }; + for q in ["0", "5", "19", "19.5"] { + let g = read_g2(&[gq(rec(POS, REF, ALT, "1/1", "."), q)], false); + assert_eq!((g.status, g.copies), ("not_read", None), "G2 1/1 at GQ {q}"); + assert!(g.text.contains("below 20"), "{g:?}"); + let g = read_g2(&[gq(block(36_266_100), q)], true); + assert_eq!((g.status, g.copies), ("not_read", None), "block at GQ {q}"); + } + let mut lowqual = block(36_266_100); + lowqual[6] = "LowQual".into(); + let g = read_g2(&[lowqual], true); + assert_eq!((g.status, g.copies), ("not_read", None), "{g:?}"); + // Controls: GQ 20, a missing GQ and a RefCall block stay readable. + for q in ["20", "."] { + let g = read_g2(&[gq(rec(POS, REF, ALT, "1/1", "."), q)], false); + assert_eq!( + (g.status, g.copies), + ("called", Some(2)), + "G2 1/1 at GQ {q}" + ); + let g = read_g2(&[gq(block(36_266_100), q)], true); + assert_eq!((g.status, g.copies), ("called", Some(0)), "block at GQ {q}"); + } + let mut refcall = block(36_266_100); + refcall[6] = "RefCall".into(); + assert_eq!(read_g2(&[refcall], true).copies, Some(0)); + // Two G2 copies at GQ 5 no longer read G2/G2 when G1 is not read. + let weak = read_g2(&[gq(rec(POS, REF, ALT, "1/1", "."), "5")], false); + assert_eq!(combine(".", "low_gq", &weak).0, "."); + } + #[test] fn records_outside_the_span_are_skipped() { let mut chr21 = rec(POS, REF, ALT, "1/1", "."); From 2da15c10884310452f507f84839e1e05640547a6 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:45:28 +0100 Subject: [PATCH 248/255] test(e2e): a missing GQ stays readable in the traits-panel-1 oracle The case's own expectation took a GQ of . as below 20, while step 42 reads it. The header comments now name . as a readable FILTER. Bead: D10 --- scripts/42-traits.sh | 2 +- tests/e2e/traits-panel-1.sh | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/42-traits.sh b/scripts/42-traits.sh index 68dbc3dc4..82af25118 100755 --- a/scripts/42-traits.sh +++ b/scripts/42-traits.sh @@ -22,7 +22,7 @@ # # Output: traits/_traits.tsv (rsid, gene, chrom, pos, trait, genotype, # status, reading, ancestry, note, pmids). status: called, not_covered, -# low_gq (GQ below 20, or a FILTER other than PASS or RefCall: not read), or +# low_gq (GQ below 20, or a FILTER other than PASS, . or RefCall: not read), or # other_allele (an allele the table does not describe). # traits/_genotype_rules.tsv: one line per rule (docs/42-traits.md). set -euo pipefail diff --git a/tests/e2e/traits-panel-1.sh b/tests/e2e/traits-panel-1.sh index 81be07124..542cedb58 100644 --- a/tests/e2e/traits-panel-1.sh +++ b/tests/e2e/traits-panel-1.sh @@ -9,7 +9,7 @@ # - scripts/42-traits.sh: one row per table row plus the APOE line; the # table positions inside the fixture's slices carry the genotype of the # GIAB v4.2.1 truth and are read (called), or low_gq when the gVCF's GQ -# there is below 20 or its FILTER is not PASS or RefCall; every position +# there is below 20 or its FILTER is not PASS, . or RefCall; every position # outside them is not_covered (no reads, never taken for the reference); # - a planted alternate allele: a copy of the gVCF with CYP2A6*2 (rs1801272) # homozygous gives that row the table's hom_alt reading, not HG002's; @@ -68,13 +68,14 @@ check_ge "table positions inside the fixture's slices" "$IN" 2 status_of() { awk -F'\t' -v r="$1" '$1 == r { print $7 }' "$OUT"; } geno_of() { awk -F'\t' -v r="$1" '$1 == r { print $6 }' "$OUT"; } # What step 42 should make of the gVCF's call at a position: low_gq below GQ -# 20 or with a FILTER other than PASS or RefCall, else called. +# 20 or with a FILTER other than PASS, . or RefCall, else called. A missing +# GQ (.) stays readable. GENOS="${G}/${SAMPLE}/traits/${SAMPLE}_traits_genotypes.tsv" check "the genotypes table has the GQ and FILTER columns" \ test "$(awk -F'\t' 'NF != 7' "$GENOS" | wc -l | tr -d ' ')" -eq 0 -a -s "$GENOS" want_status() { awk -F'\t' -v c="$1" -v p="$2" '$1 == c && $2 == p && length($4) <= 1 { - weak = ($7 != "." && $7 != "PASS" && $7 != "RefCall") || ($6 ~ /^[0-9.]+$/ && $6 + 0 < 20) + weak = ($7 != "." && $7 != "PASS" && $7 != "RefCall") || ($6 ~ /^[0-9]+(\.[0-9]+)?$/ && $6 + 0 < 20) print (weak ? "low_gq" : "called") " (GQ " $6 ", FILTER " $7 ")"; exit }' "$GENOS" } OUTSIDE_NC=0 OUTSIDE=0 From b8fc06ac0c5de607273fa014afafc03e51c58a77 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:09:58 +0100 Subject: [PATCH 249/255] fix(og-dev): list every fixture kind in the usage of an unknown kind An unknown or mistyped kind printed only the comphet and trim-pharmcat lines, so the eight e2e kinds were missing from the error a user sees. The usage error now prints the same kinds as --help, and a unit test checks every kind is named. --- rust/crates/og-dev/src/fixture.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/rust/crates/og-dev/src/fixture.rs b/rust/crates/og-dev/src/fixture.rs index 9db7a3a1c..5af801391 100644 --- a/rust/crates/og-dev/src/fixture.rs +++ b/rust/crates/og-dev/src/fixture.rs @@ -30,7 +30,7 @@ const USAGE: &str = "usage: og dev fixture comphet OUT_DIR [--swap]\n og d /// Runs `og dev fixture ARGS`; returns the exit code. pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { let usage = |err: &mut dyn Write, msg: &str| { - let _ = writeln!(err, "{USAGE}og dev fixture: error: {msg}"); + let _ = writeln!(err, "{USAGE}{}og dev fixture: error: {msg}", e2e::USAGE); 2 }; let result = match args.first().map(String::as_str) { @@ -644,6 +644,31 @@ mod tests { fs::remove_dir_all(d).unwrap(); } + #[test] + fn an_unknown_kind_lists_every_kind_in_its_usage() { + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&["bogus".into()], &mut out, &mut err), 2); + let err = String::from_utf8(err).unwrap(); + assert!(err.contains("unknown fixture 'bogus'"), "{err}"); + for kind in [ + "comphet", + "trim-pharmcat", + "append-n", + "rfc1", + "str-extra", + "tas2r38", + "mito-sam", + "hba-tiles", + "cut-haplotype", + "contains", + ] { + assert!( + err.contains(&format!("og dev fixture {kind} ")), + "{kind}: {err}" + ); + } + } + #[test] fn annotation_slices_clamp_extreme_counts_without_narrowing() { let items = vec![Value::Null; 3]; From 18a4408591e40017df8cf34759459dbd6030c255 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:45:51 +0100 Subject: [PATCH 250/255] fix(live): let the gate's answer alone decide whether a page keeps results closed An adult whose family.json had not answered or failed got a page with results marked opened and a close button that did nothing. person.js now asks only the gate (state.json's child rule). A person closed as another family's child gets their buttons under that family's members, not in its chapter of children. --- assets/live/family-page.js | 12 ++++++------ assets/live/person.js | 16 ++++++---------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/assets/live/family-page.js b/assets/live/family-page.js index a3a8072bb..b02455086 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -152,6 +152,12 @@ add(ul, li); }); add(s, ul); + // A person whose report is closed and whom this family lists as a parent or partner (a child in another family) + // gets their buttons here, after the members: this family's chapter of children does not name them. + (f.gated || []).forEach(function (name) { + if (f.state.children.some(function (c) { return c.child === name; })) { return; } + add(s, el('h3', {}, name), U.familyControls(name, fetchFamily)); + }); return s; } @@ -405,12 +411,6 @@ } add(s, d); }); - // A person whose report is closed and whom this family lists as a parent or partner (a child in another family) - // gets their buttons here too, after the children. - (f.gated || []).forEach(function (name) { - if (f.state.children.some(function (c) { return c.child === name; })) { return; } - add(s, el('h3', {}, name), U.familyControls(name, fetchFamily)); - }); return s; } diff --git a/assets/live/person.js b/assets/live/person.js index 252563771..b0b8fe956 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -104,15 +104,11 @@ var ms = (family && family.data.state && family.data.state.members) || []; return ms.filter(function (m) { return m.sample === personName(); })[0] || null; } - // A child of the family: a member whose father or mother is listed. - function isChild() { var m = me(); return !!(m && (m.father || m.mother)); } - // A page whose results to act on were opened on purpose (core.js releases a child's report, or a person whose - // family roles are not known, only once each is opened): the map marks them opened, with a chip to close them. - // The gate's own answer counts too: a child in any family, by the server's one rule, not only in the first. - function closedPage() { - return isChild() || familyStatus === 'pending' || familyStatus === 'failed' || (familyStatus === 'known' && !me()) || - !!(U.gateChild && U.gateChild()); - } + // A page whose results to act on were opened on purpose: core.js releases a child's report, by the server's one rule + // (state.json's family plugin: a child in any family, everyone while a family's members cannot be read), only once + // each is opened, and the map marks them opened, with a chip to close them. That answer alone decides: an adult's + // page is the page without the gate whatever family.json answers, or before it does. + function closedPage() { return !!(U.gateChild && U.gateChild()); } function memberUrl(s) { var r = family && family.data.runs && family.data.runs[s]; return r && r.url ? r.url : null; @@ -326,7 +322,7 @@ // This renderer receives a child's report only once the shared data gate (core.js) has released it, with every // result to act on opened, so it closes no card row itself. The charts (plugins/charts.js) and the search box // (plugins/search.js) still ask it, as before the gate, and on a page that keeps results closed until opened (a - // child's, or one whose family roles are not known) get the same answers: before results.json has answered, + // child's, by the gate's rule) get the same answers: before results.json has answered, // every mark and hit is left out; then only a tag (results.veil) that names no finding of these results, since every // result to act on here is open. Asked with no key, the search hears that the page keeps results closed, and says so. function tagClosed(id) { return closedPage() && !findingById(id); } From e2f411094713eefd80be00b0a11bf8755987d275 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:45:52 +0100 Subject: [PATCH 251/255] test(live): check an adult's page when family.json fails, a late copied answer, and where a partner's buttons sit --- tests/test_owngenome_live_family.py | 100 ++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/tests/test_owngenome_live_family.py b/tests/test_owngenome_live_family.py index 72b22f7b6..3c8522fe8 100755 --- a/tests/test_owngenome_live_family.py +++ b/tests/test_owngenome_live_family.py @@ -39,14 +39,17 @@ holds the child's results; and an adult's and a no-act child's pages make the requests of the page without the gate, in its order, the event stream included, and are the same page (the genome map drawn with OG); an adult's and a no-act child's page at a section address give the outcome main's view files give in each order - (marks kept when the family answers first, cleared when the results do, the map's tooltip hidden="hidden"); + (marks kept when the family answers first, cleared when the results do, the map's tooltip hidden="hidden"), also + when a copied answer read as JSON comes late; an adult with a result to act on whose family.json answers 502 or + drops the connection gets the page it gets when the family answers, nothing closed and nothing left out of search; each member run of family.json says whether it is a child by the one rule; a no-act child with a stale summary.json keeps its numbers on the home page; a family file that cannot be read gives a fixed 500, and while it cannot be read the people that family lists are closed alike on the server, in state.json, on the finding route, on their page and home tile, and reopen once it reads, all with no family.json request; on the family page a gated child reads "Report closed" and, once opened there, the count summary.json gives; an adult's member line is as without the gate; a child of one family is closed on another family's page where - it is a partner, keeps that role there, and stays a child when that family sorts first. + it is a partner, keeps that role there with its buttons under the members, and stays a child when that family + sorts first. Checks 2 and 4 are run again on a copy of the fixture with two parents swapped and must fail there: a check that cannot fail is no check. @@ -702,6 +705,16 @@ def run_(req, m): return keep +SLOW_CLONE = """(function () { + var clone = Response.prototype.clone; + Response.prototype.clone = function () { + var copy = clone.call(this), json = copy.json.bind(copy); + copy.json = function () { return json().then(function (v) { return new Promise(function (done) { setTimeout(function () { done(v); }, 1000); }); }); }; + return copy; + }; +}())""" + + def live_browser_checks(live, port, work, run, real): """In live mode (state.json, then the event stream): an adult's and a no-act child's page make the requests of the page without the gate, in its order; a parent's page is the same whichever card holds a child's results.""" @@ -782,6 +795,10 @@ def capture(route, text): # stays hidden="hidden". So a change to a shared view file shows here, not only a change the gate makes. live.summary = lambda name, d, st: NO_ACT if name == "father_a" else adult(name, d, st) marks = [] + # A copied answer read as JSON (Response.clone().json()) resolves 1 s late, after the family's answer held back + # 0.5 s: a gate that checked a child's results on such a copy, not on the text it read, would hand them to the + # view after the family's answer, and the results-first order would give the family-first outcome. + slow = b.call("Page.addScriptToEvaluateOnNewDocument", {"source": SLOW_CLONE}, b.session)["identifier"] for first_, held in (("results.json", "/family.json"), ("family.json", "/results.json")): keep_routes = slow_routes(live, {held: 0.5, "/map.json": 1.0}) try: @@ -796,10 +813,12 @@ def capture(route, text): marks.append((first_, route, got, got == want)) finally: live.app.routes[:] = keep_routes + b.call("Page.removeScriptToEvaluateOnNewDocument", {"identifier": slow}, b.session) live.summary = adult check("live mode: an adult's and a no-act child's page at a section address give main's outcome in each order: " "the marks stay when the family answers first, clear when the results do, and the map's tooltip stays " "hidden=\"hidden\"", all(m[3] for m in marks), [m for m in marks if not m[3]] or marks[:2]) + family_fetch_failure_checks(b, base, live, source, adult) # A parent's page, whichever card holds the child's two results to act on. # The map's document is held back, as above, so the tooltip's end state does not depend on the race. seen = [] @@ -825,6 +844,77 @@ def capture(route, text): print(" live gate browser checks: %.2f s" % (demo.time.time() - t0)) +def failing_routes(live, suffix, how): + """The server's routes, with every answer whose path ends with SUFFIX replaced: a 502 when HOW is "502", else a + connection dropped with no answer. Returns the routes to put back.""" + import urllib.parse + keep = list(live.app.routes) + + def wrap(fn): + def run_(req, m): + if not urllib.parse.urlsplit(req.path).path.endswith(suffix): + return fn(req, m) + if how == "502": + return req.send_body(502, "text/plain; charset=utf-8", b"bad gateway\n") + req.close_connection = True + return None + return run_ + live.app.routes[:] = [(a, rx, wrap(fn), auth) for a, rx, fn, auth in keep] + return keep + + +# What a page shows of its results to act on beyond the DOM: whether the search leaves them out, and the +# map's chips for them (closed ones, or ones opened with a button to close them again). +ACT_JS = ("[String(PGP.searchHidden(null)), Array.prototype.map.call(document.querySelectorAll('.gm-chip'), function (c) { return c.textContent; }).join(' | ')]") + + +def without_family(page): + """A person's page with the parts family.json draws taken out: the kicker's family name, the role in the family and + the rail's family group.""" + page = re.sub(r'(
    [^<]*?)(?: · )?(?:Family [^<]*)?<', r'\1<', page) + page = re.sub(r'(?:(?:son|daughter|child|father|mother|parent|partner) of )[^<]*', '', page) + return re.sub(r'
    The family
    .*?
    ', '', page, flags=re.S) + + +def family_fetch_failure_checks(b, base, live, source, adult): + """An adult with results to act on whose family.json fails (a 502 or a dropped connection) gets the page it gets when + the family answers, apart from the family's own parts (the rail's link and the role): the gate's answer from + state.json alone decides whether a page keeps results closed, so nothing is closed, nothing is marked opened with a + button that cannot close it, and the search leaves nothing out. The map's document comes last, as above.""" + live.summary = lambda name, d, st: ACT_SUMMARY if name == "gf_a" else adult(name, d, st) + pages = {} + try: + for how in ("ok", "502", "drop"): + keep_routes = failing_routes(live, "/family.json", how) if how != "ok" else list(live.app.routes) + slow_routes(live, {"/map.json": 0.5}) + try: + with open(os.path.join(live.cfg.assets, "core.js"), "w") as f: + f.write(source) + b.go("about:blank") + assert b.wait("location.href === 'about:blank'") + b.go(base + "/runs/gf_a/") + assert b.wait("document.documentElement.dataset.pgpStatus === 'ready' && " + "!!document.querySelector('#results .card')"), how + ": gf_a never drawn" + if how == "ok": + assert b.wait("!!document.querySelector('#rail a[href*=family_a]')"), "family never drawn" + b.wait("/^(drawn|failed|nodecompress|unavailable)$/.test((document.getElementById('genome-map') || " + "{dataset: {}}).dataset.pgpMap || '')", 15) + time.sleep(1.0) + page, acts = b.eval("[%s, %s]" % (PAGE_JS, ACT_JS)) + pages[how] = [without_family(page), acts, page != without_family(page)] + finally: + live.app.routes[:] = keep_routes + finally: + live.summary = adult + for how in ("502", "drop"): + got, want = pages[how], pages["ok"] + check("live mode: an adult with results to act on whose family.json %s gets the page it gets when the family " + "answers: no result closed or marked opened, the search leaves nothing out" + % ("answers 502" if how == "502" else "drops the connection"), + got[:2] == want[:2] and want[2] and want[1][0] == "false" and "Opened" not in want[1][1], + (got[1], want[1], differences(got[0], want[0], 12))) + + def family_gate_checks(b, base, live, run): """The family page: a gated child's member line is the fixed closed line, never "Nothing flagged"; once the child's results are opened there, the line and the pedigree's mark give the count family.json gives without the gate (the @@ -882,9 +972,11 @@ def look(who): got = b.wait("document.documentElement.dataset.pgpStatus === 'ready' && !!document.querySelector('.person-card') && " "[document.querySelector('[data-gate-child=father_a]') ? 'closed' : 'open', " "/One child|children:/.test(document.getElementById('family-lead').textContent) ? 'child counted' : 'no child', " - "/father_a and zara are not screened as a couple yet/.test(document.body.textContent) ? 'couple line' : 'no couple line'].join()", 15) + "/father_a and zara are not screened as a couple yet/.test(document.body.textContent) ? 'couple line' : 'no couple line', " + "((document.querySelector('[data-gate-child=father_a]') || document.body).parentElement.closest('section') || {}).id].join()", 15) check("family page: a person who is a child in another family is closed on this family's page too, keeps the " - "partner role (no child counted) and the couple's line", got == "closed,no child,couple line", got) + "partner role (no child counted) and the couple's line, and has its buttons under the members, not in the " + "chapter of this family's children", got == "closed,no child,couple line,ch-members", got) first = os.path.join(live.root, "family_0") os.rename(z, first) z = first From 50a7dbd5380aeadda43077affe06b0da5bb9bc9d Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:45:52 +0100 Subject: [PATCH 252/255] docs(live): say the gate's answer alone closes a page, name the reads iframe, and fix the producer comments --- CHANGELOG.md | 2 +- bin/live_results.py | 18 +++++++++++------- docs/live-site.md | 10 +++++----- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1fe2971df..c3aadf98a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -211,7 +211,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Each output is written whole or not at all: a run that stops leaves no part-written table behind. A read-only `family_plan.tsv` is replaced; a folder that cannot be written still stops the plan. - Step 38, step 39 with a families file, `run-all.sh` with `FAMILIES` and the family workflow now need `og` (`scripts/setup.sh --og`). -- A child's live report now has one gate where the page reads its data, instead of a veil on each place that could name a result. While any of a child's results to act on is unopened, the page shows a fixed explanation, the number of such results, one "Show the result" button each, and the own record of each result opened on purpose; nothing else of the child's report reaches the page, and the page is the same whichever result is closed. Which result a button opens is drawn at random on each load, and the address names buttons (`#f/closed-1`), never a result. The server answers every finding address of such a child with the same page. Opening the last result shows the full report with its own addresses. Closing one again closes the report and that result, and the others stay opened; a reload or Back closes them all. The cost: such a child sees none of the rest of their report until every result to act on is opened. The home page shows a gated child's tile with fixed text, the count and the run's progress; the family page shows the child's buttons and leaves out the child's rows, stretches, couple screen and cells (the side by side reads "closed"), while the parents' values stay. Adults and children with no result to act on see the same pages as before, with the same requests in the same order. Two things differ for them. Another member's page, a parent's for example, no longer shows the flagged-card count of a child whose report is closed, because that count changes with which cards the unopened results flag; the family page reads "Report closed" for that child instead, and shows the count `family.json` gave before once it opens the child's results. And the home tile of a child with no result to act on now, whose `summary.json` is older than a finished step and still names one, counts from the summary its `results.json` reads. A person any family lists as a child is closed on their page, the home page and every family page that lists them, whatever the other families say; a family where they are a partner keeps that role. One deliberate difference shows only while a file of a family folder cannot be read: the people that family may list are treated as children, closed on their pages, the home page, the family page and the finding addresses, until the file reads again (everyone while its members cannot be read, its own members while another of its files cannot be read). The server works this out from the files on each request. APOE keeps its own warning. These outputs change, and no field is renamed or removed: `family.json` gives `attention: null` for such a child and a new `child` field for each member run; `family.json` answers 500 with a fixed line when a family file cannot be read, where it dropped the connection; `/runs.json` counts a child's results from the summary its `results.json` reads when either summary has a result to act on, also while `summary.json` is older than a finished step, and gives each family `unreadable`; `state.json`'s family plugin adds `child` for a member; and `results.json` adds `saved_attention` for such a child, the flagged-card count of `summary.json`. `results.json` keeps its `veil` field, which the gate does not read. The gate guards the page in the browser: the static files still reach a browser that asks for them. +- A child's live report now has one gate where the page reads its data, instead of a veil on each place that could name a result. While any of a child's results to act on is unopened, the page shows a fixed explanation, the number of such results, one "Show the result" button each, and the own record of each result opened on purpose; nothing else of the child's report reaches the page, and the page is the same whichever result is closed. Which result a button opens is drawn at random on each load, and the address names buttons (`#f/closed-1`), never a result. The server answers every finding address of such a child with the same page. Opening the last result shows the full report with its own addresses. Closing one again closes the report and that result, and the others stay opened; a reload or Back closes them all. The cost: such a child sees none of the rest of their report until every result to act on is opened. The home page shows a gated child's tile with fixed text, the count and the run's progress; the family page shows the child's buttons and leaves out the child's rows, stretches, couple screen and cells (the side by side reads "closed"), while the parents' values stay. Adults and children with no result to act on see the same pages as before, with the same requests in the same order. Two things differ for them. Another member's page, a parent's for example, no longer shows the flagged-card count of a child whose report is closed, because that count changes with which cards the unopened results flag; the family page reads "Report closed" for that child instead, and shows the count `family.json` gave before once it opens the child's results. And the home tile of a child with no result to act on now, whose `summary.json` is older than a finished step and still names one, counts from the summary its `results.json` reads. A person any family lists as a child is closed on their page, the home page and every family page that lists them, whatever the other families say; a family where they are a partner keeps that role, with their buttons under its members. Whether a page keeps results closed now comes from the server's answer in `state.json` alone: an adult whose family's `family.json` has not answered yet, or fails, gets their normal page, where before such a page kept their results to act on closed until opened. One deliberate difference shows only while a file of a family folder cannot be read: the people that family may list are treated as children, closed on their pages, the home page, the family page and the finding addresses, until the file reads again (everyone while its members cannot be read, or while its members file names no one; its own members while another of its files cannot be read). The server works this out from the files on each request. APOE keeps its own warning. These outputs change, and no field is renamed or removed: `family.json` gives `attention: null` for such a child and a new `child` field for each member run; `family.json` answers 500 with a fixed line when a family file cannot be read, where it dropped the connection; `/runs.json` counts a child's results from the summary its `results.json` reads when either summary has a result to act on, also while `summary.json` is older than a finished step, and gives each family `unreadable`; `state.json`'s family plugin adds `child` for a member; and `results.json` adds `saved_attention` for such a child, the flagged-card count of `summary.json`. `results.json` keeps its `veil` field, which the gate does not read. The gate guards the page in the browser: the static files still reach a browser that asks for them. - Step 40, the 23andMe and AncestryDNA raw-data files, now runs on the Rust `og` binary: `og consumer export`, `og consumer build-sites` and `og consumer check` replace `bin/export_consumer.py`. On a synthetic gVCF of 552 MB (32 million records) with 617,044 sites it takes about 2 s and 43 MB in a Linux container on 4 cores, where the Python took 30 s and 660 MB; the files hold the same rows. `setup.sh --consumer-sites` reads dbSNP on 4 cores without the grep pass. The zips are now the same bytes for the same day (`--date`, else `SOURCE_DATE_EPOCH`, else today). Fixes: a call of three or more alleles stops the export with the site, instead of a file the strict reader refuses; two dbSNP records of one rsID at one position have their ALT alleles joined before the strand test; a bad site row or an unreadable zip is reported with the file and line instead of a traceback; a good export removes the `.txt` files a failed one left. The musl builds of `og` use the mimalloc allocator, which keeps a long multi-threaded read at its live memory. Step 40 needs `og`: `scripts/setup.sh ` builds it, and `scripts/setup.sh --og` builds only it. diff --git a/bin/live_results.py b/bin/live_results.py index e9e1af597..975b8a6ff 100644 --- a/bin/live_results.py +++ b/bin/live_results.py @@ -22,7 +22,9 @@ Beside the cards, "veil" tags the rows that name a result to act on outside its own card (every card's table rows and lines, the viewer links, the charts' marks, the other findings and the lead's names) with -that result's id, so a child's page closes them with it (veil() below). +that result's id (veil() below). A child's page no longer closes them one by one: one gate in +assets/live/core.js holds the whole report while a result to act on is unopened, and of "veil" the page +still reads only the charts' marks. Nothing here reads a file or computes anything about genetics; the adapters pick fields the summary already holds. A section id with no adapter falls back to its @@ -793,8 +795,9 @@ class _Row: cannot read (a field of the wrong type, a stars value that is not a number) costs that row only: it becomes a finding on its card that says the row could not be read, and every other row and card still renders. Its words name no gene or variant: the row is unreadable. Its level is the highest the rule - can give (level "act" for a rule that can give one): a child's page veils a card that holds a finding - to act on, and the card's table may still draw the row the rule could not read.""" + can give (level "act" for a rule that can give one): a child's report stays closed while a finding to act + on is unopened (the gate in assets/live/core.js), and the card's table may still draw the row the rule could + not read.""" def __init__(self, out, card_id, level="review"): self.out, self.card_id, self.level = out, card_id, level @@ -1137,10 +1140,11 @@ def lead(found, levels, summary): # --- the veil's tags ----------------------------------------------------------------------------- -# A child's page keeps each result to act on closed (assets/live/person.js veiled()). The page closes a -# finding's own card with it, but other places can name the same gene or variant: any card's table and -# lines, the charts' marks and their titles, another finding (a symptom ranking), the lead, the search index. Each such row carries the id of the -# result it belongs to, so the page closes it with that result and opens it with it. The rule: a row +# Places outside the own card of a result to act on can name the same gene or variant: any card's table and +# lines, the charts' marks and their titles, another finding (a symptom ranking), the lead, the search index. +# Each such row carries the id of the result it belongs to. A child's page no longer closes them one by one: +# one gate (assets/live/core.js) holds the whole report while a result to act on is unopened, and of these +# tags the page still reads only the charts' marks (veil() below). The rule: a row # belongs to a closed result when it names the result's gene, or its position falls inside the result's # position or span; a row that is the result's own position or span belongs to that result first, else # the first such result in finding order. The veil lives in the browser: the files still hold every row. diff --git a/docs/live-site.md b/docs/live-site.md index f6e6c7740..d49ac0161 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -62,7 +62,7 @@ A stopped run's page says so under the name and in the summary line, with the da While the run goes, the page still opens on the report being written. A short band under the name says how far the run is ("23 of 27 are done, about 8–13 hours remain"), with the steps done as a number and a bar, the time elapsed, the time to go from the usual step times and the retries, and links to the step timeline. Each section and each line of the contents says whether it is written, being written or waiting. The run itself, below, closes the page after section 10, folded until opened (or reached through `#progress`, the band or the contents); a page with no results yet keeps it open. -**A child's report gate.** A child's report stays closed while any of the child's results to act on in adult life (the findings of level `act`) is unopened. A child is a person any family lists with a father or a mother, whatever the other families that list them say. While a family folder cannot be read, a person it might list is treated as a child too, so the gate fails closed: everyone while its members cannot be read, and the people it lists while another of its files cannot be read (then `family.json` answers 500 with a fixed line). The server works this out from the files on each request, so once the files read again the people open again, with no other request needed. A person whose state cannot be read is treated as a child, and a child whose results cannot be read stays closed too. The server gives the answer to the person page in `state.json` (the family plugin's `child`, or `unknown` for a person no readable family lists), and the home page, the family page and the finding addresses decide by the same rule. While the gate is shut, the page shows only: +**A child's report gate.** A child's report stays closed while any of the child's results to act on in adult life (the findings of level `act`) is unopened. A child is a person any family lists with a father or a mother, whatever the other families that list them say. While a family folder cannot be read, a person it might list is treated as a child too, so the gate fails closed: everyone while its members cannot be read (a members file that names no one counts as one that cannot be read), and the people it lists while another of its files cannot be read (then `family.json` answers 500 with a fixed line). The server works this out from the files on each request, so once the files read again the people open again, with no other request needed. A person whose state cannot be read is treated as a child, and a child whose results cannot be read stays closed too. The server gives the answer to the person page in `state.json` (the family plugin's `child`, or `unknown` for a person no readable family lists), and the home page, the family page and the finding addresses decide by the same rule. That answer alone decides whether a page keeps results closed: an adult's page is their normal page even while `family.json` has not answered or fails. While the gate is shut, the page shows only: - a fixed explanation and the number of results to act on ("1 result to act on", "2 results to act on"); - one "Show the result" button per closed result, all in one block; @@ -80,9 +80,9 @@ One count is left out for a child whose report is closed. Other members' pages, The home page decides from `/runs.json` alone. A gated child is a child (the rule above, read from the families of `/runs.json`) whose entry counts or names a result to act on. The server counts a child's results from the summary its `results.json` reads. While `summary.json` is older than a finished step, that is the step files read again, so the home page closes a child whenever the child's page does. A child with no result to act on in either summary keeps the numbers of `summary.json`, as before. Its tile and attention rows show the fixed text and the count of results to act on, and link to the run's page. The run's progress (its state, the steps done, planned, running and failed, and its times) stays, because it says how far the run is, never what it found. -On the family page, chapter IV shows a gated child's buttons only, and each child opens on its own. A gated person whom this family lists as a parent or partner, because another family lists them as a child, gets their buttons in chapter IV too, after the children. The family view gets the family without that child's report data: no de novo or compound-het rows, stretches, trio status or trio files of the child, no relationship or shared-segment row with the child, and no couple screen of a couple the child is part of (the couple keeps its names and whether it was screened; a screened couple's line says its screen stays closed). The side by side keeps a row only when an open member has a value in it, in a fixed order, and the child's cell reads "closed". Plan rows that name the child are left out. The parents' other plan rows and the couple reports of couples with no gated partner stay. Other reports of the family folder are left out while a child is gated, because they can hold the child's data. A member with no run here has no report, so the gate does not close anything of theirs. The pedigree and the members' progress stay. +On the family page, chapter IV shows a gated child's buttons only, and each child opens on its own. A gated person whom this family lists as a parent or partner, because another family lists them as a child, gets their buttons under the members, after the member list. The family view gets the family without that child's report data: no de novo or compound-het rows, stretches, trio status or trio files of the child, no relationship or shared-segment row with the child, and no couple screen of a couple the child is part of (the couple keeps its names and whether it was screened; a screened couple's line says its screen stays closed). The side by side keeps a row only when an open member has a value in it, in a fixed order, and the child's cell reads "closed". Plan rows that name the child are left out. The parents' other plan rows and the couple reports of couples with no gated partner stay. Other reports of the family folder are left out while a child is gated, because they can hold the child's data. A member with no run here has no report, so the gate does not close anything of theirs. The pedigree and the members' progress stay. -The gate holds the page's `fetch` and `EventSource`. A view or plugin reads a run's files through them. `XMLHttpRequest`, an `` and `import()` are not held, so the views use them only on what the gate has released. The gate guards the page in the browser: its document, attributes, accessibility text, title, address, canvas and drawings, storage and the requests its views make. The static files (`results.json`, `map.json`, the charts, the search index and `family.json`) still reach a browser that asks for them whole: the gate is not an access control for them. +The gate holds the page's `fetch` and `EventSource`. A view or plugin reads a run's files through them. `XMLHttpRequest`, an ``, an `
    {half}LOCUS_B20/75full_mutationraw score only
    chr", page)) >= 50) - # 9. run_delta with the additive field + # 9. og run delta with the additive field prev = copy.deepcopy(s) del prev["sections"]["mito"]["data"]["heteroplasmic_sites"] - check("9. run_delta: no change for the added heteroplasmic_sites", run_delta.compare(prev, {}, s, {}) == [], - run_delta.compare(prev, {}, s, {})) + got = run_delta(prev, s) + check("9. og run delta: no change for the added heteroplasmic_sites", got == [], got) moved = copy.deepcopy(s) moved["sections"]["clinvar"]["data"]["hits"][0]["significance"] = "Benign" - check("9. red control: a reclassified hit is a change", run_delta.compare(prev, {}, moved, {}) != []) + check("9. red control: a reclassified hit is a change", run_delta(prev, moved) not in ([], None)) + + +def run_delta(prev, cur): + """The changes og run delta lists between two summaries (no manifests), or None if it failed.""" + with tempfile.TemporaryDirectory() as work: + os.makedirs(os.path.join(work, "prev")) + for path, summ in ((os.path.join(work, "prev", "summary.json"), prev), (os.path.join(work, "cur.json"), cur)): + with open(path, "w") as f: + json.dump(summ, f) + out = os.path.join(work, "delta") + p = subprocess.run([OG, "run", "delta", "--sample", "S", "--summary", os.path.join(work, "cur.json"), + "--previous", os.path.join(work, "prev"), "--out", out], capture_output=True, text=True) + if p.returncode: + print(p.stderr) + return None + with open(out + ".json") as f: + return json.load(f)["changes"] def page_problems(b, base, run): diff --git a/tests/test_run_state.py b/tests/test_run_state.py index dd4bf83a3..03feaec3d 100755 --- a/tests/test_run_state.py +++ b/tests/test_run_state.py @@ -233,7 +233,7 @@ def catalogue_checks(steps, sections, run_all_set): out["the recorded steps are what run-all.sh records"] = (recorded == want, sorted(recorded ^ want)) out["RUN_ALL_STEPS is the recorded steps"] = (set(run_all_set) == recorded, sorted(set(run_all_set) ^ recorded)) owner = {k: s["step"] for s in steps for k in s["sections"]} - keys = {k: step for k, _, step, _ in sections} + keys = {k: step for k, _, step in sections} wrong = [(k, step, owner.get(k)) for k, step in keys.items() if owner.get(k) != step] out["every SECTIONS key belongs to the row of its step"] = (not wrong and set(owner) <= set(keys), (wrong, sorted(set(owner) - set(keys)))) diff --git a/tests/test_str_ranges.py b/tests/test_str_ranges.py index 8c34556a1..6482d4152 100644 --- a/tests/test_str_ranges.py +++ b/tests/test_str_ranges.py @@ -9,14 +9,14 @@ no_thresholds; intermediate_min and intermediate_max are STRchive's own. Controls: a table with one row dropped, or one row twice, is caught. 2. bin/str_intermediate_ranges.tsv, the GeneReviews ranges the report names - (HTT, FMR1, DMPK): the columns bin/collect_summary.py reads, one row per + (HTT, FMR1, DMPK): the columns og summary collect reads, one row per locus, whole numbers with min <= max ending right below the next class, a label and a GeneReviews citation; docs/interpreting-results.md prints the same range in the locus's row; every locus has a row in str_ranges.tsv, and the range lies above its normal_max and below its pathogenic_min, so a count the report calls intermediate is one Stranger calls pre_mutation - (control: a planted HTT range of 20-35 is caught); collect_summary reads - every row. + (control: a planted HTT range of 20-35 is caught). That og reads every + row is og-summary's own test (collect::group_c::tests). 3. The "Key Disease Thresholds" table of docs/09-str-expansions.md: each row's normal maximum, pre-mutation band and full-mutation minimum are the catalog's (control: the table with HTT's minimum changed to 37 is caught). @@ -31,7 +31,6 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) sys.path.insert(0, os.path.join(REPO, "tests")) -import collect_summary # noqa: E402 import test_str_catalog # noqa: E402 TSV = os.path.join(REPO, "bin", "str_intermediate_ranges.tsv") @@ -130,7 +129,7 @@ def part_generated(cat, cat_path): def part_subset(gen): head, body = table(TSV) - check("the header is the columns collect_summary.py reads", head == COLUMNS, head) + check("the header is the columns og summary collect reads", head == COLUMNS, head) check("at least HTT, FMR1 and DMPK are listed", {"HTT", "FMR1", "DMPK"} <= {r[0] for r in body}, [r[0] for r in body]) loci = [r[0] for r in body] @@ -162,8 +161,6 @@ def inside(locus, lo, hi): f"({g.get('normal_max')}) and below its pathogenic minimum ({g.get('pathogenic_min')})", inside(d["locus"], lo, hi), g) check("control: a planted HTT range of 20-35 is outside STRchive's pre-mutation band", not inside("HTT", 20, 35)) - got = collect_summary.str_ranges(TSV) - check("collect_summary.str_ranges reads every row", sorted(got) == sorted(loci), sorted(got)) def threshold_rows(text): diff --git a/tests/test_traits_report.py b/tests/test_traits_report.py index cf55a9599..07fb6de3b 100644 --- a/tests/test_traits_report.py +++ b/tests/test_traits_report.py @@ -13,7 +13,6 @@ import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(REPO, "bin")) -import traits_report as tr # noqa: E402 TABLE = os.path.join(REPO, "assets", "traits.tsv") RULES = os.path.join(REPO, "assets", "genotype_rules.tsv") CHECK = os.path.join(REPO, "scripts", "ci", "check-rule-alleles.sh") @@ -21,6 +20,36 @@ SMN_HEADER = "chrom\tstart\tend\tlocus\tagCN_filter\tagCN\tagCN_qual\tpsCN_filter\tpsCN\tpsCN_qual\thomologous_regions\n" SMN1_NOT_PROOF = "This marker does not prove 2+0, and its absence does not rule 2+0 out." FAILS = 0 +# The traits table's columns (og traits table --out). +OUT_COLUMNS = ["rsid", "gene", "chrom", "pos", "trait", "genotype", "status", "reading", "ancestry", + "note", "pmids"] +# The copy-aware markers, pinned here as the test's own copy: og cn +# marker-regions must print the same copies (rust/crates/og-cn/src/markers.rs). +MARKERS = { + # KIV-2 4925G>A: the last base of the second KIV-2 exon (its donor splice + # site), G>A on LPA's minus strand, so C>T on the plus strand. GRCh38 holds + # it at the end of each of the six KIV-2 units (exons 5, 7, ..., 15 of + # ENST00000316300.10; NM_005577.4:c.733G>A at the first, p.Ala245Thr) and + # at the near-identical last exon of KIV-1 (exon 3, c.391): seven copies, + # the block og cn call --locus LPA_KIV2 counts as N + 2 copies (N KIV-2 repeats and + # one KIV-1 per chromosome). dbSNP names the first KIV-2 copy only. + "LPA_KIV2_4925": { + "name": "LPA KIV-2 4925G>A", "chrom": "chr6", "ref": "C", "alt": "T", + "dbsnp": ("rs1484919544", "chr6", 160640667, "C", "T"), + "copies": (160646214, 160640667, 160635123, 160629574, 160624030, 160618484, 160612932), + "context": "GACTCCTTACCTTGTTCGGAA", + "min_depth": 100, "min_alt": 5, "min_fraction": 0.008, "max_absent": 2, + }, + # SMN1 g.27134T>G is NM_000344.4:c.*3+80T>G in intron 7 (plus strand). + # SMN2 holds the same 21 bases 875,420 bases upstream, as at exon 7. + "SMN1_27134": { + "name": "SMN1 g.27134T>G", "chrom": "chr5", "ref": "T", "alt": "G", + "dbsnp": ("rs143838139", "chr5", 70952074, "T", "G"), + "copies": (70952074, 70076654), + "context": "TTTTTGAACATTTAAAAAGTT", + "min_depth": 10, "min_alt": 3, "min_fraction": 0.05, "max_absent": 1, + }, +} APOE_GOLDEN = { ("T/T", "C/C"): "e3d7512b649f1d23f5acbcaa08da19bd6a009d5b39ac57cb6856591c50e7cfda", ("T/T", "C/T"): "62e910d8ec7ac45ab15e9c9ed85c93b128ac41703d1149a332082dc0d7b5f389", @@ -120,7 +149,7 @@ def marker_errors(name, m=None, dbsnp=None): is one of its copies with its ref and alt, on the plus strand and not swapped; every copy sits in the middle of the same context, whose middle base is the ref.""" - m = tr.MARKERS[name] if m is None else m + m = MARKERS[name] if m is None else m rs, chrom, pos, ref, alt = m["dbsnp"] if dbsnp is None else dbsnp comp = {"A": "T", "C": "G", "G": "C", "T": "A"} errs = [] @@ -143,7 +172,7 @@ def marker_errors(name, m=None, dbsnp=None): def marker_checks(): """11. The copy-aware markers.""" - for name, m in tr.MARKERS.items(): + for name, m in MARKERS.items(): check(f"marker {name}: the pinned dbSNP record agrees with the marker", marker_errors(name) == [], marker_errors(name)) rs, chrom, pos, ref, alt = m["dbsnp"] @@ -159,16 +188,16 @@ def marker_checks(): e = marker_errors(name, dict(m, context=m["context"][:10] + alt + m["context"][11:])) check(f"marker {name} control: a context whose middle base is not the ref is caught", any("middle base" in x for x in e), e) - check("LPA_KIV2_4925: seven copies (six KIV-2 units and KIV-1)", len(tr.MARKERS["LPA_KIV2_4925"]["copies"]) == 7) + check("LPA_KIV2_4925: seven copies (six KIV-2 units and KIV-1)", len(MARKERS["LPA_KIV2_4925"]["copies"]) == 7) if not OG: print("skip: OG is not set (OG=rust/target/release/og): og cn marker-regions and marker-read did not run") return - for name, m in tr.MARKERS.items(): + for name, m in MARKERS.items(): res = og_cn("marker-regions", name) check(f"marker {name}: og cn marker-regions prints one region per copy of MARKERS", res.returncode == 0 and res.stdout.split() == [f"{m['chrom']}:{p}-{p}" for p in m["copies"]], res) - sm = tr.MARKERS["SMN1_27134"] + sm = MARKERS["SMN1_27134"] work = tempfile.mkdtemp() try: def pileup(t_reads, g_reads, extra=""): @@ -305,11 +334,8 @@ def plain_tsv_checks(): check(f"plain TSV control: {name} is refused", res.returncode == 1, res.stderr) check(f"plain TSV control: {name} preserves both outputs", previous == (open(out, "rb").read(), open(rout, "rb").read())) - try: - _, collected = cs.sec_traits(work, "S") - readable = bool(collected["rows"]) - except cs.Unreadable: - readable = False + sec = cs.collect("S", work)["sections"]["traits"] + readable = sec["state"] == "ok" and bool(sec["data"]["rows"]) check(f"plain TSV control: {name} leaves collector-readable output", readable) @@ -348,7 +374,7 @@ def table(records): [("44908684", "T", "C", a), ("44908822", "C", "T", b)] if gt != "."] traits, lines = table(records) apoe = next(r for r in traits if "+" in r["rsid"]) - got = hashlib.sha256("\t".join(apoe[k] for k in tr.OUT_COLUMNS).encode()).hexdigest() + got = hashlib.sha256("\t".join(apoe[k] for k in OUT_COLUMNS).encode()).hexdigest() check(f"APOE golden {a}, {b}", got == want, got) for rid, cases in MULTI_SNP.items(): for gts, label in cases: @@ -378,7 +404,7 @@ def table(records): # Collect and render the actual CLI output, independently for each rule. records = [(s[1], s[2], s[3], s[4], gt) for s, gt in zip(sites, gts)] _, lines = table(records) - read = cs.genotype_rules(work, "S") or [] + read = cs.collect("S", work)["sections"]["traits"]["data"]["rules"] or [] page = "\n".join(rr.rules_html({"rules": read}, "S")) got = next(r for r in lines if r["rule"] == rid) check(f"HTML report: {rid} with result {want}", rr.E(got["trait"]) in page and rr.E(want) in page, page[:300]) diff --git a/tests/test_traits_table.py b/tests/test_traits_table.py index 3d2566b27..984d426f1 100644 --- a/tests/test_traits_table.py +++ b/tests/test_traits_table.py @@ -7,6 +7,7 @@ positive, wrong-base and missing-contig controls without network access. """ import argparse +import csv import json import os import shutil @@ -16,9 +17,6 @@ import time REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import traits_report as tr # noqa: E402 - TABLE = os.path.join(REPO, "assets", "traits.tsv") FAILS = 0 NC = {f"NC_0000{n:02d}": f"chr{n}" for n in range(1, 23)} @@ -122,13 +120,20 @@ def online_errors(rows): return errs +def read_table(path): + """The rows of a traits table as dicts (its schema is og traits check's to judge).""" + with open(path, encoding="utf-8") as f: + return list(csv.DictReader((line for line in f if line.strip() and not line.startswith("#")), + delimiter="\t", quoting=csv.QUOTE_NONE)) + + def schema(path): return subprocess.run([os.environ.get("OG", os.path.join(REPO, "bin", "og")), "traits", "check", "--table", path], capture_output=True, text=True) def schema_controls(): - good = tr.read_table(TABLE) + good = read_table(TABLE) check("the table has at least 30 rows", len(good) >= 30, len(good)) work = tempfile.mkdtemp() try: @@ -174,13 +179,9 @@ def main(argv=None): return 0 schema_controls() else: - try: - rows = tr.read_table(a.table) - except tr.TableError as e: - check("the table can be read", False, e) - rows = [] res = schema(a.table) check(f"{a.table} passes the schema check", res.returncode == 0, res.stderr) + rows = read_table(a.table) if res.returncode == 0 else [] if a.fasta and rows: e, n = fasta_errors(a.fasta, rows) print(f" ref checked against {a.fasta}: {n} rows (the others are on contigs it lacks)") diff --git a/tests/test_viral_report.py b/tests/test_viral_report.py index 96d14505d..7d16316a9 100644 --- a/tests/test_viral_report.py +++ b/tests/test_viral_report.py @@ -206,6 +206,20 @@ def drop_genome(d): import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 +class Unreadable(Exception): + """A section og summary collect could not read (its state is unreadable).""" + + +def sec(d, s, key): + """(source path, data) of section KEY of the summary og summary collect writes + for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" + x = cs.collect(s, d)["sections"][key] + if x["state"] == "unreadable": + raise Unreadable(x["note"]) + return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] + + + def og_report(d): """og viral report on the inputs in d, into d/S/viral/S_viral.tsv.""" @@ -218,12 +232,12 @@ def og_report(d): write_inputs(d, {B: flat(15), EBV: flat(1), PHIX: flat(3)}, {B: 400, EBV: 30, PHIX: 20}) os.makedirs(f"{d}/S/viral") og_report(d) - path, data = cs.sec_viral(f"{d}/S", "S") + path, data = sec(f"{d}/S", "S", "viral") check("8 the summary finds the table", path.endswith("viral/S_viral.tsv"), True) check("8 the summary's HHV-6 line", data["hhv6"], "inherited integrated HHV-6B likely") check("8 the summary lists what was found", [x["virus"] for x in data["found"]], ["HHV-6B", "Human gammaherpesvirus 4", "Escherichia phage phiX174"]) - check("8 no table: no section", cs.sec_viral(f"{d}/none", "S"), (None, {})) + check("8 no table: no section", sec(f"{d}/none", "S", "viral"), (None, {})) lines = rr.viral_lines(data) check("8 the text block opens with the HHV-6 line", lines[0], "HHV-6: inherited integrated HHV-6B likely") check("8 and ends with the note", lines[-1], rr.VIRAL_NOTE) @@ -232,9 +246,9 @@ def og_report(d): with open(f"{d}/S/viral/S_viral.tsv", "w") as f: f.write("virus\tcall\n") try: - cs.sec_viral(f"{d}/S", "S") + sec(f"{d}/S", "S", "viral") check("8 a table without its columns is unreadable", "read", "Unreadable") - except cs.Unreadable: + except Unreadable: check("8 a table without its columns is unreadable", "Unreadable", "Unreadable") # 8. the summary's HHV-6 line without a human depth, and with one but no integration @@ -244,7 +258,7 @@ def og_report(d): write_inputs(d, {EBV: flat(1)}, {EBV: 30}, human=human) os.makedirs(f"{d}/S/viral") og_report(d) - data = cs.sec_viral(f"{d}/S", "S")[1] + data = sec(f"{d}/S", "S", "viral")[1] check(f"8 the summary's HHV-6 line reads {want!r}", data["hhv6"], want) if human: check("8 no human depth: the text line leaves out copies per cell", rr.viral_lines(data)[1], @@ -255,7 +269,7 @@ def og_report(d): write_inputs(d, {HCV: lambda i: 1 if i < 50 else 0}, {HCV: 5}) os.makedirs(f"{d}/S/viral") og_report(d) - data = cs.sec_viral(f"{d}/S", "S")[1] + data = sec(f"{d}/S", "S", "viral")[1] check("8 a genome with 5 reads is listed", [(x["virus"], x["call"], x["reads"]) for x in data["found"]], [("Hepatitis C virus genotype 1", CALL_NONE, "5")]) check("8 and the text does not say no read", any(ln.startswith("No read") for ln in rr.viral_lines(data)), False) diff --git a/tests/test_vntyper_report.py b/tests/test_vntyper_report.py index c4341a997..8c1ac8384 100644 --- a/tests/test_vntyper_report.py +++ b/tests/test_vntyper_report.py @@ -192,7 +192,8 @@ def sample(name, kestrel, **kw): s = collect_summary.collect("S", sd) return s, render_report.text_report(s), render_report.html_report(s) - line = collect_summary.VNTYPER_NOT_ASSESSED + line = ("MUC1 VNTR frameshifts (ADTKD-MUC1): step 52 (VNtyper, opt-in) gave no reading; " + "no standard caller sees them") s, txt, html = sample("pos", call()) sec = s["sections"].get("vntyper", {}) check("6. the section is read", sec.get("state") == "ok" and sec["data"]["status"] == "positive", sec) diff --git a/tests/test_ystr_report.py b/tests/test_ystr_report.py index 77c49ea30..64e4815df 100644 --- a/tests/test_ystr_report.py +++ b/tests/test_ystr_report.py @@ -305,19 +305,33 @@ def good(gb="0", dp="20"): import collect_summary as cs # noqa: E402 import render_report as rr # noqa: E402 +class Unreadable(Exception): + """A section og summary collect could not read (its state is unreadable).""" + + +def sec(d, s, key): + """(source path, data) of section KEY of the summary og summary collect writes + for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" + x = cs.collect(s, d)["sections"][key] + if x["state"] == "unreadable": + raise Unreadable(x["note"]) + return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] + + + sd = os.path.join(TMP, "sample") os.makedirs(os.path.join(sd, "y_str")) v = vcf({"DYS393": good("4"), "DYS390": ("0", "0", "0.6", "20", "0", "0")}) subprocess.run([OG, "y", "str-report", "--panel", PANEL, "--vcf", v, "--tsv", os.path.join(sd, "y_str", "S1_y_str.tsv"), "--csv", os.path.join(sd, "y_str", "S1_y_str.csv")], check=True, capture_output=True) -path, data = cs.sec_y_str(sd, "S1") +path, data = sec(sd, "S1", "y_str") check("7 collect_summary finds the table", os.path.basename(path or ""), "S1_y_str.tsv") check("7 counts per confidence", (data["high"], data["low"], data["not_callable"], data["no_call"]), (1, 1, sum(1 for r in panel if r["kind"] in ("multicopy", "absent")) + 3, # + DYS452, DYS389II, DYS448: too long 111 - 2 - sum(1 for r in panel if r["kind"] in ("multicopy", "absent")) - 3)) check("7 the markers keep their order", [x["marker"] for x in data["markers"]], names) -check("7 a sample without the step has no section", cs.sec_y_str(TMP, "nobody"), (None, {})) +check("7 a sample without the step has no section", sec(TMP, "nobody", "y_str"), (None, {})) html = "\n".join(rr.ystr_body(data)) check("7 the card lists DYS393 at 13", "DYS39313highDYS39024lowGBA1, GBAP1