From 1cf722ff40ea7007d1452989a5373a9bab83854c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:53:18 +0100 Subject: [PATCH 01/25] fix: score detection f1 as 0 when no real bout is matched calculate_precision_recall_f1 returned NaN for F1 whenever TP was 0, so a recording with real bouts and no match (nothing predicted, or only false bouts) left the mean over recordings. F1 is now 0 when TP = 0 and FP + FN > 0, the value of 2 TP / (2 TP + FP + FN). It stays NaN for TP = FP = FN = 0 and every other value is unchanged. The MATLAB function BANOScalculatePrecisionRecallF1 gets the same rule. The goldens move to golden_python_v110.json and golden_matlab_v110.json (the v0.2.0 files are kept). Two values change: Recording_8 mount F1 goes from null to 0 and Recording_8 overall F1 from 0.6062 to 0.4546. --- .../BANOS/BANOScalculatePrecisionRecallF1.m | 12 +- src/banos/core/metrics.py | 9 +- tests/fixtures/golden_matlab_v110.json | 1 + tests/fixtures/golden_python_v110.json | 802 ++++++++++++++++++ .../matlab/generate_matlab_fixtures.m | 10 +- .../python/generate_golden_fixtures.py | 14 +- tests/test_metrics.py | 27 + tests/test_parity.py | 8 +- tests/test_pipeline.py | 58 ++ tests/test_regression.py | 29 +- 10 files changed, 934 insertions(+), 36 deletions(-) create mode 100644 tests/fixtures/golden_matlab_v110.json create mode 100644 tests/fixtures/golden_python_v110.json diff --git a/matlab/BANOS/BANOScalculatePrecisionRecallF1.m b/matlab/BANOS/BANOScalculatePrecisionRecallF1.m index fcd2bc5..2e2e1a9 100644 --- a/matlab/BANOS/BANOScalculatePrecisionRecallF1.m +++ b/matlab/BANOS/BANOScalculatePrecisionRecallF1.m @@ -1,7 +1,10 @@ function [precision, recall, f1] = BANOScalculatePrecisionRecallF1(tp, fp, fn) % Calculates Precision, Recall, and F1 score at the bout level. % - % Returns NaN (not 0) when the denominator is zero, matching Python behaviour. + % Precision and recall are NaN (not 0) when their denominator is zero, + % matching Python behaviour. F1 is 0 when no bout is matched but some + % bouts were predicted or missed (TP = 0 and FP + FN > 0), and NaN only + % when all three counts are zero. % NaN values propagate correctly through BANOSaggregateMetrics (nanmean). % % Arguments: @@ -12,7 +15,8 @@ % Returns: % precision - TP / (TP + FP), or NaN if TP + FP = 0. % recall - TP / (TP + FN), or NaN if TP + FN = 0. - % f1 - Harmonic mean of precision and recall, or NaN if undefined. + % f1 - Harmonic mean of precision and recall; 0 if TP = 0 and + % FP + FN > 0; NaN if TP = FP = FN = 0. if (tp + fp) > 0 precision = tp / (tp + fp); @@ -26,7 +30,9 @@ recall = NaN; end - if ~isnan(precision) && ~isnan(recall) && (precision + recall) > 0 + if tp == 0 && (fp + fn) > 0 + f1 = 0; + elseif ~isnan(precision) && ~isnan(recall) && (precision + recall) > 0 f1 = 2 * precision * recall / (precision + recall); else f1 = NaN; diff --git a/src/banos/core/metrics.py b/src/banos/core/metrics.py index bdd10d5..5dc93b9 100644 --- a/src/banos/core/metrics.py +++ b/src/banos/core/metrics.py @@ -29,11 +29,16 @@ def identify_bouts(sequence: pd.Series | list) -> list[tuple[int, int]]: def calculate_precision_recall_f1(n_tp: int, n_fp: int, n_fn: int) -> tuple[float, float, float]: """Compute Detection Accuracy: Precision, Recall, F1. - Returns NaN for any metric where the denominator is zero. + Precision and recall are NaN when their denominator is zero. + F1 is 0 when no bout is matched but some bouts were predicted or missed + (``n_tp == 0`` and ``n_fp + n_fn > 0``), and NaN only when all three + counts are zero. """ precision = n_tp / (n_tp + n_fp) if (n_tp + n_fp) > 0 else float("nan") recall = n_tp / (n_tp + n_fn) if (n_tp + n_fn) > 0 else float("nan") - if not math.isnan(precision) and not math.isnan(recall) and (precision + recall) > 0: + if n_tp == 0 and n_fp + n_fn > 0: + f1_score = 0.0 + elif not math.isnan(precision) and not math.isnan(recall) and (precision + recall) > 0: f1_score = 2 * precision * recall / (precision + recall) else: f1_score = float("nan") diff --git a/tests/fixtures/golden_matlab_v110.json b/tests/fixtures/golden_matlab_v110.json new file mode 100644 index 0000000..a0936cd --- /dev/null +++ b/tests/fixtures/golden_matlab_v110.json @@ -0,0 +1 @@ +{"Recording_1":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.98294071362580027},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.99086091350216854},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.99306915975344445}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.98294071362580027},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.99086091350216854},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.99306915975344445}},"overall_metrics":{"precision":0.92826086956521736,"recall":0.78129251700680269,"f1_score":0.836455036759336,"so":0.761228298078524,"tp":0.30623014745659,"ic":0.99171769672035337}},"Recording_2":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98905093606628935},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99464840013584821},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99192990230311961}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98905093606628935},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99464840013584821},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99192990230311961}},"overall_metrics":{"precision":0.9671052631578948,"recall":0.83992445266728333,"f1_score":0.897300738477209,"so":0.82594559036003834,"tp":0.30908214265027689,"ic":0.99390730962631435}},"Recording_3":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98496628121015162},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.9917129219739137},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99621162045778}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98496628121015162},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.9917129219739137},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99621162045778}},"overall_metrics":{"precision":0.95948072383949645,"recall":0.85096664987969339,"f1_score":0.89862686038270723,"so":0.723715686180092,"tp":0.26887555753525666,"ic":0.99322270591046136}},"Recording_4":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.99528949732999117},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99394902176918432}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.99528949732999117},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99394902176918432}},"overall_metrics":{"precision":0.975,"recall":0.61568627450980384,"f1_score":0.72916666666666674,"so":0.70708735677240686,"tp":0.27571526135052643,"ic":0.9965302523591395}},"Recording_5":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98991370716597},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99654108838242228},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.994024222522601}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98991370716597},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99654108838242228},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.994024222522601}},"overall_metrics":{"precision":1,"recall":0.89210526315789473,"f1_score":0.94145869368256319,"so":0.83673046008612484,"tp":0.27928892486843937,"ic":0.99511975451774837}},"Recording_6":{"file_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.99366643311730019},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.99039202618472566},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.99188832402173766}},"group_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.99366643311730019},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.99039202618472566},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.99188832402173766}},"overall_metrics":{"precision":0.91408268733850129,"recall":0.74567099567099571,"f1_score":0.79250474383301706,"so":0.624464020496978,"tp":0.26406773460888022,"ic":0.9939866958309409}},"Recording_7":{"file_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98947078465360738},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.99454387011498291},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.995759546530849}},"group_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98947078465360738},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.99454387011498291},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.995759546530849}},"overall_metrics":{"precision":1,"recall":0.58390957446808511,"f1_score":0.70652984743264124,"so":0.66449505594099667,"tp":0.27653014329623343,"ic":0.99494355032485982}},"Recording_8":{"file_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99528213381542252},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":1},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99510532135592722}},"group_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99528213381542252},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":1},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99510532135592722}},"overall_metrics":{"precision":0.89166666666666661,"recall":0.34672064777327932,"f1_score":0.45462962962962961,"so":0.53056180162368516,"tp":0.038138242319112579,"ic":0.99557663443524391}},"Recording_9":{"file_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98938636025336613},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99863004362050167},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99466920484634314}},"group_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98938636025336613},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99863004362050167},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99466920484634314}},"overall_metrics":{"precision":0.79839767616191892,"recall":0.65381944444444451,"f1_score":0.712354862119013,"so":0.68272952732383074,"tp":0.27780938219974027,"ic":0.99567140218005268}},"Recording_10":{"file_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.99141187589349233},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.99506116351626372},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.998292947043181}},"group_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.99141187589349233},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.99506116351626372},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.998292947043181}},"overall_metrics":{"precision":0.89910714285714288,"recall":0.47575971177944865,"f1_score":0.58347469607011593,"so":0.66680627484973121,"tp":0.28592277410718,"ic":0.99619149661323425}}} diff --git a/tests/fixtures/golden_python_v110.json b/tests/fixtures/golden_python_v110.json new file mode 100644 index 0000000..37ed23f --- /dev/null +++ b/tests/fixtures/golden_python_v110.json @@ -0,0 +1,802 @@ +{ + "Recording_1": { + "file_metrics": { + "Recording_1": { + "attack": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "investigation": { + "precision": 0.8, + "recall": 0.7346938775510204, + "f1_score": 0.7659574468085106, + "so": 0.5444095073699892, + "tp": 0.0846356832664267, + "ic": 0.9829407136258003 + }, + "mount": { + "precision": 1.0, + "recall": 0.5333333333333333, + "f1_score": 0.6956521739130436, + "so": 0.7332766872436964, + "tp": 0.07847125492506643, + "ic": 0.9908609135021688 + }, + "other": { + "precision": 0.9130434782608695, + "recall": 0.8571428571428571, + "f1_score": 0.8842105263157894, + "so": 0.7672269977004106, + "tp": 0.061813651634866906, + "ic": 0.9930691597534443 + } + } + }, + "group_metrics": { + "attack": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "investigation": { + "precision": 0.8, + "recall": 0.7346938775510204, + "f1_score": 0.7659574468085106, + "so": 0.5444095073699892, + "tp": 0.0846356832664267, + "ic": 0.9829407136258003 + }, + "mount": { + "precision": 1.0, + "recall": 0.5333333333333333, + "f1_score": 0.6956521739130436, + "so": 0.7332766872436964, + "tp": 0.07847125492506643, + "ic": 0.9908609135021688 + }, + "other": { + "precision": 0.9130434782608695, + "recall": 0.8571428571428571, + "f1_score": 0.8842105263157894, + "so": 0.7672269977004106, + "tp": 0.061813651634866906, + "ic": 0.9930691597534443 + } + }, + "overall_metrics": { + "precision": 0.9282608695652174, + "recall": 0.7812925170068027, + "f1_score": 0.836455036759336, + "so": 0.7612282980785241, + "tp": 0.30623014745659, + "ic": 0.9917176967203534 + } + }, + "Recording_2": { + "file_metrics": { + "Recording_2": { + "attack": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "investigation": { + "precision": 0.8947368421052632, + "recall": 0.7391304347826086, + "f1_score": 0.8095238095238095, + "so": 0.6609424826096988, + "tp": 0.04615109869237036, + "ic": 0.9890509360662894 + }, + "mount": { + "precision": 1.0, + "recall": 0.8333333333333334, + "f1_score": 0.9090909090909091, + "so": 0.931754332584827, + "tp": 0.1451695616211745, + "ic": 0.9946484001358482 + }, + "other": { + "precision": 0.9736842105263158, + "recall": 0.7872340425531915, + "f1_score": 0.8705882352941177, + "so": 0.7110855462456277, + "tp": 0.04500791028756257, + "ic": 0.9919299023031196 + } + } + }, + "group_metrics": { + "attack": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "investigation": { + "precision": 0.8947368421052632, + "recall": 0.7391304347826086, + "f1_score": 0.8095238095238095, + "so": 0.6609424826096988, + "tp": 0.04615109869237036, + "ic": 0.9890509360662894 + }, + "mount": { + "precision": 1.0, + "recall": 0.8333333333333334, + "f1_score": 0.9090909090909091, + "so": 0.931754332584827, + "tp": 0.1451695616211745, + "ic": 0.9946484001358482 + }, + "other": { + "precision": 0.9736842105263158, + "recall": 0.7872340425531915, + "f1_score": 0.8705882352941177, + "so": 0.7110855462456277, + "tp": 0.04500791028756257, + "ic": 0.9919299023031196 + } + }, + "overall_metrics": { + "precision": 0.9671052631578947, + "recall": 0.8399244526672833, + "f1_score": 0.897300738477209, + "so": 0.8259455903600383, + "tp": 0.30908214265027684, + "ic": 0.9939073096263142 + } + }, + "Recording_3": { + "file_metrics": { + "Recording_3": { + "attack": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "investigation": { + "precision": 0.9354838709677419, + "recall": 0.7837837837837838, + "f1_score": 0.8529411764705881, + "so": 0.4563103530678653, + "tp": 0.02232158655731043, + "ic": 0.9849662812101514 + }, + "mount": { + "precision": 1.0, + "recall": 0.7391304347826086, + "f1_score": 0.85, + "so": 0.6745042152422358, + "tp": 0.028794827943741816, + "ic": 0.9917129219739135 + }, + "other": { + "precision": 0.9024390243902439, + "recall": 0.8809523809523809, + "f1_score": 0.8915662650602411, + "so": 0.7640481764102668, + "tp": 0.024385815639974426, + "ic": 0.9962116204577801 + } + } + }, + "group_metrics": { + "attack": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "investigation": { + "precision": 0.9354838709677419, + "recall": 0.7837837837837838, + "f1_score": 0.8529411764705881, + "so": 0.4563103530678653, + "tp": 0.02232158655731043, + "ic": 0.9849662812101514 + }, + "mount": { + "precision": 1.0, + "recall": 0.7391304347826086, + "f1_score": 0.85, + "so": 0.6745042152422358, + "tp": 0.028794827943741816, + "ic": 0.9917129219739135 + }, + "other": { + "precision": 0.9024390243902439, + "recall": 0.8809523809523809, + "f1_score": 0.8915662650602411, + "so": 0.7640481764102668, + "tp": 0.024385815639974426, + "ic": 0.9962116204577801 + } + }, + "overall_metrics": { + "precision": 0.9594807238394965, + "recall": 0.8509666498796933, + "f1_score": 0.8986268603827072, + "so": 0.723715686180092, + "tp": 0.26887555753525666, + "ic": 0.9932227059104612 + } + }, + "Recording_4": { + "file_metrics": { + "Recording_4": { + "attack": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "investigation": { + "precision": 0.9, + "recall": 0.5294117647058824, + "f1_score": 0.6666666666666667, + "so": 0.39147100794903517, + "tp": 0.014025218005513157, + "ic": 0.9968824903373824 + }, + "mount": { + "precision": 1.0, + "recall": 0.3333333333333333, + "f1_score": 0.5, + "so": 0.7011185281643236, + "tp": 0.04008717445788384, + "ic": 0.9952894973299912 + }, + "other": { + "precision": 1.0, + "recall": 0.6, + "f1_score": 0.7499999999999999, + "so": 0.7357598909762686, + "tp": 0.04874865293870875, + "ic": 0.9939490217691843 + } + } + }, + "group_metrics": { + "attack": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "investigation": { + "precision": 0.9, + "recall": 0.5294117647058824, + "f1_score": 0.6666666666666667, + "so": 0.39147100794903517, + "tp": 0.014025218005513157, + "ic": 0.9968824903373824 + }, + "mount": { + "precision": 1.0, + "recall": 0.3333333333333333, + "f1_score": 0.5, + "so": 0.7011185281643236, + "tp": 0.04008717445788384, + "ic": 0.9952894973299912 + }, + "other": { + "precision": 1.0, + "recall": 0.6, + "f1_score": 0.7499999999999999, + "so": 0.7357598909762686, + "tp": 0.04874865293870875, + "ic": 0.9939490217691843 + } + }, + "overall_metrics": { + "precision": 0.975, + "recall": 0.615686274509804, + "f1_score": 0.7291666666666666, + "so": 0.7070873567724069, + "tp": 0.27571526135052643, + "ic": 0.9965302523591395 + } + }, + "Recording_5": { + "file_metrics": { + "Recording_5": { + "attack": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "investigation": { + "precision": 1.0, + "recall": 0.8, + "f1_score": 0.888888888888889, + "so": 0.6589591312525748, + "tp": 0.035333351984487595, + "ic": 0.98991370716597 + }, + "mount": { + "precision": 1.0, + "recall": 0.9, + "f1_score": 0.9473684210526316, + "so": 0.9100237928302215, + "tp": 0.03596010219174562, + "ic": 0.9965410883824223 + }, + "other": { + "precision": 1.0, + "recall": 0.868421052631579, + "f1_score": 0.9295774647887324, + "so": 0.7779389162617028, + "tp": 0.045862245297524175, + "ic": 0.9940242225226008 + } + } + }, + "group_metrics": { + "attack": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "investigation": { + "precision": 1.0, + "recall": 0.8, + "f1_score": 0.888888888888889, + "so": 0.6589591312525748, + "tp": 0.035333351984487595, + "ic": 0.98991370716597 + }, + "mount": { + "precision": 1.0, + "recall": 0.9, + "f1_score": 0.9473684210526316, + "so": 0.9100237928302215, + "tp": 0.03596010219174562, + "ic": 0.9965410883824223 + }, + "other": { + "precision": 1.0, + "recall": 0.868421052631579, + "f1_score": 0.9295774647887324, + "so": 0.7779389162617028, + "tp": 0.045862245297524175, + "ic": 0.9940242225226008 + } + }, + "overall_metrics": { + "precision": 1.0, + "recall": 0.8921052631578947, + "f1_score": 0.9414586936825633, + "so": 0.8367304600861247, + "tp": 0.27928892486843937, + "ic": 0.9951197545177483 + } + }, + "Recording_6": { + "file_metrics": { + "Recording_6": { + "attack": { + "precision": 1.0, + "recall": 0.36363636363636365, + "f1_score": 0.5333333333333333, + "so": 0.3554875922085192, + "tp": 0.014142932226192012, + "ic": 0.9936664331173002 + }, + "investigation": { + "precision": 0.7674418604651163, + "recall": 0.7857142857142857, + "f1_score": 0.7764705882352941, + "so": 0.5395797274871901, + "tp": 0.020316910620777756, + "ic": 0.9903920261847257 + }, + "mount": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "other": { + "precision": 0.8888888888888888, + "recall": 0.8333333333333334, + "f1_score": 0.8602150537634408, + "so": 0.6027887622922026, + "tp": 0.021811095588551047, + "ic": 0.9918883240217377 + } + } + }, + "group_metrics": { + "attack": { + "precision": 1.0, + "recall": 0.36363636363636365, + "f1_score": 0.5333333333333333, + "so": 0.3554875922085192, + "tp": 0.014142932226192012, + "ic": 0.9936664331173002 + }, + "investigation": { + "precision": 0.7674418604651163, + "recall": 0.7857142857142857, + "f1_score": 0.7764705882352941, + "so": 0.5395797274871901, + "tp": 0.020316910620777756, + "ic": 0.9903920261847257 + }, + "mount": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "other": { + "precision": 0.8888888888888888, + "recall": 0.8333333333333334, + "f1_score": 0.8602150537634408, + "so": 0.6027887622922026, + "tp": 0.021811095588551047, + "ic": 0.9918883240217377 + } + }, + "overall_metrics": { + "precision": 0.9140826873385013, + "recall": 0.7456709956709957, + "f1_score": 0.7925047438330171, + "so": 0.624464020496978, + "tp": 0.2640677346088802, + "ic": 0.9939866958309409 + } + }, + "Recording_7": { + "file_metrics": { + "Recording_7": { + "attack": { + "precision": 1.0, + "recall": 0.525, + "f1_score": 0.6885245901639345, + "so": 0.3906951258181009, + "tp": 0.030228614492615706, + "ic": 0.9894707846536077 + }, + "investigation": { + "precision": 1.0, + "recall": 0.3, + "f1_score": 0.4615384615384615, + "so": 0.536495310012478, + "tp": 0.031330852383483966, + "ic": 0.9945438701149829 + }, + "mount": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "other": { + "precision": 1.0, + "recall": 0.5106382978723404, + "f1_score": 0.676056338028169, + "so": 0.7307897879334082, + "tp": 0.044561106308833955, + "ic": 0.9957595465308487 + } + } + }, + "group_metrics": { + "attack": { + "precision": 1.0, + "recall": 0.525, + "f1_score": 0.6885245901639345, + "so": 0.3906951258181009, + "tp": 0.030228614492615706, + "ic": 0.9894707846536077 + }, + "investigation": { + "precision": 1.0, + "recall": 0.3, + "f1_score": 0.4615384615384615, + "so": 0.536495310012478, + "tp": 0.031330852383483966, + "ic": 0.9945438701149829 + }, + "mount": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "other": { + "precision": 1.0, + "recall": 0.5106382978723404, + "f1_score": 0.676056338028169, + "so": 0.7307897879334082, + "tp": 0.044561106308833955, + "ic": 0.9957595465308487 + } + }, + "overall_metrics": { + "precision": 1.0, + "recall": 0.5839095744680851, + "f1_score": 0.7065298474326412, + "so": 0.6644950559409968, + "tp": 0.27653014329623343, + "ic": 0.9949435503248598 + } + }, + "Recording_8": { + "file_metrics": { + "Recording_8": { + "attack": { + "precision": 0.875, + "recall": 0.3684210526315789, + "f1_score": 0.5185185185185185, + "so": 0.2683638230814003, + "tp": 0.020422876058778405, + "ic": 0.9919190825696258 + }, + "investigation": { + "precision": 1.0, + "recall": 0.5384615384615384, + "f1_score": 0.7000000000000001, + "so": 0.5910507486515022, + "tp": 0.02527092835048601, + "ic": 0.9952821338154225 + }, + "mount": { + "precision": null, + "recall": 0.0, + "f1_score": 0.0, + "so": null, + "tp": null, + "ic": 1.0 + }, + "other": { + "precision": 0.8, + "recall": 0.48, + "f1_score": 0.6, + "so": 0.7322708331381532, + "tp": 0.06872092254807333, + "ic": 0.995105321355927 + } + } + }, + "group_metrics": { + "attack": { + "precision": 0.875, + "recall": 0.3684210526315789, + "f1_score": 0.5185185185185185, + "so": 0.2683638230814003, + "tp": 0.020422876058778405, + "ic": 0.9919190825696258 + }, + "investigation": { + "precision": 1.0, + "recall": 0.5384615384615384, + "f1_score": 0.7000000000000001, + "so": 0.5910507486515022, + "tp": 0.02527092835048601, + "ic": 0.9952821338154225 + }, + "mount": { + "precision": null, + "recall": 0.0, + "f1_score": 0.0, + "so": null, + "tp": null, + "ic": 1.0 + }, + "other": { + "precision": 0.8, + "recall": 0.48, + "f1_score": 0.6, + "so": 0.7322708331381532, + "tp": 0.06872092254807333, + "ic": 0.995105321355927 + } + }, + "overall_metrics": { + "precision": 0.8916666666666666, + "recall": 0.3467206477732793, + "f1_score": 0.4546296296296296, + "so": 0.5305618016236853, + "tp": 0.03813824231911258, + "ic": 0.9955766344352438 + } + }, + "Recording_9": { + "file_metrics": { + "Recording_9": { + "attack": { + "precision": 0.9565217391304348, + "recall": 0.7333333333333333, + "f1_score": 0.8301886792452831, + "so": 0.4241002824392379, + "tp": 0.0205898063721443, + "ic": 0.989386360253366 + }, + "investigation": { + "precision": 0.375, + "recall": 0.1875, + "f1_score": 0.25, + "so": 0.631678281068525, + "tp": 0.03480952380952381, + "ic": 0.9986300436205017 + }, + "mount": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "other": { + "precision": 0.8620689655172413, + "recall": 0.6944444444444444, + "f1_score": 0.7692307692307692, + "so": 0.6751395457875596, + "tp": 0.05583819861729283, + "ic": 0.9946692048463431 + } + } + }, + "group_metrics": { + "attack": { + "precision": 0.9565217391304348, + "recall": 0.7333333333333333, + "f1_score": 0.8301886792452831, + "so": 0.4241002824392379, + "tp": 0.0205898063721443, + "ic": 0.989386360253366 + }, + "investigation": { + "precision": 0.375, + "recall": 0.1875, + "f1_score": 0.25, + "so": 0.631678281068525, + "tp": 0.03480952380952381, + "ic": 0.9986300436205017 + }, + "mount": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "other": { + "precision": 0.8620689655172413, + "recall": 0.6944444444444444, + "f1_score": 0.7692307692307692, + "so": 0.6751395457875596, + "tp": 0.05583819861729283, + "ic": 0.9946692048463431 + } + }, + "overall_metrics": { + "precision": 0.798397676161919, + "recall": 0.6538194444444444, + "f1_score": 0.712354862119013, + "so": 0.6827295273238306, + "tp": 0.2778093821997402, + "ic": 0.9956714021800527 + } + }, + "Recording_10": { + "file_metrics": { + "Recording_10": { + "attack": { + "precision": 1.0, + "recall": 0.2857142857142857, + "f1_score": 0.4444444444444445, + "so": 0.18490328157186495, + "tp": 0.011189008329645374, + "ic": 0.9914118758934924 + }, + "investigation": { + "precision": 0.625, + "recall": 0.2631578947368421, + "f1_score": 0.37037037037037035, + "so": 0.6572072189744603, + "tp": 0.06468253968253967, + "ic": 0.9950611635162638 + }, + "mount": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "other": { + "precision": 0.9714285714285714, + "recall": 0.3541666666666667, + "f1_score": 0.5190839694656488, + "so": 0.8251145988525997, + "tp": 0.06781954841653498, + "ic": 0.998292947043181 + } + } + }, + "group_metrics": { + "attack": { + "precision": 1.0, + "recall": 0.2857142857142857, + "f1_score": 0.4444444444444445, + "so": 0.18490328157186495, + "tp": 0.011189008329645374, + "ic": 0.9914118758934924 + }, + "investigation": { + "precision": 0.625, + "recall": 0.2631578947368421, + "f1_score": 0.37037037037037035, + "so": 0.6572072189744603, + "tp": 0.06468253968253967, + "ic": 0.9950611635162638 + }, + "mount": { + "precision": 1.0, + "recall": 1.0, + "f1_score": 1.0, + "so": 1.0, + "tp": 1.0, + "ic": 1.0 + }, + "other": { + "precision": 0.9714285714285714, + "recall": 0.3541666666666667, + "f1_score": 0.5190839694656488, + "so": 0.8251145988525997, + "tp": 0.06781954841653498, + "ic": 0.998292947043181 + } + }, + "overall_metrics": { + "precision": 0.8991071428571429, + "recall": 0.47575971177944865, + "f1_score": 0.5834746960701159, + "so": 0.6668062748497312, + "tp": 0.28592277410718003, + "ic": 0.9961914966132344 + } + } +} diff --git a/tests/fixtures/matlab/generate_matlab_fixtures.m b/tests/fixtures/matlab/generate_matlab_fixtures.m index ce98f95..15279f0 100644 --- a/tests/fixtures/matlab/generate_matlab_fixtures.m +++ b/tests/fixtures/matlab/generate_matlab_fixtures.m @@ -1,8 +1,8 @@ %% generate_matlab_fixtures.m % Generates a single JSON reference file from the Matlab BANOS implementation, -% containing all 10 recordings in the same structure as golden_python_current.json. +% containing all 10 recordings in the same structure as golden_python_v110.json. % -% Output: tests/fixtures/golden_matlab_v020.json +% Output: tests/fixtures/golden_matlab_v110.json % % Run from the project root using any of: % matlab -batch "run('tests/fixtures/matlab/generate_matlab_fixtures.m')" @@ -13,7 +13,7 @@ addpath(fullfile(scriptDir, '..', '..', '..', 'matlab', 'BANOS')); datasetDir = fullfile(scriptDir, '..', '..', '..', 'data', 'dataset_human_vs_human'); -outFile = fullfile(scriptDir, '..', 'golden_matlab_v020.json'); +outFile = fullfile(scriptDir, '..', 'golden_matlab_v110.json'); fprintf('Generating Matlab BANOS fixtures...\n'); fprintf('Dataset dir: %s\n', datasetDir); @@ -67,12 +67,12 @@ clear singleData; end -% Save all recordings as a single JSON (matches golden_python_current.json structure) +% Save all recordings as a single JSON (matches golden_python_v110.json structure) fid = fopen(outFile, 'w'); if fid == -1 error('Cannot open output file: %s', outFile); end -fprintf(fid, '%s', jsonencode(allResults)); +fprintf(fid, '%s\n', jsonencode(allResults)); fclose(fid); fprintf('\nDone! Saved to: %s\n', outFile); diff --git a/tests/fixtures/python/generate_golden_fixtures.py b/tests/fixtures/python/generate_golden_fixtures.py index 7131db4..2c1b613 100644 --- a/tests/fixtures/python/generate_golden_fixtures.py +++ b/tests/fixtures/python/generate_golden_fixtures.py @@ -1,11 +1,10 @@ # tests/fixtures/python/generate_golden_fixtures.py -"""Generate golden output fixtures from the current BANOS code (v0.2.0). +"""Generate golden output fixtures from the current BANOS code (v1.1.0). -These fixtures freeze the current (corrected) behaviour for regression testing. -They reflect three improvements over archive v0.1.5: - 1. Score 1.0 for correct absence (both GT and machine have no bouts). - 2. Score 0.0 for false detection (machine predicts bouts when GT has none). - 3. IC switch-counting correctly includes the last adjacent frame pair. +These fixtures freeze the current behaviour for regression testing. +Changes since the v0.2.0 fixtures (golden_python_v020.json, kept as an archive): + 1. Detection F1 is 0, not NaN, when a recording has real bouts and no + predicted bout matches any of them. Run once to regenerate after a deliberate algorithm change: python tests/fixtures/python/generate_golden_fixtures.py @@ -24,7 +23,7 @@ from banos import aggregate_metrics, calculate_banos_for_each_file, preprocess_data DATASET_DIR = Path(__file__).parent.parent.parent.parent / "data" / "dataset_human_vs_human" -OUT_FILE = Path(__file__).parent.parent / "golden_python_v020.json" +OUT_FILE = Path(__file__).parent.parent / "golden_python_v110.json" def load_recording(n: int): @@ -68,6 +67,7 @@ def make_json_serializable(obj): OUT_FILE.parent.mkdir(parents=True, exist_ok=True) with open(OUT_FILE, "w") as f: json.dump(results, f, indent=2) + f.write("\n") print(f"\nSaved golden fixtures to {OUT_FILE}") print(f"Total recordings: {len(results)}") diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 9175540..712b53e 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -17,6 +17,10 @@ correctly excludes NaN values so they do not affect recordings where the behavior actually occurred. + F1 is the exception: with no matched bout (TP = 0) but some predicted or + missed bouts (FP + FN > 0) it is 0, the value of the count form + 2 TP / (2 TP + FP + FN). It is NaN only when TP = FP = FN = 0. + IC switch-counting range: count_switches_within_bout uses range(gt_start, gt_end) — inclusive of gt_end-1, checking all adjacent pairs (i, i+1) up to (gt_end-1, gt_end). This was a bug @@ -88,6 +92,29 @@ def test_prf1_balanced(): assert abs(f1 - 2 / 3) < 1e-10 +def test_prf1_nothing_predicted_f1_is_zero(): + """Real bouts, nothing predicted: precision is 0/0, but F1 is 0, not NaN.""" + p, r, f1 = calculate_precision_recall_f1(0, 0, 3) + assert math.isnan(p) + assert r == 0.0 + assert f1 == 0.0 + + +def test_prf1_only_false_bouts_f1_is_zero(): + """Only false bouts and no real bout: recall is 0/0, but F1 is 0, not NaN.""" + p, r, f1 = calculate_precision_recall_f1(0, 3, 0) + assert p == 0.0 + assert math.isnan(r) + assert f1 == 0.0 + + +def test_prf1_false_and_missed_bouts_f1_is_zero(): + """Predicted and real bouts that never overlap: P = R = 0 and F1 = 0.""" + p, r, f1 = calculate_precision_recall_f1(0, 2, 2) + assert p == 0.0 and r == 0.0 + assert f1 == 0.0 + + # --- calculate_tiou --- diff --git a/tests/test_parity.py b/tests/test_parity.py index 58ae3db..eeef04f 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -9,11 +9,11 @@ FIXTURE FILE ------------ -tests/fixtures/golden_matlab_v020.json — generated by: +tests/fixtures/golden_matlab_v110.json — generated by: matlab -batch "run('tests/fixtures/matlab/generate_matlab_fixtures.m')" The file contains all 10 recordings in a single JSON (same structure as -golden_python_current.json), keyed by "Recording_N". +golden_python_v110.json), keyed by "Recording_N". TOLERANCE --------- @@ -40,7 +40,7 @@ from banos import aggregate_metrics, calculate_banos_for_each_file, preprocess_data DATASET_DIR = Path(__file__).parent.parent / "data" / "dataset_human_vs_human" -MATLAB_FIXTURE = Path(__file__).parent / "fixtures" / "golden_matlab_v020.json" +MATLAB_FIXTURE = Path(__file__).parent / "fixtures" / "golden_matlab_v110.json" TOLERANCE = 1e-6 # floating point tolerance for Matlab vs Python comparisons METRICS = ["precision", "recall", "f1_score", "so", "tp", "ic"] @@ -48,7 +48,7 @@ # Load MATLAB fixture once (skip all tests if absent) if not MATLAB_FIXTURE.exists(): pytest.skip( - "Matlab fixture not found: tests/fixtures/golden_matlab_v020.json — " + "Matlab fixture not found: tests/fixtures/golden_matlab_v110.json — " "run: matlab -batch \"run('tests/fixtures/matlab/generate_matlab_fixtures.m')\"", allow_module_level=True, ) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 6bd912c..63c30aa 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -39,6 +39,25 @@ } ) +# Hand-made videos of 60 frames with three real bouts of one behavior ("evt"). +N_FRAMES = 60 +REAL_BOUTS = [(5, 9), (25, 29), (45, 49)] + + +def _frames(bouts: list[tuple[int, int]]) -> list[int]: + """0/1 sequence of N_FRAMES frames, 1 on every frame of the (start, end) bouts.""" + sequence = [0] * N_FRAMES + for start, end in bouts: + sequence[start : end + 1] = [1] * (end - start + 1) + return sequence + + +def _video( + pred_bouts: list[tuple[int, int]], gt_bouts: list[tuple[int, int]] = REAL_BOUTS +) -> tuple[pd.DataFrame, pd.DataFrame]: + """(pred_df, gt_df) pair of one video from its predicted and real bouts.""" + return pd.DataFrame({"evt": _frames(pred_bouts)}), pd.DataFrame({"evt": _frames(gt_bouts)}) + def test_score_single_file_returns_dict(): result = banos.score(PRED, GT) @@ -128,3 +147,42 @@ def test_aggregate_metrics_excludes_nan(): def test_score_version(): assert isinstance(banos.__version__, str) assert banos.__version__ != "" + + +def test_file_with_only_false_bouts_scores_f1_zero(): + """A file whose predicted bouts all miss the real bouts scores F1 0 and stays in the mean. + + Both models predict video 1 exactly. In video 2, model C predicts three + bouts that overlap no real bout (TP 0, FP 3, FN 3: P 0, R 0, F1 0) and + model D finds two of the three real bouts (TP 2, FN 1: P 1, R 2/3, F1 0.8). + Per-file means: C (1 + 0) / 2 = 0.5, D (1 + 0.8) / 2 = 0.9. Before 1.1.0, + C's video 2 scored F1 NaN, left the mean, and C got 1.0. + """ + false_bouts = [(15, 19), (35, 39), (52, 56)] + c_video2 = banos.score(*_video(false_bouts))["evt"] + assert c_video2["precision"] == 0.0 + assert c_video2["recall"] == 0.0 + assert c_video2["f1_score"] == 0.0 + + group_c, _ = banos.score({"v1": _video(REAL_BOUTS), "v2": _video(false_bouts)}) + group_d, _ = banos.score({"v1": _video(REAL_BOUTS), "v2": _video(REAL_BOUTS[:2])}) + assert abs(group_c["evt"]["f1_score"] - 0.5) < 1e-10 + assert abs(group_d["evt"]["f1_score"] - 0.9) < 1e-10 + + +def test_silent_model_on_bundled_data(all_recordings): + """A model that predicts nothing scores F1 0 on every recording with the behavior. + + Its only non-zero F1 cells are the correct absences (1.0). In the bundled + data, attack is absent from 5 of the 10 recordings and mount from 4, while + investigation and other are in all 10. Per-file means: attack 5/10 = 0.5, + mount 4/10 = 0.4, investigation 0, other 0; overall F1 = 9 cells of 40 = + 0.225. Before 1.1.0 the recordings with the behavior scored F1 NaN and left + the means: attack, mount and overall F1 were 1.0. + """ + data = {f"Recording_{n}": (gt * 0, gt) for n, (_, gt) in all_recordings.items()} + group, overall = banos.score(data) + expected = {"attack": 0.5, "investigation": 0.0, "mount": 0.4, "other": 0.0} + for behavior, f1 in expected.items(): + assert abs(group[behavior]["f1_score"] - f1) < 1e-10, behavior + assert abs(overall["f1_score"] - 0.225) < 1e-10 diff --git a/tests/test_regression.py b/tests/test_regression.py index 51cce7f..4ea6837 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -7,15 +7,14 @@ If any refactoring introduces an unintended numerical change — even a tiny floating-point difference — these tests catch it immediately. -The golden fixtures in tests/fixtures/golden_python_v020.json represent the -CORRECTED v0.2.0 behaviour (three improvements over v0.1.5): - 1. Score 1.0 for correct absence (both GT and machine have no bouts). - 2. Score 0.0 for false detection (machine predicts bouts when GT has none). - 3. IC switch-counting covers all adjacent frame pairs within each GT bout - (the original code missed the last pair, giving inflated IC values). +The golden fixtures in tests/fixtures/golden_python_v110.json represent the +v1.1.0 behaviour. Changes since v0.2.0 that move values of the bundled data: + 1. Detection F1 is 0, not NaN, when a recording has real bouts and no + predicted bout matches any of them (Recording_8 mount). -The archive fixtures (tests/fixtures/golden_python_v015.json) preserve -the original v0.1.5 behaviour for reference and comparison. +The archive fixtures (tests/fixtures/golden_python_v020.json and +golden_python_v015.json) preserve the v0.2.0 and v0.1.5 behaviour for +reference and comparison. JSON null = Python NaN ---------------------- @@ -27,8 +26,8 @@ ----------------- If you make a deliberate algorithm change that affects output values, update the fixtures and document why: - python scripts/generate_golden_fixtures.py - # Then commit the updated tests/fixtures/golden_python_v020.json with a + python tests/fixtures/python/generate_golden_fixtures.py + # Then commit the updated tests/fixtures/golden_python_v110.json with a # clear commit message explaining the change. """ @@ -42,7 +41,7 @@ from banos import aggregate_metrics, calculate_banos_for_each_file, preprocess_data DATASET_DIR = Path(__file__).parent.parent / "data" / "dataset_human_vs_human" -GOLDEN_FILE = Path(__file__).parent / "fixtures" / "golden_python_v020.json" +GOLDEN_FILE = Path(__file__).parent / "fixtures" / "golden_python_v110.json" TOLERANCE = 1e-10 with open(GOLDEN_FILE) as f: @@ -77,7 +76,7 @@ def test_regression_overall_metrics(n): If this fails after a code change, the change affected numerical output. Decide whether the change is a bug fix or a regression, then either: - Fix the regression (no fixture update needed), OR - - Deliberately update: python scripts/generate_golden_fixtures.py + - Deliberately update: python tests/fixtures/python/generate_golden_fixtures.py """ pred, gt = load_recording(n) data, _ = preprocess_data({f"Recording_{n}": (pred.copy(), gt.copy())}) @@ -96,9 +95,9 @@ def test_regression_overall_metrics(n): def test_regression_group_metrics(n): """Per-behavior metrics for each recording must match golden fixtures (tol=1e-10). - `null` in the JSON fixture means NaN — the behavior had no GT bouts in that - recording and the metric is undefined. The `_close()` helper treats `null` - and `float('nan')` as equivalent. + `null` in the JSON fixture means NaN — the metric is undefined for that + behavior in that recording (e.g. precision when nothing is predicted). + The `_close()` helper treats `null` and `float('nan')` as equivalent. """ pred, gt = load_recording(n) data, _ = preprocess_data({f"Recording_{n}": (pred.copy(), gt.copy())}) From a250f4141fe8c594f2d29df21cf5928248aa278c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:54:35 +0100 Subject: [PATCH 02/25] fix: read predictions by position in intra-bout continuity count_switches_within_bout read pred_sequence[i] on a pandas Series, which is a lookup by index label, while bouts are frame positions. Data indexed other than 0..n-1 (frames numbered from 1, reversed or float labels, strings on pandas 3) got a wrong IC or a KeyError. count_switches_within_bout and calculate_ic now read a Series by position, and compute_behavior_metrics passes the column values. No golden value changes; the MATLAB code has no index labels. --- src/banos/core/metrics.py | 18 ++++++++++++++++-- src/banos/pipeline.py | 2 +- tests/test_metrics.py | 24 ++++++++++++++++++++++++ tests/test_pipeline.py | 26 ++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/banos/core/metrics.py b/src/banos/core/metrics.py index 5dc93b9..d0e2ca0 100644 --- a/src/banos/core/metrics.py +++ b/src/banos/core/metrics.py @@ -5,9 +5,15 @@ import math +import numpy as np import pandas as pd +def _by_position(sequence: pd.Series | np.ndarray | list) -> np.ndarray | list: + """Return ``sequence`` indexable by frame position (a Series gives its values).""" + return sequence.to_numpy() if isinstance(sequence, pd.Series) else sequence + + def identify_bouts(sequence: pd.Series | list) -> list[tuple[int, int]]: """Identify continuous runs of 1s in a binary sequence. @@ -88,12 +94,16 @@ def calculate_tp_metric( return sum(scores) / len(scores) if scores else float("nan") -def count_switches_within_bout(pred_sequence: pd.Series | list, gt_start: int, gt_end: int) -> int: +def count_switches_within_bout( + pred_sequence: pd.Series | np.ndarray | list, gt_start: int, gt_end: int +) -> int: """Count label transitions within [gt_start, gt_end] (inclusive). Compares pred[i] with pred[i+1] for i in [gt_start, gt_end-1]. This includes the transition at the last frame boundary (gt_end-1 → gt_end). + Frames are read by position: the index labels of a Series are ignored. """ + pred_sequence = _by_position(pred_sequence) switches = 0 for i in range(gt_start, gt_end): # gt_end inclusive: checks pairs up to (gt_end-1, gt_end) if pred_sequence[i] != pred_sequence[i + 1]: @@ -101,13 +111,17 @@ def count_switches_within_bout(pred_sequence: pd.Series | list, gt_start: int, g return switches -def calculate_ic(pred_sequence: pd.Series | list, gt_bouts: list[tuple[int, int]]) -> float: +def calculate_ic( + pred_sequence: pd.Series | np.ndarray | list, gt_bouts: list[tuple[int, int]] +) -> float: """Intra-bout Continuity: 1 - (switches / gt_length) averaged over GT bouts. + Frames are read by position: the index labels of a Series are ignored. Returns NaN if there are no GT bouts. """ if not gt_bouts: return float("nan") + pred_sequence = _by_position(pred_sequence) ic_scores = [] for gt in gt_bouts: gt_length = gt[1] - gt[0] diff --git a/src/banos/pipeline.py b/src/banos/pipeline.py index 5b61fa7..6f0e2a6 100644 --- a/src/banos/pipeline.py +++ b/src/banos/pipeline.py @@ -60,7 +60,7 @@ def compute_behavior_metrics( precision, recall, f1_score = calculate_precision_recall_f1(n_tp, n_fp, n_fn) so = calculate_so(pred_bouts, gt_bouts) tp_m = calculate_tp_metric(pred_bouts, gt_bouts) - ic = calculate_ic(pred_matrix[behavior], gt_bouts) + ic = calculate_ic(pred_matrix[behavior].to_numpy(), gt_bouts) metrics[behavior] = { "precision": precision, "recall": recall, diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 712b53e..9459bc7 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -29,6 +29,9 @@ import math +import pandas as pd +import pytest + from banos.core.metrics import ( calculate_ic, calculate_precision_recall_f1, @@ -215,6 +218,27 @@ def test_switches_subset_range(): assert count_switches_within_bout([0, 1, 0, 1, 0], 1, 3) == 2 +SERIES_INDEXES = { + "1-based": [1, 2, 3, 4, 5], + "reversed": [4, 3, 2, 1, 0], + "float": [0.0, 0.04, 0.08, 0.12, 0.16], + "string": ["f0", "f1", "f2", "f3", "f4"], +} + + +@pytest.mark.parametrize("kind", SERIES_INDEXES) +def test_switches_and_ic_read_series_by_position(kind): + """A Series is read by frame position; its index labels are ignored. + + pred=[1,1,0,0,0], gt=(0,2): one switch (frames 1-2) in length 2 -> IC 0.5. + Read by label, the 1-based and float indexes raise KeyError and the + reversed one reads frames 4, 3, 2, which hold no switch. + """ + pred = pd.Series([1, 1, 0, 0, 0], index=SERIES_INDEXES[kind]) + assert count_switches_within_bout(pred, 0, 2) == 1 + assert calculate_ic(pred, [(0, 2)]) == 0.5 + + # --- calculate_ic --- diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 63c30aa..318cc78 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -149,6 +149,32 @@ def test_score_version(): assert banos.__version__ != "" +FRAME_INDEXES = { + "1-based": range(1, N_FRAMES + 1), + "reversed": range(N_FRAMES - 1, -1, -1), + "float": [frame / 30 for frame in range(N_FRAMES)], + "string": [f"frame_{frame}" for frame in range(N_FRAMES)], +} + + +@pytest.mark.parametrize("kind", FRAME_INDEXES) +def test_score_reads_frames_by_position_whatever_the_index(kind): + """The DataFrame index is ignored: every metric reads the frames by position. + + Real bouts (5,9), (25,29), (45,49); predicted (5,9), (25,27), (45,49). + The second real bout has one switch (frames 27-28) in length 4, so + IC = (1 + 3/4 + 1) / 3 = 11/12 with the default index, and must stay so + with frames numbered from 1, reversed labels, times in seconds or strings. + Read by label, IC was 2/3 (1-based), 1.0 (reversed) or a KeyError. + """ + pred, gt = _video([(5, 9), (25, 27), (45, 49)]) + expected = banos.score(pred, gt) + assert abs(expected["evt"]["ic"] - 11 / 12) < 1e-10 + + index = pd.Index(FRAME_INDEXES[kind]) + assert banos.score(pred.set_axis(index), gt.set_axis(index)) == expected + + def test_file_with_only_false_bouts_scores_f1_zero(): """A file whose predicted bouts all miss the real bouts scores F1 0 and stays in the mean. From 9ca7b63564dce2994f728584fa139b4bb674ec86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:55:57 +0100 Subject: [PATCH 03/25] fix: average intra-bout continuity over found real bouts only calculate_ic averaged 1 - switches / (length - 1) over every real bout. A missed real bout has no predicted frame, hence no switch, and scored 1.0: missing a bout raised IC, and a model that predicted nothing got IC 1.0. IC now averages over the real bouts that contain at least one predicted frame (the overlap rule of SO and TP) and is NaN when no real bout is found. One-frame real bouts stay out of IC. The MATLAB function BANOScalculateIC gets the same rule. 39 golden IC values change, in every recording. Recording_8 mount IC goes from 1.0 to null: its two real bouts are both missed. --- matlab/BANOS/BANOScalculateIC.m | 15 +- src/banos/core/metrics.py | 11 +- tests/fixtures/golden_matlab_v110.json | 2 +- tests/fixtures/golden_python_v110.json | 136 +++++++++--------- .../python/generate_golden_fixtures.py | 2 + tests/test_metrics.py | 16 +++ tests/test_pipeline.py | 5 + tests/test_regression.py | 2 + 8 files changed, 111 insertions(+), 78 deletions(-) diff --git a/matlab/BANOS/BANOScalculateIC.m b/matlab/BANOS/BANOScalculateIC.m index e4cd775..3a7f5be 100644 --- a/matlab/BANOS/BANOScalculateIC.m +++ b/matlab/BANOS/BANOScalculateIC.m @@ -1,9 +1,11 @@ function ic = BANOScalculateIC(predSequence, gtBouts) % Calculates the Intra-bout Continuity (IC) for each ground truth bout. % - % IC = 1 - (switches / boutLength) averaged over all GT bouts. - % Zero-length bouts are excluded from the average (contribute NaN). - % Returns NaN if gtBouts is empty or all bouts are zero-length + % IC = 1 - (switches / boutLength) averaged over the found GT bouts. + % A GT bout is found when at least one of its frames is predicted (value 1). + % Missed bouts and zero-length bouts are excluded from the average + % (contribute NaN). + % Returns NaN if gtBouts is empty or no found bout is longer than zero % (matches Python behaviour — NaN propagates through nanmean). % % Arguments: @@ -11,16 +13,17 @@ % gtBouts - Cell array of ground truth bouts {[start, end], ...}. % % Returns: - % ic - The average IC across GT bouts, or NaN if undefined. + % ic - The average IC across found GT bouts, or NaN if undefined. icScores = []; for i = 1:length(gtBouts) boutLength = gtBouts{i}(2) - gtBouts{i}(1); switches = BANOScountSwitchesWithinBout(predSequence, gtBouts{i}(1), gtBouts{i}(2)); - if boutLength > 0 + found = any(predSequence(gtBouts{i}(1):gtBouts{i}(2)) == 1); + if boutLength > 0 && found icScores(end + 1) = 1 - (switches / boutLength); else - icScores(end + 1) = NaN; % zero-length bout: undefined, excluded from mean + icScores(end + 1) = NaN; % zero-length or missed bout: excluded from mean end end diff --git a/src/banos/core/metrics.py b/src/banos/core/metrics.py index d0e2ca0..3663022 100644 --- a/src/banos/core/metrics.py +++ b/src/banos/core/metrics.py @@ -114,10 +114,14 @@ def count_switches_within_bout( def calculate_ic( pred_sequence: pd.Series | np.ndarray | list, gt_bouts: list[tuple[int, int]] ) -> float: - """Intra-bout Continuity: 1 - (switches / gt_length) averaged over GT bouts. + """Intra-bout Continuity: 1 - (switches / gt_length) averaged over found GT bouts. + A GT bout is found when at least one of its frames is predicted (value 1). + Missed GT bouts are left out of the average, and so are one-frame GT bouts + (gt_length = end - start = 0, no frame pair to compare). Frames are read by position: the index labels of a Series are ignored. - Returns NaN if there are no GT bouts. + Returns NaN if there are no GT bouts or none of them is found and longer + than one frame. """ if not gt_bouts: return float("nan") @@ -125,7 +129,8 @@ def calculate_ic( ic_scores = [] for gt in gt_bouts: gt_length = gt[1] - gt[0] - if gt_length > 0: + found = any(pred_sequence[i] == 1 for i in range(gt[0], gt[1] + 1)) + if gt_length > 0 and found: switches = count_switches_within_bout(pred_sequence, gt[0], gt[1]) ic_scores.append(1.0 - (switches / gt_length)) else: diff --git a/tests/fixtures/golden_matlab_v110.json b/tests/fixtures/golden_matlab_v110.json index a0936cd..28582b7 100644 --- a/tests/fixtures/golden_matlab_v110.json +++ b/tests/fixtures/golden_matlab_v110.json @@ -1 +1 @@ -{"Recording_1":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.98294071362580027},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.99086091350216854},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.99306915975344445}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.98294071362580027},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.99086091350216854},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.99306915975344445}},"overall_metrics":{"precision":0.92826086956521736,"recall":0.78129251700680269,"f1_score":0.836455036759336,"so":0.761228298078524,"tp":0.30623014745659,"ic":0.99171769672035337}},"Recording_2":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98905093606628935},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99464840013584821},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99192990230311961}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98905093606628935},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99464840013584821},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99192990230311961}},"overall_metrics":{"precision":0.9671052631578948,"recall":0.83992445266728333,"f1_score":0.897300738477209,"so":0.82594559036003834,"tp":0.30908214265027689,"ic":0.99390730962631435}},"Recording_3":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98496628121015162},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.9917129219739137},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99621162045778}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98496628121015162},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.9917129219739137},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99621162045778}},"overall_metrics":{"precision":0.95948072383949645,"recall":0.85096664987969339,"f1_score":0.89862686038270723,"so":0.723715686180092,"tp":0.26887555753525666,"ic":0.99322270591046136}},"Recording_4":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.99528949732999117},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99394902176918432}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.99528949732999117},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99394902176918432}},"overall_metrics":{"precision":0.975,"recall":0.61568627450980384,"f1_score":0.72916666666666674,"so":0.70708735677240686,"tp":0.27571526135052643,"ic":0.9965302523591395}},"Recording_5":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98991370716597},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99654108838242228},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.994024222522601}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98991370716597},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99654108838242228},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.994024222522601}},"overall_metrics":{"precision":1,"recall":0.89210526315789473,"f1_score":0.94145869368256319,"so":0.83673046008612484,"tp":0.27928892486843937,"ic":0.99511975451774837}},"Recording_6":{"file_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.99366643311730019},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.99039202618472566},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.99188832402173766}},"group_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.99366643311730019},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.99039202618472566},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.99188832402173766}},"overall_metrics":{"precision":0.91408268733850129,"recall":0.74567099567099571,"f1_score":0.79250474383301706,"so":0.624464020496978,"tp":0.26406773460888022,"ic":0.9939866958309409}},"Recording_7":{"file_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98947078465360738},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.99454387011498291},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.995759546530849}},"group_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98947078465360738},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.99454387011498291},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.995759546530849}},"overall_metrics":{"precision":1,"recall":0.58390957446808511,"f1_score":0.70652984743264124,"so":0.66449505594099667,"tp":0.27653014329623343,"ic":0.99494355032485982}},"Recording_8":{"file_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99528213381542252},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":1},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99510532135592722}},"group_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99528213381542252},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":1},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99510532135592722}},"overall_metrics":{"precision":0.89166666666666661,"recall":0.34672064777327932,"f1_score":0.45462962962962961,"so":0.53056180162368516,"tp":0.038138242319112579,"ic":0.99557663443524391}},"Recording_9":{"file_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98938636025336613},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99863004362050167},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99466920484634314}},"group_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98938636025336613},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99863004362050167},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99466920484634314}},"overall_metrics":{"precision":0.79839767616191892,"recall":0.65381944444444451,"f1_score":0.712354862119013,"so":0.68272952732383074,"tp":0.27780938219974027,"ic":0.99567140218005268}},"Recording_10":{"file_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.99141187589349233},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.99506116351626372},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.998292947043181}},"group_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.99141187589349233},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.99506116351626372},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.998292947043181}},"overall_metrics":{"precision":0.89910714285714288,"recall":0.47575971177944865,"f1_score":0.58347469607011593,"so":0.66680627484973121,"tp":0.28592277410718,"ic":0.99619149661323425}}} +{"Recording_1":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.9805603480852142},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.98286421281656633},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.992439083367394}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.9805603480852142},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.98286421281656633},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.992439083367394}},"overall_metrics":{"precision":0.92826086956521736,"recall":0.78129251700680269,"f1_score":0.836455036759336,"so":0.761228298078524,"tp":0.30623014745659,"ic":0.98896591106729359}},"Recording_2":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98771568436705637},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99357808016301785},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99074891239625884}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98771568436705637},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99357808016301785},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99074891239625884}},"overall_metrics":{"precision":0.9671052631578948,"recall":0.83992445266728333,"f1_score":0.897300738477209,"so":0.82594559036003834,"tp":0.30908214265027689,"ic":0.99301066923158332}},"Recording_3":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98261726264923754},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.98941095585555594},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99591253786234157}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98261726264923754},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.98941095585555594},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99591253786234157}},"overall_metrics":{"precision":0.95948072383949645,"recall":0.85096664987969339,"f1_score":0.89862686038270723,"so":0.723715686180092,"tp":0.26887555753525666,"ic":0.99198518909178379}},"Recording_4":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.98586849198997339},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99092353265377653}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.98586849198997339},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99092353265377653}},"overall_metrics":{"precision":0.975,"recall":0.61568627450980384,"f1_score":0.72916666666666674,"so":0.70708735677240686,"tp":0.27571526135052643,"ic":0.99341862874528308}},"Recording_5":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98823265836029839},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99615676486935811},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.993672706200401}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98823265836029839},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99615676486935811},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.993672706200401}},"overall_metrics":{"precision":1,"recall":0.89210526315789473,"f1_score":0.94145869368256319,"so":0.83673046008612484,"tp":0.27928892486843937,"ic":0.99451553235751433}},"Recording_6":{"file_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.98838846071505027},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.98965295127585839},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.99046878072554168}},"group_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.98838846071505027},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.98965295127585839},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.99046878072554168}},"overall_metrics":{"precision":0.91408268733850129,"recall":0.74567099567099571,"f1_score":0.79250474383301706,"so":0.624464020496978,"tp":0.26406773460888022,"ic":0.99212754817911253}},"Recording_7":{"file_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98891661542484988},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.98181290038327618},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.99318498549600709}},"group_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98891661542484988},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.98181290038327618},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.99318498549600709}},"overall_metrics":{"precision":1,"recall":0.58390957446808511,"f1_score":0.70652984743264124,"so":0.66449505594099667,"tp":0.27653014329623343,"ic":0.99097862532603331}},"Recording_8":{"file_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99318530440005481},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":null},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99102642248586637}},"group_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99318530440005481},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":null},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99102642248586637}},"overall_metrics":{"precision":0.89166666666666661,"recall":0.34672064777327932,"f1_score":0.45462962962962961,"so":0.53056180162368516,"tp":0.038138242319112579,"ic":0.992043603151849}},"Recording_9":{"file_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98862824312860653},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99269356597600877},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99282392960084676}},"group_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98862824312860653},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99269356597600877},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99282392960084676}},"overall_metrics":{"precision":0.79839767616191892,"recall":0.65381944444444451,"f1_score":0.712354862119013,"so":0.68272952732383074,"tp":0.27780938219974027,"ic":0.99353643467636554}},"Recording_10":{"file_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.9908056553683271},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.98123242136180233},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.99554269505719462}},"group_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.9908056553683271},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.98123242136180233},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.99554269505719462}},"overall_metrics":{"precision":0.89910714285714288,"recall":0.47575971177944865,"f1_score":0.58347469607011593,"so":0.66680627484973121,"tp":0.28592277410718,"ic":0.99189519294683093}}} diff --git a/tests/fixtures/golden_python_v110.json b/tests/fixtures/golden_python_v110.json index 37ed23f..d111c44 100644 --- a/tests/fixtures/golden_python_v110.json +++ b/tests/fixtures/golden_python_v110.json @@ -16,7 +16,7 @@ "f1_score": 0.7659574468085106, "so": 0.5444095073699892, "tp": 0.0846356832664267, - "ic": 0.9829407136258003 + "ic": 0.9805603480852142 }, "mount": { "precision": 1.0, @@ -24,7 +24,7 @@ "f1_score": 0.6956521739130436, "so": 0.7332766872436964, "tp": 0.07847125492506643, - "ic": 0.9908609135021688 + "ic": 0.9828642128165663 }, "other": { "precision": 0.9130434782608695, @@ -32,7 +32,7 @@ "f1_score": 0.8842105263157894, "so": 0.7672269977004106, "tp": 0.061813651634866906, - "ic": 0.9930691597534443 + "ic": 0.9924390833673938 } } }, @@ -51,7 +51,7 @@ "f1_score": 0.7659574468085106, "so": 0.5444095073699892, "tp": 0.0846356832664267, - "ic": 0.9829407136258003 + "ic": 0.9805603480852142 }, "mount": { "precision": 1.0, @@ -59,7 +59,7 @@ "f1_score": 0.6956521739130436, "so": 0.7332766872436964, "tp": 0.07847125492506643, - "ic": 0.9908609135021688 + "ic": 0.9828642128165663 }, "other": { "precision": 0.9130434782608695, @@ -67,7 +67,7 @@ "f1_score": 0.8842105263157894, "so": 0.7672269977004106, "tp": 0.061813651634866906, - "ic": 0.9930691597534443 + "ic": 0.9924390833673938 } }, "overall_metrics": { @@ -76,7 +76,7 @@ "f1_score": 0.836455036759336, "so": 0.7612282980785241, "tp": 0.30623014745659, - "ic": 0.9917176967203534 + "ic": 0.9889659110672936 } }, "Recording_2": { @@ -96,7 +96,7 @@ "f1_score": 0.8095238095238095, "so": 0.6609424826096988, "tp": 0.04615109869237036, - "ic": 0.9890509360662894 + "ic": 0.9877156843670564 }, "mount": { "precision": 1.0, @@ -104,7 +104,7 @@ "f1_score": 0.9090909090909091, "so": 0.931754332584827, "tp": 0.1451695616211745, - "ic": 0.9946484001358482 + "ic": 0.9935780801630179 }, "other": { "precision": 0.9736842105263158, @@ -112,7 +112,7 @@ "f1_score": 0.8705882352941177, "so": 0.7110855462456277, "tp": 0.04500791028756257, - "ic": 0.9919299023031196 + "ic": 0.9907489123962591 } } }, @@ -131,7 +131,7 @@ "f1_score": 0.8095238095238095, "so": 0.6609424826096988, "tp": 0.04615109869237036, - "ic": 0.9890509360662894 + "ic": 0.9877156843670564 }, "mount": { "precision": 1.0, @@ -139,7 +139,7 @@ "f1_score": 0.9090909090909091, "so": 0.931754332584827, "tp": 0.1451695616211745, - "ic": 0.9946484001358482 + "ic": 0.9935780801630179 }, "other": { "precision": 0.9736842105263158, @@ -147,7 +147,7 @@ "f1_score": 0.8705882352941177, "so": 0.7110855462456277, "tp": 0.04500791028756257, - "ic": 0.9919299023031196 + "ic": 0.9907489123962591 } }, "overall_metrics": { @@ -156,7 +156,7 @@ "f1_score": 0.897300738477209, "so": 0.8259455903600383, "tp": 0.30908214265027684, - "ic": 0.9939073096263142 + "ic": 0.9930106692315833 } }, "Recording_3": { @@ -176,7 +176,7 @@ "f1_score": 0.8529411764705881, "so": 0.4563103530678653, "tp": 0.02232158655731043, - "ic": 0.9849662812101514 + "ic": 0.9826172626492377 }, "mount": { "precision": 1.0, @@ -184,7 +184,7 @@ "f1_score": 0.85, "so": 0.6745042152422358, "tp": 0.028794827943741816, - "ic": 0.9917129219739135 + "ic": 0.9894109558555562 }, "other": { "precision": 0.9024390243902439, @@ -192,7 +192,7 @@ "f1_score": 0.8915662650602411, "so": 0.7640481764102668, "tp": 0.024385815639974426, - "ic": 0.9962116204577801 + "ic": 0.9959125378623417 } } }, @@ -211,7 +211,7 @@ "f1_score": 0.8529411764705881, "so": 0.4563103530678653, "tp": 0.02232158655731043, - "ic": 0.9849662812101514 + "ic": 0.9826172626492377 }, "mount": { "precision": 1.0, @@ -219,7 +219,7 @@ "f1_score": 0.85, "so": 0.6745042152422358, "tp": 0.028794827943741816, - "ic": 0.9917129219739135 + "ic": 0.9894109558555562 }, "other": { "precision": 0.9024390243902439, @@ -227,7 +227,7 @@ "f1_score": 0.8915662650602411, "so": 0.7640481764102668, "tp": 0.024385815639974426, - "ic": 0.9962116204577801 + "ic": 0.9959125378623417 } }, "overall_metrics": { @@ -236,7 +236,7 @@ "f1_score": 0.8986268603827072, "so": 0.723715686180092, "tp": 0.26887555753525666, - "ic": 0.9932227059104612 + "ic": 0.9919851890917839 } }, "Recording_4": { @@ -264,7 +264,7 @@ "f1_score": 0.5, "so": 0.7011185281643236, "tp": 0.04008717445788384, - "ic": 0.9952894973299912 + "ic": 0.9858684919899735 }, "other": { "precision": 1.0, @@ -272,7 +272,7 @@ "f1_score": 0.7499999999999999, "so": 0.7357598909762686, "tp": 0.04874865293870875, - "ic": 0.9939490217691843 + "ic": 0.9909235326537765 } } }, @@ -299,7 +299,7 @@ "f1_score": 0.5, "so": 0.7011185281643236, "tp": 0.04008717445788384, - "ic": 0.9952894973299912 + "ic": 0.9858684919899735 }, "other": { "precision": 1.0, @@ -307,7 +307,7 @@ "f1_score": 0.7499999999999999, "so": 0.7357598909762686, "tp": 0.04874865293870875, - "ic": 0.9939490217691843 + "ic": 0.9909235326537765 } }, "overall_metrics": { @@ -316,7 +316,7 @@ "f1_score": 0.7291666666666666, "so": 0.7070873567724069, "tp": 0.27571526135052643, - "ic": 0.9965302523591395 + "ic": 0.9934186287452831 } }, "Recording_5": { @@ -336,7 +336,7 @@ "f1_score": 0.888888888888889, "so": 0.6589591312525748, "tp": 0.035333351984487595, - "ic": 0.98991370716597 + "ic": 0.9882326583602984 }, "mount": { "precision": 1.0, @@ -344,7 +344,7 @@ "f1_score": 0.9473684210526316, "so": 0.9100237928302215, "tp": 0.03596010219174562, - "ic": 0.9965410883824223 + "ic": 0.9961567648693581 }, "other": { "precision": 1.0, @@ -352,7 +352,7 @@ "f1_score": 0.9295774647887324, "so": 0.7779389162617028, "tp": 0.045862245297524175, - "ic": 0.9940242225226008 + "ic": 0.9936727062004008 } } }, @@ -371,7 +371,7 @@ "f1_score": 0.888888888888889, "so": 0.6589591312525748, "tp": 0.035333351984487595, - "ic": 0.98991370716597 + "ic": 0.9882326583602984 }, "mount": { "precision": 1.0, @@ -379,7 +379,7 @@ "f1_score": 0.9473684210526316, "so": 0.9100237928302215, "tp": 0.03596010219174562, - "ic": 0.9965410883824223 + "ic": 0.9961567648693581 }, "other": { "precision": 1.0, @@ -387,7 +387,7 @@ "f1_score": 0.9295774647887324, "so": 0.7779389162617028, "tp": 0.045862245297524175, - "ic": 0.9940242225226008 + "ic": 0.9936727062004008 } }, "overall_metrics": { @@ -396,7 +396,7 @@ "f1_score": 0.9414586936825633, "so": 0.8367304600861247, "tp": 0.27928892486843937, - "ic": 0.9951197545177483 + "ic": 0.9945155323575143 } }, "Recording_6": { @@ -408,7 +408,7 @@ "f1_score": 0.5333333333333333, "so": 0.3554875922085192, "tp": 0.014142932226192012, - "ic": 0.9936664331173002 + "ic": 0.9883884607150503 }, "investigation": { "precision": 0.7674418604651163, @@ -416,7 +416,7 @@ "f1_score": 0.7764705882352941, "so": 0.5395797274871901, "tp": 0.020316910620777756, - "ic": 0.9903920261847257 + "ic": 0.9896529512758584 }, "mount": { "precision": 1.0, @@ -432,7 +432,7 @@ "f1_score": 0.8602150537634408, "so": 0.6027887622922026, "tp": 0.021811095588551047, - "ic": 0.9918883240217377 + "ic": 0.9904687807255417 } } }, @@ -443,7 +443,7 @@ "f1_score": 0.5333333333333333, "so": 0.3554875922085192, "tp": 0.014142932226192012, - "ic": 0.9936664331173002 + "ic": 0.9883884607150503 }, "investigation": { "precision": 0.7674418604651163, @@ -451,7 +451,7 @@ "f1_score": 0.7764705882352941, "so": 0.5395797274871901, "tp": 0.020316910620777756, - "ic": 0.9903920261847257 + "ic": 0.9896529512758584 }, "mount": { "precision": 1.0, @@ -467,7 +467,7 @@ "f1_score": 0.8602150537634408, "so": 0.6027887622922026, "tp": 0.021811095588551047, - "ic": 0.9918883240217377 + "ic": 0.9904687807255417 } }, "overall_metrics": { @@ -476,7 +476,7 @@ "f1_score": 0.7925047438330171, "so": 0.624464020496978, "tp": 0.2640677346088802, - "ic": 0.9939866958309409 + "ic": 0.9921275481791125 } }, "Recording_7": { @@ -488,7 +488,7 @@ "f1_score": 0.6885245901639345, "so": 0.3906951258181009, "tp": 0.030228614492615706, - "ic": 0.9894707846536077 + "ic": 0.9889166154248502 }, "investigation": { "precision": 1.0, @@ -496,7 +496,7 @@ "f1_score": 0.4615384615384615, "so": 0.536495310012478, "tp": 0.031330852383483966, - "ic": 0.9945438701149829 + "ic": 0.9818129003832762 }, "mount": { "precision": 1.0, @@ -512,7 +512,7 @@ "f1_score": 0.676056338028169, "so": 0.7307897879334082, "tp": 0.044561106308833955, - "ic": 0.9957595465308487 + "ic": 0.993184985496007 } } }, @@ -523,7 +523,7 @@ "f1_score": 0.6885245901639345, "so": 0.3906951258181009, "tp": 0.030228614492615706, - "ic": 0.9894707846536077 + "ic": 0.9889166154248502 }, "investigation": { "precision": 1.0, @@ -531,7 +531,7 @@ "f1_score": 0.4615384615384615, "so": 0.536495310012478, "tp": 0.031330852383483966, - "ic": 0.9945438701149829 + "ic": 0.9818129003832762 }, "mount": { "precision": 1.0, @@ -547,7 +547,7 @@ "f1_score": 0.676056338028169, "so": 0.7307897879334082, "tp": 0.044561106308833955, - "ic": 0.9957595465308487 + "ic": 0.993184985496007 } }, "overall_metrics": { @@ -556,7 +556,7 @@ "f1_score": 0.7065298474326412, "so": 0.6644950559409968, "tp": 0.27653014329623343, - "ic": 0.9949435503248598 + "ic": 0.9909786253260333 } }, "Recording_8": { @@ -576,7 +576,7 @@ "f1_score": 0.7000000000000001, "so": 0.5910507486515022, "tp": 0.02527092835048601, - "ic": 0.9952821338154225 + "ic": 0.9931853044000548 }, "mount": { "precision": null, @@ -584,7 +584,7 @@ "f1_score": 0.0, "so": null, "tp": null, - "ic": 1.0 + "ic": null }, "other": { "precision": 0.8, @@ -592,7 +592,7 @@ "f1_score": 0.6, "so": 0.7322708331381532, "tp": 0.06872092254807333, - "ic": 0.995105321355927 + "ic": 0.9910264224858661 } } }, @@ -611,7 +611,7 @@ "f1_score": 0.7000000000000001, "so": 0.5910507486515022, "tp": 0.02527092835048601, - "ic": 0.9952821338154225 + "ic": 0.9931853044000548 }, "mount": { "precision": null, @@ -619,7 +619,7 @@ "f1_score": 0.0, "so": null, "tp": null, - "ic": 1.0 + "ic": null }, "other": { "precision": 0.8, @@ -627,7 +627,7 @@ "f1_score": 0.6, "so": 0.7322708331381532, "tp": 0.06872092254807333, - "ic": 0.995105321355927 + "ic": 0.9910264224858661 } }, "overall_metrics": { @@ -636,7 +636,7 @@ "f1_score": 0.4546296296296296, "so": 0.5305618016236853, "tp": 0.03813824231911258, - "ic": 0.9955766344352438 + "ic": 0.9920436031518488 } }, "Recording_9": { @@ -648,7 +648,7 @@ "f1_score": 0.8301886792452831, "so": 0.4241002824392379, "tp": 0.0205898063721443, - "ic": 0.989386360253366 + "ic": 0.9886282431286064 }, "investigation": { "precision": 0.375, @@ -656,7 +656,7 @@ "f1_score": 0.25, "so": 0.631678281068525, "tp": 0.03480952380952381, - "ic": 0.9986300436205017 + "ic": 0.9926935659760088 }, "mount": { "precision": 1.0, @@ -672,7 +672,7 @@ "f1_score": 0.7692307692307692, "so": 0.6751395457875596, "tp": 0.05583819861729283, - "ic": 0.9946692048463431 + "ic": 0.9928239296008466 } } }, @@ -683,7 +683,7 @@ "f1_score": 0.8301886792452831, "so": 0.4241002824392379, "tp": 0.0205898063721443, - "ic": 0.989386360253366 + "ic": 0.9886282431286064 }, "investigation": { "precision": 0.375, @@ -691,7 +691,7 @@ "f1_score": 0.25, "so": 0.631678281068525, "tp": 0.03480952380952381, - "ic": 0.9986300436205017 + "ic": 0.9926935659760088 }, "mount": { "precision": 1.0, @@ -707,7 +707,7 @@ "f1_score": 0.7692307692307692, "so": 0.6751395457875596, "tp": 0.05583819861729283, - "ic": 0.9946692048463431 + "ic": 0.9928239296008466 } }, "overall_metrics": { @@ -716,7 +716,7 @@ "f1_score": 0.712354862119013, "so": 0.6827295273238306, "tp": 0.2778093821997402, - "ic": 0.9956714021800527 + "ic": 0.9935364346763654 } }, "Recording_10": { @@ -728,7 +728,7 @@ "f1_score": 0.4444444444444445, "so": 0.18490328157186495, "tp": 0.011189008329645374, - "ic": 0.9914118758934924 + "ic": 0.9908056553683272 }, "investigation": { "precision": 0.625, @@ -736,7 +736,7 @@ "f1_score": 0.37037037037037035, "so": 0.6572072189744603, "tp": 0.06468253968253967, - "ic": 0.9950611635162638 + "ic": 0.9812324213618023 }, "mount": { "precision": 1.0, @@ -752,7 +752,7 @@ "f1_score": 0.5190839694656488, "so": 0.8251145988525997, "tp": 0.06781954841653498, - "ic": 0.998292947043181 + "ic": 0.9955426950571944 } } }, @@ -763,7 +763,7 @@ "f1_score": 0.4444444444444445, "so": 0.18490328157186495, "tp": 0.011189008329645374, - "ic": 0.9914118758934924 + "ic": 0.9908056553683272 }, "investigation": { "precision": 0.625, @@ -771,7 +771,7 @@ "f1_score": 0.37037037037037035, "so": 0.6572072189744603, "tp": 0.06468253968253967, - "ic": 0.9950611635162638 + "ic": 0.9812324213618023 }, "mount": { "precision": 1.0, @@ -787,7 +787,7 @@ "f1_score": 0.5190839694656488, "so": 0.8251145988525997, "tp": 0.06781954841653498, - "ic": 0.998292947043181 + "ic": 0.9955426950571944 } }, "overall_metrics": { @@ -796,7 +796,7 @@ "f1_score": 0.5834746960701159, "so": 0.6668062748497312, "tp": 0.28592277410718003, - "ic": 0.9961914966132344 + "ic": 0.9918951929468309 } } } diff --git a/tests/fixtures/python/generate_golden_fixtures.py b/tests/fixtures/python/generate_golden_fixtures.py index 2c1b613..cd4f10b 100644 --- a/tests/fixtures/python/generate_golden_fixtures.py +++ b/tests/fixtures/python/generate_golden_fixtures.py @@ -5,6 +5,8 @@ Changes since the v0.2.0 fixtures (golden_python_v020.json, kept as an archive): 1. Detection F1 is 0, not NaN, when a recording has real bouts and no predicted bout matches any of them. + 2. IC averages over the real bouts that contain a predicted frame; missed + bouts no longer score 1.0. Run once to regenerate after a deliberate algorithm change: python tests/fixtures/python/generate_golden_fixtures.py diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 9459bc7..0bdc319 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -11,6 +11,7 @@ - No GT bouts -> recall denominator is 0 -> recall = NaN, not 0 - No overlapping bouts -> SO/TP have no inputs -> NaN, not 0 - Single-frame GT bout -> IC denominator is 0 -> NaN, not 0 + - No GT bout found (no predicted frame inside) -> IC has no bout to average -> NaN WHY NaN and not 0: returning 0 would bias the group/overall averages downward by treating "undefined" as "worst possible score". nanmean() in aggregate_metrics @@ -267,3 +268,18 @@ def test_ic_zero_length_bout_excluded(): result = calculate_ic([1], [(0, 0)]) # gt_length = 0, so this bout is skipped -> no valid scores -> NaN assert math.isnan(result) + + +def test_ic_leaves_missed_bouts_out(): + """IC averages over the found GT bouts only; a missed bout does not score 1.0. + + pred=[1,1,0,0,0,0,0], gt=(0,2) and (4,6). Bout (0,2) is found: one switch + in length 2 -> 0.5. Bout (4,6) has no predicted frame and is left out, so + IC = 0.5 (it was (0.5 + 1.0) / 2 = 0.75 when a missed bout scored 1.0). + """ + assert calculate_ic([1, 1, 0, 0, 0, 0, 0], [(0, 2), (4, 6)]) == 0.5 + + +def test_ic_nan_when_no_bout_is_found(): + """Nothing predicted inside any GT bout: IC has nothing to average (NaN, not 1.0).""" + assert math.isnan(calculate_ic([0, 0, 0, 0, 0], [(0, 1), (3, 4)])) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 318cc78..2856559 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -205,6 +205,9 @@ def test_silent_model_on_bundled_data(all_recordings): mount 4/10 = 0.4, investigation 0, other 0; overall F1 = 9 cells of 40 = 0.225. Before 1.1.0 the recordings with the behavior scored F1 NaN and left the means: attack, mount and overall F1 were 1.0. + + IC has no found bout to average in any recording with the behavior, so + investigation and other (present in all 10) have IC NaN, not 1.0. """ data = {f"Recording_{n}": (gt * 0, gt) for n, (_, gt) in all_recordings.items()} group, overall = banos.score(data) @@ -212,3 +215,5 @@ def test_silent_model_on_bundled_data(all_recordings): for behavior, f1 in expected.items(): assert abs(group[behavior]["f1_score"] - f1) < 1e-10, behavior assert abs(overall["f1_score"] - 0.225) < 1e-10 + assert math.isnan(group["investigation"]["ic"]) + assert math.isnan(group["other"]["ic"]) diff --git a/tests/test_regression.py b/tests/test_regression.py index 4ea6837..c656e73 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -11,6 +11,8 @@ v1.1.0 behaviour. Changes since v0.2.0 that move values of the bundled data: 1. Detection F1 is 0, not NaN, when a recording has real bouts and no predicted bout matches any of them (Recording_8 mount). + 2. IC averages over the real bouts that contain a predicted frame; missed + bouts no longer score 1.0 (39 IC values, Recording_8 mount IC is NaN). The archive fixtures (tests/fixtures/golden_python_v020.json and golden_python_v015.json) preserve the v0.2.0 and v0.1.5 behaviour for From 94e36b46fcf7158023731b9cccb4e4f879427c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:01:22 +0100 Subject: [PATCH 04/25] feat: add missed_bouts option to score missed real bouts as zero SO and TP average over the overlapping (pred, GT) bout pairs and IC over the found real bouts, so a real bout with no predicted frame inside it leaves all three averages. The new missed_bouts option keeps this as the default ("exclude") or adds one term equal to 0 per missed real bout to SO, TP and IC ("zero"); one-frame real bouts stay out of IC. Precision, recall, F1 and the absent-behavior rule do not depend on it. calculate_so_terms, calculate_tp_terms and calculate_ic_terms return the terms behind each average; calculate_so, calculate_tp_metric and calculate_ic return their mean. They take the option as a keyword, as do compute_behavior_metrics, calculate_banos_for_each_file and score(), which check it on entry: an unknown value raises even when no column reaches the metrics. MATLAB takes it as the 'missed_bouts' name-value pair, read and validated by the new BANOSparseOptions: an unknown option name or value now raises an error. BANOScalculateSO, BANOScalculateTP and BANOScalculateIC return their terms as a second output. Default results are unchanged; six MATLAB golden IC values move by one or two units in the last place, IC being now the mean of its terms instead of a nanmean. --- matlab/BANOS/BANOS_score.m | 52 +++--- matlab/BANOS/BANOScalculateBANOSForEachFile.m | 9 +- matlab/BANOS/BANOScalculateIC.m | 40 +++-- matlab/BANOS/BANOScalculateSO.m | 29 +++- matlab/BANOS/BANOScalculateTP.m | 30 +++- matlab/BANOS/BANOScomputeBehaviorMetrics.m | 15 +- matlab/BANOS/BANOSparseOptions.m | 56 ++++++ matlab/BANOS/Contents.m | 1 + src/banos/__init__.py | 18 +- src/banos/core/__init__.py | 6 + src/banos/core/metrics.py | 161 +++++++++++++----- src/banos/pipeline.py | 20 ++- tests/fixtures/golden_matlab_v110.json | 2 +- tests/test_metrics.py | 91 ++++++++++ tests/test_pipeline.py | 70 ++++++++ 15 files changed, 484 insertions(+), 116 deletions(-) create mode 100644 matlab/BANOS/BANOSparseOptions.m diff --git a/matlab/BANOS/BANOS_score.m b/matlab/BANOS/BANOS_score.m index 76341f1..b0c0e17 100644 --- a/matlab/BANOS/BANOS_score.m +++ b/matlab/BANOS/BANOS_score.m @@ -20,8 +20,14 @@ % overallMetrics (single average across all behaviors and recordings). % % NAME-VALUE OPTIONS: -% 'matching' - 'greedy' (default). 'optimal' is not yet implemented in -% MATLAB; use the Python package for optimal matching. +% 'matching' - 'greedy' (default). 'optimal' is not yet implemented in +% MATLAB; use the Python package for optimal matching. +% 'missed_bouts' - how SO, TP and IC count a GT bout with no predicted +% frame inside it: 'exclude' (default) leaves it out of +% their averages, 'zero' adds a term equal to 0 to each. +% Precision, recall and F1 do not depend on it. +% Option names and values are case-insensitive. An unknown name or value +% raises an error (BANOS:unknownOption, BANOS:invalidOption). % % NaN VALUES: % A behavior with no ground-truth bouts returns NaN for all metrics. @@ -44,24 +50,19 @@ % BANOSaggregateMetrics, Contents % --- Parse name-value options ------------------------------------------ - matching = 'greedy'; - i = 1; - while i <= length(varargin) - if ischar(varargin{i}) || isstring(varargin{i}) - key = lower(char(varargin{i})); - if strcmp(key, 'matching') - if i + 1 > length(varargin) - error('BANOS_score: ''matching'' option requires a value.'); - end - matching = lower(char(varargin{i + 1})); - i = i + 2; - continue; - end + % Single recording: the first cell array argument is the ground truth; + % the remaining arguments are name-value options. + gt = []; + if iscell(data) + gtIndex = find(cellfun(@iscell, varargin), 1); + if ~isempty(gtIndex) + gt = varargin{gtIndex}; + varargin(gtIndex) = []; end - i = i + 1; end + options = BANOSparseOptions(varargin, {'matching', 'missed_bouts'}); - if strcmp(matching, 'optimal') + if strcmp(options.matching, 'optimal') error('BANOS_score: optimal matching is not yet implemented in MATLAB. Use the Python package (banos.score(..., matching=''optimal'')).'); end @@ -69,28 +70,23 @@ if isstruct(data) % Multi-recording: data is already a properly formatted struct [preprocessed, ~] = BANOSpreprocessData(data); - fileMetrics = BANOScalculateBANOSForEachFile(preprocessed); + fileMetrics = BANOScalculateBANOSForEachFile(preprocessed, ... + 'missed_bouts', options.missed_bouts); [groupMetrics, overallMetrics] = BANOSaggregateMetrics(fileMetrics); varargout{1} = groupMetrics; varargout{2} = overallMetrics; elseif iscell(data) - % Single-recording: data = pred cell array, varargin{1} = gt cell array - % (after matching option parsing, the first non-option arg is gt) - gt = []; - for j = 1:length(varargin) - if iscell(varargin{j}) - gt = varargin{j}; - break; - end - end + % Single-recording: data = pred cell array, gt = first cell array in + % varargin (taken out above, before the options were parsed) if isempty(gt) error('BANOS_score: single-recording mode requires two cell array arguments: BANOS_score(pred, gt).'); end tmpData.single_recording = {data, gt}; [preprocessed, ~] = BANOSpreprocessData(tmpData); - fileMetrics = BANOScalculateBANOSForEachFile(preprocessed); + fileMetrics = BANOScalculateBANOSForEachFile(preprocessed, ... + 'missed_bouts', options.missed_bouts); varargout{1} = fileMetrics.single_recording; else diff --git a/matlab/BANOS/BANOScalculateBANOSForEachFile.m b/matlab/BANOS/BANOScalculateBANOSForEachFile.m index 35894cc..3f27055 100644 --- a/matlab/BANOS/BANOScalculateBANOSForEachFile.m +++ b/matlab/BANOS/BANOScalculateBANOSForEachFile.m @@ -1,12 +1,16 @@ -function banosMetrics = BANOScalculateBANOSForEachFile(preprocessedData) +function banosMetrics = BANOScalculateBANOSForEachFile(preprocessedData, varargin) % Calculates BANOS metrics for each behavior in each file. % % Arguments: % preprocessedData - A struct with preprocessed prediction and ground truth data. + % 'missed_bouts' - (name-value) how SO, TP and IC count a GT bout that no + % predicted bout overlaps: 'exclude' (default) or 'zero' + % (see BANOScalculateSO). % % Returns: % banosMetrics - A struct containing BANOS metrics for each behavior in each file. + options = BANOSparseOptions(varargin, {'missed_bouts'}); fileNames = fieldnames(preprocessedData); banosMetrics = struct(); @@ -15,7 +19,8 @@ extractMatrix = preprocessedData.(fileName); predMatrix = extractMatrix{1}; gtMatrix = extractMatrix{2}; - behaviorMetrics = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix); + behaviorMetrics = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix, ... + 'missed_bouts', options.missed_bouts); banosMetrics.(fileName) = behaviorMetrics; end end diff --git a/matlab/BANOS/BANOScalculateIC.m b/matlab/BANOS/BANOScalculateIC.m index 3a7f5be..c6042d9 100644 --- a/matlab/BANOS/BANOScalculateIC.m +++ b/matlab/BANOS/BANOScalculateIC.m @@ -1,38 +1,42 @@ -function ic = BANOScalculateIC(predSequence, gtBouts) +function [ic, icTerms] = BANOScalculateIC(predSequence, gtBouts, varargin) % Calculates the Intra-bout Continuity (IC) for each ground truth bout. % % IC = 1 - (switches / boutLength) averaged over the found GT bouts. % A GT bout is found when at least one of its frames is predicted (value 1). - % Missed bouts and zero-length bouts are excluded from the average - % (contribute NaN). - % Returns NaN if gtBouts is empty or no found bout is longer than zero - % (matches Python behaviour — NaN propagates through nanmean). + % Zero-length bouts (boutLength = 0) are always excluded from the average. + % Returns NaN if there is no term (matches Python behaviour — NaN + % propagates through nanmean). % % Arguments: % predSequence - The sequence of predictions (1-based index vector). % gtBouts - Cell array of ground truth bouts {[start, end], ...}. + % 'missed_bouts' - (name-value) how a missed GT bout is counted: + % 'exclude' (default) leaves it out, 'zero' adds a term + % equal to 0. % % Returns: - % ic - The average IC across found GT bouts, or NaN if undefined. + % ic - The average of the terms, or NaN if undefined. + % icTerms - Row vector of the terms, one per found GT bout longer than + % zero, and 0 for each missed one with 'zero'. - icScores = []; + options = BANOSparseOptions(varargin, {'missed_bouts'}); + icTerms = []; for i = 1:length(gtBouts) boutLength = gtBouts{i}(2) - gtBouts{i}(1); - switches = BANOScountSwitchesWithinBout(predSequence, gtBouts{i}(1), gtBouts{i}(2)); - found = any(predSequence(gtBouts{i}(1):gtBouts{i}(2)) == 1); - if boutLength > 0 && found - icScores(end + 1) = 1 - (switches / boutLength); - else - icScores(end + 1) = NaN; % zero-length or missed bout: excluded from mean + if boutLength == 0 + continue; % zero-length bout: no frame pair, no term + end + if any(predSequence(gtBouts{i}(1):gtBouts{i}(2)) == 1) + switches = BANOScountSwitchesWithinBout(predSequence, gtBouts{i}(1), gtBouts{i}(2)); + icTerms(end + 1) = 1 - (switches / boutLength); + elseif strcmp(options.missed_bouts, 'zero') + icTerms(end + 1) = 0; % missed bout end end - % nanmean excludes NaN entries from both numerator and denominator, - % matching Python's: sum(valid) / len(valid). - % Returns NaN when icScores is empty or all values are NaN. - if isempty(icScores) + if isempty(icTerms) ic = NaN; else - ic = nanmean(icScores); + ic = mean(icTerms); end end diff --git a/matlab/BANOS/BANOScalculateSO.m b/matlab/BANOS/BANOScalculateSO.m index ac41cb5..4a93174 100644 --- a/matlab/BANOS/BANOScalculateSO.m +++ b/matlab/BANOS/BANOScalculateSO.m @@ -1,26 +1,39 @@ -function so = BANOScalculateSO(predBouts, gtBouts) - % Calculates the Segment Overlap (SO) for each true positive bout. +function [so, soTerms] = BANOScalculateSO(predBouts, gtBouts, varargin) + % Calculates the Segment Overlap (SO) over the overlapping bout pairs. + % + % SO is the average tIoU of every overlapping (predicted, GT) bout pair. % % Arguments: % predBouts - Cell array of predicted bouts. % gtBouts - Cell array of ground truth bouts. + % 'missed_bouts' - (name-value) how a GT bout that no predicted bout + % overlaps is counted: 'exclude' (default) leaves it + % out, 'zero' adds a term equal to 0. % % Returns: - % so - The average SO for the true positive bouts. + % so - The average of the terms, or NaN if there is no term. + % soTerms - Row vector of the terms: the tIoU of each overlapping pair, + % GT bout by GT bout, and 0 for each missed GT bout with 'zero'. - tiouScores = []; + options = BANOSparseOptions(varargin, {'missed_bouts'}); + soTerms = []; for i = 1:length(gtBouts) + found = false; for j = 1:length(predBouts) if BANOSoverlaps(predBouts{j}, gtBouts{i}) - tiouScores(end + 1) = BANOScalculateTiou(predBouts{j}, gtBouts{i}); + soTerms(end + 1) = BANOScalculateTiou(predBouts{j}, gtBouts{i}); + found = true; end end + if ~found && strcmp(options.missed_bouts, 'zero') + soTerms(end + 1) = 0; + end end - % Return NaN if no overlapping bouts were found (matches Python behaviour). + % Return NaN if there is no term (matches Python behaviour). % NaN propagates correctly through BANOSaggregateMetrics (nanmean). - if isempty(tiouScores) + if isempty(soTerms) so = NaN; else - so = mean(tiouScores); + so = mean(soTerms); end end diff --git a/matlab/BANOS/BANOScalculateTP.m b/matlab/BANOS/BANOScalculateTP.m index 904827b..86ad21e 100644 --- a/matlab/BANOS/BANOScalculateTP.m +++ b/matlab/BANOS/BANOScalculateTP.m @@ -1,27 +1,41 @@ -function tp = BANOScalculateTP(predBouts, gtBouts) - % Calculates the Temporal Precision (TP) for each true positive bout. +function [tp, tpTerms] = BANOScalculateTP(predBouts, gtBouts, varargin) + % Calculates the Temporal Precision (TP) over the overlapping bout pairs. + % + % TP is the average of 1 / (1 + |start deviation| + |end deviation|) + % over every overlapping (predicted, GT) bout pair. % % Arguments: % predBouts - Cell array of predicted bouts. % gtBouts - Cell array of ground truth bouts. + % 'missed_bouts' - (name-value) how a GT bout that no predicted bout + % overlaps is counted: 'exclude' (default) leaves it + % out, 'zero' adds a term equal to 0. % % Returns: - % tp - The average TP for the true positive bouts. + % tp - The average of the terms, or NaN if there is no term. + % tpTerms - Row vector of the terms, one per overlapping pair, GT bout + % by GT bout, and 0 for each missed GT bout with 'zero'. - tpScores = []; + options = BANOSparseOptions(varargin, {'missed_bouts'}); + tpTerms = []; for i = 1:length(gtBouts) + found = false; for j = 1:length(predBouts) if BANOSoverlaps(predBouts{j}, gtBouts{i}) startDiff = abs(predBouts{j}(1) - gtBouts{i}(1)); endDiff = abs(predBouts{j}(2) - gtBouts{i}(2)); - tpScores(end + 1) = 1 / (1 + startDiff + endDiff); + tpTerms(end + 1) = 1 / (1 + startDiff + endDiff); + found = true; end end + if ~found && strcmp(options.missed_bouts, 'zero') + tpTerms(end + 1) = 0; + end end - % Return NaN if no overlapping bouts were found (matches Python behaviour). - if isempty(tpScores) + % Return NaN if there is no term (matches Python behaviour). + if isempty(tpTerms) tp = NaN; else - tp = mean(tpScores); + tp = mean(tpTerms); end end diff --git a/matlab/BANOS/BANOScomputeBehaviorMetrics.m b/matlab/BANOS/BANOScomputeBehaviorMetrics.m index a5ee719..9b741e9 100644 --- a/matlab/BANOS/BANOScomputeBehaviorMetrics.m +++ b/matlab/BANOS/BANOScomputeBehaviorMetrics.m @@ -1,5 +1,11 @@ -function behaviorMetrics = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix) +function behaviorMetrics = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix, varargin) % Computes metrics for each behavior in the prediction and ground truth matrices. + % + % Name-value option: + % 'missed_bouts' - how SO, TP and IC count a GT bout that no predicted + % bout overlaps: 'exclude' (default) or 'zero' + % (see BANOScalculateSO). + options = BANOSparseOptions(varargin, {'missed_bouts'}); behaviorMetrics = struct(); for behavior = 1:size(predMatrix, 2) @@ -10,9 +16,10 @@ if ~isempty(gtBouts) [tp, fp, fn] = BANOScalculateTpFpFn(predBouts, gtBouts); [precision, recall, f1] = BANOScalculatePrecisionRecallF1(tp, fp, fn); - so = BANOScalculateSO(predBouts, gtBouts); - tp = BANOScalculateTP(predBouts, gtBouts); - ic = BANOScalculateIC(cell2mat(predMatrix(2:end, behavior)), gtBouts); + so = BANOScalculateSO(predBouts, gtBouts, 'missed_bouts', options.missed_bouts); + tp = BANOScalculateTP(predBouts, gtBouts, 'missed_bouts', options.missed_bouts); + ic = BANOScalculateIC(cell2mat(predMatrix(2:end, behavior)), gtBouts, ... + 'missed_bouts', options.missed_bouts); elseif isempty(predBouts) % Correct absence: system correctly detected nothing precision = 1.0; recall = 1.0; f1 = 1.0; diff --git a/matlab/BANOS/BANOSparseOptions.m b/matlab/BANOS/BANOSparseOptions.m new file mode 100644 index 0000000..81fffe5 --- /dev/null +++ b/matlab/BANOS/BANOSparseOptions.m @@ -0,0 +1,56 @@ +function options = BANOSparseOptions(args, names) + % Reads and validates BANOS name-value options. + % + % Arguments: + % args - Cell array of name-value pairs, e.g. {'missed_bouts', 'zero'}. + % Names and values are case-insensitive. + % names - Cell array of the option names the caller accepts. + % + % Returns: + % options - Struct with one field per accepted name, holding the given + % value in lowercase, or the default value. + % + % Options and their values (the first value is the default): + % 'matching' - 'greedy', 'optimal' + % 'missed_bouts' - 'exclude', 'zero' + % + % Errors: + % BANOS:unknownOption - a name the caller does not accept. + % BANOS:missingOptionValue - a name without a value. + % BANOS:invalidOption - a value that is not allowed for its name. + + allowedValues = struct( ... + 'matching', {{'greedy', 'optimal'}}, ... + 'missed_bouts', {{'exclude', 'zero'}}); + + options = struct(); + for k = 1:numel(names) + values = allowedValues.(names{k}); + options.(names{k}) = values{1}; + end + accepted = strjoin(strcat('''', names, ''''), ', '); + + for i = 1:2:numel(args) + name = args{i}; + if ~(ischar(name) || (isstring(name) && isscalar(name))) + error('BANOS:unknownOption', ... + 'Option names must be text. Accepted options: %s.', accepted); + end + name = lower(char(name)); + if ~any(strcmp(name, names)) + error('BANOS:unknownOption', ... + 'Unknown option ''%s''. Accepted options: %s.', name, accepted); + end + if i + 1 > numel(args) + error('BANOS:missingOptionValue', 'Option ''%s'' requires a value.', name); + end + value = args{i + 1}; + allowed = allowedValues.(name); + if ~(ischar(value) || (isstring(value) && isscalar(value))) ... + || ~any(strcmpi(char(value), allowed)) + error('BANOS:invalidOption', 'Option ''%s'' must be %s.', ... + name, strjoin(strcat('''', allowed, ''''), ' or ')); + end + options.(name) = lower(char(value)); + end +end diff --git a/matlab/BANOS/Contents.m b/matlab/BANOS/Contents.m index 3f2cb56..f44cd2a 100644 --- a/matlab/BANOS/Contents.m +++ b/matlab/BANOS/Contents.m @@ -25,6 +25,7 @@ % BANOSisLogicalColumn - Test whether a column contains only 0 and 1. % BANOSmatchHeaders - Retain only common behavior columns in pred and GT. % BANOSoverlaps - Test whether two bout intervals overlap. +% BANOSparseOptions - Read and validate name-value options. % % Setup: % setup - Add this toolbox to the MATLAB path. diff --git a/src/banos/__init__.py b/src/banos/__init__.py index de9905c..eabfd42 100644 --- a/src/banos/__init__.py +++ b/src/banos/__init__.py @@ -18,6 +18,7 @@ import pandas as pd +from banos.core.metrics import _check_missed_bouts from banos.core.preprocessing import preprocess_data from banos.pipeline import ( aggregate_metrics, @@ -40,6 +41,7 @@ def score( gt: pd.DataFrame | None = None, *, matching: str = "greedy", + missed_bouts: str = "exclude", ) -> dict | tuple[dict, dict]: """Compute BANOS metrics for one or multiple annotation pairs. @@ -54,6 +56,11 @@ def score( Bout matching algorithm. 'greedy': first-match wins (fast, default, backward compatible) 'optimal': Hungarian algorithm (scientifically optimal, requires scipy) + missed_bouts : {'exclude', 'zero'} + How SO, TP and IC count a GT bout with no predicted frame inside it. + 'exclude': left out of their averages (default) + 'zero': one term equal to 0 in each of them + Precision, recall and F1 do not depend on this option. Returns ------- @@ -66,16 +73,23 @@ def score( ----- Metrics return ``NaN`` when ground truth has no bouts for a given behavior. """ + _check_missed_bouts(missed_bouts) + if isinstance(data, pd.DataFrame): if gt is None: raise ValueError("When 'data' is a DataFrame, 'gt' must be provided.") data_dict = {"single": (data, gt)} data_dict, _ = preprocess_data(data_dict) return compute_behavior_metrics( - data_dict["single"][0], data_dict["single"][1], matching=matching + data_dict["single"][0], + data_dict["single"][1], + matching=matching, + missed_bouts=missed_bouts, ) # Multi-file path data_dict, _ = preprocess_data(dict(data)) - file_metrics = calculate_banos_for_each_file(data_dict, matching=matching) + file_metrics = calculate_banos_for_each_file( + data_dict, matching=matching, missed_bouts=missed_bouts + ) return aggregate_metrics(file_metrics) diff --git a/src/banos/core/__init__.py b/src/banos/core/__init__.py index cadc8da..d96007c 100644 --- a/src/banos/core/__init__.py +++ b/src/banos/core/__init__.py @@ -2,10 +2,13 @@ from banos.core.matching import calculate_tp_fp_fn from banos.core.metrics import ( calculate_ic, + calculate_ic_terms, calculate_precision_recall_f1, calculate_so, + calculate_so_terms, calculate_tiou, calculate_tp_metric, + calculate_tp_terms, count_switches_within_bout, identify_bouts, ) @@ -21,8 +24,11 @@ "identify_bouts", "calculate_tiou", "calculate_so", + "calculate_so_terms", "calculate_tp_metric", + "calculate_tp_terms", "calculate_ic", + "calculate_ic_terms", "count_switches_within_bout", "calculate_precision_recall_f1", "is_logical_column", diff --git a/src/banos/core/metrics.py b/src/banos/core/metrics.py index 3663022..0a6898c 100644 --- a/src/banos/core/metrics.py +++ b/src/banos/core/metrics.py @@ -4,6 +4,7 @@ from __future__ import annotations import math +from collections.abc import Callable import numpy as np import pandas as pd @@ -60,38 +61,98 @@ def calculate_tiou(pred_bout: tuple[int, int], gt_bout: tuple[int, int]) -> floa return intersection / union if union != 0 else 0.0 -def calculate_so(pred_bouts: list[tuple[int, int]], gt_bouts: list[tuple[int, int]]) -> float: +def _check_missed_bouts(missed_bouts: str) -> None: + if missed_bouts not in ("exclude", "zero"): + raise ValueError(f"Unknown missed_bouts option: {missed_bouts!r}. Use 'exclude' or 'zero'.") + + +def _mean(terms: list[float]) -> float: + return sum(terms) / len(terms) if terms else float("nan") + + +def _pair_terms( + pred_bouts: list[tuple[int, int]], + gt_bouts: list[tuple[int, int]], + pair_term: Callable[[tuple[int, int], tuple[int, int]], float], + missed_bouts: str, +) -> list[float]: + """Apply ``pair_term`` to every overlapping (pred, GT) bout pair, GT bout by GT bout. + + A GT bout that no predicted bout overlaps is missed: it adds one term + equal to 0 with ``missed_bouts="zero"`` and no term with ``"exclude"``. + """ + _check_missed_bouts(missed_bouts) + terms: list[float] = [] + for gt in gt_bouts: + gt_terms = [ + pair_term(pred, gt) for pred in pred_bouts if pred[1] >= gt[0] and pred[0] <= gt[1] + ] + if not gt_terms and missed_bouts == "zero": + gt_terms = [0.0] + terms.extend(gt_terms) + return terms + + +def _boundary_precision(pred_bout: tuple[int, int], gt_bout: tuple[int, int]) -> float: + return 1.0 / (1.0 + abs(pred_bout[0] - gt_bout[0]) + abs(pred_bout[1] - gt_bout[1])) + + +def calculate_so_terms( + pred_bouts: list[tuple[int, int]], + gt_bouts: list[tuple[int, int]], + *, + missed_bouts: str = "exclude", +) -> list[float]: + """Segment Overlap terms: the tIoU of every overlapping bout pair. + + A GT bout that no predicted bout overlaps adds no term + (``missed_bouts="exclude"``, default) or one term equal to 0 (``"zero"``). + """ + return _pair_terms(pred_bouts, gt_bouts, calculate_tiou, missed_bouts) + + +def calculate_so( + pred_bouts: list[tuple[int, int]], + gt_bouts: list[tuple[int, int]], + *, + missed_bouts: str = "exclude", +) -> float: """Segment Overlap: average tIoU over all overlapping bout pairs. - Returns NaN if there are no GT bouts. + With ``missed_bouts="zero"``, each GT bout that no predicted bout overlaps + adds a 0 to the average (see ``calculate_so_terms``). + Returns NaN if there are no GT bouts, or no overlapping pair with "exclude". """ - if not gt_bouts: - return float("nan") - scores = [ - calculate_tiou(pred, gt) - for gt in gt_bouts - for pred in pred_bouts - if pred[1] >= gt[0] and pred[0] <= gt[1] - ] - return sum(scores) / len(scores) if scores else float("nan") + return _mean(calculate_so_terms(pred_bouts, gt_bouts, missed_bouts=missed_bouts)) + + +def calculate_tp_terms( + pred_bouts: list[tuple[int, int]], + gt_bouts: list[tuple[int, int]], + *, + missed_bouts: str = "exclude", +) -> list[float]: + """Temporal Precision terms: 1 / (1 + |Δstart| + |Δend|) of every overlapping bout pair. + + A GT bout that no predicted bout overlaps adds no term + (``missed_bouts="exclude"``, default) or one term equal to 0 (``"zero"``). + """ + return _pair_terms(pred_bouts, gt_bouts, _boundary_precision, missed_bouts) def calculate_tp_metric( - pred_bouts: list[tuple[int, int]], gt_bouts: list[tuple[int, int]] + pred_bouts: list[tuple[int, int]], + gt_bouts: list[tuple[int, int]], + *, + missed_bouts: str = "exclude", ) -> float: """Temporal Precision: 1 / (1 + |Δstart| + |Δend|) averaged over overlapping bouts. - Returns NaN if there are no GT bouts. + With ``missed_bouts="zero"``, each GT bout that no predicted bout overlaps + adds a 0 to the average (see ``calculate_tp_terms``). + Returns NaN if there are no GT bouts, or no overlapping pair with "exclude". """ - if not gt_bouts: - return float("nan") - scores = [ - 1.0 / (1.0 + abs(pred[0] - gt[0]) + abs(pred[1] - gt[1])) - for gt in gt_bouts - for pred in pred_bouts - if pred[1] >= gt[0] and pred[0] <= gt[1] - ] - return sum(scores) / len(scores) if scores else float("nan") + return _mean(calculate_tp_terms(pred_bouts, gt_bouts, missed_bouts=missed_bouts)) def count_switches_within_bout( @@ -111,29 +172,49 @@ def count_switches_within_bout( return switches -def calculate_ic( - pred_sequence: pd.Series | np.ndarray | list, gt_bouts: list[tuple[int, int]] -) -> float: - """Intra-bout Continuity: 1 - (switches / gt_length) averaged over found GT bouts. +def calculate_ic_terms( + pred_sequence: pd.Series | np.ndarray | list, + gt_bouts: list[tuple[int, int]], + *, + missed_bouts: str = "exclude", +) -> list[float]: + """Intra-bout Continuity terms: 1 - (switches / gt_length) for each found GT bout. A GT bout is found when at least one of its frames is predicted (value 1). - Missed GT bouts are left out of the average, and so are one-frame GT bouts - (gt_length = end - start = 0, no frame pair to compare). + A missed GT bout adds no term (``missed_bouts="exclude"``, default) or one + term equal to 0 (``"zero"``). One-frame GT bouts (gt_length = end - start + = 0, no frame pair to compare) add no term. Frames are read by position: the index labels of a Series are ignored. - Returns NaN if there are no GT bouts or none of them is found and longer - than one frame. """ - if not gt_bouts: - return float("nan") + _check_missed_bouts(missed_bouts) pred_sequence = _by_position(pred_sequence) - ic_scores = [] + terms: list[float] = [] for gt in gt_bouts: gt_length = gt[1] - gt[0] - found = any(pred_sequence[i] == 1 for i in range(gt[0], gt[1] + 1)) - if gt_length > 0 and found: + if gt_length == 0: + continue + if any(pred_sequence[i] == 1 for i in range(gt[0], gt[1] + 1)): switches = count_switches_within_bout(pred_sequence, gt[0], gt[1]) - ic_scores.append(1.0 - (switches / gt_length)) - else: - ic_scores.append(float("nan")) - valid = [v for v in ic_scores if not math.isnan(v)] - return sum(valid) / len(valid) if valid else float("nan") + terms.append(1.0 - (switches / gt_length)) + elif missed_bouts == "zero": + terms.append(0.0) + return terms + + +def calculate_ic( + pred_sequence: pd.Series | np.ndarray | list, + gt_bouts: list[tuple[int, int]], + *, + missed_bouts: str = "exclude", +) -> float: + """Intra-bout Continuity: 1 - (switches / gt_length) averaged over found GT bouts. + + A GT bout is found when at least one of its frames is predicted (value 1). + Missed GT bouts are left out of the average (``missed_bouts="exclude"``, + default) or count as 0 (``"zero"``); one-frame GT bouts are always left + out (see ``calculate_ic_terms``). + Frames are read by position: the index labels of a Series are ignored. + Returns NaN if there is no term (no GT bout longer than one frame, or none + found with "exclude"). + """ + return _mean(calculate_ic_terms(pred_sequence, gt_bouts, missed_bouts=missed_bouts)) diff --git a/src/banos/pipeline.py b/src/banos/pipeline.py index 6f0e2a6..58d6398 100644 --- a/src/banos/pipeline.py +++ b/src/banos/pipeline.py @@ -9,6 +9,7 @@ from banos.core.matching import calculate_tp_fp_fn from banos.core.metrics import ( + _check_missed_bouts, calculate_ic, calculate_precision_recall_f1, calculate_so, @@ -21,13 +22,19 @@ def compute_behavior_metrics( pred_matrix: pd.DataFrame, gt_matrix: pd.DataFrame, matching: str = "greedy", + *, + missed_bouts: str = "exclude", ) -> dict: """Compute all BANOS metrics for each behavior column. + ``missed_bouts`` sets how SO, TP and IC count a GT bout that no predicted + bout overlaps: left out ('exclude', default) or a term equal to 0 ('zero'). + Returns ------- {behavior: {precision, recall, f1_score, so, tp, ic}} """ + _check_missed_bouts(missed_bouts) metrics: dict = {} for behavior in pred_matrix.columns: pred_bouts = identify_bouts(pred_matrix[behavior]) @@ -58,9 +65,9 @@ def compute_behavior_metrics( n_tp, n_fp, n_fn = calculate_tp_fp_fn(pred_bouts, gt_bouts, matching=matching) precision, recall, f1_score = calculate_precision_recall_f1(n_tp, n_fp, n_fn) - so = calculate_so(pred_bouts, gt_bouts) - tp_m = calculate_tp_metric(pred_bouts, gt_bouts) - ic = calculate_ic(pred_matrix[behavior].to_numpy(), gt_bouts) + so = calculate_so(pred_bouts, gt_bouts, missed_bouts=missed_bouts) + tp_m = calculate_tp_metric(pred_bouts, gt_bouts, missed_bouts=missed_bouts) + ic = calculate_ic(pred_matrix[behavior].to_numpy(), gt_bouts, missed_bouts=missed_bouts) metrics[behavior] = { "precision": precision, "recall": recall, @@ -72,10 +79,13 @@ def compute_behavior_metrics( return metrics -def calculate_banos_for_each_file(preprocessed_data: dict, matching: str = "greedy") -> dict: +def calculate_banos_for_each_file( + preprocessed_data: dict, matching: str = "greedy", *, missed_bouts: str = "exclude" +) -> dict: """Compute BANOS metrics for each file in the preprocessed data dict.""" + _check_missed_bouts(missed_bouts) return { - file_name: compute_behavior_metrics(pred, gt, matching=matching) + file_name: compute_behavior_metrics(pred, gt, matching=matching, missed_bouts=missed_bouts) for file_name, (pred, gt) in preprocessed_data.items() } diff --git a/tests/fixtures/golden_matlab_v110.json b/tests/fixtures/golden_matlab_v110.json index 28582b7..3e0663f 100644 --- a/tests/fixtures/golden_matlab_v110.json +++ b/tests/fixtures/golden_matlab_v110.json @@ -1 +1 @@ -{"Recording_1":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.9805603480852142},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.98286421281656633},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.992439083367394}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.9805603480852142},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.98286421281656633},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.992439083367394}},"overall_metrics":{"precision":0.92826086956521736,"recall":0.78129251700680269,"f1_score":0.836455036759336,"so":0.761228298078524,"tp":0.30623014745659,"ic":0.98896591106729359}},"Recording_2":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98771568436705637},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99357808016301785},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99074891239625884}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98771568436705637},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99357808016301785},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99074891239625884}},"overall_metrics":{"precision":0.9671052631578948,"recall":0.83992445266728333,"f1_score":0.897300738477209,"so":0.82594559036003834,"tp":0.30908214265027689,"ic":0.99301066923158332}},"Recording_3":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98261726264923754},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.98941095585555594},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99591253786234157}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98261726264923754},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.98941095585555594},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99591253786234157}},"overall_metrics":{"precision":0.95948072383949645,"recall":0.85096664987969339,"f1_score":0.89862686038270723,"so":0.723715686180092,"tp":0.26887555753525666,"ic":0.99198518909178379}},"Recording_4":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.98586849198997339},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99092353265377653}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.98586849198997339},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99092353265377653}},"overall_metrics":{"precision":0.975,"recall":0.61568627450980384,"f1_score":0.72916666666666674,"so":0.70708735677240686,"tp":0.27571526135052643,"ic":0.99341862874528308}},"Recording_5":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98823265836029839},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99615676486935811},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.993672706200401}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98823265836029839},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99615676486935811},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.993672706200401}},"overall_metrics":{"precision":1,"recall":0.89210526315789473,"f1_score":0.94145869368256319,"so":0.83673046008612484,"tp":0.27928892486843937,"ic":0.99451553235751433}},"Recording_6":{"file_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.98838846071505027},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.98965295127585839},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.99046878072554168}},"group_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.98838846071505027},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.98965295127585839},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.99046878072554168}},"overall_metrics":{"precision":0.91408268733850129,"recall":0.74567099567099571,"f1_score":0.79250474383301706,"so":0.624464020496978,"tp":0.26406773460888022,"ic":0.99212754817911253}},"Recording_7":{"file_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98891661542484988},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.98181290038327618},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.99318498549600709}},"group_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98891661542484988},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.98181290038327618},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.99318498549600709}},"overall_metrics":{"precision":1,"recall":0.58390957446808511,"f1_score":0.70652984743264124,"so":0.66449505594099667,"tp":0.27653014329623343,"ic":0.99097862532603331}},"Recording_8":{"file_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99318530440005481},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":null},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99102642248586637}},"group_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99318530440005481},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":null},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99102642248586637}},"overall_metrics":{"precision":0.89166666666666661,"recall":0.34672064777327932,"f1_score":0.45462962962962961,"so":0.53056180162368516,"tp":0.038138242319112579,"ic":0.992043603151849}},"Recording_9":{"file_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98862824312860653},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99269356597600877},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99282392960084676}},"group_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98862824312860653},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99269356597600877},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99282392960084676}},"overall_metrics":{"precision":0.79839767616191892,"recall":0.65381944444444451,"f1_score":0.712354862119013,"so":0.68272952732383074,"tp":0.27780938219974027,"ic":0.99353643467636554}},"Recording_10":{"file_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.9908056553683271},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.98123242136180233},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.99554269505719462}},"group_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.9908056553683271},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.98123242136180233},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.99554269505719462}},"overall_metrics":{"precision":0.89910714285714288,"recall":0.47575971177944865,"f1_score":0.58347469607011593,"so":0.66680627484973121,"tp":0.28592277410718,"ic":0.99189519294683093}}} +{"Recording_1":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.98056034808521408},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.98286421281656633},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.99243908336739384}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.98056034808521408},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.98286421281656633},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.99243908336739384}},"overall_metrics":{"precision":0.92826086956521736,"recall":0.78129251700680269,"f1_score":0.836455036759336,"so":0.761228298078524,"tp":0.30623014745659,"ic":0.98896591106729359}},"Recording_2":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98771568436705637},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99357808016301785},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99074891239625906}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98771568436705637},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99357808016301785},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99074891239625906}},"overall_metrics":{"precision":0.9671052631578948,"recall":0.83992445266728333,"f1_score":0.897300738477209,"so":0.82594559036003834,"tp":0.30908214265027689,"ic":0.99301066923158332}},"Recording_3":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98261726264923754},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.98941095585555594},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99591253786234157}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98261726264923754},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.98941095585555594},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99591253786234157}},"overall_metrics":{"precision":0.95948072383949645,"recall":0.85096664987969339,"f1_score":0.89862686038270723,"so":0.723715686180092,"tp":0.26887555753525666,"ic":0.99198518909178379}},"Recording_4":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.98586849198997339},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99092353265377653}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.98586849198997339},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99092353265377653}},"overall_metrics":{"precision":0.975,"recall":0.61568627450980384,"f1_score":0.72916666666666674,"so":0.70708735677240686,"tp":0.27571526135052643,"ic":0.99341862874528308}},"Recording_5":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98823265836029839},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99615676486935811},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.993672706200401}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98823265836029839},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99615676486935811},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.993672706200401}},"overall_metrics":{"precision":1,"recall":0.89210526315789473,"f1_score":0.94145869368256319,"so":0.83673046008612484,"tp":0.27928892486843937,"ic":0.99451553235751433}},"Recording_6":{"file_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.98838846071505027},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.98965295127585839},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.9904687807255419}},"group_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.98838846071505027},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.98965295127585839},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.9904687807255419}},"overall_metrics":{"precision":0.91408268733850129,"recall":0.74567099567099571,"f1_score":0.79250474383301706,"so":0.624464020496978,"tp":0.26406773460888022,"ic":0.99212754817911264}},"Recording_7":{"file_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98891661542484988},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.98181290038327618},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.99318498549600709}},"group_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98891661542484988},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.98181290038327618},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.99318498549600709}},"overall_metrics":{"precision":1,"recall":0.58390957446808511,"f1_score":0.70652984743264124,"so":0.66449505594099667,"tp":0.27653014329623343,"ic":0.99097862532603331}},"Recording_8":{"file_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99318530440005481},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":null},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99102642248586637}},"group_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99318530440005481},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":null},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99102642248586637}},"overall_metrics":{"precision":0.89166666666666661,"recall":0.34672064777327932,"f1_score":0.45462962962962961,"so":0.53056180162368516,"tp":0.038138242319112579,"ic":0.992043603151849}},"Recording_9":{"file_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98862824312860653},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99269356597600877},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99282392960084676}},"group_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98862824312860653},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99269356597600877},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99282392960084676}},"overall_metrics":{"precision":0.79839767616191892,"recall":0.65381944444444451,"f1_score":0.712354862119013,"so":0.68272952732383074,"tp":0.27780938219974027,"ic":0.99353643467636554}},"Recording_10":{"file_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.99080565536832721},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.98123242136180233},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.99554269505719462}},"group_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.99080565536832721},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.98123242136180233},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.99554269505719462}},"overall_metrics":{"precision":0.89910714285714288,"recall":0.47575971177944865,"f1_score":0.58347469607011593,"so":0.66680627484973121,"tp":0.28592277410718,"ic":0.99189519294683093}}} diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 0bdc319..6dd3ac7 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -35,10 +35,13 @@ from banos.core.metrics import ( calculate_ic, + calculate_ic_terms, calculate_precision_recall_f1, calculate_so, + calculate_so_terms, calculate_tiou, calculate_tp_metric, + calculate_tp_terms, count_switches_within_bout, identify_bouts, ) @@ -283,3 +286,91 @@ def test_ic_leaves_missed_bouts_out(): def test_ic_nan_when_no_bout_is_found(): """Nothing predicted inside any GT bout: IC has nothing to average (NaN, not 1.0).""" assert math.isnan(calculate_ic([0, 0, 0, 0, 0], [(0, 1), (3, 4)])) + + +# --- missed_bouts: 'exclude' (default) or 'zero' --- +# +# GT bouts (0,4), (10,14), (20,24), (30,30); predicted bouts (0,4), (11,14). +# (0,4) is found exactly, (10,14) is found one frame late, (20,24) and the +# one-frame bout (30,30) are missed. + +MISSED_PRED_BOUTS = [(0, 4), (11, 14)] +MISSED_GT_BOUTS = [(0, 4), (10, 14), (20, 24), (30, 30)] +MISSED_PRED_SEQUENCE = [1] * 5 + [0] * 6 + [1] * 4 + [0] * 16 # frames 0..30 + + +def test_so_missed_bouts(): + """SO terms: tIoU 1 and 4/5 for the found bouts, plus one 0 per missed bout with 'zero'. + + exclude: (1 + 0.8) / 2 = 0.9; zero: (1 + 0.8 + 0 + 0) / 4 = 0.45. + """ + assert calculate_so_terms(MISSED_PRED_BOUTS, MISSED_GT_BOUTS) == [1.0, 0.8] + assert calculate_so_terms(MISSED_PRED_BOUTS, MISSED_GT_BOUTS, missed_bouts="zero") == [ + 1.0, + 0.8, + 0.0, + 0.0, + ] + assert abs(calculate_so(MISSED_PRED_BOUTS, MISSED_GT_BOUTS) - 0.9) < 1e-10 + assert abs(calculate_so(MISSED_PRED_BOUTS, MISSED_GT_BOUTS, missed_bouts="zero") - 0.45) < 1e-10 + + +def test_tp_metric_missed_bouts(): + """TP terms: 1 and 1/(1 + 1 + 0) = 0.5 for the found bouts, plus one 0 per missed bout. + + exclude: (1 + 0.5) / 2 = 0.75; zero: (1 + 0.5 + 0 + 0) / 4 = 0.375. + """ + assert calculate_tp_terms(MISSED_PRED_BOUTS, MISSED_GT_BOUTS) == [1.0, 0.5] + assert calculate_tp_terms(MISSED_PRED_BOUTS, MISSED_GT_BOUTS, missed_bouts="zero") == [ + 1.0, + 0.5, + 0.0, + 0.0, + ] + assert calculate_tp_metric(MISSED_PRED_BOUTS, MISSED_GT_BOUTS) == 0.75 + assert calculate_tp_metric(MISSED_PRED_BOUTS, MISSED_GT_BOUTS, missed_bouts="zero") == 0.375 + + +def test_ic_missed_bouts(): + """IC terms: 1 for (0,4), 1 - 1/4 for (10,14) (switch at frames 10-11), 0 for (20,24) with 'zero'. + + The missed one-frame bout (30,30) adds no IC term in either mode. + exclude: (1 + 0.75) / 2 = 0.875; zero: (1 + 0.75 + 0) / 3 = 1.75 / 3. + """ + assert calculate_ic_terms(MISSED_PRED_SEQUENCE, MISSED_GT_BOUTS) == [1.0, 0.75] + assert calculate_ic_terms(MISSED_PRED_SEQUENCE, MISSED_GT_BOUTS, missed_bouts="zero") == [ + 1.0, + 0.75, + 0.0, + ] + assert calculate_ic(MISSED_PRED_SEQUENCE, MISSED_GT_BOUTS) == 0.875 + zero = calculate_ic(MISSED_PRED_SEQUENCE, MISSED_GT_BOUTS, missed_bouts="zero") + assert abs(zero - 1.75 / 3) < 1e-10 + + +def test_missed_bouts_zero_when_nothing_overlaps(): + """With 'zero', real bouts and no overlap give SO = TP = IC = 0 instead of NaN.""" + pred_bouts, gt_bouts, pred_sequence = [(0, 1)], [(5, 8)], [1, 1, 0, 0, 0, 0, 0, 0, 0] + assert math.isnan(calculate_so(pred_bouts, gt_bouts)) + assert calculate_so(pred_bouts, gt_bouts, missed_bouts="zero") == 0.0 + assert math.isnan(calculate_tp_metric(pred_bouts, gt_bouts)) + assert calculate_tp_metric(pred_bouts, gt_bouts, missed_bouts="zero") == 0.0 + assert math.isnan(calculate_ic(pred_sequence, gt_bouts)) + assert calculate_ic(pred_sequence, gt_bouts, missed_bouts="zero") == 0.0 + + +@pytest.mark.parametrize( + "function, first_argument", + [ + (calculate_so, []), + (calculate_so_terms, []), + (calculate_tp_metric, []), + (calculate_tp_terms, []), + (calculate_ic, [0, 1]), + (calculate_ic_terms, [0, 1]), + ], +) +def test_invalid_missed_bouts_raises(function, first_argument): + """An unknown missed_bouts value raises ValueError, even without any bout.""" + with pytest.raises(ValueError, match="Unknown missed_bouts option"): + function(first_argument, [], missed_bouts="drop") diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 2856559..baabcb5 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -175,6 +175,76 @@ def test_score_reads_frames_by_position_whatever_the_index(kind): assert banos.score(pred.set_axis(index), gt.set_axis(index)) == expected +def test_score_missed_bouts_zero(): + """missed_bouts='zero' adds a 0 term per missed real bout to SO, TP and IC only. + + Real bouts (5,9), (25,29), (45,49); predicted (5,9) and (26,29): the + third real bout is missed. Terms of the found bouts: SO 1 and 4/5, + TP 1 and 1/2, IC 1 and 3/4 (one switch, frames 25-26, in length 4). + exclude: SO 0.9, TP 0.75, IC 0.875; zero: SO 1.8/3, TP 1.5/3, IC 1.75/3. + Detection is the same in both modes: TP 2, FN 1 -> P 1, R 2/3, F1 0.8. + """ + video = _video([(5, 9), (26, 29)]) + exclude = banos.score(*video)["evt"] + zero = banos.score(*video, missed_bouts="zero")["evt"] + expected_exclude = {"so": 0.9, "tp": 0.75, "ic": 0.875} + expected_zero = {"so": 1.8 / 3, "tp": 1.5 / 3, "ic": 1.75 / 3} + for key in expected_exclude: + assert abs(exclude[key] - expected_exclude[key]) < 1e-10, key + assert abs(zero[key] - expected_zero[key]) < 1e-10, key + for key in ("precision", "recall", "f1_score"): + assert exclude[key] == zero[key] + assert abs(zero["f1_score"] - 0.8) < 1e-10 + + group, _ = banos.score({"v1": video, "v2": video}, missed_bouts="zero") + assert abs(group["evt"]["so"] - 0.6) < 1e-10 + + +def test_missed_bouts_zero_keeps_absent_behavior_rule(): + """With no real bout in a file, 'zero' keeps 1.0 (nothing predicted) or 0.0 (false bouts).""" + absent = banos.score(*_video([], []), missed_bouts="zero")["evt"] + false_detection = banos.score(*_video([(5, 9)], []), missed_bouts="zero")["evt"] + assert set(absent.values()) == {1.0} + assert set(false_detection.values()) == {0.0} + + +def test_score_invalid_missed_bouts_raises(): + with pytest.raises(ValueError, match="Unknown missed_bouts option"): + banos.score(PRED, GT, missed_bouts="drop") + + +NO_COLUMN = pd.DataFrame(index=range(N_FRAMES)) + + +@pytest.mark.parametrize( + "call", + [ + pytest.param( + lambda: banos.score(NO_COLUMN, NO_COLUMN, missed_bouts="drop"), id="no-column" + ), + pytest.param(lambda: banos.score(PRED * 0.5, GT, missed_bouts="drop"), id="all-dropped"), + pytest.param(lambda: banos.score({}, missed_bouts="drop"), id="no-file"), + pytest.param( + lambda: banos.compute_behavior_metrics(NO_COLUMN, NO_COLUMN, missed_bouts="drop"), + id="compute_behavior_metrics", + ), + pytest.param( + lambda: banos.calculate_banos_for_each_file({}, missed_bouts="drop"), + id="calculate_banos_for_each_file", + ), + ], +) +def test_invalid_missed_bouts_raises_without_any_column(call): + """An unknown missed_bouts raises even when no column reaches the metrics. + + No behavior column, every column dropped by preprocessing (values 0 and + 0.5), or no file at all: the option is checked on entry, not only by the + metric functions. + """ + with pytest.raises(ValueError, match="Unknown missed_bouts option"): + call() + + def test_file_with_only_false_bouts_scores_f1_zero(): """A file whose predicted bouts all miss the real bouts scores F1 0 and stays in the mean. From 1921e13fb7a351fcc2d514c01cb7ecb16d9fd213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:49:34 +0100 Subject: [PATCH 05/25] feat: add pooled aggregation of bout counts and terms over files score() averages per-file metrics (aggregation="per_file", default): each recording weighs the same whatever its number of bouts, and a recording without the behavior scores 1.0 or 0.0 on all six metrics. The new aggregation="pooled" sums, per behavior, the TP, FP and FN counts of all recordings for precision, recall and F1, and averages all their SO, TP and IC terms (under the chosen missed_bouts). A recording without the behavior adds its false bouts to FP and no term; a behavior with no real and no predicted bout anywhere is NaN. overall_metrics is the mean over the behaviors. The result has the same form in both modes. compute_behavior_components and calculate_components_for_each_file return the per-file counts and terms, and pool_components combines them. compute_behavior_metrics now derives its metrics from the components, with unchanged results. score() checks matching, missed_bouts and aggregation on entry, and the pipeline functions check matching and missed_bouts on entry, so an unknown value raises even when no column reaches the bout matching. In MATLAB, BANOS_score takes the 'aggregation' option, BANOScomputeBehaviorMetrics and BANOScalculateBANOSForEachFile return the components as a second output, and BANOSpoolComponents pools them. --- matlab/BANOS/BANOS_score.m | 37 ++- matlab/BANOS/BANOScalculateBANOSForEachFile.m | 9 +- matlab/BANOS/BANOScomputeBehaviorMetrics.m | 29 +- matlab/BANOS/BANOSparseOptions.m | 4 +- matlab/BANOS/BANOSpoolComponents.m | 69 +++++ matlab/BANOS/Contents.m | 1 + src/banos/__init__.py | 49 +++- src/banos/core/matching.py | 8 +- src/banos/pipeline.py | 204 ++++++++++---- tests/test_pipeline.py | 258 ++++++++++++++++++ 10 files changed, 586 insertions(+), 82 deletions(-) create mode 100644 matlab/BANOS/BANOSpoolComponents.m diff --git a/matlab/BANOS/BANOS_score.m b/matlab/BANOS/BANOS_score.m index b0c0e17..eda0e37 100644 --- a/matlab/BANOS/BANOS_score.m +++ b/matlab/BANOS/BANOS_score.m @@ -26,13 +26,23 @@ % frame inside it: 'exclude' (default) leaves it out of % their averages, 'zero' adds a term equal to 0 to each. % Precision, recall and F1 do not depend on it. +% 'aggregation' - how the metrics are combined: 'per_file' (default) +% scores each recording on its own, then averages; +% 'pooled' scores each behavior once on the bout counts +% and terms of all recordings (see BANOSpoolComponents). +% The outputs have the same form in both modes. % Option names and values are case-insensitive. An unknown name or value % raises an error (BANOS:unknownOption, BANOS:invalidOption). % % NaN VALUES: -% A behavior with no ground-truth bouts returns NaN for all metrics. -% NaN values are correctly excluded from group and overall averages -% (via nanmean), so they do not bias results. +% With 'per_file', a behavior with no ground-truth bouts in a recording +% scores 1.0 on all metrics if nothing is predicted there and 0.0 +% otherwise; groupMetrics and overallMetrics average with nanmean. +% With 'pooled', this 1.0/0.0 rule does not apply and overallMetrics +% averages the pooled behaviors with nanmean. +% A metric is NaN when it is undefined: precision when nothing is +% predicted, recall when there is no GT bout (pooled), SO, TP and IC when +% they have no term (no overlap, or no found GT bout). % % EXAMPLE — single recording: % pred = [{'event1','event2'}; num2cell(predData)]; @@ -47,7 +57,7 @@ % disp(overallMet.f1_score) % % See also: BANOSpreprocessData, BANOScalculateBANOSForEachFile, -% BANOSaggregateMetrics, Contents +% BANOSaggregateMetrics, BANOSpoolComponents, Contents % --- Parse name-value options ------------------------------------------ % Single recording: the first cell array argument is the ground truth; @@ -60,7 +70,8 @@ varargin(gtIndex) = []; end end - options = BANOSparseOptions(varargin, {'matching', 'missed_bouts'}); + options = BANOSparseOptions(varargin, {'matching', 'missed_bouts', 'aggregation'}); + pooled = strcmp(options.aggregation, 'pooled'); if strcmp(options.matching, 'optimal') error('BANOS_score: optimal matching is not yet implemented in MATLAB. Use the Python package (banos.score(..., matching=''optimal'')).'); @@ -70,9 +81,13 @@ if isstruct(data) % Multi-recording: data is already a properly formatted struct [preprocessed, ~] = BANOSpreprocessData(data); - fileMetrics = BANOScalculateBANOSForEachFile(preprocessed, ... + [fileMetrics, fileComponents] = BANOScalculateBANOSForEachFile(preprocessed, ... 'missed_bouts', options.missed_bouts); - [groupMetrics, overallMetrics] = BANOSaggregateMetrics(fileMetrics); + if pooled + [groupMetrics, overallMetrics] = BANOSpoolComponents(fileComponents); + else + [groupMetrics, overallMetrics] = BANOSaggregateMetrics(fileMetrics); + end varargout{1} = groupMetrics; varargout{2} = overallMetrics; @@ -85,9 +100,13 @@ tmpData.single_recording = {data, gt}; [preprocessed, ~] = BANOSpreprocessData(tmpData); - fileMetrics = BANOScalculateBANOSForEachFile(preprocessed, ... + [fileMetrics, fileComponents] = BANOScalculateBANOSForEachFile(preprocessed, ... 'missed_bouts', options.missed_bouts); - varargout{1} = fileMetrics.single_recording; + if pooled + varargout{1} = BANOSpoolComponents(fileComponents); + else + varargout{1} = fileMetrics.single_recording; + end else error('BANOS_score: first argument must be a cell array (single recording) or a struct (multiple recordings).'); diff --git a/matlab/BANOS/BANOScalculateBANOSForEachFile.m b/matlab/BANOS/BANOScalculateBANOSForEachFile.m index 3f27055..e6fda0f 100644 --- a/matlab/BANOS/BANOScalculateBANOSForEachFile.m +++ b/matlab/BANOS/BANOScalculateBANOSForEachFile.m @@ -1,4 +1,4 @@ -function banosMetrics = BANOScalculateBANOSForEachFile(preprocessedData, varargin) +function [banosMetrics, fileComponents] = BANOScalculateBANOSForEachFile(preprocessedData, varargin) % Calculates BANOS metrics for each behavior in each file. % % Arguments: @@ -9,18 +9,23 @@ % % Returns: % banosMetrics - A struct containing BANOS metrics for each behavior in each file. + % fileComponents - A struct containing the metric components (bout counts + % and terms) for each behavior in each file, to pool with + % BANOSpoolComponents (see BANOScomputeBehaviorMetrics). options = BANOSparseOptions(varargin, {'missed_bouts'}); fileNames = fieldnames(preprocessedData); banosMetrics = struct(); + fileComponents = struct(); for i = 1:length(fileNames) fileName = fileNames{i}; extractMatrix = preprocessedData.(fileName); predMatrix = extractMatrix{1}; gtMatrix = extractMatrix{2}; - behaviorMetrics = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix, ... + [behaviorMetrics, behaviorComponents] = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix, ... 'missed_bouts', options.missed_bouts); banosMetrics.(fileName) = behaviorMetrics; + fileComponents.(fileName) = behaviorComponents; end end diff --git a/matlab/BANOS/BANOScomputeBehaviorMetrics.m b/matlab/BANOS/BANOScomputeBehaviorMetrics.m index 9b741e9..57b89b3 100644 --- a/matlab/BANOS/BANOScomputeBehaviorMetrics.m +++ b/matlab/BANOS/BANOScomputeBehaviorMetrics.m @@ -1,25 +1,38 @@ -function behaviorMetrics = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix, varargin) +function [behaviorMetrics, behaviorComponents] = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix, varargin) % Computes metrics for each behavior in the prediction and ground truth matrices. % % Name-value option: % 'missed_bouts' - how SO, TP and IC count a GT bout that no predicted % bout overlaps: 'exclude' (default) or 'zero' % (see BANOScalculateSO). + % + % Returns: + % behaviorMetrics - Struct with one field per behavior holding .precision, + % .recall, .f1_score, .so, .tp and .ic. + % behaviorComponents - Struct with one field per behavior holding what the + % metrics are computed from: the bout counts .n_tp, + % .n_fp, .n_fn and the terms .so_terms, .tp_terms, + % .ic_terms averaged by SO, TP and IC. Summed over + % files by BANOSpoolComponents. options = BANOSparseOptions(varargin, {'missed_bouts'}); behaviorMetrics = struct(); + behaviorComponents = struct(); for behavior = 1:size(predMatrix, 2) behaviorName = predMatrix{1,behavior}; - predBouts = BANOSidentifyBouts(cell2mat(predMatrix(2:end, behavior))); + predSequence = cell2mat(predMatrix(2:end, behavior)); + predBouts = BANOSidentifyBouts(predSequence); gtBouts = BANOSidentifyBouts(cell2mat(gtMatrix(2:end, behavior))); + [nTp, nFp, nFn] = BANOScalculateTpFpFn(predBouts, gtBouts); + [so, soTerms] = BANOScalculateSO(predBouts, gtBouts, 'missed_bouts', options.missed_bouts); + [tp, tpTerms] = BANOScalculateTP(predBouts, gtBouts, 'missed_bouts', options.missed_bouts); + [ic, icTerms] = BANOScalculateIC(predSequence, gtBouts, 'missed_bouts', options.missed_bouts); + behaviorComponents.(behaviorName) = struct('n_tp', nTp, 'n_fp', nFp, 'n_fn', nFn, ... + 'so_terms', soTerms, 'tp_terms', tpTerms, 'ic_terms', icTerms); + if ~isempty(gtBouts) - [tp, fp, fn] = BANOScalculateTpFpFn(predBouts, gtBouts); - [precision, recall, f1] = BANOScalculatePrecisionRecallF1(tp, fp, fn); - so = BANOScalculateSO(predBouts, gtBouts, 'missed_bouts', options.missed_bouts); - tp = BANOScalculateTP(predBouts, gtBouts, 'missed_bouts', options.missed_bouts); - ic = BANOScalculateIC(cell2mat(predMatrix(2:end, behavior)), gtBouts, ... - 'missed_bouts', options.missed_bouts); + [precision, recall, f1] = BANOScalculatePrecisionRecallF1(nTp, nFp, nFn); elseif isempty(predBouts) % Correct absence: system correctly detected nothing precision = 1.0; recall = 1.0; f1 = 1.0; diff --git a/matlab/BANOS/BANOSparseOptions.m b/matlab/BANOS/BANOSparseOptions.m index 81fffe5..82a8d6a 100644 --- a/matlab/BANOS/BANOSparseOptions.m +++ b/matlab/BANOS/BANOSparseOptions.m @@ -13,6 +13,7 @@ % Options and their values (the first value is the default): % 'matching' - 'greedy', 'optimal' % 'missed_bouts' - 'exclude', 'zero' + % 'aggregation' - 'per_file', 'pooled' % % Errors: % BANOS:unknownOption - a name the caller does not accept. @@ -21,7 +22,8 @@ allowedValues = struct( ... 'matching', {{'greedy', 'optimal'}}, ... - 'missed_bouts', {{'exclude', 'zero'}}); + 'missed_bouts', {{'exclude', 'zero'}}, ... + 'aggregation', {{'per_file', 'pooled'}}); options = struct(); for k = 1:numel(names) diff --git a/matlab/BANOS/BANOSpoolComponents.m b/matlab/BANOS/BANOSpoolComponents.m new file mode 100644 index 0000000..9f888b6 --- /dev/null +++ b/matlab/BANOS/BANOSpoolComponents.m @@ -0,0 +1,69 @@ +function [groupMetrics, overallMetrics] = BANOSpoolComponents(fileComponents) + % Pools the metric components of all files and scores each behavior once. + % + % For each behavior, the TP, FP and FN counts are summed over the files and + % give precision, recall and F1 (BANOScalculatePrecisionRecallF1); SO, TP + % and IC are the means of all their terms over all files. A file without + % the behavior adds its false bouts to FP and no SO, TP or IC term: the + % 1.0/0.0 absent-behavior rule of the per-file metrics does not apply. + % A behavior with no GT bout and no predicted bout in any file is NaN on + % all six metrics. + % + % Arguments: + % fileComponents - Struct with one field per file, each holding the + % behavior components returned as second output by + % BANOScalculateBANOSForEachFile. + % + % Returns: + % groupMetrics - Pooled metrics for each behavior. + % overallMetrics - Mean of the pooled metrics over the behaviors (nanmean). + + pooled = struct(); + fileNames = fieldnames(fileComponents); + for i = 1:length(fileNames) + components = fileComponents.(fileNames{i}); + behaviors = fieldnames(components); + for j = 1:length(behaviors) + behavior = behaviors{j}; + c = components.(behavior); + if ~isfield(pooled, behavior) + pooled.(behavior) = struct('n_tp', 0, 'n_fp', 0, 'n_fn', 0, ... + 'so_terms', [], 'tp_terms', [], 'ic_terms', []); + end + p = pooled.(behavior); + p.n_tp = p.n_tp + c.n_tp; + p.n_fp = p.n_fp + c.n_fp; + p.n_fn = p.n_fn + c.n_fn; + p.so_terms = [p.so_terms, c.so_terms]; + p.tp_terms = [p.tp_terms, c.tp_terms]; + p.ic_terms = [p.ic_terms, c.ic_terms]; + pooled.(behavior) = p; + end + end + + groupMetrics = struct(); + overallMetrics = struct('precision', [], 'recall', [], 'f1_score', [], 'so', [], 'tp', [], 'ic', []); + metricFields = fieldnames(overallMetrics); + behaviors = fieldnames(pooled); + for j = 1:length(behaviors) + p = pooled.(behaviors{j}); + [precision, recall, f1] = BANOScalculatePrecisionRecallF1(p.n_tp, p.n_fp, p.n_fn); + % Mean of each term list; NaN when a list is empty (no term) + terms = {p.so_terms, p.tp_terms, p.ic_terms}; + termMeans = NaN(1, 3); + for k = 1:3 + if ~isempty(terms{k}) + termMeans(k) = mean(terms{k}); + end + end + groupMetrics.(behaviors{j}) = struct('precision', precision, 'recall', recall, ... + 'f1_score', f1, 'so', termMeans(1), 'tp', termMeans(2), 'ic', termMeans(3)); + for k = 1:length(metricFields) + overallMetrics.(metricFields{k})(end + 1) = groupMetrics.(behaviors{j}).(metricFields{k}); + end + end + + for k = 1:length(metricFields) + overallMetrics.(metricFields{k}) = nanmean(overallMetrics.(metricFields{k})); + end +end diff --git a/matlab/BANOS/Contents.m b/matlab/BANOS/Contents.m index f44cd2a..761285e 100644 --- a/matlab/BANOS/Contents.m +++ b/matlab/BANOS/Contents.m @@ -8,6 +8,7 @@ % BANOSpreprocessData - Fill NaN, drop non-binary columns, align headers. % BANOScalculateBANOSForEachFile - Compute metrics for each recording. % BANOSaggregateMetrics - Average metrics across recordings (nanmean). +% BANOSpoolComponents - Pool bout counts and terms across recordings. % BANOScomputeBehaviorMetrics - Per-behavior metric computation for one recording. % % Metric functions: diff --git a/src/banos/__init__.py b/src/banos/__init__.py index eabfd42..e2d4a39 100644 --- a/src/banos/__init__.py +++ b/src/banos/__init__.py @@ -18,12 +18,15 @@ import pandas as pd -from banos.core.metrics import _check_missed_bouts from banos.core.preprocessing import preprocess_data from banos.pipeline import ( + _check_options, aggregate_metrics, calculate_banos_for_each_file, + calculate_components_for_each_file, + compute_behavior_components, compute_behavior_metrics, + pool_components, ) __all__ = [ @@ -32,6 +35,9 @@ "compute_behavior_metrics", "calculate_banos_for_each_file", "aggregate_metrics", + "compute_behavior_components", + "calculate_components_for_each_file", + "pool_components", "__version__", ] @@ -42,6 +48,7 @@ def score( *, matching: str = "greedy", missed_bouts: str = "exclude", + aggregation: str = "per_file", ) -> dict | tuple[dict, dict]: """Compute BANOS metrics for one or multiple annotation pairs. @@ -61,25 +68,54 @@ def score( 'exclude': left out of their averages (default) 'zero': one term equal to 0 in each of them Precision, recall and F1 do not depend on this option. + aggregation : {'per_file', 'pooled'} + How the metrics are combined. + 'per_file': each file is scored on its own, then averaged (default) + 'pooled': each behavior is scored once on the bout counts and terms + of all files (see ``pool_components``) Returns ------- Single file : dict ``{behavior: {precision, recall, f1_score, so, tp, ic}}`` Multi file : tuple[dict, dict] - ``(group_metrics, overall_metrics)`` + ``(group_metrics, overall_metrics)``: ``group_metrics`` has the same + form as the single-file dict; ``overall_metrics`` is + ``{metric: value}``. Notes ----- - Metrics return ``NaN`` when ground truth has no bouts for a given behavior. + With ``aggregation='per_file'``, a behavior with no GT bout in a file scores + 1.0 on all six metrics if nothing is predicted there and 0.0 otherwise. + ``group_metrics`` averages each behavior over the files and + ``overall_metrics`` averages every (file, behavior) cell, NaN excluded. + With ``aggregation='pooled'``, this 1.0/0.0 rule does not apply and + ``overall_metrics`` averages the pooled behaviors, NaN excluded. + A metric is NaN when it is undefined: precision when nothing is predicted, + recall when there is no GT bout (pooled), SO, TP and IC when they have no + term (no overlap, or no found GT bout). + + An unknown ``matching``, ``missed_bouts`` or ``aggregation`` value raises + ValueError before the data is read. """ - _check_missed_bouts(missed_bouts) + _check_options(matching, missed_bouts) + if aggregation not in ("per_file", "pooled"): + raise ValueError(f"Unknown aggregation: {aggregation!r}. Use 'per_file' or 'pooled'.") if isinstance(data, pd.DataFrame): if gt is None: raise ValueError("When 'data' is a DataFrame, 'gt' must be provided.") data_dict = {"single": (data, gt)} data_dict, _ = preprocess_data(data_dict) + if aggregation == "pooled": + components = compute_behavior_components( + data_dict["single"][0], + data_dict["single"][1], + matching=matching, + missed_bouts=missed_bouts, + ) + group_metrics, _ = pool_components({"single": components}) + return group_metrics return compute_behavior_metrics( data_dict["single"][0], data_dict["single"][1], @@ -89,6 +125,11 @@ def score( # Multi-file path data_dict, _ = preprocess_data(dict(data)) + if aggregation == "pooled": + file_components = calculate_components_for_each_file( + data_dict, matching=matching, missed_bouts=missed_bouts + ) + return pool_components(file_components) file_metrics = calculate_banos_for_each_file( data_dict, matching=matching, missed_bouts=missed_bouts ) diff --git a/src/banos/core/matching.py b/src/banos/core/matching.py index aff9e95..5f059b7 100644 --- a/src/banos/core/matching.py +++ b/src/banos/core/matching.py @@ -8,6 +8,11 @@ def _overlaps(pred: tuple[int, int], gt: tuple[int, int]) -> bool: return pred[1] >= gt[0] and pred[0] <= gt[1] +def _check_matching(matching: str) -> None: + if matching not in ("greedy", "optimal"): + raise ValueError(f"Unknown matching algorithm: {matching!r}. Use 'greedy' or 'optimal'.") + + def calculate_tp_fp_fn( pred_bouts: list[tuple[int, int]], gt_bouts: list[tuple[int, int]], @@ -25,8 +30,7 @@ def calculate_tp_fp_fn( ------- (true_positives, false_positives, false_negatives) """ - if matching not in ("greedy", "optimal"): - raise ValueError(f"Unknown matching algorithm: {matching!r}. Use 'greedy' or 'optimal'.") + _check_matching(matching) if not gt_bouts: return 0, len(pred_bouts), 0 diff --git a/src/banos/pipeline.py b/src/banos/pipeline.py index 58d6398..d01eaf0 100644 --- a/src/banos/pipeline.py +++ b/src/banos/pipeline.py @@ -7,83 +7,140 @@ import pandas as pd -from banos.core.matching import calculate_tp_fp_fn +from banos.core.matching import _check_matching, calculate_tp_fp_fn from banos.core.metrics import ( _check_missed_bouts, - calculate_ic, + _mean, + calculate_ic_terms, calculate_precision_recall_f1, - calculate_so, - calculate_tp_metric, + calculate_so_terms, + calculate_tp_terms, identify_bouts, ) +METRIC_KEYS = ["precision", "recall", "f1_score", "so", "tp", "ic"] -def compute_behavior_metrics( + +def _check_options(matching: str, missed_bouts: str) -> None: + """Raise ValueError for an unknown matching or missed_bouts value.""" + _check_matching(matching) + _check_missed_bouts(missed_bouts) + + +def _nanmean(values: list) -> float: + valid = [ + v for v in values if v is not None and not math.isnan(v) + ] # None guard for JSON round-trip safety + return _mean(valid) + + +def _metrics_from_components(components: dict) -> dict: + """The six metrics of one behavior from its bout counts and terms.""" + precision, recall, f1_score = calculate_precision_recall_f1( + components["n_tp"], components["n_fp"], components["n_fn"] + ) + return { + "precision": precision, + "recall": recall, + "f1_score": f1_score, + "so": _mean(components["so_terms"]), + "tp": _mean(components["tp_terms"]), + "ic": _mean(components["ic_terms"]), + } + + +def compute_behavior_components( pred_matrix: pd.DataFrame, gt_matrix: pd.DataFrame, matching: str = "greedy", *, missed_bouts: str = "exclude", ) -> dict: - """Compute all BANOS metrics for each behavior column. + """Compute the components of the BANOS metrics for each behavior column. + + The components are what the metrics are computed from: the bout counts + behind precision, recall and F1, and the terms that SO, TP and IC + average. Summed over files, they give the pooled metrics + (see ``pool_components``). ``missed_bouts`` sets how SO, TP and IC count a GT bout that no predicted bout overlaps: left out ('exclude', default) or a term equal to 0 ('zero'). Returns ------- - {behavior: {precision, recall, f1_score, so, tp, ic}} + {behavior: {n_tp, n_fp, n_fn, so_terms, tp_terms, ic_terms}} + n_tp, n_fp, n_fn : matched, false and missed bout counts + so_terms, tp_terms, ic_terms : lists of the terms of SO, TP and IC """ - _check_missed_bouts(missed_bouts) - metrics: dict = {} + _check_options(matching, missed_bouts) + components: dict = {} for behavior in pred_matrix.columns: pred_bouts = identify_bouts(pred_matrix[behavior]) gt_bouts = identify_bouts(gt_matrix[behavior]) - - if not gt_bouts: - if not pred_bouts: - # Correct absence: system correctly detected nothing - metrics[behavior] = { - "precision": 1.0, - "recall": 1.0, - "f1_score": 1.0, - "so": 1.0, - "tp": 1.0, - "ic": 1.0, - } - else: - # False detection: system predicted bouts that don't exist in GT - metrics[behavior] = { - "precision": 0.0, - "recall": 0.0, - "f1_score": 0.0, - "so": 0.0, - "tp": 0.0, - "ic": 0.0, - } - continue - n_tp, n_fp, n_fn = calculate_tp_fp_fn(pred_bouts, gt_bouts, matching=matching) - precision, recall, f1_score = calculate_precision_recall_f1(n_tp, n_fp, n_fn) - so = calculate_so(pred_bouts, gt_bouts, missed_bouts=missed_bouts) - tp_m = calculate_tp_metric(pred_bouts, gt_bouts, missed_bouts=missed_bouts) - ic = calculate_ic(pred_matrix[behavior].to_numpy(), gt_bouts, missed_bouts=missed_bouts) - metrics[behavior] = { - "precision": precision, - "recall": recall, - "f1_score": f1_score, - "so": so, - "tp": tp_m, - "ic": ic, + components[behavior] = { + "n_tp": n_tp, + "n_fp": n_fp, + "n_fn": n_fn, + "so_terms": calculate_so_terms(pred_bouts, gt_bouts, missed_bouts=missed_bouts), + "tp_terms": calculate_tp_terms(pred_bouts, gt_bouts, missed_bouts=missed_bouts), + "ic_terms": calculate_ic_terms( + pred_matrix[behavior].to_numpy(), gt_bouts, missed_bouts=missed_bouts + ), } + return components + + +def compute_behavior_metrics( + pred_matrix: pd.DataFrame, + gt_matrix: pd.DataFrame, + matching: str = "greedy", + *, + missed_bouts: str = "exclude", +) -> dict: + """Compute all BANOS metrics for each behavior column. + + ``missed_bouts`` sets how SO, TP and IC count a GT bout that no predicted + bout overlaps: left out ('exclude', default) or a term equal to 0 ('zero'). + + Returns + ------- + {behavior: {precision, recall, f1_score, so, tp, ic}} + """ + components = compute_behavior_components( + pred_matrix, gt_matrix, matching=matching, missed_bouts=missed_bouts + ) + metrics: dict = {} + for behavior, behavior_components in components.items(): + if behavior_components["n_tp"] + behavior_components["n_fn"] == 0: + # No GT bout: 1.0 for a correct absence (system correctly detected + # nothing), 0.0 for a false detection (system predicted bouts that + # don't exist in GT) + value = 1.0 if behavior_components["n_fp"] == 0 else 0.0 + metrics[behavior] = dict.fromkeys(METRIC_KEYS, value) + else: + metrics[behavior] = _metrics_from_components(behavior_components) return metrics +def calculate_components_for_each_file( + preprocessed_data: dict, matching: str = "greedy", *, missed_bouts: str = "exclude" +) -> dict: + """Compute the BANOS metric components for each file in the preprocessed data dict.""" + _check_options(matching, missed_bouts) + return { + file_name: compute_behavior_components( + pred, gt, matching=matching, missed_bouts=missed_bouts + ) + for file_name, (pred, gt) in preprocessed_data.items() + } + + def calculate_banos_for_each_file( preprocessed_data: dict, matching: str = "greedy", *, missed_bouts: str = "exclude" ) -> dict: """Compute BANOS metrics for each file in the preprocessed data dict.""" - _check_missed_bouts(missed_bouts) + _check_options(matching, missed_bouts) return { file_name: compute_behavior_metrics(pred, gt, matching=matching, missed_bouts=missed_bouts) for file_name, (pred, gt) in preprocessed_data.items() @@ -100,27 +157,62 @@ def aggregate_metrics(file_metrics: dict) -> tuple[dict, dict]: group_metrics : {behavior: {metric: mean_across_files}} overall_metrics: {metric: mean_across_behaviors_and_files} """ - metric_keys = ["precision", "recall", "f1_score", "so", "tp", "ic"] group: dict = {} - overall: dict = {k: [] for k in metric_keys} + overall: dict = {k: [] for k in METRIC_KEYS} for behaviors in file_metrics.values(): for behavior, m in behaviors.items(): if behavior not in group: - group[behavior] = {k: [] for k in metric_keys} - for key in metric_keys: + group[behavior] = {k: [] for k in METRIC_KEYS} + for key in METRIC_KEYS: val = m.get(key, float("nan")) group[behavior][key].append(val) overall[key].append(val) - def _nanmean(values: list) -> float: - valid = [ - v for v in values if v is not None and not math.isnan(v) - ] # None guard for JSON round-trip safety - return sum(valid) / len(valid) if valid else float("nan") - for behavior in group: - for key in metric_keys: + for key in METRIC_KEYS: group[behavior][key] = _nanmean(group[behavior][key]) return group, {key: _nanmean(vals) for key, vals in overall.items()} + + +def pool_components(file_components: dict) -> tuple[dict, dict]: + """Pool the metric components of all files and score each behavior once. + + For each behavior, the TP, FP and FN counts are summed over the files and + give precision, recall and F1; SO, TP and IC are the means of all their + terms over all files. A file without the behavior adds its false bouts to + FP and no SO, TP or IC term: the 1.0/0.0 absent-behavior rule of the + per-file metrics does not apply. A behavior with no GT bout and no + predicted bout in any file is NaN on all six metrics. + + Parameters + ---------- + file_components : {file_name: {behavior: components}} + As returned by ``calculate_components_for_each_file``. + + Returns + ------- + group_metrics : {behavior: {metric: pooled_value}} + overall_metrics: {metric: mean_of_pooled_values_across_behaviors} (NaN excluded) + """ + pooled: dict = {} + for behaviors in file_components.values(): + for behavior, components in behaviors.items(): + if behavior not in pooled: + pooled[behavior] = { + "n_tp": 0, + "n_fp": 0, + "n_fn": 0, + "so_terms": [], + "tp_terms": [], + "ic_terms": [], + } + for key in ("n_tp", "n_fp", "n_fn"): + pooled[behavior][key] += components[key] + for key in ("so_terms", "tp_terms", "ic_terms"): + pooled[behavior][key].extend(components[key]) + + group = {behavior: _metrics_from_components(p) for behavior, p in pooled.items()} + overall = {key: _nanmean([m[key] for m in group.values()]) for key in METRIC_KEYS} + return group, overall diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index baabcb5..d986acb 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -10,6 +10,8 @@ Pipeline stages tested together: preprocess_data -> calculate_banos_for_each_file -> aggregate_metrics + preprocess_data -> calculate_components_for_each_file -> pool_components + (the second chain is aggregation='pooled') The test_aggregate_metrics_excludes_nan test is critical: it confirms that recordings where a behavior has no GT bouts (NaN metrics) do NOT pull down @@ -22,8 +24,12 @@ import pytest import banos +from banos.core.matching import calculate_tp_fp_fn from banos.pipeline import ( aggregate_metrics, + calculate_components_for_each_file, + compute_behavior_components, + pool_components, ) PRED = pd.DataFrame( @@ -265,6 +271,17 @@ def test_file_with_only_false_bouts_scores_f1_zero(): assert abs(group_c["evt"]["f1_score"] - 0.5) < 1e-10 assert abs(group_d["evt"]["f1_score"] - 0.9) < 1e-10 + # Pooled: C has TP 3, FP 3, FN 3 -> F1 0.5; D has TP 5, FP 0, FN 1 -> + # P 1, R 5/6, F1 10/11. + pooled_c, _ = banos.score( + {"v1": _video(REAL_BOUTS), "v2": _video(false_bouts)}, aggregation="pooled" + ) + pooled_d, _ = banos.score( + {"v1": _video(REAL_BOUTS), "v2": _video(REAL_BOUTS[:2])}, aggregation="pooled" + ) + assert abs(pooled_c["evt"]["f1_score"] - 0.5) < 1e-10 + assert abs(pooled_d["evt"]["f1_score"] - 10 / 11) < 1e-10 + def test_silent_model_on_bundled_data(all_recordings): """A model that predicts nothing scores F1 0 on every recording with the behavior. @@ -287,3 +304,244 @@ def test_silent_model_on_bundled_data(all_recordings): assert abs(overall["f1_score"] - 0.225) < 1e-10 assert math.isnan(group["investigation"]["ic"]) assert math.isnan(group["other"]["ic"]) + + # Pooled: TP 0, FP 0, FN > 0 for every behavior, so F1 is 0 everywhere; + # SO, TP and IC have no term (NaN), or only 0 terms with missed_bouts='zero'. + pooled, pooled_overall = banos.score(data, aggregation="pooled") + assert {m["f1_score"] for m in pooled.values()} == {0.0} + assert pooled_overall["f1_score"] == 0.0 + assert all(math.isnan(m["so"]) for m in pooled.values()) + pooled_zero, _ = banos.score(data, aggregation="pooled", missed_bouts="zero") + assert {m["so"] for m in pooled_zero.values()} == {0.0} + + +# --- aggregation='pooled' --- + + +def test_pooled_counts_and_terms_equal_hand_sums(): + """Pooled metrics come from the bout counts and terms summed over the files. + + Video a: predicted (5,9) and (26,29) -> TP 2, FP 0, FN 1; SO terms 1, 4/5; + TP terms 1, 1/2; IC terms 1, 3/4 (the third real bout is missed). + Video b: predicted (15,19) and (45,49) -> TP 1, FP 1, FN 2; one term of 1 + for SO, TP and IC (bout 3 found exactly, bouts 1 and 2 missed). + Pooled: TP 3, FP 1, FN 3 -> P 3/4, R 1/2, F1 0.6; SO (1 + 0.8 + 1) / 3, + TP (1 + 0.5 + 1) / 3, IC (1 + 0.75 + 1) / 3. With missed_bouts='zero', + the three missed bouts add three 0 terms: SO = 2.8 / 6. + """ + data = {"a": _video([(5, 9), (26, 29)]), "b": _video([(15, 19), (45, 49)])} + preprocessed, _ = banos.preprocess_data(data) + components = calculate_components_for_each_file(preprocessed) + assert components["a"]["evt"] == { + "n_tp": 2, + "n_fp": 0, + "n_fn": 1, + "so_terms": [1.0, 0.8], + "tp_terms": [1.0, 0.5], + "ic_terms": [1.0, 0.75], + } + assert components["b"]["evt"] == compute_behavior_components(*preprocessed["b"])["evt"] + assert components["b"]["evt"] == { + "n_tp": 1, + "n_fp": 1, + "n_fn": 2, + "so_terms": [1.0], + "tp_terms": [1.0], + "ic_terms": [1.0], + } + + group, overall = pool_components(components) + expected = { + "precision": 0.75, + "recall": 0.5, + "f1_score": 0.6, + "so": 2.8 / 3, + "tp": 2.5 / 3, + "ic": 2.75 / 3, + } + for key, value in expected.items(): + assert abs(group["evt"][key] - value) < 1e-10, key + assert overall == group["evt"] + assert banos.score(data, aggregation="pooled") == (group, overall) + + zero, _ = banos.score(data, aggregation="pooled", missed_bouts="zero") + assert abs(zero["evt"]["so"] - 2.8 / 6) < 1e-10 + + +def test_pooled_absent_behavior(): + """In pooled mode a file without the behavior adds its false bouts to FP, nothing else. + + "evt" has the real bouts of video a only (P 1, R 2/3, F1 0.8); "rare" + has no real bout anywhere. + - Video c has neither behavior and predicts nothing: the pooled values + do not move (per file, its correct absences would score 1.0). + - "rare" with nothing predicted anywhere is NaN on all six metrics. + - Video d predicts one false bout of each: "evt" gets FP 1, so + P = R = F1 = 2/3 and SO, TP, IC are unchanged; "rare" gets P 0, + F1 0, and R, SO, TP, IC NaN. + overall_metrics is the mean over the behaviors, NaN excluded. + """ + + def video(evt_pred, evt_gt, rare_pred): + pred = pd.DataFrame({"evt": _frames(evt_pred), "rare": _frames(rare_pred)}) + gt = pd.DataFrame({"evt": _frames(evt_gt), "rare": _frames([])}) + return pred, gt + + a = video([(5, 9), (26, 29)], REAL_BOUTS, []) + c = video([], [], []) + d = video([(15, 19)], [], [(40, 42)]) + + alone, _ = banos.score({"a": a}, aggregation="pooled") + group, overall = banos.score({"a": a, "c": c}, aggregation="pooled") + assert group["evt"] == alone["evt"] + assert abs(group["evt"]["f1_score"] - 0.8) < 1e-10 + assert all(math.isnan(value) for value in group["rare"].values()) + assert overall == group["evt"] + + per_file, _ = banos.score({"a": a, "c": c}) + assert abs(per_file["evt"]["f1_score"] - 0.9) < 1e-10 # (0.8 + 1.0) / 2 + assert per_file["rare"]["f1_score"] == 1.0 + + group, overall = banos.score({"a": a, "c": c, "d": d}, aggregation="pooled") + evt, rare = group["evt"], group["rare"] + for key in ("precision", "recall", "f1_score"): + assert abs(evt[key] - 2 / 3) < 1e-10, key + for key in ("so", "tp", "ic"): + assert evt[key] == alone["evt"][key], key + assert rare["precision"] == 0.0 + assert rare["f1_score"] == 0.0 + assert all(math.isnan(rare[key]) for key in ("recall", "so", "tp", "ic")) + assert abs(overall["f1_score"] - 1 / 3) < 1e-10 # (2/3 + 0) / 2 + assert abs(overall["recall"] - 2 / 3) < 1e-10 # rare's NaN recall is excluded + assert overall["so"] == evt["so"] + + +def test_pooled_ranks_models_by_bouts_found(): + """Model A finds 6 of 12 real bouts, model B 3 of 12: both modes rank A first. + + Four videos with real bouts (5,9), (25,29), (45,49). A finds all three in + video 1 and the first one in videos 2-4; B finds all three in video 1 and + nothing else. + Per file: A (1 + 3 * 0.5) / 4 = 0.625 (F1 0.5 from P 1, R 1/3), + B (1 + 0 + 0 + 0) / 4 = 0.25. Pooled: A TP 6, FN 6 -> F1 2/3; + B TP 3, FN 9 -> F1 0.4. Before 1.1.0, B's three empty videos scored F1 NaN + and left the mean: B got 1.0 and ranked above A. + """ + model_a = {"v1": _video(REAL_BOUTS)} | {f"v{k}": _video(REAL_BOUTS[:1]) for k in (2, 3, 4)} + model_b = {"v1": _video(REAL_BOUTS)} | {f"v{k}": _video([]) for k in (2, 3, 4)} + expected = {"per_file": (0.625, 0.25), "pooled": (2 / 3, 0.4)} + for aggregation, (f1_a, f1_b) in expected.items(): + group_a, _ = banos.score(model_a, aggregation=aggregation) + group_b, _ = banos.score(model_b, aggregation=aggregation) + assert abs(group_a["evt"]["f1_score"] - f1_a) < 1e-10, aggregation + assert abs(group_b["evt"]["f1_score"] - f1_b) < 1e-10, aggregation + + +def test_pooled_single_file_returns_dict_per_behavior(): + """A single DataFrame pair gives a dict per behavior in both aggregation modes. + + PRED and GT have real bouts of both behaviors, so pooling one file changes + no value. Without real bouts, per file scores 1.0 (correct absence) and + pooled NaN. + """ + assert banos.score(PRED, GT, aggregation="pooled") == banos.score(PRED, GT) + absent = banos.score(*_video([], []), aggregation="pooled")["evt"] + assert all(math.isnan(value) for value in absent.values()) + + +def test_score_invalid_aggregation_raises(): + with pytest.raises(ValueError, match="Unknown aggregation"): + banos.score(PRED, GT, aggregation="pool") + with pytest.raises(ValueError, match="Unknown aggregation"): + banos.score({"f1": (PRED, GT)}, aggregation="pool") + + +@pytest.mark.parametrize( + "call, message", + [ + pytest.param( + lambda: banos.score(NO_COLUMN, NO_COLUMN, matching="best"), + "Unknown matching", + id="score-no-column-matching", + ), + pytest.param( + lambda: banos.score(PRED * 0.5, GT, matching="best"), + "Unknown matching", + id="score-all-dropped-matching", + ), + pytest.param( + lambda: banos.score({}, matching="best"), + "Unknown matching", + id="score-no-file-matching", + ), + pytest.param( + lambda: banos.score({}, aggregation="pool"), + "Unknown aggregation", + id="score-no-file-aggregation", + ), + pytest.param( + lambda: compute_behavior_components(NO_COLUMN, NO_COLUMN, matching="best"), + "Unknown matching", + id="compute_behavior_components-matching", + ), + pytest.param( + lambda: compute_behavior_components(NO_COLUMN, NO_COLUMN, missed_bouts="drop"), + "Unknown missed_bouts", + id="compute_behavior_components-missed_bouts", + ), + pytest.param( + lambda: banos.compute_behavior_metrics(NO_COLUMN, NO_COLUMN, matching="best"), + "Unknown matching", + id="compute_behavior_metrics-matching", + ), + pytest.param( + lambda: calculate_components_for_each_file({}, matching="best"), + "Unknown matching", + id="calculate_components_for_each_file-matching", + ), + pytest.param( + lambda: calculate_components_for_each_file({}, missed_bouts="drop"), + "Unknown missed_bouts", + id="calculate_components_for_each_file-missed_bouts", + ), + pytest.param( + lambda: banos.calculate_banos_for_each_file({}, matching="best"), + "Unknown matching", + id="calculate_banos_for_each_file-matching", + ), + ], +) +def test_invalid_options_raise_without_any_column(call, message): + """Unknown matching, missed_bouts and aggregation values raise even when + no column reaches the bout matching or the metrics (no column, every + column dropped, no file).""" + with pytest.raises(ValueError, match=message): + call() + + +@pytest.mark.parametrize("aggregation", ["per_file", "pooled"]) +@pytest.mark.parametrize("several_files", [False, True]) +def test_score_passes_matching_to_bout_matching(monkeypatch, aggregation, several_files): + """Every score() path hands the matching option to calculate_tp_fp_fn. + + Greedy and optimal matching count the same true positives on bouts read + from frame sequences: such bouts never overlap one another, on either + side, and then taking the first free overlapping real bout is already a + maximum matching. No input of score() can tell the two apart by its + numbers, so this test records the option that reaches the bout matching. + """ + seen = [] + + def spy(pred_bouts, gt_bouts, matching="greedy"): + seen.append(matching) + return calculate_tp_fp_fn(pred_bouts, gt_bouts, matching=matching) + + monkeypatch.setattr("banos.pipeline.calculate_tp_fp_fn", spy) + if several_files: + banos.score( + {"f1": (PRED, GT), "f2": (PRED, GT)}, matching="optimal", aggregation=aggregation + ) + else: + banos.score(PRED, GT, matching="optimal", aggregation=aggregation) + assert seen + assert set(seen) == {"optimal"} From 2ce6419fe3df35f423b875fe0c8c891d4718ca8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:50:26 +0100 Subject: [PATCH 06/25] fix: check input lengths and warn when columns are dropped preprocess_data, and so score(), now raises ValueError when the prediction and the ground truth of a file differ in number of frames (a shorter prediction used to raise KeyError, a longer one was scored) or when a file has no frame. It emits a UserWarning naming the file and the column for each column dropped (values other than 0 and 1, such as probabilities or inf, or a column missing or invalid on the other side) and for each column whose NaN values are replaced by 0; score() used to discard the drop report without a word. The warnings point at the line that called preprocess_data() or score(). Valid input gives the same results and no warning. BANOSpreprocessData raises and warns the same way, with the identifiers BANOS:lengthMismatch, BANOS:noFrame, BANOS:columnDropped and BANOS:nanReplaced. --- matlab/BANOS/BANOSpreprocessData.m | 56 ++++++++++++++++++-- src/banos/__init__.py | 10 ++-- src/banos/core/preprocessing.py | 53 +++++++++++++++++++ tests/test_pipeline.py | 58 +++++++++++++++++++++ tests/test_preprocessing.py | 84 +++++++++++++++++++++++++++++- 5 files changed, 253 insertions(+), 8 deletions(-) diff --git a/matlab/BANOS/BANOSpreprocessData.m b/matlab/BANOS/BANOSpreprocessData.m index d94a72c..23920fa 100644 --- a/matlab/BANOS/BANOSpreprocessData.m +++ b/matlab/BANOS/BANOSpreprocessData.m @@ -2,9 +2,19 @@ % Preprocesses prediction and ground truth data for BANOS computation. % % Steps performed for each file: - % 1. Replace NaN values with 0. - % 2. Drop columns whose values are not strictly binary (0/1). - % 3. Retain only behavior columns whose names appear in BOTH pred and gt. + % 1. Check that pred and gt have the same number of frames (rows 2+), + % and at least one. + % 2. Replace NaN values with 0. + % 3. Drop columns whose values are not strictly binary (0/1). + % 4. Retain only behavior columns whose names appear in BOTH pred and gt. + % + % Errors and warnings: + % BANOS:lengthMismatch (error) - pred and gt differ in number of frames. + % BANOS:noFrame (error) - a file has no frame. + % BANOS:nanReplaced (warning) - NaN values of a column replaced by 0. + % BANOS:columnDropped (warning) - a column dropped (values other than + % 0 and 1, or missing or invalid on the other side). + % Each message names the file and the column. % % Arguments: % dataDict - struct; each field is a recording name whose value is @@ -27,6 +37,18 @@ predMatrix = extractMatrix{1}; gtMatrix = extractMatrix{2}; + % --- Check the number of frames (rows 2+) --- + nPredFrames = max(size(predMatrix, 1) - 1, 0); + nGtFrames = max(size(gtMatrix, 1) - 1, 0); + if nPredFrames ~= nGtFrames + error('BANOS:lengthMismatch', ... + 'File ''%s'': the prediction has %d frames but the ground truth has %d.', ... + fileName, nPredFrames, nGtFrames); + end + if nPredFrames == 0 + error('BANOS:noFrame', 'File ''%s'' has no frame.', fileName); + end + % --- Separate headers (row 1) from data (rows 2+) --- predHeaders = predMatrix(1, :); gtHeaders = gtMatrix(1, :); @@ -34,12 +56,30 @@ gtData = cell2mat(gtMatrix(2:end, :)); % --- Replace NaNs with zeros --- + sides = {'prediction', 'ground truth'}; + nanCounts = {sum(isnan(predData), 1), sum(isnan(gtData), 1)}; + sideHeaders = {predHeaders, gtHeaders}; + for s = 1:2 + for k = find(nanCounts{s} > 0) + warning('BANOS:nanReplaced', ... + 'File ''%s'': %d NaN value(s) in %s column ''%s'' replaced by 0.', ... + fileName, nanCounts{s}(k), sides{s}, sideHeaders{s}{k}); + end + end predData(isnan(predData)) = 0; gtData(isnan(gtData)) = 0; % --- Drop non-logical columns from data, track which indices --- [predData, predDropped] = BANOSdropNonLogicalColumns(predData); [gtData, gtDropped] = BANOSdropNonLogicalColumns(gtData); + sideDropped = {predDropped, gtDropped}; + for s = 1:2 + for k = sideDropped{s} + warning('BANOS:columnDropped', ... + 'File ''%s'': %s column ''%s'' dropped (values other than 0 and 1).', ... + fileName, sides{s}, sideHeaders{s}{k}); + end + end % Update header arrays to reflect dropped columns predKeepIdx = setdiff(1:length(predHeaders), predDropped); @@ -47,6 +87,16 @@ predHeaders = predHeaders(predKeepIdx); gtHeaders = gtHeaders(gtKeepIdx); + % --- Warn about columns present on one side only --- + oneSide = {setdiff(predHeaders, gtHeaders, 'stable'), setdiff(gtHeaders, predHeaders, 'stable')}; + for s = 1:2 + for k = 1:numel(oneSide{s}) + warning('BANOS:columnDropped', ... + 'File ''%s'': %s column ''%s'' dropped (missing or invalid in the %s).', ... + fileName, sides{s}, oneSide{s}{k}, sides{3 - s}); + end + end + % --- Retain only columns present in both pred and gt --- commonHeaders = intersect(predHeaders, gtHeaders, 'stable'); [~, predIdx] = ismember(commonHeaders, predHeaders); diff --git a/src/banos/__init__.py b/src/banos/__init__.py index e2d4a39..e5e2c22 100644 --- a/src/banos/__init__.py +++ b/src/banos/__init__.py @@ -18,7 +18,7 @@ import pandas as pd -from banos.core.preprocessing import preprocess_data +from banos.core.preprocessing import _preprocess_data, preprocess_data from banos.pipeline import ( _check_options, aggregate_metrics, @@ -97,6 +97,10 @@ def score( An unknown ``matching``, ``missed_bouts`` or ``aggregation`` value raises ValueError before the data is read. + + The input goes through ``preprocess_data``: a ValueError is raised when the + prediction and the ground truth of a file differ in length or a file has no + frame, and a UserWarning names each column dropped and each NaN replaced by 0. """ _check_options(matching, missed_bouts) if aggregation not in ("per_file", "pooled"): @@ -106,7 +110,7 @@ def score( if gt is None: raise ValueError("When 'data' is a DataFrame, 'gt' must be provided.") data_dict = {"single": (data, gt)} - data_dict, _ = preprocess_data(data_dict) + data_dict, _ = _preprocess_data(data_dict) if aggregation == "pooled": components = compute_behavior_components( data_dict["single"][0], @@ -124,7 +128,7 @@ def score( ) # Multi-file path - data_dict, _ = preprocess_data(dict(data)) + data_dict, _ = _preprocess_data(dict(data)) if aggregation == "pooled": file_components = calculate_components_for_each_file( data_dict, matching=matching, missed_bouts=missed_bouts diff --git a/src/banos/core/preprocessing.py b/src/banos/core/preprocessing.py index 24c47a3..bd6fba8 100644 --- a/src/banos/core/preprocessing.py +++ b/src/banos/core/preprocessing.py @@ -3,6 +3,8 @@ from __future__ import annotations +import warnings + import pandas as pd @@ -39,10 +41,19 @@ def match_headers( return pred_matrix[common], gt_matrix[common], dropped +def _warn(message: str) -> None: + # Level 4 is the line that called preprocess_data() or score(): the levels + # below are _warn, _preprocess_data, then preprocess_data or score. + warnings.warn(message, UserWarning, stacklevel=4) + + def preprocess_data(data_dict: dict[str, tuple[pd.DataFrame, pd.DataFrame]]) -> tuple[dict, dict]: """Preprocess prediction and ground truth DataFrames. Fills NaN with 0, drops non-binary columns, aligns headers. + Each NaN replacement and each dropped column (values other than 0 and 1, + or a column present on one side only) emits a UserWarning naming the file + and the column. Parameters ---------- @@ -52,14 +63,56 @@ def preprocess_data(data_dict: dict[str, tuple[pd.DataFrame, pd.DataFrame]]) -> ------- data_dict : cleaned data dropped_info : per-file info about dropped columns + + Raises + ------ + ValueError + If the prediction and the ground truth of a file differ in length + (number of rows, i.e. frames), or a file has no frame. """ + return _preprocess_data(data_dict) + + +def _preprocess_data(data_dict: dict[str, tuple[pd.DataFrame, pd.DataFrame]]) -> tuple[dict, dict]: + # Body of preprocess_data(). score() calls it directly, so that the + # warnings point at the caller's line in both cases (see _warn). data_dict = dict(data_dict) # avoid mutating caller's dict dropped_info: dict = {} for file_name, (pred, gt) in data_dict.items(): + if len(pred) != len(gt): + raise ValueError( + f"File {file_name!r}: the prediction has {len(pred)} frames " + f"but the ground truth has {len(gt)}." + ) + if len(pred) == 0: + raise ValueError(f"File {file_name!r} has no frame.") + for side, matrix in (("prediction", pred), ("ground truth", gt)): + for column, n_nan in matrix.isna().sum().items(): + if n_nan: + _warn( + f"File {file_name!r}: {n_nan} NaN value(s) in {side} column " + f"{column!r} replaced by 0." + ) pred = pred.fillna(0) gt = gt.fillna(0) pred, pred_dropped = drop_non_logical_columns(pred) gt, gt_dropped = drop_non_logical_columns(gt) + for side, dropped in (("prediction", pred_dropped), ("ground truth", gt_dropped)): + for column in dropped: + _warn( + f"File {file_name!r}: {side} column {column!r} dropped " + "(values other than 0 and 1)." + ) + for column in pred.columns.difference(gt.columns, sort=False): + _warn( + f"File {file_name!r}: prediction column {column!r} dropped " + "(missing or invalid in the ground truth)." + ) + for column in gt.columns.difference(pred.columns, sort=False): + _warn( + f"File {file_name!r}: ground truth column {column!r} dropped " + "(missing or invalid in the prediction)." + ) pred, gt, header_dropped = match_headers(pred, gt) data_dict[file_name] = (pred, gt) dropped_info[file_name] = { diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index d986acb..9e46296 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -19,6 +19,8 @@ """ import math +import warnings +from pathlib import Path import pandas as pd import pytest @@ -449,6 +451,62 @@ def test_pooled_single_file_returns_dict_per_behavior(): assert all(math.isnan(value) for value in absent.values()) +def test_score_reports_dropped_columns_and_rejects_length_mismatch(): + """score() no longer drops columns silently, and checks the lengths. + + A probability column (0 or 0.2) is dropped with a warning naming the file + and the column, in single- and multi-file calls; before 1.1.0 the second + file silently left the mean. A prediction shorter or longer than its + ground truth raises ValueError (it used to raise KeyError or be scored). + """ + pred, gt = _video(REAL_BOUTS) + probability = pred * 0.2 + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + assert banos.score(probability, gt) == {} + group, _ = banos.score({"v1": (pred, gt), "v2": (probability, gt)}) + assert [str(w.message) for w in caught if w.category is UserWarning] == [ + "File 'single': prediction column 'evt' dropped (values other than 0 and 1).", + "File 'single': ground truth column 'evt' dropped (missing or invalid in the prediction).", + "File 'v2': prediction column 'evt' dropped (values other than 0 and 1).", + "File 'v2': ground truth column 'evt' dropped (missing or invalid in the prediction).", + ] + assert group["evt"]["f1_score"] == 1.0 # from v1 alone + with pytest.raises(ValueError, match="File 'single': the prediction has 45 frames"): + banos.score(pred.iloc[:45], gt) + with pytest.raises(ValueError, match="File 'v2': the prediction has 60 frames"): + banos.score({"v1": (pred, gt), "v2": (pred, gt.iloc[:45])}) + + +@pytest.mark.parametrize( + "call", + [ + pytest.param(lambda data: banos.preprocess_data(data), id="preprocess_data"), + pytest.param(lambda data: banos.score(*data["v"]), id="score-single"), + pytest.param( + lambda data: banos.score(*data["v"], aggregation="pooled"), id="score-single-pooled" + ), + pytest.param(lambda data: banos.score(data), id="score-several"), + pytest.param( + lambda data: banos.score(data, aggregation="pooled"), id="score-several-pooled" + ), + ], +) +def test_warnings_point_at_the_caller(call): + """Warnings name the caller's line, through preprocess_data() and every score() path. + + Python shows a warning with the file and line it is attributed to; one + stack level too few would name a line of banos instead of this file. + """ + pred, gt = _video(REAL_BOUTS) + data = {"v": (pred * 0.2, gt)} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + call(data) + assert caught + assert {Path(w.filename).resolve() for w in caught} == {Path(__file__).resolve()} + + def test_score_invalid_aggregation_raises(): with pytest.raises(ValueError, match="Unknown aggregation"): banos.score(PRED, GT, aggregation="pool") diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index 730873c..ab8a58f 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -14,9 +14,15 @@ - Column dropping: non-binary columns are removed and their names reported. - Header alignment: only behaviors present in both pred and GT are kept. - NaN filling: missing values are replaced with 0 before further processing. +- Input checks: ValueError when a file's prediction and ground truth differ in + length or have no frame; a UserWarning for each NaN replacement and each + dropped column, naming the file and the column. """ +import warnings + import pandas as pd +import pytest from banos.core.preprocessing import ( drop_non_logical_columns, @@ -98,7 +104,8 @@ def test_match_headers_all_common(): def test_preprocess_data_fills_nan(): pred = pd.DataFrame({"a": [float("nan"), 1.0, 0.0]}) gt = pd.DataFrame({"a": [1.0, float("nan"), 0.0]}) - result, _ = preprocess_data({"f1": (pred, gt)}) + with pytest.warns(UserWarning, match="replaced by 0"): + result, _ = preprocess_data({"f1": (pred, gt)}) pred_out, gt_out = result["f1"] assert pred_out["a"].isna().sum() == 0 assert gt_out["a"].isna().sum() == 0 @@ -122,7 +129,80 @@ def test_preprocess_data_does_not_mutate_caller_dict(): def test_preprocess_data_drops_non_logical(): pred = pd.DataFrame({"a": [0, 1], "b": [0, 2]}) gt = pd.DataFrame({"a": [1, 0], "b": [1, 1]}) - result, info = preprocess_data({"f1": (pred, gt)}) + with pytest.warns(UserWarning, match="dropped"): + result, info = preprocess_data({"f1": (pred, gt)}) pred_out, _ = result["f1"] assert "b" not in pred_out.columns assert "b" in info["f1"]["pred_dropped"] + + +# --- input checks: errors and warnings --- + + +def _preprocess_with_warnings(data_dict: dict) -> tuple[dict, dict, list[str]]: + """Run preprocess_data and also return the messages of the UserWarnings it emits.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result, info = preprocess_data(data_dict) + messages = [str(w.message) for w in caught if w.category is UserWarning] + return result, info, messages + + +def test_preprocess_data_raises_on_length_mismatch(): + """Prediction and ground truth of a file must have the same number of frames (rows).""" + pred = pd.DataFrame({"a": [0, 1, 0]}) + gt = pd.DataFrame({"a": [0, 1, 0, 0]}) + with pytest.raises(ValueError, match="File 'f1': the prediction has 3 frames but"): + preprocess_data({"ok": (gt, gt.copy()), "f1": (pred, gt)}) + + +def test_preprocess_data_raises_on_file_without_frame(): + empty = pd.DataFrame({"a": []}) + with pytest.raises(ValueError, match="File 'f1' has no frame"): + preprocess_data({"f1": (empty, empty.copy())}) + + +def test_preprocess_data_warns_for_each_nan_column(): + """Each column with NaN replaced by 0 is reported with its file, side and count.""" + pred = pd.DataFrame({"a": [float("nan"), 1.0, float("nan")], "b": [0, 1, 0]}) + gt = pd.DataFrame({"a": [1.0, 0.0, 0.0], "b": [0.0, float("nan"), 1.0]}) + result, _, messages = _preprocess_with_warnings({"f1": (pred, gt)}) + assert messages == [ + "File 'f1': 2 NaN value(s) in prediction column 'a' replaced by 0.", + "File 'f1': 1 NaN value(s) in ground truth column 'b' replaced by 0.", + ] + assert result["f1"][0]["a"].tolist() == [0.0, 1.0, 0.0] + + +def test_preprocess_data_warns_for_each_dropped_column(): + """Each dropped column is reported: values other than 0/1 (a probability, inf) + or a column missing, or dropped, on the other side.""" + pred = pd.DataFrame( + {"a": [0, 1, 0], "prob": [0.2, 0.9, 0.1], "only_pred": [0, 1, 1], "inf": [0, 1, 0]} + ) + gt = pd.DataFrame( + {"a": [0, 1, 1], "prob": [0, 1, 0], "only_gt": [1, 0, 0], "inf": [0, float("inf"), 1]} + ) + result, info, messages = _preprocess_with_warnings({"f1": (pred, gt)}) + assert list(result["f1"][0].columns) == ["a"] + assert list(result["f1"][1].columns) == ["a"] + assert info["f1"]["pred_dropped"] == ["prob"] + assert info["f1"]["gt_dropped"] == ["inf"] + assert messages == [ + "File 'f1': prediction column 'prob' dropped (values other than 0 and 1).", + "File 'f1': ground truth column 'inf' dropped (values other than 0 and 1).", + "File 'f1': prediction column 'only_pred' dropped (missing or invalid in the ground truth).", + "File 'f1': prediction column 'inf' dropped (missing or invalid in the ground truth).", + "File 'f1': ground truth column 'prob' dropped (missing or invalid in the prediction).", + "File 'f1': ground truth column 'only_gt' dropped (missing or invalid in the prediction).", + ] + + +def test_preprocess_data_valid_input_emits_no_warning(): + """Binary columns (int, float or bool) on both sides, same length: no warning.""" + pred = pd.DataFrame({"a": [0, 1, 1], "b": [True, False, True]}) + gt = pd.DataFrame({"a": [0.0, 1.0, 0.0], "b": [1, 0, 0]}) + with warnings.catch_warnings(): + warnings.simplefilter("error") + _, info = preprocess_data({"f1": (pred, gt)}) + assert info["f1"] == {"pred_dropped": [], "gt_dropped": [], "header_dropped": []} From 4eea9182e68c356b9f519c051e5d6a4cc996973a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 22:13:03 +0100 Subject: [PATCH 07/25] test: add golden values for the ten recordings scored together The per-recording goldens score each recording alone, so the means across recordings and the pooled metrics were never pinned. The generators now also score the 10 bundled recordings together with score() and BANOS_score, once per combination of missed_bouts and aggregation, under "all_recordings" in golden_python_v110.json and golden_matlab_v110.json. test_regression checks these values at 1e-10 and test_parity checks Python against MATLAB at 1e-6. --- tests/fixtures/golden_matlab_v110.json | 2 +- tests/fixtures/golden_python_v110.json | 178 ++++++++++++++++++ .../matlab/generate_matlab_fixtures.m | 19 ++ .../python/generate_golden_fixtures.py | 26 ++- tests/test_parity.py | 40 +++- tests/test_regression.py | 42 ++++- 6 files changed, 301 insertions(+), 6 deletions(-) diff --git a/tests/fixtures/golden_matlab_v110.json b/tests/fixtures/golden_matlab_v110.json index 3e0663f..049172c 100644 --- a/tests/fixtures/golden_matlab_v110.json +++ b/tests/fixtures/golden_matlab_v110.json @@ -1 +1 @@ -{"Recording_1":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.98056034808521408},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.98286421281656633},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.99243908336739384}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.98056034808521408},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.98286421281656633},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.99243908336739384}},"overall_metrics":{"precision":0.92826086956521736,"recall":0.78129251700680269,"f1_score":0.836455036759336,"so":0.761228298078524,"tp":0.30623014745659,"ic":0.98896591106729359}},"Recording_2":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98771568436705637},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99357808016301785},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99074891239625906}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98771568436705637},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99357808016301785},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99074891239625906}},"overall_metrics":{"precision":0.9671052631578948,"recall":0.83992445266728333,"f1_score":0.897300738477209,"so":0.82594559036003834,"tp":0.30908214265027689,"ic":0.99301066923158332}},"Recording_3":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98261726264923754},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.98941095585555594},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99591253786234157}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98261726264923754},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.98941095585555594},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99591253786234157}},"overall_metrics":{"precision":0.95948072383949645,"recall":0.85096664987969339,"f1_score":0.89862686038270723,"so":0.723715686180092,"tp":0.26887555753525666,"ic":0.99198518909178379}},"Recording_4":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.98586849198997339},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99092353265377653}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.98586849198997339},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99092353265377653}},"overall_metrics":{"precision":0.975,"recall":0.61568627450980384,"f1_score":0.72916666666666674,"so":0.70708735677240686,"tp":0.27571526135052643,"ic":0.99341862874528308}},"Recording_5":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98823265836029839},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99615676486935811},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.993672706200401}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98823265836029839},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99615676486935811},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.993672706200401}},"overall_metrics":{"precision":1,"recall":0.89210526315789473,"f1_score":0.94145869368256319,"so":0.83673046008612484,"tp":0.27928892486843937,"ic":0.99451553235751433}},"Recording_6":{"file_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.98838846071505027},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.98965295127585839},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.9904687807255419}},"group_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.98838846071505027},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.98965295127585839},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.9904687807255419}},"overall_metrics":{"precision":0.91408268733850129,"recall":0.74567099567099571,"f1_score":0.79250474383301706,"so":0.624464020496978,"tp":0.26406773460888022,"ic":0.99212754817911264}},"Recording_7":{"file_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98891661542484988},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.98181290038327618},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.99318498549600709}},"group_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98891661542484988},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.98181290038327618},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.99318498549600709}},"overall_metrics":{"precision":1,"recall":0.58390957446808511,"f1_score":0.70652984743264124,"so":0.66449505594099667,"tp":0.27653014329623343,"ic":0.99097862532603331}},"Recording_8":{"file_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99318530440005481},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":null},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99102642248586637}},"group_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99318530440005481},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":null},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99102642248586637}},"overall_metrics":{"precision":0.89166666666666661,"recall":0.34672064777327932,"f1_score":0.45462962962962961,"so":0.53056180162368516,"tp":0.038138242319112579,"ic":0.992043603151849}},"Recording_9":{"file_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98862824312860653},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99269356597600877},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99282392960084676}},"group_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98862824312860653},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99269356597600877},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99282392960084676}},"overall_metrics":{"precision":0.79839767616191892,"recall":0.65381944444444451,"f1_score":0.712354862119013,"so":0.68272952732383074,"tp":0.27780938219974027,"ic":0.99353643467636554}},"Recording_10":{"file_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.99080565536832721},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.98123242136180233},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.99554269505719462}},"group_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.99080565536832721},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.98123242136180233},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.99554269505719462}},"overall_metrics":{"precision":0.89910714285714288,"recall":0.47575971177944865,"f1_score":0.58347469607011593,"so":0.66680627484973121,"tp":0.28592277410718,"ic":0.99189519294683093}}} +{"Recording_1":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.98056034808521408},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.98286421281656633},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.99243908336739384}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.8,"recall":0.73469387755102045,"f1_score":0.76595744680851063,"so":0.54440950736998917,"tp":0.0846356832664267,"ic":0.98056034808521408},"mount":{"precision":1,"recall":0.53333333333333333,"f1_score":0.69565217391304357,"so":0.73327668724369643,"tp":0.07847125492506643,"ic":0.98286421281656633},"other":{"precision":0.91304347826086951,"recall":0.8571428571428571,"f1_score":0.88421052631578945,"so":0.76722699770041058,"tp":0.061813651634866906,"ic":0.99243908336739384}},"overall_metrics":{"precision":0.92826086956521736,"recall":0.78129251700680269,"f1_score":0.836455036759336,"so":0.761228298078524,"tp":0.30623014745659,"ic":0.98896591106729359}},"Recording_2":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98771568436705637},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99357808016301785},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99074891239625906}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.89473684210526316,"recall":0.73913043478260865,"f1_score":0.80952380952380953,"so":0.6609424826096989,"tp":0.046151098692370357,"ic":0.98771568436705637},"mount":{"precision":1,"recall":0.83333333333333337,"f1_score":0.90909090909090906,"so":0.931754332584827,"tp":0.1451695616211745,"ic":0.99357808016301785},"other":{"precision":0.97368421052631582,"recall":0.78723404255319152,"f1_score":0.87058823529411766,"so":0.7110855462456277,"tp":0.045007910287562576,"ic":0.99074891239625906}},"overall_metrics":{"precision":0.9671052631578948,"recall":0.83992445266728333,"f1_score":0.897300738477209,"so":0.82594559036003834,"tp":0.30908214265027689,"ic":0.99301066923158332}},"Recording_3":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98261726264923754},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.98941095585555594},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99591253786234157}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.93548387096774188,"recall":0.78378378378378377,"f1_score":0.85294117647058809,"so":0.45631035306786549,"tp":0.022321586557310428,"ic":0.98261726264923754},"mount":{"precision":1,"recall":0.73913043478260865,"f1_score":0.85,"so":0.67450421524223569,"tp":0.028794827943741812,"ic":0.98941095585555594},"other":{"precision":0.90243902439024393,"recall":0.88095238095238093,"f1_score":0.89156626506024106,"so":0.76404817641026657,"tp":0.024385815639974422,"ic":0.99591253786234157}},"overall_metrics":{"precision":0.95948072383949645,"recall":0.85096664987969339,"f1_score":0.89862686038270723,"so":0.723715686180092,"tp":0.26887555753525666,"ic":0.99198518909178379}},"Recording_4":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.98586849198997339},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99092353265377653}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":0.9,"recall":0.52941176470588236,"f1_score":0.66666666666666674,"so":0.39147100794903517,"tp":0.014025218005513157,"ic":0.99688249033738241},"mount":{"precision":1,"recall":0.33333333333333331,"f1_score":0.5,"so":0.70111852816432363,"tp":0.040087174457883831,"ic":0.98586849198997339},"other":{"precision":1,"recall":0.6,"f1_score":0.74999999999999989,"so":0.73575989097626859,"tp":0.048748652938708746,"ic":0.99092353265377653}},"overall_metrics":{"precision":0.975,"recall":0.61568627450980384,"f1_score":0.72916666666666674,"so":0.70708735677240686,"tp":0.27571526135052643,"ic":0.99341862874528308}},"Recording_5":{"file_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98823265836029839},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99615676486935811},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.993672706200401}},"group_metrics":{"attack":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"investigation":{"precision":1,"recall":0.8,"f1_score":0.888888888888889,"so":0.65895913125257488,"tp":0.035333351984487595,"ic":0.98823265836029839},"mount":{"precision":1,"recall":0.9,"f1_score":0.94736842105263164,"so":0.91002379283022172,"tp":0.035960102191745627,"ic":0.99615676486935811},"other":{"precision":1,"recall":0.868421052631579,"f1_score":0.92957746478873238,"so":0.77793891626170275,"tp":0.045862245297524168,"ic":0.993672706200401}},"overall_metrics":{"precision":1,"recall":0.89210526315789473,"f1_score":0.94145869368256319,"so":0.83673046008612484,"tp":0.27928892486843937,"ic":0.99451553235751433}},"Recording_6":{"file_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.98838846071505027},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.98965295127585839},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.9904687807255419}},"group_metrics":{"attack":{"precision":1,"recall":0.36363636363636365,"f1_score":0.53333333333333333,"so":0.35548759220851922,"tp":0.014142932226192012,"ic":0.98838846071505027},"investigation":{"precision":0.76744186046511631,"recall":0.7857142857142857,"f1_score":0.77647058823529413,"so":0.53957972748719008,"tp":0.02031691062077776,"ic":0.98965295127585839},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.88888888888888884,"recall":0.83333333333333337,"f1_score":0.86021505376344076,"so":0.60278876229220257,"tp":0.021811095588551047,"ic":0.9904687807255419}},"overall_metrics":{"precision":0.91408268733850129,"recall":0.74567099567099571,"f1_score":0.79250474383301706,"so":0.624464020496978,"tp":0.26406773460888022,"ic":0.99212754817911264}},"Recording_7":{"file_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98891661542484988},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.98181290038327618},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.99318498549600709}},"group_metrics":{"attack":{"precision":1,"recall":0.525,"f1_score":0.68852459016393452,"so":0.39069512581810084,"tp":0.030228614492615703,"ic":0.98891661542484988},"investigation":{"precision":1,"recall":0.3,"f1_score":0.46153846153846151,"so":0.536495310012478,"tp":0.031330852383483966,"ic":0.98181290038327618},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":1,"recall":0.51063829787234039,"f1_score":0.676056338028169,"so":0.7307897879334081,"tp":0.044561106308833941,"ic":0.99318498549600709}},"overall_metrics":{"precision":1,"recall":0.58390957446808511,"f1_score":0.70652984743264124,"so":0.66449505594099667,"tp":0.27653014329623343,"ic":0.99097862532603331}},"Recording_8":{"file_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99318530440005481},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":null},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99102642248586637}},"group_metrics":{"attack":{"precision":0.875,"recall":0.36842105263157893,"f1_score":0.51851851851851849,"so":0.26836382308140022,"tp":0.020422876058778405,"ic":0.9919190825696258},"investigation":{"precision":1,"recall":0.53846153846153844,"f1_score":0.70000000000000007,"so":0.5910507486515022,"tp":0.02527092835048601,"ic":0.99318530440005481},"mount":{"precision":null,"recall":0,"f1_score":0,"so":null,"tp":null,"ic":null},"other":{"precision":0.8,"recall":0.48,"f1_score":0.6,"so":0.73227083313815311,"tp":0.068720922548073327,"ic":0.99102642248586637}},"overall_metrics":{"precision":0.89166666666666661,"recall":0.34672064777327932,"f1_score":0.45462962962962961,"so":0.53056180162368516,"tp":0.038138242319112579,"ic":0.992043603151849}},"Recording_9":{"file_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98862824312860653},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99269356597600877},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99282392960084676}},"group_metrics":{"attack":{"precision":0.95652173913043481,"recall":0.73333333333333328,"f1_score":0.83018867924528306,"so":0.424100282439238,"tp":0.020589806372144304,"ic":0.98862824312860653},"investigation":{"precision":0.375,"recall":0.1875,"f1_score":0.25,"so":0.631678281068525,"tp":0.034809523809523811,"ic":0.99269356597600877},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.86206896551724133,"recall":0.69444444444444442,"f1_score":0.76923076923076916,"so":0.67513954578756,"tp":0.055838198617292827,"ic":0.99282392960084676}},"overall_metrics":{"precision":0.79839767616191892,"recall":0.65381944444444451,"f1_score":0.712354862119013,"so":0.68272952732383074,"tp":0.27780938219974027,"ic":0.99353643467636554}},"Recording_10":{"file_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.99080565536832721},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.98123242136180233},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.99554269505719462}},"group_metrics":{"attack":{"precision":1,"recall":0.2857142857142857,"f1_score":0.44444444444444448,"so":0.18490328157186497,"tp":0.011189008329645372,"ic":0.99080565536832721},"investigation":{"precision":0.625,"recall":0.26315789473684209,"f1_score":0.37037037037037035,"so":0.65720721897446033,"tp":0.064682539682539675,"ic":0.98123242136180233},"mount":{"precision":1,"recall":1,"f1_score":1,"so":1,"tp":1,"ic":1},"other":{"precision":0.97142857142857142,"recall":0.35416666666666669,"f1_score":0.51908396946564883,"so":0.82511459885259986,"tp":0.06781954841653498,"ic":0.99554269505719462}},"overall_metrics":{"precision":0.89910714285714288,"recall":0.47575971177944865,"f1_score":0.58347469607011593,"so":0.66680627484973121,"tp":0.28592277410718,"ic":0.99189519294683093}},"all_recordings":{"exclude_per_file":{"group_metrics":{"attack":{"precision":0.98315217391304355,"recall":0.7276105035315561,"f1_score":0.80150095657055132,"so":0.66235501051191226,"tp":0.5096573237479376,"ic":0.994865805720646},"investigation":{"precision":0.829766257353812,"recall":0.56618535797359615,"f1_score":0.654235740850259,"so":0.56681037684433189,"tp":0.037887769335291942,"ic":0.987458558719619},"mount":{"precision":1,"recall":0.73391304347826092,"f1_score":0.79021115040565848,"so":0.8834086173405894,"tp":0.48094254679329018,"ic":0.99420872285494133},"other":{"precision":0.9311553139012132,"recall":0.68663330755967933,"f1_score":0.77505286219469072,"so":0.73221630555982,"tp":0.048456914727792295,"ic":0.99267435858456277}},"overall_metrics":{"precision":0.9343778833764278,"recall":0.67858555313577318,"f1_score":0.75525017750528978,"so":0.70678190987758827,"tp":0.26380776921153409,"ic":0.99225296758827564}},"exclude_pooled":{"group_metrics":{"attack":{"precision":0.975609756097561,"recall":0.418848167539267,"f1_score":0.58608058608058611,"so":0.28221499894768604,"tp":0.017892953838825207,"ic":0.99008918314299021},"investigation":{"precision":0.84615384615384615,"recall":0.65845070422535212,"f1_score":0.74059405940594059,"so":0.56118339150599472,"tp":0.040843958498742394,"ic":0.98677021109585328},"mount":{"precision":1,"recall":0.60465116279069764,"f1_score":0.75362318840579712,"so":0.75902040206560628,"tp":0.0570522939982909,"ic":0.98863902556706129},"other":{"precision":0.93146417445482865,"recall":0.66004415011037532,"f1_score":0.77260981912144711,"so":0.72972716864506215,"tp":0.046268588052302044,"ic":0.99284303851706734}},"overall_metrics":{"precision":0.938306944176559,"recall":0.585498546166423,"f1_score":0.71322691325344278,"so":0.58303649029108739,"tp":0.040514448597040134,"ic":0.98958536458074309}},"zero_per_file":{"group_metrics":{"attack":{"precision":0.98315217391304355,"recall":0.7276105035315561,"f1_score":0.80150095657055132,"so":0.64019653023213274,"tp":0.50865228106262994,"ic":0.93187083493412037},"investigation":{"precision":0.829766257353812,"recall":0.56618535797359615,"f1_score":0.654235740850259,"so":0.378197550178644,"tp":0.024841101231380485,"ic":0.67800634715154251},"mount":{"precision":1,"recall":0.73391304347826092,"f1_score":0.79021115040565848,"so":0.67481432973885869,"tp":0.42310875368414608,"ic":0.73516615169765176},"other":{"precision":0.9311553139012132,"recall":0.68663330755967933,"f1_score":0.77505286219469072,"so":0.5312769706084941,"tp":0.033673866992742066,"ic":0.74164232059319424}},"overall_metrics":{"precision":0.9343778833764278,"recall":0.67858555313577318,"f1_score":0.75525017750528978,"so":0.55612134518953238,"tp":0.24756900074272456,"ic":0.77167141359412728}},"zero_pooled":{"group_metrics":{"attack":{"precision":0.975609756097561,"recall":0.418848167539267,"f1_score":0.58608058608058611,"so":0.26005151735493581,"tp":0.016487748039964584,"ic":0.91233348813176063},"investigation":{"precision":0.84615384615384615,"recall":0.65845070422535212,"f1_score":0.74059405940594059,"so":0.43910139054679587,"tp":0.031958606123577378,"ic":0.77134854529323749},"mount":{"precision":1,"recall":0.60465116279069764,"f1_score":0.75362318840579712,"so":0.46776838731950149,"tp":0.035160134673365331,"ic":0.60927753901225867},"other":{"precision":0.93146417445482865,"recall":0.66004415011037532,"f1_score":0.77260981912144711,"so":0.51682579597483269,"tp":0.032769507394336553,"ic":0.71240445816144926}},"overall_metrics":{"precision":0.938306944176559,"recall":0.585498546166423,"f1_score":0.71322691325344278,"so":0.42093677279901642,"tp":0.029093999057810963,"ic":0.75134100764967648}}}} diff --git a/tests/fixtures/golden_python_v110.json b/tests/fixtures/golden_python_v110.json index d111c44..03b91f9 100644 --- a/tests/fixtures/golden_python_v110.json +++ b/tests/fixtures/golden_python_v110.json @@ -798,5 +798,183 @@ "tp": 0.28592277410718003, "ic": 0.9918951929468309 } + }, + "all_recordings": { + "exclude_per_file": { + "group_metrics": { + "attack": { + "precision": 0.9831521739130435, + "recall": 0.7276105035315561, + "f1_score": 0.8015009565705513, + "so": 0.6623550105119123, + "tp": 0.5096573237479376, + "ic": 0.994865805720646 + }, + "investigation": { + "precision": 0.829766257353812, + "recall": 0.5661853579735961, + "f1_score": 0.654235740850259, + "so": 0.5668103768443319, + "tp": 0.03788776933529195, + "ic": 0.987458558719619 + }, + "mount": { + "precision": 1.0, + "recall": 0.7339130434782609, + "f1_score": 0.7902111504056585, + "so": 0.8834086173405894, + "tp": 0.4809425467932902, + "ic": 0.9942087228549413 + }, + "other": { + "precision": 0.9311553139012132, + "recall": 0.6866333075596793, + "f1_score": 0.7750528621946908, + "so": 0.73221630555982, + "tp": 0.048456914727792295, + "ic": 0.9926743585845628 + } + }, + "overall_metrics": { + "precision": 0.9343778833764279, + "recall": 0.6785855531357731, + "f1_score": 0.7552501775052899, + "so": 0.7067819098775884, + "tp": 0.2638077692115341, + "ic": 0.9922529675882756 + } + }, + "exclude_pooled": { + "group_metrics": { + "attack": { + "precision": 0.975609756097561, + "recall": 0.418848167539267, + "f1_score": 0.5860805860805861, + "so": 0.28221499894768604, + "tp": 0.017892953838825207, + "ic": 0.9900891831429903 + }, + "investigation": { + "precision": 0.8461538461538461, + "recall": 0.6584507042253521, + "f1_score": 0.7405940594059406, + "so": 0.5611833915059947, + "tp": 0.04084395849874241, + "ic": 0.9867702110958532 + }, + "mount": { + "precision": 1.0, + "recall": 0.6046511627906976, + "f1_score": 0.7536231884057971, + "so": 0.7590204020656063, + "tp": 0.057052293998290905, + "ic": 0.9886390255670613 + }, + "other": { + "precision": 0.9314641744548287, + "recall": 0.6600441501103753, + "f1_score": 0.7726098191214471, + "so": 0.7297271686450624, + "tp": 0.04626858805230205, + "ic": 0.9928430385170676 + } + }, + "overall_metrics": { + "precision": 0.9383069441765589, + "recall": 0.585498546166423, + "f1_score": 0.7132269132534428, + "so": 0.5830364902910874, + "tp": 0.04051444859704014, + "ic": 0.9895853645807431 + } + }, + "zero_per_file": { + "group_metrics": { + "attack": { + "precision": 0.9831521739130435, + "recall": 0.7276105035315561, + "f1_score": 0.8015009565705513, + "so": 0.6401965302321327, + "tp": 0.5086522810626298, + "ic": 0.9318708349341206 + }, + "investigation": { + "precision": 0.829766257353812, + "recall": 0.5661853579735961, + "f1_score": 0.654235740850259, + "so": 0.378197550178644, + "tp": 0.024841101231380485, + "ic": 0.6780063471515425 + }, + "mount": { + "precision": 1.0, + "recall": 0.7339130434782609, + "f1_score": 0.7902111504056585, + "so": 0.6748143297388587, + "tp": 0.4231087536841461, + "ic": 0.7351661516976518 + }, + "other": { + "precision": 0.9311553139012132, + "recall": 0.6866333075596793, + "f1_score": 0.7750528621946908, + "so": 0.531276970608494, + "tp": 0.033673866992742066, + "ic": 0.7416423205931941 + } + }, + "overall_metrics": { + "precision": 0.9343778833764279, + "recall": 0.6785855531357731, + "f1_score": 0.7552501775052899, + "so": 0.5561213451895324, + "tp": 0.24756900074272462, + "ic": 0.7716714135941273 + } + }, + "zero_pooled": { + "group_metrics": { + "attack": { + "precision": 0.975609756097561, + "recall": 0.418848167539267, + "f1_score": 0.5860805860805861, + "so": 0.2600515173549358, + "tp": 0.016487748039964587, + "ic": 0.9123334881317607 + }, + "investigation": { + "precision": 0.8461538461538461, + "recall": 0.6584507042253521, + "f1_score": 0.7405940594059406, + "so": 0.43910139054679587, + "tp": 0.03195860612357739, + "ic": 0.7713485452932374 + }, + "mount": { + "precision": 1.0, + "recall": 0.6046511627906976, + "f1_score": 0.7536231884057971, + "so": 0.46776838731950154, + "tp": 0.035160134673365324, + "ic": 0.6092775390122587 + }, + "other": { + "precision": 0.9314641744548287, + "recall": 0.6600441501103753, + "f1_score": 0.7726098191214471, + "so": 0.5168257959748328, + "tp": 0.032769507394336546, + "ic": 0.7124044581614494 + } + }, + "overall_metrics": { + "precision": 0.9383069441765589, + "recall": 0.585498546166423, + "f1_score": 0.7132269132534428, + "so": 0.42093677279901653, + "tp": 0.029093999057810963, + "ic": 0.7513410076496765 + } + } } } diff --git a/tests/fixtures/matlab/generate_matlab_fixtures.m b/tests/fixtures/matlab/generate_matlab_fixtures.m index 15279f0..2a2cc3b 100644 --- a/tests/fixtures/matlab/generate_matlab_fixtures.m +++ b/tests/fixtures/matlab/generate_matlab_fixtures.m @@ -1,6 +1,10 @@ %% generate_matlab_fixtures.m % Generates a single JSON reference file from the Matlab BANOS implementation, % containing all 10 recordings in the same structure as golden_python_v110.json. +% Each Recording_N entry scores one recording alone, with the default options. +% The all_recordings entry scores the 10 recordings together with BANOS_score, +% once per combination of 'missed_bouts' and 'aggregation', keyed +% _ (e.g. zero_pooled). % % Output: tests/fixtures/golden_matlab_v110.json % @@ -63,10 +67,25 @@ fprintf('Processed %s (F1=%.4f, SO=%.4f, IC=%.4f)\n', ... recName, overallMetrics.f1_score, overallMetrics.so, overallMetrics.ic); + % Keep the recording for the all_recordings entry + allData.(recName) = {predMatrix, gtMatrix}; + % Reset for next recording clear singleData; end +% Score the 10 recordings together, for every combination of options +combinations = {'exclude', 'per_file'; 'exclude', 'pooled'; 'zero', 'per_file'; 'zero', 'pooled'}; +for c = 1:size(combinations, 1) + key = sprintf('%s_%s', combinations{c, 1}, combinations{c, 2}); + [groupMetrics, overallMetrics] = BANOS_score(allData, ... + 'missed_bouts', combinations{c, 1}, 'aggregation', combinations{c, 2}); + allResults.all_recordings.(key).group_metrics = groupMetrics; + allResults.all_recordings.(key).overall_metrics = overallMetrics; + fprintf('all_recordings %s (F1=%.4f, SO=%.4f, IC=%.4f)\n', ... + key, overallMetrics.f1_score, overallMetrics.so, overallMetrics.ic); +end + % Save all recordings as a single JSON (matches golden_python_v110.json structure) fid = fopen(outFile, 'w'); if fid == -1 diff --git a/tests/fixtures/python/generate_golden_fixtures.py b/tests/fixtures/python/generate_golden_fixtures.py index cd4f10b..685ea2e 100644 --- a/tests/fixtures/python/generate_golden_fixtures.py +++ b/tests/fixtures/python/generate_golden_fixtures.py @@ -8,6 +8,11 @@ 2. IC averages over the real bouts that contain a predicted frame; missed bouts no longer score 1.0. +Each "Recording_N" entry scores one recording alone, with the default options. +The "all_recordings" entry scores the 10 recordings together with banos.score(), +once per combination of missed_bouts and aggregation, keyed +"_" (e.g. "zero_pooled"). + Run once to regenerate after a deliberate algorithm change: python tests/fixtures/python/generate_golden_fixtures.py @@ -22,10 +27,16 @@ import pandas as pd -from banos import aggregate_metrics, calculate_banos_for_each_file, preprocess_data +from banos import aggregate_metrics, calculate_banos_for_each_file, preprocess_data, score DATASET_DIR = Path(__file__).parent.parent.parent.parent / "data" / "dataset_human_vs_human" OUT_FILE = Path(__file__).parent.parent / "golden_python_v110.json" +OPTION_COMBINATIONS = [ + ("exclude", "per_file"), + ("exclude", "pooled"), + ("zero", "per_file"), + ("zero", "pooled"), +] def load_recording(n: int): @@ -65,6 +76,17 @@ def make_json_serializable(obj): "overall_metrics": make_json_serializable(overall_m), } print(f"Recording_{n}: {overall_m}") + n_recordings = len(results) + + all_data = {f"Recording_{n}": load_recording(n) for n in range(1, 11)} + results["all_recordings"] = {} + for missed_bouts, aggregation in OPTION_COMBINATIONS: + group_m, overall_m = score(all_data, missed_bouts=missed_bouts, aggregation=aggregation) + results["all_recordings"][f"{missed_bouts}_{aggregation}"] = { + "group_metrics": make_json_serializable(group_m), + "overall_metrics": make_json_serializable(overall_m), + } + print(f"all_recordings {missed_bouts}_{aggregation}: {overall_m}") OUT_FILE.parent.mkdir(parents=True, exist_ok=True) with open(OUT_FILE, "w") as f: @@ -72,4 +94,4 @@ def make_json_serializable(obj): f.write("\n") print(f"\nSaved golden fixtures to {OUT_FILE}") - print(f"Total recordings: {len(results)}") + print(f"Total recordings: {n_recordings}") diff --git a/tests/test_parity.py b/tests/test_parity.py index eeef04f..7daca4d 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -13,7 +13,9 @@ matlab -batch "run('tests/fixtures/matlab/generate_matlab_fixtures.m')" The file contains all 10 recordings in a single JSON (same structure as -golden_python_v110.json), keyed by "Recording_N". +golden_python_v110.json), keyed by "Recording_N", plus the 10 recordings +scored together for each combination of missed_bouts and aggregation +("all_recordings"). TOLERANCE --------- @@ -37,13 +39,19 @@ import pandas as pd import pytest -from banos import aggregate_metrics, calculate_banos_for_each_file, preprocess_data +from banos import aggregate_metrics, calculate_banos_for_each_file, preprocess_data, score DATASET_DIR = Path(__file__).parent.parent / "data" / "dataset_human_vs_human" MATLAB_FIXTURE = Path(__file__).parent / "fixtures" / "golden_matlab_v110.json" TOLERANCE = 1e-6 # floating point tolerance for Matlab vs Python comparisons METRICS = ["precision", "recall", "f1_score", "so", "tp", "ic"] +OPTION_COMBINATIONS = [ + ("exclude", "per_file"), + ("exclude", "pooled"), + ("zero", "per_file"), + ("zero", "pooled"), +] # Load MATLAB fixture once (skip all tests if absent) if not MATLAB_FIXTURE.exists(): @@ -136,3 +144,31 @@ def test_parity_group_metrics(n): assert _values_match(actual, expected), ( f"{rec_name} {behavior} {metric}: Matlab={expected}, Python={actual}" ) + + +@pytest.mark.parametrize("missed_bouts, aggregation", OPTION_COMBINATIONS) +def test_parity_all_recordings(missed_bouts, aggregation): + """Python must match Matlab within 1e-6 on the 10 recordings scored together. + + One case per combination of missed_bouts and aggregation (per-behavior + and overall metrics). + """ + matlab_result = _MATLAB_ALL["all_recordings"][f"{missed_bouts}_{aggregation}"] + data = {f"Recording_{n}": load_recording(n) for n in range(1, 11)} + group, overall = score(data, missed_bouts=missed_bouts, aggregation=aggregation) + + assert set(group) == set(matlab_result["group_metrics"]) + for behavior, metrics in matlab_result["group_metrics"].items(): + for metric in METRICS: + expected = metrics[metric] + actual = group[behavior][metric] + assert _values_match(actual, expected), ( + f"{missed_bouts}/{aggregation} {behavior} {metric}: " + f"Matlab={expected}, Python={actual}" + ) + for metric in METRICS: + expected = matlab_result["overall_metrics"][metric] + actual = overall[metric] + assert _values_match(actual, expected), ( + f"{missed_bouts}/{aggregation} overall {metric}: Matlab={expected}, Python={actual}" + ) diff --git a/tests/test_regression.py b/tests/test_regression.py index c656e73..3bc6cd3 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -14,6 +14,11 @@ 2. IC averages over the real bouts that contain a predicted frame; missed bouts no longer score 1.0 (39 IC values, Recording_8 mount IC is NaN). +Each "Recording_N" entry scores one recording alone, with the default options. +The "all_recordings" entry scores the 10 recordings together, for each of the +four combinations of missed_bouts and aggregation: these values pin the means +across recordings and the pooled metrics. + The archive fixtures (tests/fixtures/golden_python_v020.json and golden_python_v015.json) preserve the v0.2.0 and v0.1.5 behaviour for reference and comparison. @@ -40,11 +45,17 @@ import pandas as pd import pytest -from banos import aggregate_metrics, calculate_banos_for_each_file, preprocess_data +from banos import aggregate_metrics, calculate_banos_for_each_file, preprocess_data, score DATASET_DIR = Path(__file__).parent.parent / "data" / "dataset_human_vs_human" GOLDEN_FILE = Path(__file__).parent / "fixtures" / "golden_python_v110.json" TOLERANCE = 1e-10 +OPTION_COMBINATIONS = [ + ("exclude", "per_file"), + ("exclude", "pooled"), + ("zero", "per_file"), + ("zero", "pooled"), +] with open(GOLDEN_FILE) as f: GOLDEN = json.load(f) @@ -113,3 +124,32 @@ def test_regression_group_metrics(n): assert _close(actual, expected), ( f"Recording_{n} {behavior} {metric}: expected {expected}, got {actual}" ) + + +@pytest.mark.parametrize("missed_bouts, aggregation", OPTION_COMBINATIONS) +def test_regression_all_recordings(missed_bouts, aggregation): + """The 10 recordings scored together must match the golden values (tol=1e-10). + + One case per combination of missed_bouts and aggregation. These pin the + means across recordings (per_file) and the pooled metrics, which the + per-recording goldens do not. Every behavior and metric is compared. + """ + data = {f"Recording_{n}": load_recording(n) for n in range(1, 11)} + group, overall = score(data, missed_bouts=missed_bouts, aggregation=aggregation) + + golden = GOLDEN["all_recordings"][f"{missed_bouts}_{aggregation}"] + assert set(group) == set(golden["group_metrics"]) + for behavior, metrics in golden["group_metrics"].items(): + assert set(group[behavior]) == set(metrics) + for metric, expected in metrics.items(): + actual = group[behavior][metric] + assert _close(actual, expected), ( + f"{missed_bouts}/{aggregation} {behavior} {metric}: " + f"expected {expected}, got {actual}" + ) + assert set(overall) == set(golden["overall_metrics"]) + for metric, expected in golden["overall_metrics"].items(): + actual = overall[metric] + assert _close(actual, expected), ( + f"{missed_bouts}/{aggregation} overall {metric}: expected {expected}, got {actual}" + ) From a30bb898af8ca96975b8d3bd353dc97923d193c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:07:34 +0100 Subject: [PATCH 08/25] test: make the mutation and version tests able to fail test_preprocess_data_does_not_mutate_caller_dict now compares the caller's dict and DataFrames before and after preprocess_data; it compared the dict's id with itself and could not fail. It runs twice, once with the prediction and once with the ground truth as the side that needs every cleaning step, so that a step applied in place to either side fails it. test_score_version compares banos.__version__ with the version in pyproject.toml. Remove tests/test_comparison_report.py. It looked for golden_matlab_current.json, which does not exist, so it was always skipped. Its check, the per-behavior metrics of each recording against Matlab at 1e-6, is made by test_parity_group_metrics, which also requires NaN on both sides where it accepted a Python NaN against any Matlab number. test_parity_group_metrics now also fails when Python and Matlab report different behaviors, as the removed test did; it skipped a behavior missing on the Python side and ignored an extra one. The removed test was the only user of pytest-html, which leaves the dev dependencies; the lock drops pytest-html and pytest-metadata. Correct test docstrings that said NaN comes from behaviors with no GT bout (such a recording scores 1.0 or 0.0), that optimal matching maximizes total overlap (it maximizes the number of matched pairs), or that pointed to the removed test. generate_archive_fixtures.py now states that the v0.1.5 fixture is frozen, how it was produced, and that the BANOS 0.1.5 package from PyPI writes the same file; like the other generator, it ends the file with a newline. --- pyproject.toml | 2 - .../python/generate_archive_fixtures.py | 21 +- tests/test_comparison_report.py | 193 ------------------ tests/test_matching.py | 2 +- tests/test_metrics.py | 13 +- tests/test_parity.py | 30 ++- tests/test_pipeline.py | 30 ++- tests/test_preprocessing.py | 33 ++- uv.lock | 30 --- 9 files changed, 89 insertions(+), 265 deletions(-) delete mode 100644 tests/test_comparison_report.py diff --git a/pyproject.toml b/pyproject.toml index d8f9017..930cf4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ dependencies = [ dev = [ "pytest>=7.4", "pytest-cov>=4.1", - "pytest-html>=4.0", "pre-commit>=3.5", "ruff>=0.4", ] @@ -77,7 +76,6 @@ dev = [ "nbformat>=5.10.4", "pytest>=7.4", "pytest-cov>=4.1", - "pytest-html>=4.2.0", "pre-commit>=3.5", "ruff>=0.4", ] diff --git a/tests/fixtures/python/generate_archive_fixtures.py b/tests/fixtures/python/generate_archive_fixtures.py index 479c351..79d419e 100644 --- a/tests/fixtures/python/generate_archive_fixtures.py +++ b/tests/fixtures/python/generate_archive_fixtures.py @@ -1,10 +1,17 @@ -"""Run the ORIGINAL archive/python/BANOS/BANOS.py code against the 10 recordings. +"""Record of how the frozen v0.1.5 fixture, golden_python_v015.json, was made. -Saves output to tests/fixtures/golden_python_archive.json. -This represents v0.1.5 behaviour (before the Phase 4 refactoring). +The fixture holds the output of the original BANOS 0.1.5 code on the 10 +bundled recordings, each scored alone with that code's preprocess_data, +calculate_banos_for_each_file and aggregate_metrics. It is an archive, kept +for comparison with later versions (see tests/fixtures/compare_fixtures.py), +and is not regenerated: the code it records no longer changes. -Run from the project root: - python tests/fixtures/python/generate_archive_fixtures.py +It was written by this script, which imported the 0.1.5 module from a copy at +archive/python/BANOS/BANOS.py. That folder is not part of this repository, so +the import below only works where the BANOS 0.1.5 package from PyPI (module +BANOS.BANOS) is installed. Run that way from the project root, the script +writes the committed file again, unchanged: + uv run --isolated --no-project --with "BANOS==0.1.5" python tests/fixtures/python/generate_archive_fixtures.py """ import json @@ -13,7 +20,8 @@ from pathlib import Path # --------------------------------------------------------------------------- -# Import from archive — NOT from the installed banos package +# Import the 0.1.5 module (archive copy if present, else the installed +# BANOS 0.1.5 package) — NOT the banos package of this repository # --------------------------------------------------------------------------- PROJECT_ROOT = Path(__file__).parent.parent.parent.parent sys.path.insert(0, str(PROJECT_ROOT / "archive" / "python")) @@ -69,6 +77,7 @@ def make_json_serializable(obj): OUT_FILE.parent.mkdir(parents=True, exist_ok=True) with open(OUT_FILE, "w") as f: json.dump(results, f, indent=2) + f.write("\n") print(f"\nSaved archive fixtures to {OUT_FILE}") print(f"Total recordings: {len(results)}") diff --git a/tests/test_comparison_report.py b/tests/test_comparison_report.py deleted file mode 100644 index b321957..0000000 --- a/tests/test_comparison_report.py +++ /dev/null @@ -1,193 +0,0 @@ -# tests/test_comparison_report.py -"""Python vs Matlab output comparison — generates HTML side-by-side tables. - -PURPOSE -------- -This module produces a human-readable HTML report (via pytest-html) that shows -Python and Matlab metric values side-by-side for every recording, every behavior, -and every metric. - -It serves two goals: - 1. Visually confirm Python and Matlab agree (or document where they intentionally differ). - 2. Provide a permanent audit trail: if you change either implementation, running - `pytest --html=reports/test_report.html` gives you an updated comparison. - -RUNNING -------- -Requires Matlab fixtures to be pre-generated: - matlab -batch "run('tests/fixtures/matlab/generate_matlab_fixtures.m')" - -If no Matlab fixtures are found, all tests in this module are automatically -skipped — safe for CI where Matlab is not installed. - -EXPECTED DIVERGENCES (orange rows, not test failures) ------------------------------------------------------- -The Python and Matlab implementations have one known intentional difference: - - - Python returns NaN when a behavior has no GT bouts (mathematically undefined). - - The current Matlab implementation may return 0 in the same case (legacy choice). - -These rows are marked orange ("expected divergence") rather than red (failure). -All other differences are marked red and cause the test to fail. - -TOLERANCE ---------- -Numeric values are compared with tolerance 1e-6 (matching test_parity.py). -""" - -import json -import math -from pathlib import Path - -import pandas as pd -import pytest - -from banos import aggregate_metrics, calculate_banos_for_each_file, preprocess_data - -DATASET_DIR = Path(__file__).parent.parent / "data" / "dataset_human_vs_human" -MATLAB_FIXTURE = Path(__file__).parent / "fixtures" / "golden_matlab_current.json" -TOLERANCE = 1e-6 -METRICS = ["precision", "recall", "f1_score", "so", "tp", "ic"] - - -# --------------------------------------------------------------------------- -# Skip entire module if Matlab fixture is absent -# --------------------------------------------------------------------------- -if not MATLAB_FIXTURE.exists(): - pytest.skip( - "Matlab fixture not found: tests/fixtures/golden_matlab_current.json. " - "Run: matlab -batch \"run('tests/fixtures/matlab/generate_matlab_fixtures.m')\"", - allow_module_level=True, - ) - -with open(MATLAB_FIXTURE, encoding="utf-8") as _f: - _MATLAB_ALL = json.load(_f) - - -def load_recording(n: int): - """Load pred/gt DataFrames for Recording N.""" - rec_dir = DATASET_DIR / f"Recording_{n}" - pred_path = rec_dir / "humanAnnotation_2.csv" - if not pred_path.exists(): - raise FileNotFoundError(f"No pred file for Recording_{n}") - pred = pd.read_csv(pred_path) - gt_path = rec_dir / "humanAnnotation_1.csv" - if not gt_path.exists(): - raise FileNotFoundError(f"No GT file for Recording_{n}") - return pred, pd.read_csv(gt_path) - - -def _is_nan(v) -> bool: - return v is None or (isinstance(v, float) and math.isnan(v)) - - -def _fmt(v) -> str: - if _is_nan(v): - return "NaN" - return f"{v:.6f}" - - -def _build_html_table(rec_name: str, python_group: dict, matlab_group: dict) -> str: - """Build an HTML comparison table for one recording.""" - rows = [] - all_behaviors = sorted(set(python_group) | set(matlab_group)) - unexpected_diffs = 0 - - for behavior in all_behaviors: - py_bm = python_group.get(behavior, {}) - ml_bm = matlab_group.get(behavior, {}) - for metric in METRICS: - py_val = py_bm.get(metric) - ml_val = ml_bm.get(metric) - py_nan = _is_nan(py_val) - ml_nan = _is_nan(ml_val) - - if py_nan and ml_nan: - status = "NaN/NaN" - color = "#d4edda" # green — both undefined - diff_str = "—" - elif py_nan and not ml_nan: - # Python undefined, Matlab numeric (expected divergence: Matlab returns 0) - status = "expected divergence (Py=NaN, Ml=0)" - color = "#fff3cd" # orange - diff_str = f"Py=NaN, Ml={_fmt(ml_val)}" - elif not py_nan and ml_nan: - # Should not happen — flag as unexpected - status = "UNEXPECTED (Py=numeric, Ml=NaN)" - color = "#f8d7da" # red - diff_str = f"Py={_fmt(py_val)}, Ml=NaN" - unexpected_diffs += 1 - else: - diff = abs(py_val - ml_val) - if diff <= TOLERANCE: - status = "OK" - color = "#d4edda" # green - diff_str = f"{diff:.2e}" - else: - status = f"DIFF > {TOLERANCE:.0e}" - color = "#f8d7da" # red - diff_str = f"{diff:.2e}" - unexpected_diffs += 1 - - rows.append( - f"" - f"{behavior}{metric}" - f"{_fmt(py_val)}{_fmt(ml_val)}" - f"{diff_str}{status}" - "" - ) - - header = ( - "" - "" - "" - "" - ) - footer = "
BehaviorMetricPythonMatlabDiffStatus
" - legend = ( - "

" - "Green = agree  " - "Orange = expected divergence (NaN vs 0)  " - "Red = unexpected mismatch" - "

" - ) - return header + "".join(rows) + footer + legend, unexpected_diffs - - -@pytest.mark.parametrize("n", range(1, 11)) -def test_comparison_python_vs_matlab(n, extras): - """Compare Python and Matlab per-behavior metrics for Recording N. - - Produces an HTML table in the pytest report showing values side-by-side. - Orange rows = known Python(NaN) vs Matlab(0) divergence — not failures. - Red rows = unexpected mismatches — cause this test to fail. - """ - rec_name_key = f"Recording_{n}" - if rec_name_key not in _MATLAB_ALL: - pytest.skip(f"Recording_{n} not in Matlab fixture") - - matlab_result = _MATLAB_ALL[rec_name_key] - - # Compute Python output - pred, gt = load_recording(n) - rec_name = rec_name_key - data, _ = preprocess_data({rec_name: (pred.copy(), gt.copy())}) - file_metrics = calculate_banos_for_each_file(data) - group_py, _ = aggregate_metrics(file_metrics) - - matlab_group = matlab_result.get("group_metrics", {}) - - table_html, unexpected_diffs = _build_html_table(rec_name, group_py, matlab_group) - - # Inject into HTML report - try: - from pytest_html import extras as html_extras - - extras.append(html_extras.html(f"

{rec_name} — Python vs Matlab

{table_html}")) - except ImportError: - pass # pytest-html not installed; test still runs, just no HTML extra - - assert unexpected_diffs == 0, ( - f"{rec_name}: {unexpected_diffs} unexpected mismatches between Python and Matlab. " - "See the HTML report for details." - ) diff --git a/tests/test_matching.py b/tests/test_matching.py index 2c38672..9c662f6 100644 --- a/tests/test_matching.py +++ b/tests/test_matching.py @@ -11,7 +11,7 @@ that overlaps. Fast (O(n*m)) but can make suboptimal early commitments. Optimal matching: Hungarian algorithm (scipy.optimize.linear_sum_assignment) -finds the globally best assignment, maximizing total overlap. Slower (O(n^3)) +finds the assignment with the most overlapping (matched) pairs. Slower (O(n^3)) but scientifically preferable when bouts are densely overlapping. The test_optimal_beats_greedy test documents the exact scenario where greedy diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 6dd3ac7..c2e8340 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -14,9 +14,13 @@ - No GT bout found (no predicted frame inside) -> IC has no bout to average -> NaN WHY NaN and not 0: returning 0 would bias the group/overall averages downward - by treating "undefined" as "worst possible score". nanmean() in aggregate_metrics - correctly excludes NaN values so they do not affect recordings where the behavior - actually occurred. + by treating "undefined" as "worst possible score"; aggregate_metrics leaves + NaN values out of its averages (nanmean). In the pipeline, a recording with no + GT bout of a behavior never gets these NaN: compute_behavior_metrics scores it + 1.0 (nothing predicted) or 0.0 (false bouts). The NaN cells that reach the + averages come from recordings that have GT bouts: precision when nothing is + predicted, SO/TP when no bout overlaps, IC when no GT bout longer than one + frame is found. F1 is the exception: with no matched bout (TP = 0) but some predicted or missed bouts (FP + FN > 0) it is 0, the value of the count form @@ -155,7 +159,8 @@ def test_so_returns_nan_for_empty_gt(): """SO is undefined (NaN) when GT has no bouts — not 0. WHY: 'no GT bouts' means the metric is inapplicable, not that overlap is zero. - NaN causes aggregate_metrics to exclude this recording from the SO average. + The pipeline does not average this NaN: a recording with no GT bout of a + behavior scores 1.0 or 0.0 per file, and adds no SO term when pooled. """ assert math.isnan(calculate_so([(0, 5)], [])) diff --git a/tests/test_parity.py b/tests/test_parity.py index 7daca4d..4f5e722 100644 --- a/tests/test_parity.py +++ b/tests/test_parity.py @@ -24,12 +24,17 @@ NaN HANDLING ------------ -Both Python and Matlab v0.2.0 return 1.0 for correct absence (both have no bouts) -and 0.0 for false detection (machine predicts bouts, GT has none). No NaN expected -for absent behaviors in v0.2.0. The _values_match() helper still handles None/NaN -for any residual edge cases during Matlab JSON encoding. - -See also: tests/test_comparison_report.py for a visual HTML comparison table. +Both Python and Matlab return 1.0 for correct absence (both have no bouts) +and 0.0 for false detection (machine predicts bouts, GT has none), so a +behavior absent from a recording is never NaN with per-file aggregation. +A metric is NaN where it is undefined, in both implementations: precision when +nothing is predicted, SO and TP when no bout overlaps, IC when no GT bout longer +than one frame is found (and, pooled, recall of a behavior with no GT bout). +Matlab writes NaN as JSON null; _values_match() requires NaN on both sides or +numbers on both sides. + +See also: tests/fixtures/compare_fixtures.py for side-by-side tables of the +fixture files (text or HTML). """ import json @@ -83,7 +88,7 @@ def _values_match(actual, expected) -> bool: Matlab encodes NaN via jsonencode as JSON null; Python's json.load() reads null as None. Both Python NaN and Matlab null/NaN are treated as equivalent - 'undefined' values — a behavior with no GT bouts in both implementations. + 'undefined' values; NaN on one side only is a mismatch. """ expected_nan = expected is None or (isinstance(expected, float) and math.isnan(expected)) actual_nan = actual is None or (isinstance(actual, float) and math.isnan(actual)) @@ -121,7 +126,10 @@ def test_parity_overall_metrics(n): @pytest.mark.parametrize("n", range(1, 11)) def test_parity_group_metrics(n): - """Python per-behavior metrics must match Matlab output within 1e-6.""" + """Python per-behavior metrics must match Matlab output within 1e-6. + + Both must report the same behaviors: one missing on either side is a mismatch. + """ rec_name = f"Recording_{n}" if rec_name not in _MATLAB_ALL: pytest.skip(f"{rec_name} not in Matlab fixture") @@ -133,9 +141,11 @@ def test_parity_group_metrics(n): group, _ = aggregate_metrics(file_metrics) matlab_group = matlab_result.get("group_metrics", {}) + assert set(group) == set(matlab_group), ( + f"{rec_name} behaviors: Python only {sorted(set(group) - set(matlab_group))}, " + f"Matlab only {sorted(set(matlab_group) - set(group))}" + ) for behavior, metrics in matlab_group.items(): - if behavior not in group: - pytest.skip(f"Behavior '{behavior}' not in Python output for {rec_name}") for metric in METRICS: if metric not in metrics: continue diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 9e46296..776140c 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -13,12 +13,15 @@ preprocess_data -> calculate_components_for_each_file -> pool_components (the second chain is aggregation='pooled') -The test_aggregate_metrics_excludes_nan test is critical: it confirms that -recordings where a behavior has no GT bouts (NaN metrics) do NOT pull down -the group/overall averages. Using nanmean() is the correct design choice. +The test_aggregate_metrics_excludes_nan test confirms that NaN cells (a metric +undefined for a behavior in a recording, e.g. precision when nothing is +predicted there) are left out of the group/overall averages (nanmean). +A recording with no GT bout of a behavior is not NaN with the default +per-file aggregation: it scores 1.0 if nothing is predicted and 0.0 otherwise. """ import math +import tomllib import warnings from pathlib import Path @@ -101,8 +104,10 @@ def test_score_all_metrics_in_range(): """All finite metric values must lie in [0, 1]; NaN is allowed (undefined case). WHY: values outside [0, 1] indicate a formula error (e.g., division producing - values > 1). NaN is explicitly allowed because behaviors with no GT bouts - return NaN by design — that is the correct signal for 'not applicable'. + values > 1). NaN is explicitly allowed because a metric can be undefined for + a behavior that has GT bouts: precision when nothing is predicted, SO and TP + when no predicted bout overlaps a GT bout, IC when no GT bout longer than + one frame is found. """ result = banos.score(PRED, GT) for behavior, metrics in result.items(): @@ -119,8 +124,8 @@ def test_score_invalid_matching_raises(): def test_aggregate_metrics_excludes_nan(): """NaN values in file metrics must be excluded from group and overall averages. - WHY: f2 has all-NaN metrics (behavior had no GT bouts in that recording). - If nanmean() were replaced by plain mean(), the average would be pulled toward + WHY: f2 has all-NaN metrics (every metric undefined for that behavior in that + recording). If NaN were averaged as 0, the average would be pulled toward 0, making the system appear worse than it actually is. The result must equal only f1's values, as if f2 were never processed for that behavior. """ @@ -153,8 +158,15 @@ def test_aggregate_metrics_excludes_nan(): def test_score_version(): - assert isinstance(banos.__version__, str) - assert banos.__version__ != "" + """banos.__version__ is the version declared in pyproject.toml. + + WHY: the version is written in both files; a release that bumps only one + of them would publish a package whose __version__ is wrong. + """ + pyproject = Path(__file__).parent.parent / "pyproject.toml" + with open(pyproject, "rb") as f: + declared = tomllib.load(f)["project"]["version"] + assert banos.__version__ == declared FRAME_INDEXES = { diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py index ab8a58f..55582ad 100644 --- a/tests/test_preprocessing.py +++ b/tests/test_preprocessing.py @@ -7,7 +7,7 @@ If non-binary columns are silently kept, or headers are not aligned, every downstream metric will be computed on wrong data with no error raised. These tests verify that data cleaning is both correct and non-destructive -(the caller's dict object must not be replaced or mutated unexpectedly). +(the caller's dict and DataFrames are left unchanged). Key behaviours tested: - Binary detection: 0/1 int and float are accepted; 2, -1, or NaN are not. @@ -111,19 +111,32 @@ def test_preprocess_data_fills_nan(): assert gt_out["a"].isna().sum() == 0 -def test_preprocess_data_does_not_mutate_caller_dict(): - """The caller's dict object must not be replaced — only internal copies are modified. +@pytest.mark.parametrize("dirty_side", ["prediction", "ground truth"]) +def test_preprocess_data_does_not_mutate_caller_dict(dirty_side): + """The caller's dict and DataFrames are left unchanged — only copies are cleaned. WHY: user code often reuses the same dict across multiple calls. If preprocess_data - replaced the dict in-place, the second call would receive already-processed data, - causing subtle double-processing bugs. + modified the dict or its DataFrames in place, the second call would receive + already-processed data, causing subtle double-processing bugs. + One side needs every cleaning step (a NaN filled, a non-binary column dropped, + a column missing on the other side dropped) and the other side needs none. The + test runs with each side as the one to clean, so that a step applied in place to + the caller's prediction or to the caller's ground truth would show here. """ - pred = pd.DataFrame({"a": [0, 1]}) - gt = pd.DataFrame({"a": [1, 0]}) + dirty = pd.DataFrame( + {"a": [0.0, float("nan"), 1.0], "prob": [0.2, 0.9, 0.1], "one_side": [0, 1, 1]} + ) + clean = pd.DataFrame({"a": [1, 0, 0], "prob": [0, 1, 0]}) + pred, gt = (dirty, clean) if dirty_side == "prediction" else (clean, dirty) + expected_pred, expected_gt = pred.copy(), gt.copy() original = {"f1": (pred, gt)} - original_id = id(original) - preprocess_data(original) - assert id(original) == original_id # same dict object, not mutated internally + with pytest.warns(UserWarning): + result, _ = preprocess_data(original) + assert result is not original + assert list(original) == ["f1"] + assert original["f1"][0] is pred and original["f1"][1] is gt + pd.testing.assert_frame_equal(pred, expected_pred) + pd.testing.assert_frame_equal(gt, expected_gt) def test_preprocess_data_drops_non_logical(): diff --git a/uv.lock b/uv.lock index dd86c66..cd3d466 100644 --- a/uv.lock +++ b/uv.lock @@ -174,7 +174,6 @@ dev = [ { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, - { name = "pytest-html" }, { name = "ruff" }, ] notebook = [ @@ -188,7 +187,6 @@ dev = [ { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, - { name = "pytest-html" }, { name = "ruff" }, ] @@ -202,7 +200,6 @@ requires-dist = [ { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.4" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.1" }, - { name = "pytest-html", marker = "extra == 'dev'", specifier = ">=4.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4" }, { name = "scikit-learn", marker = "extra == 'calms21'", specifier = ">=1.3" }, { name = "scipy", specifier = ">=1.10" }, @@ -218,7 +215,6 @@ dev = [ { name = "pre-commit", specifier = ">=3.5" }, { name = "pytest", specifier = ">=7.4" }, { name = "pytest-cov", specifier = ">=4.1" }, - { name = "pytest-html", specifier = ">=4.2.0" }, { name = "ruff", specifier = ">=0.4" }, ] @@ -2404,32 +2400,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, ] -[[package]] -name = "pytest-html" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "jinja2" }, - { name = "pytest" }, - { name = "pytest-metadata" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c4/08/2076aa09507e51c1119d16a84c6307354d16270558f1a44fc9a2c99fdf1d/pytest_html-4.2.0.tar.gz", hash = "sha256:b6a88cba507500d8709959201e2e757d3941e859fd17cfd4ed87b16fc0c67912", size = 108634, upload-time = "2026-01-19T11:25:26.471Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/47/07046e0acedc12fe2bae79cf6c73ad67f51ae9d67df64d06b0f3eac73d36/pytest_html-4.2.0-py3-none-any.whl", hash = "sha256:ff5caf3e17a974008e5816edda61168e6c3da442b078a44f8744865862a85636", size = 23801, upload-time = "2026-01-19T11:25:25.008Z" }, -] - -[[package]] -name = "pytest-metadata" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a6/85/8c969f8bec4e559f8f2b958a15229a35495f5b4ce499f6b865eac54b878d/pytest_metadata-3.1.1.tar.gz", hash = "sha256:d2a29b0355fbc03f168aa96d41ff88b1a3b44a3b02acbe491801c98a048017c8", size = 9952, upload-time = "2024-02-12T19:38:44.887Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3e/43/7e7b2ec865caa92f67b8f0e9231a798d102724ca4c0e1f414316be1c1ef2/pytest_metadata-3.1.1-py3-none-any.whl", hash = "sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b", size = 11428, upload-time = "2024-02-12T19:38:42.531Z" }, -] - [[package]] name = "python-dateutil" version = "2.9.0.post0" From 5958086c684876e53f067b65faf93c990e5dcb6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:09:29 +0100 Subject: [PATCH 09/25] test: correct the tags and legends of the fixture comparison compare_fixtures.py tagged a value that becomes NaN as NULL->VALUE and the reverse as VALUE->NULL. The v0.1.5 / v0.2.0 legend then explained the four Recording_8 mount cells that go from 0 to NaN (precision, F1, SO, TP) as correct absence now scoring 1, and every other difference as the IC fix. Swap the tags and give each comparison a meaning for each tag. From v0.1.5 to v0.2.0: a metric with a zero denominator is NaN instead of 0; correct absence scores 1.0 instead of 0 on all six metrics; IC counts the last frame pair of a bout and leaves one-frame bouts out; the overall means move with these. Add mode 3, Python v0.2.0 against v1.1.0: one F1 from NaN to 0, one IC from 1.0 to NaN, 39 other values changed (IC and overall means), 259 equal. Only the Recording_N entries are compared; the all_recordings entry of the v1.1.0 file has no counterpart in older files. Legends use ASCII arrows, which a Windows console can print. Regenerate reports/fixture_comparison.html with the corrected script. The report now ends with a newline, as the committed file did before; the script never wrote one. tests/test_compare_fixtures.py runs the script: mode 3 must tag exactly the 41 values changed by v1.1.0, and the committed report must equal what the script writes (skipped where reports/ is absent, as in the sdist). The output depends only on the fixture files, so a fixture edited without regenerating the report, or a report edited by hand, fails the test. --- reports/fixture_comparison.html | 28 ++-- tests/fixtures/compare_fixtures.py | 200 +++++++++++++++++++---------- tests/test_compare_fixtures.py | 98 ++++++++++++++ 3 files changed, 251 insertions(+), 75 deletions(-) create mode 100644 tests/test_compare_fixtures.py diff --git a/reports/fixture_comparison.html b/reports/fixture_comparison.html index 9f6dcef..5749f35 100644 --- a/reports/fixture_comparison.html +++ b/reports/fixture_comparison.html @@ -3,18 +3,28 @@

Mode 1: Python v0.2.0 vs Matlab v0.2.0

Shows Python/Matlab parity -- should be SAME

Total: 300  |  SAME: 300  |  +VALUE->NULL: 0  |  NULL->VALUE: 0  |  -CHANGED: 0  |  -VALUE->NULL: 0

-

SAME = match   NULL->VALUE = Unexpected divergence (should not occur in v0.2.0)   CHANGED = Unexpected numeric mismatch   VALUE->NULL = unexpected

+CHANGED: 0

+

SAME = match   VALUE->NULL = Unexpected divergence (should not occur in v0.2.0)   NULL->VALUE = Unexpected divergence (should not occur in v0.2.0)   CHANGED = Unexpected numeric mismatch

LocationMetricPython v0.2.0Matlab v0.2.0Status
Recording_1/overallprecision0.9282610.928261SAME
Recording_1/overallrecall0.7812930.781293SAME
Recording_1/overallf1_score0.8364550.836455SAME
Recording_1/overallso0.7612280.761228SAME
Recording_1/overalltp0.3062300.306230SAME
Recording_1/overallic0.9917180.991718SAME
Recording_1/group/attackprecision1.0000001.000000SAME
Recording_1/group/attackrecall1.0000001.000000SAME
Recording_1/group/attackf1_score1.0000001.000000SAME
Recording_1/group/attackso1.0000001.000000SAME
Recording_1/group/attacktp1.0000001.000000SAME
Recording_1/group/attackic1.0000001.000000SAME
Recording_1/group/investigationprecision0.8000000.800000SAME
Recording_1/group/investigationrecall0.7346940.734694SAME
Recording_1/group/investigationf1_score0.7659570.765957SAME
Recording_1/group/investigationso0.5444100.544410SAME
Recording_1/group/investigationtp0.0846360.084636SAME
Recording_1/group/investigationic0.9829410.982941SAME
Recording_1/group/mountprecision1.0000001.000000SAME
Recording_1/group/mountrecall0.5333330.533333SAME
Recording_1/group/mountf1_score0.6956520.695652SAME
Recording_1/group/mountso0.7332770.733277SAME
Recording_1/group/mounttp0.0784710.078471SAME
Recording_1/group/mountic0.9908610.990861SAME
Recording_1/group/otherprecision0.9130430.913043SAME
Recording_1/group/otherrecall0.8571430.857143SAME
Recording_1/group/otherf1_score0.8842110.884211SAME
Recording_1/group/otherso0.7672270.767227SAME
Recording_1/group/othertp0.0618140.061814SAME
Recording_1/group/otheric0.9930690.993069SAME
Recording_2/overallprecision0.9671050.967105SAME
Recording_2/overallrecall0.8399240.839924SAME
Recording_2/overallf1_score0.8973010.897301SAME
Recording_2/overallso0.8259460.825946SAME
Recording_2/overalltp0.3090820.309082SAME
Recording_2/overallic0.9939070.993907SAME
Recording_2/group/attackprecision1.0000001.000000SAME
Recording_2/group/attackrecall1.0000001.000000SAME
Recording_2/group/attackf1_score1.0000001.000000SAME
Recording_2/group/attackso1.0000001.000000SAME
Recording_2/group/attacktp1.0000001.000000SAME
Recording_2/group/attackic1.0000001.000000SAME
Recording_2/group/investigationprecision0.8947370.894737SAME
Recording_2/group/investigationrecall0.7391300.739130SAME
Recording_2/group/investigationf1_score0.8095240.809524SAME
Recording_2/group/investigationso0.6609420.660942SAME
Recording_2/group/investigationtp0.0461510.046151SAME
Recording_2/group/investigationic0.9890510.989051SAME
Recording_2/group/mountprecision1.0000001.000000SAME
Recording_2/group/mountrecall0.8333330.833333SAME
Recording_2/group/mountf1_score0.9090910.909091SAME
Recording_2/group/mountso0.9317540.931754SAME
Recording_2/group/mounttp0.1451700.145170SAME
Recording_2/group/mountic0.9946480.994648SAME
Recording_2/group/otherprecision0.9736840.973684SAME
Recording_2/group/otherrecall0.7872340.787234SAME
Recording_2/group/otherf1_score0.8705880.870588SAME
Recording_2/group/otherso0.7110860.711086SAME
Recording_2/group/othertp0.0450080.045008SAME
Recording_2/group/otheric0.9919300.991930SAME
Recording_3/overallprecision0.9594810.959481SAME
Recording_3/overallrecall0.8509670.850967SAME
Recording_3/overallf1_score0.8986270.898627SAME
Recording_3/overallso0.7237160.723716SAME
Recording_3/overalltp0.2688760.268876SAME
Recording_3/overallic0.9932230.993223SAME
Recording_3/group/attackprecision1.0000001.000000SAME
Recording_3/group/attackrecall1.0000001.000000SAME
Recording_3/group/attackf1_score1.0000001.000000SAME
Recording_3/group/attackso1.0000001.000000SAME
Recording_3/group/attacktp1.0000001.000000SAME
Recording_3/group/attackic1.0000001.000000SAME
Recording_3/group/investigationprecision0.9354840.935484SAME
Recording_3/group/investigationrecall0.7837840.783784SAME
Recording_3/group/investigationf1_score0.8529410.852941SAME
Recording_3/group/investigationso0.4563100.456310SAME
Recording_3/group/investigationtp0.0223220.022322SAME
Recording_3/group/investigationic0.9849660.984966SAME
Recording_3/group/mountprecision1.0000001.000000SAME
Recording_3/group/mountrecall0.7391300.739130SAME
Recording_3/group/mountf1_score0.8500000.850000SAME
Recording_3/group/mountso0.6745040.674504SAME
Recording_3/group/mounttp0.0287950.028795SAME
Recording_3/group/mountic0.9917130.991713SAME
Recording_3/group/otherprecision0.9024390.902439SAME
Recording_3/group/otherrecall0.8809520.880952SAME
Recording_3/group/otherf1_score0.8915660.891566SAME
Recording_3/group/otherso0.7640480.764048SAME
Recording_3/group/othertp0.0243860.024386SAME
Recording_3/group/otheric0.9962120.996212SAME
Recording_4/overallprecision0.9750000.975000SAME
Recording_4/overallrecall0.6156860.615686SAME
Recording_4/overallf1_score0.7291670.729167SAME
Recording_4/overallso0.7070870.707087SAME
Recording_4/overalltp0.2757150.275715SAME
Recording_4/overallic0.9965300.996530SAME
Recording_4/group/attackprecision1.0000001.000000SAME
Recording_4/group/attackrecall1.0000001.000000SAME
Recording_4/group/attackf1_score1.0000001.000000SAME
Recording_4/group/attackso1.0000001.000000SAME
Recording_4/group/attacktp1.0000001.000000SAME
Recording_4/group/attackic1.0000001.000000SAME
Recording_4/group/investigationprecision0.9000000.900000SAME
Recording_4/group/investigationrecall0.5294120.529412SAME
Recording_4/group/investigationf1_score0.6666670.666667SAME
Recording_4/group/investigationso0.3914710.391471SAME
Recording_4/group/investigationtp0.0140250.014025SAME
Recording_4/group/investigationic0.9968820.996882SAME
Recording_4/group/mountprecision1.0000001.000000SAME
Recording_4/group/mountrecall0.3333330.333333SAME
Recording_4/group/mountf1_score0.5000000.500000SAME
Recording_4/group/mountso0.7011190.701119SAME
Recording_4/group/mounttp0.0400870.040087SAME
Recording_4/group/mountic0.9952890.995289SAME
Recording_4/group/otherprecision1.0000001.000000SAME
Recording_4/group/otherrecall0.6000000.600000SAME
Recording_4/group/otherf1_score0.7500000.750000SAME
Recording_4/group/otherso0.7357600.735760SAME
Recording_4/group/othertp0.0487490.048749SAME
Recording_4/group/otheric0.9939490.993949SAME
Recording_5/overallprecision1.0000001.000000SAME
Recording_5/overallrecall0.8921050.892105SAME
Recording_5/overallf1_score0.9414590.941459SAME
Recording_5/overallso0.8367300.836730SAME
Recording_5/overalltp0.2792890.279289SAME
Recording_5/overallic0.9951200.995120SAME
Recording_5/group/attackprecision1.0000001.000000SAME
Recording_5/group/attackrecall1.0000001.000000SAME
Recording_5/group/attackf1_score1.0000001.000000SAME
Recording_5/group/attackso1.0000001.000000SAME
Recording_5/group/attacktp1.0000001.000000SAME
Recording_5/group/attackic1.0000001.000000SAME
Recording_5/group/investigationprecision1.0000001.000000SAME
Recording_5/group/investigationrecall0.8000000.800000SAME
Recording_5/group/investigationf1_score0.8888890.888889SAME
Recording_5/group/investigationso0.6589590.658959SAME
Recording_5/group/investigationtp0.0353330.035333SAME
Recording_5/group/investigationic0.9899140.989914SAME
Recording_5/group/mountprecision1.0000001.000000SAME
Recording_5/group/mountrecall0.9000000.900000SAME
Recording_5/group/mountf1_score0.9473680.947368SAME
Recording_5/group/mountso0.9100240.910024SAME
Recording_5/group/mounttp0.0359600.035960SAME
Recording_5/group/mountic0.9965410.996541SAME
Recording_5/group/otherprecision1.0000001.000000SAME
Recording_5/group/otherrecall0.8684210.868421SAME
Recording_5/group/otherf1_score0.9295770.929577SAME
Recording_5/group/otherso0.7779390.777939SAME
Recording_5/group/othertp0.0458620.045862SAME
Recording_5/group/otheric0.9940240.994024SAME
Recording_6/overallprecision0.9140830.914083SAME
Recording_6/overallrecall0.7456710.745671SAME
Recording_6/overallf1_score0.7925050.792505SAME
Recording_6/overallso0.6244640.624464SAME
Recording_6/overalltp0.2640680.264068SAME
Recording_6/overallic0.9939870.993987SAME
Recording_6/group/attackprecision1.0000001.000000SAME
Recording_6/group/attackrecall0.3636360.363636SAME
Recording_6/group/attackf1_score0.5333330.533333SAME
Recording_6/group/attackso0.3554880.355488SAME
Recording_6/group/attacktp0.0141430.014143SAME
Recording_6/group/attackic0.9936660.993666SAME
Recording_6/group/investigationprecision0.7674420.767442SAME
Recording_6/group/investigationrecall0.7857140.785714SAME
Recording_6/group/investigationf1_score0.7764710.776471SAME
Recording_6/group/investigationso0.5395800.539580SAME
Recording_6/group/investigationtp0.0203170.020317SAME
Recording_6/group/investigationic0.9903920.990392SAME
Recording_6/group/mountprecision1.0000001.000000SAME
Recording_6/group/mountrecall1.0000001.000000SAME
Recording_6/group/mountf1_score1.0000001.000000SAME
Recording_6/group/mountso1.0000001.000000SAME
Recording_6/group/mounttp1.0000001.000000SAME
Recording_6/group/mountic1.0000001.000000SAME
Recording_6/group/otherprecision0.8888890.888889SAME
Recording_6/group/otherrecall0.8333330.833333SAME
Recording_6/group/otherf1_score0.8602150.860215SAME
Recording_6/group/otherso0.6027890.602789SAME
Recording_6/group/othertp0.0218110.021811SAME
Recording_6/group/otheric0.9918880.991888SAME
Recording_7/overallprecision1.0000001.000000SAME
Recording_7/overallrecall0.5839100.583910SAME
Recording_7/overallf1_score0.7065300.706530SAME
Recording_7/overallso0.6644950.664495SAME
Recording_7/overalltp0.2765300.276530SAME
Recording_7/overallic0.9949440.994944SAME
Recording_7/group/attackprecision1.0000001.000000SAME
Recording_7/group/attackrecall0.5250000.525000SAME
Recording_7/group/attackf1_score0.6885250.688525SAME
Recording_7/group/attackso0.3906950.390695SAME
Recording_7/group/attacktp0.0302290.030229SAME
Recording_7/group/attackic0.9894710.989471SAME
Recording_7/group/investigationprecision1.0000001.000000SAME
Recording_7/group/investigationrecall0.3000000.300000SAME
Recording_7/group/investigationf1_score0.4615380.461538SAME
Recording_7/group/investigationso0.5364950.536495SAME
Recording_7/group/investigationtp0.0313310.031331SAME
Recording_7/group/investigationic0.9945440.994544SAME
Recording_7/group/mountprecision1.0000001.000000SAME
Recording_7/group/mountrecall1.0000001.000000SAME
Recording_7/group/mountf1_score1.0000001.000000SAME
Recording_7/group/mountso1.0000001.000000SAME
Recording_7/group/mounttp1.0000001.000000SAME
Recording_7/group/mountic1.0000001.000000SAME
Recording_7/group/otherprecision1.0000001.000000SAME
Recording_7/group/otherrecall0.5106380.510638SAME
Recording_7/group/otherf1_score0.6760560.676056SAME
Recording_7/group/otherso0.7307900.730790SAME
Recording_7/group/othertp0.0445610.044561SAME
Recording_7/group/otheric0.9957600.995760SAME
Recording_8/overallprecision0.8916670.891667SAME
Recording_8/overallrecall0.3467210.346721SAME
Recording_8/overallf1_score0.6061730.606173SAME
Recording_8/overallso0.5305620.530562SAME
Recording_8/overalltp0.0381380.038138SAME
Recording_8/overallic0.9955770.995577SAME
Recording_8/group/attackprecision0.8750000.875000SAME
Recording_8/group/attackrecall0.3684210.368421SAME
Recording_8/group/attackf1_score0.5185190.518519SAME
Recording_8/group/attackso0.2683640.268364SAME
Recording_8/group/attacktp0.0204230.020423SAME
Recording_8/group/attackic0.9919190.991919SAME
Recording_8/group/investigationprecision1.0000001.000000SAME
Recording_8/group/investigationrecall0.5384620.538462SAME
Recording_8/group/investigationf1_score0.7000000.700000SAME
Recording_8/group/investigationso0.5910510.591051SAME
Recording_8/group/investigationtp0.0252710.025271SAME
Recording_8/group/investigationic0.9952820.995282SAME
Recording_8/group/mountprecisionNaNNaNSAME
Recording_8/group/mountrecall0.0000000.000000SAME
Recording_8/group/mountf1_scoreNaNNaNSAME
Recording_8/group/mountsoNaNNaNSAME
Recording_8/group/mounttpNaNNaNSAME
Recording_8/group/mountic1.0000001.000000SAME
Recording_8/group/otherprecision0.8000000.800000SAME
Recording_8/group/otherrecall0.4800000.480000SAME
Recording_8/group/otherf1_score0.6000000.600000SAME
Recording_8/group/otherso0.7322710.732271SAME
Recording_8/group/othertp0.0687210.068721SAME
Recording_8/group/otheric0.9951050.995105SAME
Recording_9/overallprecision0.7983980.798398SAME
Recording_9/overallrecall0.6538190.653819SAME
Recording_9/overallf1_score0.7123550.712355SAME
Recording_9/overallso0.6827300.682730SAME
Recording_9/overalltp0.2778090.277809SAME
Recording_9/overallic0.9956710.995671SAME
Recording_9/group/attackprecision0.9565220.956522SAME
Recording_9/group/attackrecall0.7333330.733333SAME
Recording_9/group/attackf1_score0.8301890.830189SAME
Recording_9/group/attackso0.4241000.424100SAME
Recording_9/group/attacktp0.0205900.020590SAME
Recording_9/group/attackic0.9893860.989386SAME
Recording_9/group/investigationprecision0.3750000.375000SAME
Recording_9/group/investigationrecall0.1875000.187500SAME
Recording_9/group/investigationf1_score0.2500000.250000SAME
Recording_9/group/investigationso0.6316780.631678SAME
Recording_9/group/investigationtp0.0348100.034810SAME
Recording_9/group/investigationic0.9986300.998630SAME
Recording_9/group/mountprecision1.0000001.000000SAME
Recording_9/group/mountrecall1.0000001.000000SAME
Recording_9/group/mountf1_score1.0000001.000000SAME
Recording_9/group/mountso1.0000001.000000SAME
Recording_9/group/mounttp1.0000001.000000SAME
Recording_9/group/mountic1.0000001.000000SAME
Recording_9/group/otherprecision0.8620690.862069SAME
Recording_9/group/otherrecall0.6944440.694444SAME
Recording_9/group/otherf1_score0.7692310.769231SAME
Recording_9/group/otherso0.6751400.675140SAME
Recording_9/group/othertp0.0558380.055838SAME
Recording_9/group/otheric0.9946690.994669SAME
Recording_10/overallprecision0.8991070.899107SAME
Recording_10/overallrecall0.4757600.475760SAME
Recording_10/overallf1_score0.5834750.583475SAME
Recording_10/overallso0.6668060.666806SAME
Recording_10/overalltp0.2859230.285923SAME
Recording_10/overallic0.9961910.996191SAME
Recording_10/group/attackprecision1.0000001.000000SAME
Recording_10/group/attackrecall0.2857140.285714SAME
Recording_10/group/attackf1_score0.4444440.444444SAME
Recording_10/group/attackso0.1849030.184903SAME
Recording_10/group/attacktp0.0111890.011189SAME
Recording_10/group/attackic0.9914120.991412SAME
Recording_10/group/investigationprecision0.6250000.625000SAME
Recording_10/group/investigationrecall0.2631580.263158SAME
Recording_10/group/investigationf1_score0.3703700.370370SAME
Recording_10/group/investigationso0.6572070.657207SAME
Recording_10/group/investigationtp0.0646830.064683SAME
Recording_10/group/investigationic0.9950610.995061SAME
Recording_10/group/mountprecision1.0000001.000000SAME
Recording_10/group/mountrecall1.0000001.000000SAME
Recording_10/group/mountf1_score1.0000001.000000SAME
Recording_10/group/mountso1.0000001.000000SAME
Recording_10/group/mounttp1.0000001.000000SAME
Recording_10/group/mountic1.0000001.000000SAME
Recording_10/group/otherprecision0.9714290.971429SAME
Recording_10/group/otherrecall0.3541670.354167SAME
Recording_10/group/otherf1_score0.5190840.519084SAME
Recording_10/group/otherso0.8251150.825115SAME
Recording_10/group/othertp0.0678200.067820SAME
Recording_10/group/otheric0.9982930.998293SAME

Mode 2: Python v0.1.5 vs Python v0.2.0

-

Shows ALL changes: absent-behavior fix (0→1 correct absence) + IC bug fix

+

Shows the changes of v0.2.0: undefined metrics NaN instead of 0, correct absence 1.0 instead of 0, IC fixes

Total: 300  |  SAME: 169  |  -NULL->VALUE: 4  |  -CHANGED: 127  |  -VALUE->NULL: 0

-

SAME = match   NULL->VALUE = v0.1.5=0 → v0.2.0=1.0 (correct absence now scores 1)   CHANGED = IC bug fix   VALUE->NULL = unexpected

-
LocationMetricPython v0.1.5Python v0.2.0Status
Recording_1/overallprecision0.6782610.928261CHANGED
Recording_1/overallrecall0.5312930.781293CHANGED
Recording_1/overallf1_score0.5864550.836455CHANGED
Recording_1/overallso0.5112280.761228CHANGED
Recording_1/overalltp0.0562300.306230CHANGED
Recording_1/overallic0.7379940.991718CHANGED
Recording_1/group/attackprecision0.0000001.000000CHANGED
Recording_1/group/attackrecall0.0000001.000000CHANGED
Recording_1/group/attackf1_score0.0000001.000000CHANGED
Recording_1/group/attackso0.0000001.000000CHANGED
Recording_1/group/attacktp0.0000001.000000CHANGED
Recording_1/group/attackic0.0000001.000000CHANGED
Recording_1/group/investigationprecision0.8000000.800000SAME
Recording_1/group/investigationrecall0.7346940.734694SAME
Recording_1/group/investigationf1_score0.7659570.765957SAME
Recording_1/group/investigationso0.5444100.544410SAME
Recording_1/group/investigationtp0.0846360.084636SAME
Recording_1/group/investigationic0.9876410.982941CHANGED
Recording_1/group/mountprecision1.0000001.000000SAME
Recording_1/group/mountrecall0.5333330.533333SAME
Recording_1/group/mountf1_score0.6956520.695652SAME
Recording_1/group/mountso0.7332770.733277SAME
Recording_1/group/mounttp0.0784710.078471SAME
Recording_1/group/mountic0.9908610.990861SAME
Recording_1/group/otherprecision0.9130430.913043SAME
Recording_1/group/otherrecall0.8571430.857143SAME
Recording_1/group/otherf1_score0.8842110.884211SAME
Recording_1/group/otherso0.7672270.767227SAME
Recording_1/group/othertp0.0618140.061814SAME
Recording_1/group/otheric0.9734730.993069CHANGED
Recording_2/overallprecision0.7171050.967105CHANGED
Recording_2/overallrecall0.5899240.839924CHANGED
Recording_2/overallf1_score0.6473010.897301CHANGED
Recording_2/overallso0.5759460.825946CHANGED
Recording_2/overalltp0.0590820.309082CHANGED
Recording_2/overallic0.7444980.993907CHANGED
Recording_2/group/attackprecision0.0000001.000000CHANGED
Recording_2/group/attackrecall0.0000001.000000CHANGED
Recording_2/group/attackf1_score0.0000001.000000CHANGED
Recording_2/group/attackso0.0000001.000000CHANGED
Recording_2/group/attacktp0.0000001.000000CHANGED
Recording_2/group/attackic0.0000001.000000CHANGED
Recording_2/group/investigationprecision0.8947370.894737SAME
Recording_2/group/investigationrecall0.7391300.739130SAME
Recording_2/group/investigationf1_score0.8095240.809524SAME
Recording_2/group/investigationso0.6609420.660942SAME
Recording_2/group/investigationtp0.0461510.046151SAME
Recording_2/group/investigationic0.9912190.989051CHANGED
Recording_2/group/mountprecision1.0000001.000000SAME
Recording_2/group/mountrecall0.8333330.833333SAME
Recording_2/group/mountf1_score0.9090910.909091SAME
Recording_2/group/mountso0.9317540.931754SAME
Recording_2/group/mounttp0.1451700.145170SAME
Recording_2/group/mountic0.9946480.994648SAME
Recording_2/group/otherprecision0.9736840.973684SAME
Recording_2/group/otherrecall0.7872340.787234SAME
Recording_2/group/otherf1_score0.8705880.870588SAME
Recording_2/group/otherso0.7110860.711086SAME
Recording_2/group/othertp0.0450080.045008SAME
Recording_2/group/otheric0.9921250.991930CHANGED
Recording_3/overallprecision0.7094810.959481CHANGED
Recording_3/overallrecall0.6009670.850967CHANGED
Recording_3/overallf1_score0.6486270.898627CHANGED
Recording_3/overallso0.4737160.723716CHANGED
Recording_3/overalltp0.0188760.268876CHANGED
Recording_3/overallic0.7372930.993223CHANGED
Recording_3/group/attackprecision0.0000001.000000CHANGED
Recording_3/group/attackrecall0.0000001.000000CHANGED
Recording_3/group/attackf1_score0.0000001.000000CHANGED
Recording_3/group/attackso0.0000001.000000CHANGED
Recording_3/group/attacktp0.0000001.000000CHANGED
Recording_3/group/attackic0.0000001.000000CHANGED
Recording_3/group/investigationprecision0.9354840.935484SAME
Recording_3/group/investigationrecall0.7837840.783784SAME
Recording_3/group/investigationf1_score0.8529410.852941SAME
Recording_3/group/investigationso0.4563100.456310SAME
Recording_3/group/investigationtp0.0223220.022322SAME
Recording_3/group/investigationic0.9849660.984966SAME
Recording_3/group/mountprecision1.0000001.000000SAME
Recording_3/group/mountrecall0.7391300.739130SAME
Recording_3/group/mountf1_score0.8500000.850000SAME
Recording_3/group/mountso0.6745040.674504SAME
Recording_3/group/mounttp0.0287950.028795SAME
Recording_3/group/mountic0.9917130.991713SAME
Recording_3/group/otherprecision0.9024390.902439SAME
Recording_3/group/otherrecall0.8809520.880952SAME
Recording_3/group/otherf1_score0.8915660.891566SAME
Recording_3/group/otherso0.7640480.764048SAME
Recording_3/group/othertp0.0243860.024386SAME
Recording_3/group/otheric0.9724920.996212CHANGED
Recording_4/overallprecision0.7250000.975000CHANGED
Recording_4/overallrecall0.3656860.615686CHANGED
Recording_4/overallf1_score0.4791670.729167CHANGED
Recording_4/overallso0.4570870.707087CHANGED
Recording_4/overalltp0.0257150.275715CHANGED
Recording_4/overallic0.7365910.996530CHANGED
Recording_4/group/attackprecision0.0000001.000000CHANGED
Recording_4/group/attackrecall0.0000001.000000CHANGED
Recording_4/group/attackf1_score0.0000001.000000CHANGED
Recording_4/group/attackso0.0000001.000000CHANGED
Recording_4/group/attacktp0.0000001.000000CHANGED
Recording_4/group/attackic0.0000001.000000CHANGED
Recording_4/group/investigationprecision0.9000000.900000SAME
Recording_4/group/investigationrecall0.5294120.529412SAME
Recording_4/group/investigationf1_score0.6666670.666667SAME
Recording_4/group/investigationso0.3914710.391471SAME
Recording_4/group/investigationtp0.0140250.014025SAME
Recording_4/group/investigationic0.9968820.996882SAME
Recording_4/group/mountprecision1.0000001.000000SAME
Recording_4/group/mountrecall0.3333330.333333SAME
Recording_4/group/mountf1_score0.5000000.500000SAME
Recording_4/group/mountso0.7011190.701119SAME
Recording_4/group/mounttp0.0400870.040087SAME
Recording_4/group/mountic0.9952890.995289SAME
Recording_4/group/otherprecision1.0000001.000000SAME
Recording_4/group/otherrecall0.6000000.600000SAME
Recording_4/group/otherf1_score0.7500000.750000SAME
Recording_4/group/otherso0.7357600.735760SAME
Recording_4/group/othertp0.0487490.048749SAME
Recording_4/group/otheric0.9541910.993949CHANGED
Recording_5/overallprecision0.7500001.000000CHANGED
Recording_5/overallrecall0.6421050.892105CHANGED
Recording_5/overallf1_score0.6914590.941459CHANGED
Recording_5/overallso0.5867300.836730CHANGED
Recording_5/overalltp0.0292890.279289CHANGED
Recording_5/overallic0.7321650.995120CHANGED
Recording_5/group/attackprecision0.0000001.000000CHANGED
Recording_5/group/attackrecall0.0000001.000000CHANGED
Recording_5/group/attackf1_score0.0000001.000000CHANGED
Recording_5/group/attackso0.0000001.000000CHANGED
Recording_5/group/attacktp0.0000001.000000CHANGED
Recording_5/group/attackic0.0000001.000000CHANGED
Recording_5/group/investigationprecision1.0000001.000000SAME
Recording_5/group/investigationrecall0.8000000.800000SAME
Recording_5/group/investigationf1_score0.8888890.888889SAME
Recording_5/group/investigationso0.6589590.658959SAME
Recording_5/group/investigationtp0.0353330.035333SAME
Recording_5/group/investigationic0.9902050.989914CHANGED
Recording_5/group/mountprecision1.0000001.000000SAME
Recording_5/group/mountrecall0.9000000.900000SAME
Recording_5/group/mountf1_score0.9473680.947368SAME
Recording_5/group/mountso0.9100240.910024SAME
Recording_5/group/mounttp0.0359600.035960SAME
Recording_5/group/mountic0.9965410.996541SAME
Recording_5/group/otherprecision1.0000001.000000SAME
Recording_5/group/otherrecall0.8684210.868421SAME
Recording_5/group/otherf1_score0.9295770.929577SAME
Recording_5/group/otherso0.7779390.777939SAME
Recording_5/group/othertp0.0458620.045862SAME
Recording_5/group/otheric0.9419150.994024CHANGED
Recording_6/overallprecision0.6640830.914083CHANGED
Recording_6/overallrecall0.4956710.745671CHANGED
Recording_6/overallf1_score0.5425050.792505CHANGED
Recording_6/overallso0.3744640.624464CHANGED
Recording_6/overalltp0.0140680.264068CHANGED
Recording_6/overallic0.7388210.993987CHANGED
Recording_6/group/attackprecision1.0000001.000000SAME
Recording_6/group/attackrecall0.3636360.363636SAME
Recording_6/group/attackf1_score0.5333330.533333SAME
Recording_6/group/attackso0.3554880.355488SAME
Recording_6/group/attacktp0.0141430.014143SAME
Recording_6/group/attackic0.9936660.993666SAME
Recording_6/group/investigationprecision0.7674420.767442SAME
Recording_6/group/investigationrecall0.7857140.785714SAME
Recording_6/group/investigationf1_score0.7764710.776471SAME
Recording_6/group/investigationso0.5395800.539580SAME
Recording_6/group/investigationtp0.0203170.020317SAME
Recording_6/group/investigationic0.9903920.990392SAME
Recording_6/group/mountprecision0.0000001.000000CHANGED
Recording_6/group/mountrecall0.0000001.000000CHANGED
Recording_6/group/mountf1_score0.0000001.000000CHANGED
Recording_6/group/mountso0.0000001.000000CHANGED
Recording_6/group/mounttp0.0000001.000000CHANGED
Recording_6/group/mountic0.0000001.000000CHANGED
Recording_6/group/otherprecision0.8888890.888889SAME
Recording_6/group/otherrecall0.8333330.833333SAME
Recording_6/group/otherf1_score0.8602150.860215SAME
Recording_6/group/otherso0.6027890.602789SAME
Recording_6/group/othertp0.0218110.021811SAME
Recording_6/group/otheric0.9712240.991888CHANGED
Recording_7/overallprecision0.7500001.000000CHANGED
Recording_7/overallrecall0.3339100.583910CHANGED
Recording_7/overallf1_score0.4565300.706530CHANGED
Recording_7/overallso0.4144950.664495CHANGED
Recording_7/overalltp0.0265300.276530CHANGED
Recording_7/overallic0.7344940.994944CHANGED
Recording_7/group/attackprecision1.0000001.000000SAME
Recording_7/group/attackrecall0.5250000.525000SAME
Recording_7/group/attackf1_score0.6885250.688525SAME
Recording_7/group/attackso0.3906950.390695SAME
Recording_7/group/attacktp0.0302290.030229SAME
Recording_7/group/attackic0.9894710.989471SAME
Recording_7/group/investigationprecision1.0000001.000000SAME
Recording_7/group/investigationrecall0.3000000.300000SAME
Recording_7/group/investigationf1_score0.4615380.461538SAME
Recording_7/group/investigationso0.5364950.536495SAME
Recording_7/group/investigationtp0.0313310.031331SAME
Recording_7/group/investigationic0.9945440.994544SAME
Recording_7/group/mountprecision0.0000001.000000CHANGED
Recording_7/group/mountrecall0.0000001.000000CHANGED
Recording_7/group/mountf1_score0.0000001.000000CHANGED
Recording_7/group/mountso0.0000001.000000CHANGED
Recording_7/group/mounttp0.0000001.000000CHANGED
Recording_7/group/mountic0.0000001.000000CHANGED
Recording_7/group/otherprecision1.0000001.000000SAME
Recording_7/group/otherrecall0.5106380.510638SAME
Recording_7/group/otherf1_score0.6760560.676056SAME
Recording_7/group/otherso0.7307900.730790SAME
Recording_7/group/othertp0.0445610.044561SAME
Recording_7/group/otheric0.9539600.995760CHANGED
Recording_8/overallprecision0.6687500.891667CHANGED
Recording_8/overallrecall0.3467210.346721SAME
Recording_8/overallf1_score0.4546300.606173CHANGED
Recording_8/overallso0.3979210.530562CHANGED
Recording_8/overalltp0.0286040.038138CHANGED
Recording_8/overallic0.9659430.995577CHANGED
Recording_8/group/attackprecision0.8750000.875000SAME
Recording_8/group/attackrecall0.3684210.368421SAME
Recording_8/group/attackf1_score0.5185190.518519SAME
Recording_8/group/attackso0.2683640.268364SAME
Recording_8/group/attacktp0.0204230.020423SAME
Recording_8/group/attackic0.9927960.991919CHANGED
Recording_8/group/investigationprecision1.0000001.000000SAME
Recording_8/group/investigationrecall0.5384620.538462SAME
Recording_8/group/investigationf1_score0.7000000.700000SAME
Recording_8/group/investigationso0.5910510.591051SAME
Recording_8/group/investigationtp0.0252710.025271SAME
Recording_8/group/investigationic0.9952820.995282SAME
Recording_8/group/mountprecision0.000000NaNNULL->VALUE
Recording_8/group/mountrecall0.0000000.000000SAME
Recording_8/group/mountf1_score0.000000NaNNULL->VALUE
Recording_8/group/mountso0.000000NaNNULL->VALUE
Recording_8/group/mounttp0.000000NaNNULL->VALUE
Recording_8/group/mountic1.0000001.000000SAME
Recording_8/group/otherprecision0.8000000.800000SAME
Recording_8/group/otherrecall0.4800000.480000SAME
Recording_8/group/otherf1_score0.6000000.600000SAME
Recording_8/group/otherso0.7322710.732271SAME
Recording_8/group/othertp0.0687210.068721SAME
Recording_8/group/otheric0.8756930.995105CHANGED
Recording_9/overallprecision0.5483980.798398CHANGED
Recording_9/overallrecall0.4038190.653819CHANGED
Recording_9/overallf1_score0.4623550.712355CHANGED
Recording_9/overallso0.4327300.682730CHANGED
Recording_9/overalltp0.0278090.277809CHANGED
Recording_9/overallic0.7388250.995671CHANGED
Recording_9/group/attackprecision0.9565220.956522SAME
Recording_9/group/attackrecall0.7333330.733333SAME
Recording_9/group/attackf1_score0.8301890.830189SAME
Recording_9/group/attackso0.4241000.424100SAME
Recording_9/group/attacktp0.0205900.020590SAME
Recording_9/group/attackic0.9893860.989386SAME
Recording_9/group/investigationprecision0.3750000.375000SAME
Recording_9/group/investigationrecall0.1875000.187500SAME
Recording_9/group/investigationf1_score0.2500000.250000SAME
Recording_9/group/investigationso0.6316780.631678SAME
Recording_9/group/investigationtp0.0348100.034810SAME
Recording_9/group/investigationic0.9986300.998630SAME
Recording_9/group/mountprecision0.0000001.000000CHANGED
Recording_9/group/mountrecall0.0000001.000000CHANGED
Recording_9/group/mountf1_score0.0000001.000000CHANGED
Recording_9/group/mountso0.0000001.000000CHANGED
Recording_9/group/mounttp0.0000001.000000CHANGED
Recording_9/group/mountic0.0000001.000000CHANGED
Recording_9/group/otherprecision0.8620690.862069SAME
Recording_9/group/otherrecall0.6944440.694444SAME
Recording_9/group/otherf1_score0.7692310.769231SAME
Recording_9/group/otherso0.6751400.675140SAME
Recording_9/group/othertp0.0558380.055838SAME
Recording_9/group/otheric0.9672830.994669CHANGED
Recording_10/overallprecision0.6491070.899107CHANGED
Recording_10/overallrecall0.2257600.475760CHANGED
Recording_10/overallf1_score0.3334750.583475CHANGED
Recording_10/overallso0.4168060.666806CHANGED
Recording_10/overalltp0.0359230.285923CHANGED
Recording_10/overallic0.7410640.996191CHANGED
Recording_10/group/attackprecision1.0000001.000000SAME
Recording_10/group/attackrecall0.2857140.285714SAME
Recording_10/group/attackf1_score0.4444440.444444SAME
Recording_10/group/attackso0.1849030.184903SAME
Recording_10/group/attacktp0.0111890.011189SAME
Recording_10/group/attackic0.9914120.991412SAME
Recording_10/group/investigationprecision0.6250000.625000SAME
Recording_10/group/investigationrecall0.2631580.263158SAME
Recording_10/group/investigationf1_score0.3703700.370370SAME
Recording_10/group/investigationso0.6572070.657207SAME
Recording_10/group/investigationtp0.0646830.064683SAME
Recording_10/group/investigationic0.9950610.995061SAME
Recording_10/group/mountprecision0.0000001.000000CHANGED
Recording_10/group/mountrecall0.0000001.000000CHANGED
Recording_10/group/mountf1_score0.0000001.000000CHANGED
Recording_10/group/mountso0.0000001.000000CHANGED
Recording_10/group/mounttp0.0000001.000000CHANGED
Recording_10/group/mountic0.0000001.000000CHANGED
Recording_10/group/otherprecision0.9714290.971429SAME
Recording_10/group/otherrecall0.3541670.354167SAME
Recording_10/group/otherf1_score0.5190840.519084SAME
Recording_10/group/otherso0.8251150.825115SAME
Recording_10/group/othertp0.0678200.067820SAME
Recording_10/group/otheric0.9777840.998293CHANGED

+VALUE->NULL: 4  |  +NULL->VALUE: 0  |  +CHANGED: 127

+

SAME = match   VALUE->NULL = v0.1.5=0 -> v0.2.0=NaN: a metric with a zero denominator (precision when nothing is predicted; F1, SO and TP when no bout is matched) is NaN instead of 0 and leaves the means   NULL->VALUE = Unexpected (v0.1.5 never gives NaN)   CHANGED = correct absence (no real bout, nothing predicted) scores 1.0 instead of 0 on all six metrics; IC counts the last frame pair of each bout and leaves one-frame real bouts out instead of scoring them 0; the overall means of the recordings move with these changes and with the cells that became NaN

+
LocationMetricPython v0.1.5Python v0.2.0Status
Recording_1/overallprecision0.6782610.928261CHANGED
Recording_1/overallrecall0.5312930.781293CHANGED
Recording_1/overallf1_score0.5864550.836455CHANGED
Recording_1/overallso0.5112280.761228CHANGED
Recording_1/overalltp0.0562300.306230CHANGED
Recording_1/overallic0.7379940.991718CHANGED
Recording_1/group/attackprecision0.0000001.000000CHANGED
Recording_1/group/attackrecall0.0000001.000000CHANGED
Recording_1/group/attackf1_score0.0000001.000000CHANGED
Recording_1/group/attackso0.0000001.000000CHANGED
Recording_1/group/attacktp0.0000001.000000CHANGED
Recording_1/group/attackic0.0000001.000000CHANGED
Recording_1/group/investigationprecision0.8000000.800000SAME
Recording_1/group/investigationrecall0.7346940.734694SAME
Recording_1/group/investigationf1_score0.7659570.765957SAME
Recording_1/group/investigationso0.5444100.544410SAME
Recording_1/group/investigationtp0.0846360.084636SAME
Recording_1/group/investigationic0.9876410.982941CHANGED
Recording_1/group/mountprecision1.0000001.000000SAME
Recording_1/group/mountrecall0.5333330.533333SAME
Recording_1/group/mountf1_score0.6956520.695652SAME
Recording_1/group/mountso0.7332770.733277SAME
Recording_1/group/mounttp0.0784710.078471SAME
Recording_1/group/mountic0.9908610.990861SAME
Recording_1/group/otherprecision0.9130430.913043SAME
Recording_1/group/otherrecall0.8571430.857143SAME
Recording_1/group/otherf1_score0.8842110.884211SAME
Recording_1/group/otherso0.7672270.767227SAME
Recording_1/group/othertp0.0618140.061814SAME
Recording_1/group/otheric0.9734730.993069CHANGED
Recording_2/overallprecision0.7171050.967105CHANGED
Recording_2/overallrecall0.5899240.839924CHANGED
Recording_2/overallf1_score0.6473010.897301CHANGED
Recording_2/overallso0.5759460.825946CHANGED
Recording_2/overalltp0.0590820.309082CHANGED
Recording_2/overallic0.7444980.993907CHANGED
Recording_2/group/attackprecision0.0000001.000000CHANGED
Recording_2/group/attackrecall0.0000001.000000CHANGED
Recording_2/group/attackf1_score0.0000001.000000CHANGED
Recording_2/group/attackso0.0000001.000000CHANGED
Recording_2/group/attacktp0.0000001.000000CHANGED
Recording_2/group/attackic0.0000001.000000CHANGED
Recording_2/group/investigationprecision0.8947370.894737SAME
Recording_2/group/investigationrecall0.7391300.739130SAME
Recording_2/group/investigationf1_score0.8095240.809524SAME
Recording_2/group/investigationso0.6609420.660942SAME
Recording_2/group/investigationtp0.0461510.046151SAME
Recording_2/group/investigationic0.9912190.989051CHANGED
Recording_2/group/mountprecision1.0000001.000000SAME
Recording_2/group/mountrecall0.8333330.833333SAME
Recording_2/group/mountf1_score0.9090910.909091SAME
Recording_2/group/mountso0.9317540.931754SAME
Recording_2/group/mounttp0.1451700.145170SAME
Recording_2/group/mountic0.9946480.994648SAME
Recording_2/group/otherprecision0.9736840.973684SAME
Recording_2/group/otherrecall0.7872340.787234SAME
Recording_2/group/otherf1_score0.8705880.870588SAME
Recording_2/group/otherso0.7110860.711086SAME
Recording_2/group/othertp0.0450080.045008SAME
Recording_2/group/otheric0.9921250.991930CHANGED
Recording_3/overallprecision0.7094810.959481CHANGED
Recording_3/overallrecall0.6009670.850967CHANGED
Recording_3/overallf1_score0.6486270.898627CHANGED
Recording_3/overallso0.4737160.723716CHANGED
Recording_3/overalltp0.0188760.268876CHANGED
Recording_3/overallic0.7372930.993223CHANGED
Recording_3/group/attackprecision0.0000001.000000CHANGED
Recording_3/group/attackrecall0.0000001.000000CHANGED
Recording_3/group/attackf1_score0.0000001.000000CHANGED
Recording_3/group/attackso0.0000001.000000CHANGED
Recording_3/group/attacktp0.0000001.000000CHANGED
Recording_3/group/attackic0.0000001.000000CHANGED
Recording_3/group/investigationprecision0.9354840.935484SAME
Recording_3/group/investigationrecall0.7837840.783784SAME
Recording_3/group/investigationf1_score0.8529410.852941SAME
Recording_3/group/investigationso0.4563100.456310SAME
Recording_3/group/investigationtp0.0223220.022322SAME
Recording_3/group/investigationic0.9849660.984966SAME
Recording_3/group/mountprecision1.0000001.000000SAME
Recording_3/group/mountrecall0.7391300.739130SAME
Recording_3/group/mountf1_score0.8500000.850000SAME
Recording_3/group/mountso0.6745040.674504SAME
Recording_3/group/mounttp0.0287950.028795SAME
Recording_3/group/mountic0.9917130.991713SAME
Recording_3/group/otherprecision0.9024390.902439SAME
Recording_3/group/otherrecall0.8809520.880952SAME
Recording_3/group/otherf1_score0.8915660.891566SAME
Recording_3/group/otherso0.7640480.764048SAME
Recording_3/group/othertp0.0243860.024386SAME
Recording_3/group/otheric0.9724920.996212CHANGED
Recording_4/overallprecision0.7250000.975000CHANGED
Recording_4/overallrecall0.3656860.615686CHANGED
Recording_4/overallf1_score0.4791670.729167CHANGED
Recording_4/overallso0.4570870.707087CHANGED
Recording_4/overalltp0.0257150.275715CHANGED
Recording_4/overallic0.7365910.996530CHANGED
Recording_4/group/attackprecision0.0000001.000000CHANGED
Recording_4/group/attackrecall0.0000001.000000CHANGED
Recording_4/group/attackf1_score0.0000001.000000CHANGED
Recording_4/group/attackso0.0000001.000000CHANGED
Recording_4/group/attacktp0.0000001.000000CHANGED
Recording_4/group/attackic0.0000001.000000CHANGED
Recording_4/group/investigationprecision0.9000000.900000SAME
Recording_4/group/investigationrecall0.5294120.529412SAME
Recording_4/group/investigationf1_score0.6666670.666667SAME
Recording_4/group/investigationso0.3914710.391471SAME
Recording_4/group/investigationtp0.0140250.014025SAME
Recording_4/group/investigationic0.9968820.996882SAME
Recording_4/group/mountprecision1.0000001.000000SAME
Recording_4/group/mountrecall0.3333330.333333SAME
Recording_4/group/mountf1_score0.5000000.500000SAME
Recording_4/group/mountso0.7011190.701119SAME
Recording_4/group/mounttp0.0400870.040087SAME
Recording_4/group/mountic0.9952890.995289SAME
Recording_4/group/otherprecision1.0000001.000000SAME
Recording_4/group/otherrecall0.6000000.600000SAME
Recording_4/group/otherf1_score0.7500000.750000SAME
Recording_4/group/otherso0.7357600.735760SAME
Recording_4/group/othertp0.0487490.048749SAME
Recording_4/group/otheric0.9541910.993949CHANGED
Recording_5/overallprecision0.7500001.000000CHANGED
Recording_5/overallrecall0.6421050.892105CHANGED
Recording_5/overallf1_score0.6914590.941459CHANGED
Recording_5/overallso0.5867300.836730CHANGED
Recording_5/overalltp0.0292890.279289CHANGED
Recording_5/overallic0.7321650.995120CHANGED
Recording_5/group/attackprecision0.0000001.000000CHANGED
Recording_5/group/attackrecall0.0000001.000000CHANGED
Recording_5/group/attackf1_score0.0000001.000000CHANGED
Recording_5/group/attackso0.0000001.000000CHANGED
Recording_5/group/attacktp0.0000001.000000CHANGED
Recording_5/group/attackic0.0000001.000000CHANGED
Recording_5/group/investigationprecision1.0000001.000000SAME
Recording_5/group/investigationrecall0.8000000.800000SAME
Recording_5/group/investigationf1_score0.8888890.888889SAME
Recording_5/group/investigationso0.6589590.658959SAME
Recording_5/group/investigationtp0.0353330.035333SAME
Recording_5/group/investigationic0.9902050.989914CHANGED
Recording_5/group/mountprecision1.0000001.000000SAME
Recording_5/group/mountrecall0.9000000.900000SAME
Recording_5/group/mountf1_score0.9473680.947368SAME
Recording_5/group/mountso0.9100240.910024SAME
Recording_5/group/mounttp0.0359600.035960SAME
Recording_5/group/mountic0.9965410.996541SAME
Recording_5/group/otherprecision1.0000001.000000SAME
Recording_5/group/otherrecall0.8684210.868421SAME
Recording_5/group/otherf1_score0.9295770.929577SAME
Recording_5/group/otherso0.7779390.777939SAME
Recording_5/group/othertp0.0458620.045862SAME
Recording_5/group/otheric0.9419150.994024CHANGED
Recording_6/overallprecision0.6640830.914083CHANGED
Recording_6/overallrecall0.4956710.745671CHANGED
Recording_6/overallf1_score0.5425050.792505CHANGED
Recording_6/overallso0.3744640.624464CHANGED
Recording_6/overalltp0.0140680.264068CHANGED
Recording_6/overallic0.7388210.993987CHANGED
Recording_6/group/attackprecision1.0000001.000000SAME
Recording_6/group/attackrecall0.3636360.363636SAME
Recording_6/group/attackf1_score0.5333330.533333SAME
Recording_6/group/attackso0.3554880.355488SAME
Recording_6/group/attacktp0.0141430.014143SAME
Recording_6/group/attackic0.9936660.993666SAME
Recording_6/group/investigationprecision0.7674420.767442SAME
Recording_6/group/investigationrecall0.7857140.785714SAME
Recording_6/group/investigationf1_score0.7764710.776471SAME
Recording_6/group/investigationso0.5395800.539580SAME
Recording_6/group/investigationtp0.0203170.020317SAME
Recording_6/group/investigationic0.9903920.990392SAME
Recording_6/group/mountprecision0.0000001.000000CHANGED
Recording_6/group/mountrecall0.0000001.000000CHANGED
Recording_6/group/mountf1_score0.0000001.000000CHANGED
Recording_6/group/mountso0.0000001.000000CHANGED
Recording_6/group/mounttp0.0000001.000000CHANGED
Recording_6/group/mountic0.0000001.000000CHANGED
Recording_6/group/otherprecision0.8888890.888889SAME
Recording_6/group/otherrecall0.8333330.833333SAME
Recording_6/group/otherf1_score0.8602150.860215SAME
Recording_6/group/otherso0.6027890.602789SAME
Recording_6/group/othertp0.0218110.021811SAME
Recording_6/group/otheric0.9712240.991888CHANGED
Recording_7/overallprecision0.7500001.000000CHANGED
Recording_7/overallrecall0.3339100.583910CHANGED
Recording_7/overallf1_score0.4565300.706530CHANGED
Recording_7/overallso0.4144950.664495CHANGED
Recording_7/overalltp0.0265300.276530CHANGED
Recording_7/overallic0.7344940.994944CHANGED
Recording_7/group/attackprecision1.0000001.000000SAME
Recording_7/group/attackrecall0.5250000.525000SAME
Recording_7/group/attackf1_score0.6885250.688525SAME
Recording_7/group/attackso0.3906950.390695SAME
Recording_7/group/attacktp0.0302290.030229SAME
Recording_7/group/attackic0.9894710.989471SAME
Recording_7/group/investigationprecision1.0000001.000000SAME
Recording_7/group/investigationrecall0.3000000.300000SAME
Recording_7/group/investigationf1_score0.4615380.461538SAME
Recording_7/group/investigationso0.5364950.536495SAME
Recording_7/group/investigationtp0.0313310.031331SAME
Recording_7/group/investigationic0.9945440.994544SAME
Recording_7/group/mountprecision0.0000001.000000CHANGED
Recording_7/group/mountrecall0.0000001.000000CHANGED
Recording_7/group/mountf1_score0.0000001.000000CHANGED
Recording_7/group/mountso0.0000001.000000CHANGED
Recording_7/group/mounttp0.0000001.000000CHANGED
Recording_7/group/mountic0.0000001.000000CHANGED
Recording_7/group/otherprecision1.0000001.000000SAME
Recording_7/group/otherrecall0.5106380.510638SAME
Recording_7/group/otherf1_score0.6760560.676056SAME
Recording_7/group/otherso0.7307900.730790SAME
Recording_7/group/othertp0.0445610.044561SAME
Recording_7/group/otheric0.9539600.995760CHANGED
Recording_8/overallprecision0.6687500.891667CHANGED
Recording_8/overallrecall0.3467210.346721SAME
Recording_8/overallf1_score0.4546300.606173CHANGED
Recording_8/overallso0.3979210.530562CHANGED
Recording_8/overalltp0.0286040.038138CHANGED
Recording_8/overallic0.9659430.995577CHANGED
Recording_8/group/attackprecision0.8750000.875000SAME
Recording_8/group/attackrecall0.3684210.368421SAME
Recording_8/group/attackf1_score0.5185190.518519SAME
Recording_8/group/attackso0.2683640.268364SAME
Recording_8/group/attacktp0.0204230.020423SAME
Recording_8/group/attackic0.9927960.991919CHANGED
Recording_8/group/investigationprecision1.0000001.000000SAME
Recording_8/group/investigationrecall0.5384620.538462SAME
Recording_8/group/investigationf1_score0.7000000.700000SAME
Recording_8/group/investigationso0.5910510.591051SAME
Recording_8/group/investigationtp0.0252710.025271SAME
Recording_8/group/investigationic0.9952820.995282SAME
Recording_8/group/mountprecision0.000000NaNVALUE->NULL
Recording_8/group/mountrecall0.0000000.000000SAME
Recording_8/group/mountf1_score0.000000NaNVALUE->NULL
Recording_8/group/mountso0.000000NaNVALUE->NULL
Recording_8/group/mounttp0.000000NaNVALUE->NULL
Recording_8/group/mountic1.0000001.000000SAME
Recording_8/group/otherprecision0.8000000.800000SAME
Recording_8/group/otherrecall0.4800000.480000SAME
Recording_8/group/otherf1_score0.6000000.600000SAME
Recording_8/group/otherso0.7322710.732271SAME
Recording_8/group/othertp0.0687210.068721SAME
Recording_8/group/otheric0.8756930.995105CHANGED
Recording_9/overallprecision0.5483980.798398CHANGED
Recording_9/overallrecall0.4038190.653819CHANGED
Recording_9/overallf1_score0.4623550.712355CHANGED
Recording_9/overallso0.4327300.682730CHANGED
Recording_9/overalltp0.0278090.277809CHANGED
Recording_9/overallic0.7388250.995671CHANGED
Recording_9/group/attackprecision0.9565220.956522SAME
Recording_9/group/attackrecall0.7333330.733333SAME
Recording_9/group/attackf1_score0.8301890.830189SAME
Recording_9/group/attackso0.4241000.424100SAME
Recording_9/group/attacktp0.0205900.020590SAME
Recording_9/group/attackic0.9893860.989386SAME
Recording_9/group/investigationprecision0.3750000.375000SAME
Recording_9/group/investigationrecall0.1875000.187500SAME
Recording_9/group/investigationf1_score0.2500000.250000SAME
Recording_9/group/investigationso0.6316780.631678SAME
Recording_9/group/investigationtp0.0348100.034810SAME
Recording_9/group/investigationic0.9986300.998630SAME
Recording_9/group/mountprecision0.0000001.000000CHANGED
Recording_9/group/mountrecall0.0000001.000000CHANGED
Recording_9/group/mountf1_score0.0000001.000000CHANGED
Recording_9/group/mountso0.0000001.000000CHANGED
Recording_9/group/mounttp0.0000001.000000CHANGED
Recording_9/group/mountic0.0000001.000000CHANGED
Recording_9/group/otherprecision0.8620690.862069SAME
Recording_9/group/otherrecall0.6944440.694444SAME
Recording_9/group/otherf1_score0.7692310.769231SAME
Recording_9/group/otherso0.6751400.675140SAME
Recording_9/group/othertp0.0558380.055838SAME
Recording_9/group/otheric0.9672830.994669CHANGED
Recording_10/overallprecision0.6491070.899107CHANGED
Recording_10/overallrecall0.2257600.475760CHANGED
Recording_10/overallf1_score0.3334750.583475CHANGED
Recording_10/overallso0.4168060.666806CHANGED
Recording_10/overalltp0.0359230.285923CHANGED
Recording_10/overallic0.7410640.996191CHANGED
Recording_10/group/attackprecision1.0000001.000000SAME
Recording_10/group/attackrecall0.2857140.285714SAME
Recording_10/group/attackf1_score0.4444440.444444SAME
Recording_10/group/attackso0.1849030.184903SAME
Recording_10/group/attacktp0.0111890.011189SAME
Recording_10/group/attackic0.9914120.991412SAME
Recording_10/group/investigationprecision0.6250000.625000SAME
Recording_10/group/investigationrecall0.2631580.263158SAME
Recording_10/group/investigationf1_score0.3703700.370370SAME
Recording_10/group/investigationso0.6572070.657207SAME
Recording_10/group/investigationtp0.0646830.064683SAME
Recording_10/group/investigationic0.9950610.995061SAME
Recording_10/group/mountprecision0.0000001.000000CHANGED
Recording_10/group/mountrecall0.0000001.000000CHANGED
Recording_10/group/mountf1_score0.0000001.000000CHANGED
Recording_10/group/mountso0.0000001.000000CHANGED
Recording_10/group/mounttp0.0000001.000000CHANGED
Recording_10/group/mountic0.0000001.000000CHANGED
Recording_10/group/otherprecision0.9714290.971429SAME
Recording_10/group/otherrecall0.3541670.354167SAME
Recording_10/group/otherf1_score0.5190840.519084SAME
Recording_10/group/otherso0.8251150.825115SAME
Recording_10/group/othertp0.0678200.067820SAME
Recording_10/group/otheric0.9777840.998293CHANGED

+ +

Mode 3: Python v0.2.0 vs Python v1.1.0

+

Shows the changes of v1.1.0: detection F1 0 instead of NaN when no real bout is matched, IC averaged over the found real bouts only

+

Total: 300  |  +SAME: 259  |  +VALUE->NULL: 1  |  +NULL->VALUE: 1  |  +CHANGED: 39

+

SAME = match   VALUE->NULL = v0.2.0=1.0 -> v1.1.0=NaN: IC is NaN, not 1.0, when no real bout is found   NULL->VALUE = v0.2.0=NaN -> v1.1.0=0: detection F1 is 0, not NaN, when a behavior has real bouts and none is matched   CHANGED = IC averages over the real bouts that contain a predicted frame (a missed bout no longer scores 1.0); the overall means of the recordings move with these IC changes and with the F1 change

+
LocationMetricPython v0.2.0Python v1.1.0Status
Recording_1/overallprecision0.9282610.928261SAME
Recording_1/overallrecall0.7812930.781293SAME
Recording_1/overallf1_score0.8364550.836455SAME
Recording_1/overallso0.7612280.761228SAME
Recording_1/overalltp0.3062300.306230SAME
Recording_1/overallic0.9917180.988966CHANGED
Recording_1/group/attackprecision1.0000001.000000SAME
Recording_1/group/attackrecall1.0000001.000000SAME
Recording_1/group/attackf1_score1.0000001.000000SAME
Recording_1/group/attackso1.0000001.000000SAME
Recording_1/group/attacktp1.0000001.000000SAME
Recording_1/group/attackic1.0000001.000000SAME
Recording_1/group/investigationprecision0.8000000.800000SAME
Recording_1/group/investigationrecall0.7346940.734694SAME
Recording_1/group/investigationf1_score0.7659570.765957SAME
Recording_1/group/investigationso0.5444100.544410SAME
Recording_1/group/investigationtp0.0846360.084636SAME
Recording_1/group/investigationic0.9829410.980560CHANGED
Recording_1/group/mountprecision1.0000001.000000SAME
Recording_1/group/mountrecall0.5333330.533333SAME
Recording_1/group/mountf1_score0.6956520.695652SAME
Recording_1/group/mountso0.7332770.733277SAME
Recording_1/group/mounttp0.0784710.078471SAME
Recording_1/group/mountic0.9908610.982864CHANGED
Recording_1/group/otherprecision0.9130430.913043SAME
Recording_1/group/otherrecall0.8571430.857143SAME
Recording_1/group/otherf1_score0.8842110.884211SAME
Recording_1/group/otherso0.7672270.767227SAME
Recording_1/group/othertp0.0618140.061814SAME
Recording_1/group/otheric0.9930690.992439CHANGED
Recording_2/overallprecision0.9671050.967105SAME
Recording_2/overallrecall0.8399240.839924SAME
Recording_2/overallf1_score0.8973010.897301SAME
Recording_2/overallso0.8259460.825946SAME
Recording_2/overalltp0.3090820.309082SAME
Recording_2/overallic0.9939070.993011CHANGED
Recording_2/group/attackprecision1.0000001.000000SAME
Recording_2/group/attackrecall1.0000001.000000SAME
Recording_2/group/attackf1_score1.0000001.000000SAME
Recording_2/group/attackso1.0000001.000000SAME
Recording_2/group/attacktp1.0000001.000000SAME
Recording_2/group/attackic1.0000001.000000SAME
Recording_2/group/investigationprecision0.8947370.894737SAME
Recording_2/group/investigationrecall0.7391300.739130SAME
Recording_2/group/investigationf1_score0.8095240.809524SAME
Recording_2/group/investigationso0.6609420.660942SAME
Recording_2/group/investigationtp0.0461510.046151SAME
Recording_2/group/investigationic0.9890510.987716CHANGED
Recording_2/group/mountprecision1.0000001.000000SAME
Recording_2/group/mountrecall0.8333330.833333SAME
Recording_2/group/mountf1_score0.9090910.909091SAME
Recording_2/group/mountso0.9317540.931754SAME
Recording_2/group/mounttp0.1451700.145170SAME
Recording_2/group/mountic0.9946480.993578CHANGED
Recording_2/group/otherprecision0.9736840.973684SAME
Recording_2/group/otherrecall0.7872340.787234SAME
Recording_2/group/otherf1_score0.8705880.870588SAME
Recording_2/group/otherso0.7110860.711086SAME
Recording_2/group/othertp0.0450080.045008SAME
Recording_2/group/otheric0.9919300.990749CHANGED
Recording_3/overallprecision0.9594810.959481SAME
Recording_3/overallrecall0.8509670.850967SAME
Recording_3/overallf1_score0.8986270.898627SAME
Recording_3/overallso0.7237160.723716SAME
Recording_3/overalltp0.2688760.268876SAME
Recording_3/overallic0.9932230.991985CHANGED
Recording_3/group/attackprecision1.0000001.000000SAME
Recording_3/group/attackrecall1.0000001.000000SAME
Recording_3/group/attackf1_score1.0000001.000000SAME
Recording_3/group/attackso1.0000001.000000SAME
Recording_3/group/attacktp1.0000001.000000SAME
Recording_3/group/attackic1.0000001.000000SAME
Recording_3/group/investigationprecision0.9354840.935484SAME
Recording_3/group/investigationrecall0.7837840.783784SAME
Recording_3/group/investigationf1_score0.8529410.852941SAME
Recording_3/group/investigationso0.4563100.456310SAME
Recording_3/group/investigationtp0.0223220.022322SAME
Recording_3/group/investigationic0.9849660.982617CHANGED
Recording_3/group/mountprecision1.0000001.000000SAME
Recording_3/group/mountrecall0.7391300.739130SAME
Recording_3/group/mountf1_score0.8500000.850000SAME
Recording_3/group/mountso0.6745040.674504SAME
Recording_3/group/mounttp0.0287950.028795SAME
Recording_3/group/mountic0.9917130.989411CHANGED
Recording_3/group/otherprecision0.9024390.902439SAME
Recording_3/group/otherrecall0.8809520.880952SAME
Recording_3/group/otherf1_score0.8915660.891566SAME
Recording_3/group/otherso0.7640480.764048SAME
Recording_3/group/othertp0.0243860.024386SAME
Recording_3/group/otheric0.9962120.995913CHANGED
Recording_4/overallprecision0.9750000.975000SAME
Recording_4/overallrecall0.6156860.615686SAME
Recording_4/overallf1_score0.7291670.729167SAME
Recording_4/overallso0.7070870.707087SAME
Recording_4/overalltp0.2757150.275715SAME
Recording_4/overallic0.9965300.993419CHANGED
Recording_4/group/attackprecision1.0000001.000000SAME
Recording_4/group/attackrecall1.0000001.000000SAME
Recording_4/group/attackf1_score1.0000001.000000SAME
Recording_4/group/attackso1.0000001.000000SAME
Recording_4/group/attacktp1.0000001.000000SAME
Recording_4/group/attackic1.0000001.000000SAME
Recording_4/group/investigationprecision0.9000000.900000SAME
Recording_4/group/investigationrecall0.5294120.529412SAME
Recording_4/group/investigationf1_score0.6666670.666667SAME
Recording_4/group/investigationso0.3914710.391471SAME
Recording_4/group/investigationtp0.0140250.014025SAME
Recording_4/group/investigationic0.9968820.996882SAME
Recording_4/group/mountprecision1.0000001.000000SAME
Recording_4/group/mountrecall0.3333330.333333SAME
Recording_4/group/mountf1_score0.5000000.500000SAME
Recording_4/group/mountso0.7011190.701119SAME
Recording_4/group/mounttp0.0400870.040087SAME
Recording_4/group/mountic0.9952890.985868CHANGED
Recording_4/group/otherprecision1.0000001.000000SAME
Recording_4/group/otherrecall0.6000000.600000SAME
Recording_4/group/otherf1_score0.7500000.750000SAME
Recording_4/group/otherso0.7357600.735760SAME
Recording_4/group/othertp0.0487490.048749SAME
Recording_4/group/otheric0.9939490.990924CHANGED
Recording_5/overallprecision1.0000001.000000SAME
Recording_5/overallrecall0.8921050.892105SAME
Recording_5/overallf1_score0.9414590.941459SAME
Recording_5/overallso0.8367300.836730SAME
Recording_5/overalltp0.2792890.279289SAME
Recording_5/overallic0.9951200.994516CHANGED
Recording_5/group/attackprecision1.0000001.000000SAME
Recording_5/group/attackrecall1.0000001.000000SAME
Recording_5/group/attackf1_score1.0000001.000000SAME
Recording_5/group/attackso1.0000001.000000SAME
Recording_5/group/attacktp1.0000001.000000SAME
Recording_5/group/attackic1.0000001.000000SAME
Recording_5/group/investigationprecision1.0000001.000000SAME
Recording_5/group/investigationrecall0.8000000.800000SAME
Recording_5/group/investigationf1_score0.8888890.888889SAME
Recording_5/group/investigationso0.6589590.658959SAME
Recording_5/group/investigationtp0.0353330.035333SAME
Recording_5/group/investigationic0.9899140.988233CHANGED
Recording_5/group/mountprecision1.0000001.000000SAME
Recording_5/group/mountrecall0.9000000.900000SAME
Recording_5/group/mountf1_score0.9473680.947368SAME
Recording_5/group/mountso0.9100240.910024SAME
Recording_5/group/mounttp0.0359600.035960SAME
Recording_5/group/mountic0.9965410.996157CHANGED
Recording_5/group/otherprecision1.0000001.000000SAME
Recording_5/group/otherrecall0.8684210.868421SAME
Recording_5/group/otherf1_score0.9295770.929577SAME
Recording_5/group/otherso0.7779390.777939SAME
Recording_5/group/othertp0.0458620.045862SAME
Recording_5/group/otheric0.9940240.993673CHANGED
Recording_6/overallprecision0.9140830.914083SAME
Recording_6/overallrecall0.7456710.745671SAME
Recording_6/overallf1_score0.7925050.792505SAME
Recording_6/overallso0.6244640.624464SAME
Recording_6/overalltp0.2640680.264068SAME
Recording_6/overallic0.9939870.992128CHANGED
Recording_6/group/attackprecision1.0000001.000000SAME
Recording_6/group/attackrecall0.3636360.363636SAME
Recording_6/group/attackf1_score0.5333330.533333SAME
Recording_6/group/attackso0.3554880.355488SAME
Recording_6/group/attacktp0.0141430.014143SAME
Recording_6/group/attackic0.9936660.988388CHANGED
Recording_6/group/investigationprecision0.7674420.767442SAME
Recording_6/group/investigationrecall0.7857140.785714SAME
Recording_6/group/investigationf1_score0.7764710.776471SAME
Recording_6/group/investigationso0.5395800.539580SAME
Recording_6/group/investigationtp0.0203170.020317SAME
Recording_6/group/investigationic0.9903920.989653CHANGED
Recording_6/group/mountprecision1.0000001.000000SAME
Recording_6/group/mountrecall1.0000001.000000SAME
Recording_6/group/mountf1_score1.0000001.000000SAME
Recording_6/group/mountso1.0000001.000000SAME
Recording_6/group/mounttp1.0000001.000000SAME
Recording_6/group/mountic1.0000001.000000SAME
Recording_6/group/otherprecision0.8888890.888889SAME
Recording_6/group/otherrecall0.8333330.833333SAME
Recording_6/group/otherf1_score0.8602150.860215SAME
Recording_6/group/otherso0.6027890.602789SAME
Recording_6/group/othertp0.0218110.021811SAME
Recording_6/group/otheric0.9918880.990469CHANGED
Recording_7/overallprecision1.0000001.000000SAME
Recording_7/overallrecall0.5839100.583910SAME
Recording_7/overallf1_score0.7065300.706530SAME
Recording_7/overallso0.6644950.664495SAME
Recording_7/overalltp0.2765300.276530SAME
Recording_7/overallic0.9949440.990979CHANGED
Recording_7/group/attackprecision1.0000001.000000SAME
Recording_7/group/attackrecall0.5250000.525000SAME
Recording_7/group/attackf1_score0.6885250.688525SAME
Recording_7/group/attackso0.3906950.390695SAME
Recording_7/group/attacktp0.0302290.030229SAME
Recording_7/group/attackic0.9894710.988917CHANGED
Recording_7/group/investigationprecision1.0000001.000000SAME
Recording_7/group/investigationrecall0.3000000.300000SAME
Recording_7/group/investigationf1_score0.4615380.461538SAME
Recording_7/group/investigationso0.5364950.536495SAME
Recording_7/group/investigationtp0.0313310.031331SAME
Recording_7/group/investigationic0.9945440.981813CHANGED
Recording_7/group/mountprecision1.0000001.000000SAME
Recording_7/group/mountrecall1.0000001.000000SAME
Recording_7/group/mountf1_score1.0000001.000000SAME
Recording_7/group/mountso1.0000001.000000SAME
Recording_7/group/mounttp1.0000001.000000SAME
Recording_7/group/mountic1.0000001.000000SAME
Recording_7/group/otherprecision1.0000001.000000SAME
Recording_7/group/otherrecall0.5106380.510638SAME
Recording_7/group/otherf1_score0.6760560.676056SAME
Recording_7/group/otherso0.7307900.730790SAME
Recording_7/group/othertp0.0445610.044561SAME
Recording_7/group/otheric0.9957600.993185CHANGED
Recording_8/overallprecision0.8916670.891667SAME
Recording_8/overallrecall0.3467210.346721SAME
Recording_8/overallf1_score0.6061730.454630CHANGED
Recording_8/overallso0.5305620.530562SAME
Recording_8/overalltp0.0381380.038138SAME
Recording_8/overallic0.9955770.992044CHANGED
Recording_8/group/attackprecision0.8750000.875000SAME
Recording_8/group/attackrecall0.3684210.368421SAME
Recording_8/group/attackf1_score0.5185190.518519SAME
Recording_8/group/attackso0.2683640.268364SAME
Recording_8/group/attacktp0.0204230.020423SAME
Recording_8/group/attackic0.9919190.991919SAME
Recording_8/group/investigationprecision1.0000001.000000SAME
Recording_8/group/investigationrecall0.5384620.538462SAME
Recording_8/group/investigationf1_score0.7000000.700000SAME
Recording_8/group/investigationso0.5910510.591051SAME
Recording_8/group/investigationtp0.0252710.025271SAME
Recording_8/group/investigationic0.9952820.993185CHANGED
Recording_8/group/mountprecisionNaNNaNSAME
Recording_8/group/mountrecall0.0000000.000000SAME
Recording_8/group/mountf1_scoreNaN0.000000NULL->VALUE
Recording_8/group/mountsoNaNNaNSAME
Recording_8/group/mounttpNaNNaNSAME
Recording_8/group/mountic1.000000NaNVALUE->NULL
Recording_8/group/otherprecision0.8000000.800000SAME
Recording_8/group/otherrecall0.4800000.480000SAME
Recording_8/group/otherf1_score0.6000000.600000SAME
Recording_8/group/otherso0.7322710.732271SAME
Recording_8/group/othertp0.0687210.068721SAME
Recording_8/group/otheric0.9951050.991026CHANGED
Recording_9/overallprecision0.7983980.798398SAME
Recording_9/overallrecall0.6538190.653819SAME
Recording_9/overallf1_score0.7123550.712355SAME
Recording_9/overallso0.6827300.682730SAME
Recording_9/overalltp0.2778090.277809SAME
Recording_9/overallic0.9956710.993536CHANGED
Recording_9/group/attackprecision0.9565220.956522SAME
Recording_9/group/attackrecall0.7333330.733333SAME
Recording_9/group/attackf1_score0.8301890.830189SAME
Recording_9/group/attackso0.4241000.424100SAME
Recording_9/group/attacktp0.0205900.020590SAME
Recording_9/group/attackic0.9893860.988628CHANGED
Recording_9/group/investigationprecision0.3750000.375000SAME
Recording_9/group/investigationrecall0.1875000.187500SAME
Recording_9/group/investigationf1_score0.2500000.250000SAME
Recording_9/group/investigationso0.6316780.631678SAME
Recording_9/group/investigationtp0.0348100.034810SAME
Recording_9/group/investigationic0.9986300.992694CHANGED
Recording_9/group/mountprecision1.0000001.000000SAME
Recording_9/group/mountrecall1.0000001.000000SAME
Recording_9/group/mountf1_score1.0000001.000000SAME
Recording_9/group/mountso1.0000001.000000SAME
Recording_9/group/mounttp1.0000001.000000SAME
Recording_9/group/mountic1.0000001.000000SAME
Recording_9/group/otherprecision0.8620690.862069SAME
Recording_9/group/otherrecall0.6944440.694444SAME
Recording_9/group/otherf1_score0.7692310.769231SAME
Recording_9/group/otherso0.6751400.675140SAME
Recording_9/group/othertp0.0558380.055838SAME
Recording_9/group/otheric0.9946690.992824CHANGED
Recording_10/overallprecision0.8991070.899107SAME
Recording_10/overallrecall0.4757600.475760SAME
Recording_10/overallf1_score0.5834750.583475SAME
Recording_10/overallso0.6668060.666806SAME
Recording_10/overalltp0.2859230.285923SAME
Recording_10/overallic0.9961910.991895CHANGED
Recording_10/group/attackprecision1.0000001.000000SAME
Recording_10/group/attackrecall0.2857140.285714SAME
Recording_10/group/attackf1_score0.4444440.444444SAME
Recording_10/group/attackso0.1849030.184903SAME
Recording_10/group/attacktp0.0111890.011189SAME
Recording_10/group/attackic0.9914120.990806CHANGED
Recording_10/group/investigationprecision0.6250000.625000SAME
Recording_10/group/investigationrecall0.2631580.263158SAME
Recording_10/group/investigationf1_score0.3703700.370370SAME
Recording_10/group/investigationso0.6572070.657207SAME
Recording_10/group/investigationtp0.0646830.064683SAME
Recording_10/group/investigationic0.9950610.981232CHANGED
Recording_10/group/mountprecision1.0000001.000000SAME
Recording_10/group/mountrecall1.0000001.000000SAME
Recording_10/group/mountf1_score1.0000001.000000SAME
Recording_10/group/mountso1.0000001.000000SAME
Recording_10/group/mounttp1.0000001.000000SAME
Recording_10/group/mountic1.0000001.000000SAME
Recording_10/group/otherprecision0.9714290.971429SAME
Recording_10/group/otherrecall0.3541670.354167SAME
Recording_10/group/otherf1_score0.5190840.519084SAME
Recording_10/group/otherso0.8251150.825115SAME
Recording_10/group/othertp0.0678200.067820SAME
Recording_10/group/otheric0.9982930.995543CHANGED

diff --git a/tests/fixtures/compare_fixtures.py b/tests/fixtures/compare_fixtures.py index dd3bbbc..183b1ce 100644 --- a/tests/fixtures/compare_fixtures.py +++ b/tests/fixtures/compare_fixtures.py @@ -1,17 +1,33 @@ -"""BANOS fixture comparison tool -- 2-way comparison across versions and languages. +"""BANOS fixture comparison tool -- 2-way comparisons across versions and languages. Fixture files: - golden_python_v015.json v0.1.5 original Python (0 for absent behaviors, IC bug) - golden_python_v020.json v0.2.0 Python (score-1-for-correct-absence + IC bug fixed) - golden_matlab_v020.json v0.2.0 Matlab (same semantics as Python v0.2.0) - -Two comparisons available via --mode: + golden_python_v015.json v0.1.5 original Python (archive): 0 for absent behaviors and + for undefined metrics, IC misses the last frame pair + golden_python_v020.json v0.2.0 Python (archive): 1.0 for correct absence, 0.0 for + false detection, NaN for undefined metrics, IC fixed + golden_matlab_v020.json v0.2.0 Matlab (archive, same semantics as Python v0.2.0) + golden_python_v110.json v1.1.0 Python (current): detection F1 0 when no real bout is + matched, IC averaged over the found real bouts only + +Three comparisons available via --mode: 1 Python v0.2.0 vs Matlab v0.2.0 -- shows Python/Matlab parity - 2 Python v0.1.5 vs v0.2.0 -- shows ALL changes (absent-behavior fix + IC bug fix) + 2 Python v0.1.5 vs v0.2.0 -- shows the changes of v0.2.0 + 3 Python v0.2.0 vs v1.1.0 -- shows the changes of v1.1.0 + +The Recording_N entries are compared, overall and per behavior. The +"all_recordings" entry of the v1.1.0 files has no counterpart in older files +and is not compared. + +Each pair of values (first file, second file) is tagged: + SAME equal within 1e-10, or NaN (JSON null) in both + VALUE->NULL a number in the first file, NaN in the second + NULL->VALUE NaN in the first file, a number in the second + CHANGED two different numbers +The legend of each comparison says what each tag means there. Usage: - python tests/fixtures/compare_fixtures.py # both modes, text only - python tests/fixtures/compare_fixtures.py --html # both modes + HTML report + python tests/fixtures/compare_fixtures.py # all modes, text only + python tests/fixtures/compare_fixtures.py --html # all modes + HTML report python tests/fixtures/compare_fixtures.py --mode 1 # single mode python tests/fixtures/compare_fixtures.py --mode 2 --html reports/parity.html """ @@ -19,6 +35,7 @@ from __future__ import annotations import argparse +import html import json import math from pathlib import Path @@ -30,8 +47,13 @@ "python_v015": FIXTURES / "golden_python_v015.json", "python_v020": FIXTURES / "golden_python_v020.json", "matlab_v020": FIXTURES / "golden_matlab_v020.json", + "python_v110": FIXTURES / "golden_python_v110.json", } +TAGS = ["SAME", "VALUE->NULL", "NULL->VALUE", "CHANGED"] + +# For each comparison, "meanings" maps each tag other than SAME to +# (what it means in that comparison, whether such a change is expected). COMPARISONS = [ { "id": 1, @@ -41,19 +63,69 @@ "b_key": "matlab_v020", "a_label": "Python v0.2.0", "b_label": "Matlab v0.2.0", - "null_to_val_meaning": "Unexpected divergence (should not occur in v0.2.0)", - "changed_meaning": "Unexpected numeric mismatch", + "meanings": { + "VALUE->NULL": ("Unexpected divergence (should not occur in v0.2.0)", False), + "NULL->VALUE": ("Unexpected divergence (should not occur in v0.2.0)", False), + "CHANGED": ("Unexpected numeric mismatch", False), + }, }, { "id": 2, "title": "Python v0.1.5 vs Python v0.2.0", - "subtitle": "Shows ALL changes: absent-behavior fix (0→1 correct absence) + IC bug fix", + "subtitle": ( + "Shows the changes of v0.2.0: undefined metrics NaN instead of 0, " + "correct absence 1.0 instead of 0, IC fixes" + ), "a_key": "python_v015", "b_key": "python_v020", "a_label": "Python v0.1.5", "b_label": "Python v0.2.0", - "null_to_val_meaning": "v0.1.5=0 → v0.2.0=1.0 (correct absence now scores 1)", - "changed_meaning": "IC bug fix", + "meanings": { + "VALUE->NULL": ( + "v0.1.5=0 -> v0.2.0=NaN: a metric with a zero denominator (precision " + "when nothing is predicted; F1, SO and TP when no bout is matched) is " + "NaN instead of 0 and leaves the means", + True, + ), + "NULL->VALUE": ("Unexpected (v0.1.5 never gives NaN)", False), + "CHANGED": ( + "correct absence (no real bout, nothing predicted) scores 1.0 instead " + "of 0 on all six metrics; IC counts the last frame pair of each bout " + "and leaves one-frame real bouts out instead of scoring them 0; the " + "overall means of the recordings move with these changes and with the " + "cells that became NaN", + True, + ), + }, + }, + { + "id": 3, + "title": "Python v0.2.0 vs Python v1.1.0", + "subtitle": ( + "Shows the changes of v1.1.0: detection F1 0 instead of NaN when no real " + "bout is matched, IC averaged over the found real bouts only" + ), + "a_key": "python_v020", + "b_key": "python_v110", + "a_label": "Python v0.2.0", + "b_label": "Python v1.1.0", + "meanings": { + "VALUE->NULL": ( + "v0.2.0=1.0 -> v1.1.0=NaN: IC is NaN, not 1.0, when no real bout is found", + True, + ), + "NULL->VALUE": ( + "v0.2.0=NaN -> v1.1.0=0: detection F1 is 0, not NaN, when a behavior has " + "real bouts and none is matched", + True, + ), + "CHANGED": ( + "IC averages over the real bouts that contain a predicted frame (a missed " + "bout no longer scores 1.0); the overall means of the recordings move with " + "these IC changes and with the F1 change", + True, + ), + }, }, ] @@ -61,6 +133,11 @@ TOLERANCE = 1e-10 METRICS = ["precision", "recall", "f1_score", "so", "tp", "ic"] +GREEN = "#d4edda" # SAME +YELLOW = "#fff3cd" # expected change between a number and NaN +ORANGE = "#fce8d5" # expected change between two numbers +RED = "#f8d7da" # unexpected change + def _is_nan(v) -> bool: return v is None or (isinstance(v, float) and math.isnan(v)) @@ -85,19 +162,24 @@ def _classify(a_val, b_val) -> str: if _close(a_val, b_val): return "SAME" if not _is_nan(a_val) and _is_nan(b_val): - return "NULL->VALUE" # baseline numeric, target NaN + return "VALUE->NULL" # baseline numeric, target NaN if _is_nan(a_val) and not _is_nan(b_val): - return "VALUE->NULL" # baseline NaN, target numeric + return "NULL->VALUE" # baseline NaN, target numeric return "CHANGED" +def _recordings(a_data: dict, b_data: dict) -> list[str]: + """Recording_N keys of both fixtures, in numeric order.""" + names = {key for key in (*a_data, *b_data) if key.startswith("Recording_")} + return sorted(names, key=lambda x: int(x.split("_")[1])) + + def compare_two(a_data: dict, b_data: dict) -> tuple[list[dict], dict]: """Compare two fixture dicts. Returns (rows, counts).""" - counts: dict[str, int] = {"SAME": 0, "NULL->VALUE": 0, "VALUE->NULL": 0, "CHANGED": 0} + counts: dict[str, int] = dict.fromkeys(TAGS, 0) rows: list[dict] = [] - recs = sorted(set(a_data) | set(b_data), key=lambda x: int(x.split("_")[1])) - for rec in recs: + for rec in _recordings(a_data, b_data): a_rec = a_data.get(rec, {}) b_rec = b_data.get(rec, {}) @@ -133,6 +215,12 @@ def compare_two(a_data: dict, b_data: dict) -> tuple[list[dict], dict]: return rows, counts +def _meaning(comp: dict, tag: str) -> str: + if tag == "SAME": + return "match" + return comp["meanings"][tag][0] + + def print_comparison(comp: dict, rows: list[dict], counts: dict) -> None: print(f"\n{'=' * 70}") print(f"Mode {comp['id']}: {comp['title']}") @@ -147,39 +235,31 @@ def print_comparison(comp: dict, rows: list[dict], counts: dict) -> None: ) total = sum(counts.values()) print(f"\nSUMMARY ({total} comparisons)") - print(f" SAME : {counts['SAME']}") - print(f" NULL->VALUE : {counts['NULL->VALUE']} ({comp['null_to_val_meaning']})") - print(f" VALUE->NULL : {counts['VALUE->NULL']} (baseline=NaN, target=numeric)") - print(f" CHANGED : {counts['CHANGED']} ({comp['changed_meaning']})") - - -def _row_color(tag: str, comp_id: int) -> str: - base = { - "SAME": "#d4edda", - "VALUE->NULL": "#f8d7da", - } - if tag in base: - return base[tag] - # Color meaning differs by mode - if comp_id == 1: - # Python vs Matlab: NULL->VALUE is unexpected divergence (red), CHANGED is bad (red) - return {"NULL->VALUE": "#f8d7da", "CHANGED": "#f8d7da"}.get(tag, "white") - # Mode 2: NULL->VALUE = absent-behavior fix (yellow), CHANGED = IC bug fix (orange) - return {"NULL->VALUE": "#fff3cd", "CHANGED": "#fce8d5"}.get(tag, "white") + for tag in TAGS: + print(f" {tag:13s}: {counts[tag]} ({_meaning(comp, tag)})") + + +def _row_color(tag: str, comp: dict) -> str: + if tag == "SAME": + return GREEN + _, expected = comp["meanings"][tag] + if not expected: + return RED + return ORANGE if tag == "CHANGED" else YELLOW def _build_html_section(comp: dict, rows: list[dict], counts: dict) -> str: total = sum(counts.values()) rows_html = [] for row in rows: - color = _row_color(row["tag"], comp["id"]) + color = _row_color(row["tag"], comp) rows_html.append( f"" f"{row['label']}" f"{row['metric']}" f"{_fmt(row['a'])}" f"{_fmt(row['b'])}" - f"{row['tag']}" + f"{html.escape(row['tag'])}" "" ) @@ -194,33 +274,18 @@ def _build_html_section(comp: dict, rows: list[dict], counts: dict) -> str: ) legend_items = [ - ("SAME", "match"), - ( - "NULL->VALUE", - comp["null_to_val_meaning"], - ), - ("CHANGED", comp["changed_meaning"]) - if comp["id"] == 2 - else ( - "CHANGED", - comp["changed_meaning"], - ), - ("VALUE->NULL", "unexpected"), + f"" + f"{html.escape(tag)} = {html.escape(_meaning(comp, tag))}" + for tag in TAGS ] - legend = ( - "

" - + "   ".join(f"{s} = {m}" for s, m in legend_items) - + "

" - ) + legend = "

" + "   ".join(legend_items) + "

" + summary = "  | \n".join(f"{html.escape(tag)}: {counts[tag]}" for tag in TAGS) header = f"""

Mode {comp["id"]}: {comp["title"]}

-

{comp["subtitle"]}

+

{html.escape(comp["subtitle"])}

Total: {total}  |  -SAME: {counts["SAME"]}  |  -NULL->VALUE: {counts["NULL->VALUE"]}  |  -CHANGED: {counts["CHANGED"]}  |  -VALUE->NULL: {counts["VALUE->NULL"]}

+{summary}

{legend} """ return header + table + "
" @@ -234,7 +299,7 @@ def build_html(sections: list[str]) -> str: "" - "

BANOS Fixture Comparison

" + body + "" + "

BANOS Fixture Comparison

" + body + "\n" ) @@ -250,9 +315,12 @@ def main() -> None: parser.add_argument( "--mode", type=int, - choices=[1, 2], + choices=[1, 2, 3], default=None, - help="Which comparison to run (1=Python vs Matlab, 2=v015 vs v020). Default: all.", + help=( + "Which comparison to run (1=Python vs Matlab v0.2.0, 2=v015 vs v020, " + "3=v020 vs v110). Default: all." + ), ) parser.add_argument( "--html", @@ -263,7 +331,7 @@ def main() -> None: ) args = parser.parse_args() - mode_ids = [args.mode] if args.mode else [1, 2] + mode_ids = [args.mode] if args.mode else [1, 2, 3] selected = [c for c in COMPARISONS if c["id"] in mode_ids] html_sections: list[str] = [] diff --git a/tests/test_compare_fixtures.py b/tests/test_compare_fixtures.py new file mode 100644 index 0000000..f8b4cbf --- /dev/null +++ b/tests/test_compare_fixtures.py @@ -0,0 +1,98 @@ +# tests/test_compare_fixtures.py +"""Tests of the fixture comparison script, tests/fixtures/compare_fixtures.py. + +WHY these tests matter +---------------------- +The script tags every golden value that changed between two versions and +explains each tag; its HTML output, reports/fixture_comparison.html, is +committed. No other test reads them, so swapped tags, a fixture edited without +regenerating the report, or a report edited by hand would go unnoticed. + +The report is not in the sdist (reports/ is left out), so the report test is +skipped there. +""" + +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).parent.parent +SCRIPT = ROOT / "tests" / "fixtures" / "compare_fixtures.py" +REPORT = ROOT / "reports" / "fixture_comparison.html" + +# The IC values that differ between golden_python_v020.json and +# golden_python_v110.json: the overall IC and the IC of these behaviors, per +# recording (39 values). With the two F1 values of Recording_8 (mount and +# overall), they are the 41 values changed by v1.1.0. +IC_CHANGED = { + 1: ["overall", "investigation", "mount", "other"], + 2: ["overall", "investigation", "mount", "other"], + 3: ["overall", "investigation", "mount", "other"], + 4: ["overall", "mount", "other"], + 5: ["overall", "investigation", "mount", "other"], + 6: ["overall", "attack", "investigation", "other"], + 7: ["overall", "attack", "investigation", "other"], + 8: ["overall", "investigation", "mount", "other"], + 9: ["overall", "attack", "investigation", "other"], + 10: ["overall", "attack", "investigation", "other"], +} + + +def _load_script(): + spec = importlib.util.spec_from_file_location("compare_fixtures", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_v020_to_v110_finds_the_41_changed_values(): + """Mode 3 (Python v0.2.0 vs v1.1.0) tags exactly the 41 values that v1.1.0 changed. + + Recording_8 mount: F1 NaN -> 0 (NULL->VALUE), IC 1.0 -> NaN (VALUE->NULL); + Recording_8 overall F1 and the 38 other IC values: CHANGED. The other 259 + values are SAME. + """ + script = _load_script() + comparison = next(c for c in script.COMPARISONS if c["id"] == 3) + rows, counts = script.compare_two( + script.load_fixture(comparison["a_key"]), script.load_fixture(comparison["b_key"]) + ) + + expected = {} + for n, places in IC_CHANGED.items(): + for place in places: + label = ( + f"Recording_{n}/overall" if place == "overall" else f"Recording_{n}/group/{place}" + ) + expected[(label, "ic")] = "CHANGED" + expected[("Recording_8/group/mount", "ic")] = "VALUE->NULL" + expected[("Recording_8/group/mount", "f1_score")] = "NULL->VALUE" + expected[("Recording_8/overall", "f1_score")] = "CHANGED" + assert len(expected) == 41 + + changed = {(row["label"], row["metric"]): row["tag"] for row in rows if row["tag"] != "SAME"} + assert changed == expected + assert counts == {"SAME": 259, "VALUE->NULL": 1, "NULL->VALUE": 1, "CHANGED": 39} + + +def test_committed_report_is_the_script_output(tmp_path): + """reports/fixture_comparison.html equals what the script writes now. + + The script's output depends only on the fixture files (recordings, behaviors + and metrics in a fixed order, no date or path), so it can be compared as is; + line endings are normalized by reading both files as text. + """ + if not REPORT.exists(): + pytest.skip("reports/fixture_comparison.html is not shipped in the sdist") + out = tmp_path / "fixture_comparison.html" + proc = subprocess.run( + [sys.executable, str(SCRIPT), "--html", str(out)], capture_output=True, text=True + ) + assert proc.returncode == 0, proc.stderr + assert out.read_text(encoding="utf-8") == REPORT.read_text(encoding="utf-8"), ( + "reports/fixture_comparison.html is out of date: " + "run python tests/fixtures/compare_fixtures.py --html" + ) From b8ac6316c36aad88cf79a88cd930f904a1757146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:10:03 +0100 Subject: [PATCH 10/25] ci: run the parity tests and stop persisting checkout credentials The test matrix no longer ignores tests/test_parity.py. These tests compare the Python results with the committed Matlab golden values, need no Matlab, and are the only check that both implementations agree. actions/checkout no longer leaves the job token in the git config (persist-credentials: false); no step pushes or fetches. The workflow can now be called by another workflow (workflow_call), so that the publish workflow can require the whole matrix. Job names are unchanged. --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c14665..7d17b11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: branches: [main] pull_request: branches: [main] + workflow_call: # Run by publish.yml before it publishes permissions: contents: read @@ -15,6 +16,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 @@ -40,6 +43,8 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 @@ -50,4 +55,4 @@ jobs: run: uv sync --locked --dev - name: Run tests - run: uv run pytest tests/ --ignore=tests/test_parity.py -v + run: uv run pytest tests/ -v From aece8c232280be6f18bf797902e9784fe332850d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:10:54 +0100 Subject: [PATCH 11/25] ci: publish only after full ci and when the tag matches the version publish.yml now has three jobs. check-version reads the version in pyproject.toml and fails unless the tag is v. ci calls ci.yml, so lint and the test matrix on Python 3.11 to 3.14, parity tests included, must pass. build-and-publish runs only after both: it builds with uv build and publishes with trusted publishing, as before. The workflow token is read-only by default; only the publishing job gets id-token: write, in the pypi environment. Checkouts do not persist credentials. The matrix replaces the single test run on Python 3.11 that preceded the build. --- .github/workflows/publish.yml | 36 +++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3d9b3f5..a3d6a2d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,9 +5,37 @@ on: tags: - 'v*' +permissions: + contents: read + jobs: + check-version: + name: Check tag against package version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Compare the tag with the version in pyproject.toml + env: + TAG: ${{ github.ref_name }} + run: | + version=$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])') + if [ "$TAG" != "v$version" ]; then + echo "::error::Tag $TAG does not match the package version $version (expected v$version)" + exit 1 + fi + echo "Tag $TAG matches the package version $version" + + ci: + name: CI + needs: check-version + uses: ./.github/workflows/ci.yml # Every job of ci.yml, test matrix included + build-and-publish: name: Build and publish to PyPI + needs: [check-version, ci] runs-on: ubuntu-latest environment: pypi permissions: @@ -16,18 +44,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Install uv uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: python-version: "3.11" - - name: Install dependencies - run: uv sync --locked --dev - - - name: Run tests before publishing - run: uv run pytest tests/ --ignore=tests/test_parity.py --cov=banos --cov-fail-under=80 - - name: Build package run: uv build From 065fcfeb207ac3dc6683458152634980d362c90e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:16:32 +0100 Subject: [PATCH 12/25] chore: align ruff versions, lint the quickstart and add mypy Pre-commit runs ruff v0.15.7, the version in uv.lock and CI; v0.4.4 would rewrite imports and formatting that CI considers clean. Its lint hook id becomes ruff-check, and ruff-format leaves the notebooks alone. CI lints and format-checks demo/quickstart.py too; its import order is fixed. In the two notebooks, the ruff findings are fixed in the code cells (import order, a variable named l, a trailing space); outputs, cell ids and metadata are unchanged. mypy, pandas-stubs and scipy-stubs join the dev group; the lock only gains packages, no existing version moves. The stubs are bounded to the pandas 3.0 and scipy 1.17 of the lock. A new CI job, Type check (mypy), runs mypy on src/. Its two errors are fixed without changing behavior: is_logical_column returns a bool rather than a numpy bool, and drop_non_logical_columns is annotated with list[Hashable], the column labels it returns. src/banos/py.typed tells the type checkers of users that the package is typed. --- .github/workflows/ci.yml | 23 +- .pre-commit-config.yaml | 5 +- demo/quickstart.py | 3 +- demo/tutorial_calms21_task1.ipynb | 9 +- demo/tutorial_python.ipynb | 6 +- pyproject.toml | 4 + src/banos/core/preprocessing.py | 5 +- src/banos/py.typed | 0 uv.lock | 336 +++++++++++++++++++++++++++++- 9 files changed, 371 insertions(+), 20 deletions(-) create mode 100644 src/banos/py.typed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d17b11..e09302f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,10 +28,29 @@ jobs: run: uv sync --locked --dev - name: Run ruff check - run: uv run ruff check src/ tests/ + run: uv run ruff check src/ tests/ demo/quickstart.py - name: Run ruff format check - run: uv run ruff format --check src/ tests/ + run: uv run ruff format --check src/ tests/ demo/quickstart.py + + typecheck: + name: Type check (mypy) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + python-version: "3.11" + + - name: Install dependencies + run: uv sync --locked --dev + + - name: Run mypy + run: uv run mypy src/ test: name: Test (Python ${{ matrix.python-version }}) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2bca525..53dc5f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,11 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.4 + rev: v0.15.7 # Same ruff version as uv.lock (and CI) hooks: - - id: ruff + - id: ruff-check args: [--fix] - id: ruff-format + types_or: [python, pyi] # Notebooks are not reformatted - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 hooks: diff --git a/demo/quickstart.py b/demo/quickstart.py index a59f29f..5912de3 100644 --- a/demo/quickstart.py +++ b/demo/quickstart.py @@ -7,9 +7,10 @@ from pathlib import Path -import banos import pandas as pd +import banos + DATASET = Path(__file__).parent.parent / "data" / "dataset_human_vs_human" diff --git a/demo/tutorial_calms21_task1.ipynb b/demo/tutorial_calms21_task1.ipynb index baa7e92..c8df4f4 100644 --- a/demo/tutorial_calms21_task1.ipynb +++ b/demo/tutorial_calms21_task1.ipynb @@ -68,11 +68,11 @@ } ], "source": [ + "import math\n", "import sys\n", + "import urllib.request\n", "import zipfile\n", "from pathlib import Path\n", - "import math\n", - "import urllib.request\n", "\n", "import numpy as np\n", "import pandas as pd\n", @@ -84,9 +84,10 @@ " print(\"TensorFlow not found. Install with: pip install banos[calms21]\")\n", " sys.exit(1)\n", "\n", - "import banos\n", "from sklearn.metrics import f1_score\n", "\n", + "import banos\n", + "\n", "print(f\"BANOS {banos.__version__}\")\n", "\n", "\n", @@ -288,7 +289,7 @@ "for rec in recordings[:3]:\n", " n = dataset[rec][\"gt\"].shape[0]\n", " labels, counts = np.unique(dataset[rec][\"gt\"], return_counts=True)\n", - " beh_str = \" \".join(f\"{list(VOCAB)[l]}={int(c)}\" for l, c in zip(labels, counts))\n", + " beh_str = \" \".join(f\"{list(VOCAB)[label]}={int(c)}\" for label, c in zip(labels, counts))\n", " print(f\" {rec.split('/')[-1]}: {n} frames | {beh_str}\")\n", "print(\" ...\")" ] diff --git a/demo/tutorial_python.ipynb b/demo/tutorial_python.ipynb index 627be15..53532c1 100644 --- a/demo/tutorial_python.ipynb +++ b/demo/tutorial_python.ipynb @@ -48,12 +48,12 @@ } ], "source": [ - "from pathlib import Path\n", "import math\n", + "from pathlib import Path\n", "\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "\n", "import banos\n", @@ -329,7 +329,7 @@ " per_rec_f1.append(np.nanmean(f1s))\n", "\n", "frame_f1_score = np.nanmean(per_rec_f1)\n", - "print(f\"Frame-based F1: {frame_f1_score:.3f}\") " + "print(f\"Frame-based F1: {frame_f1_score:.3f}\")" ] }, { diff --git a/pyproject.toml b/pyproject.toml index 930cf4f..a9654df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,4 +78,8 @@ dev = [ "pytest-cov>=4.1", "pre-commit>=3.5", "ruff>=0.4", + "mypy>=2.4.0", + # Stubs of the pandas and scipy versions in uv.lock (3.0 and 1.17) + "pandas-stubs>=3.0.5.260914,<3.1", + "scipy-stubs>=1.17.1.5,<1.18", ] diff --git a/src/banos/core/preprocessing.py b/src/banos/core/preprocessing.py index bd6fba8..fbd32ea 100644 --- a/src/banos/core/preprocessing.py +++ b/src/banos/core/preprocessing.py @@ -4,16 +4,17 @@ from __future__ import annotations import warnings +from collections.abc import Hashable import pandas as pd def is_logical_column(column: pd.Series) -> bool: """Return True if column contains only 0s and 1s.""" - return column.isin([0, 1]).all() + return bool(column.isin([0, 1]).all()) -def drop_non_logical_columns(matrix: pd.DataFrame) -> tuple[pd.DataFrame, list[str]]: +def drop_non_logical_columns(matrix: pd.DataFrame) -> tuple[pd.DataFrame, list[Hashable]]: """Drop columns that do not contain only binary (0/1) values. Returns diff --git a/src/banos/py.typed b/src/banos/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/uv.lock b/uv.lock index cd3d466..015893d 100644 --- a/uv.lock +++ b/uv.lock @@ -2,9 +2,12 @@ version = 1 revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.12.*' and sys_platform == 'win32'", "python_full_version < '3.12' and sys_platform == 'win32'", @@ -103,6 +106,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl", hash = "sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205", size = 68797, upload-time = "2025-10-18T17:46:45.663Z" }, ] +[[package]] +name = "ast-serialize" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/d5/5dc1ee8a80c3b20a0b63c61b43ba6bc53517faecd1def47bcd752d22268e/ast_serialize-0.13.1.tar.gz", hash = "sha256:401cc84e8888d4d76f2c8d296fa006ed1a5ff993b6aff90dd777c03973efd025", size = 955137, upload-time = "2026-10-17T00:04:17.183Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/a8/fbd136b9f70c59e75f59a8e974c84102976f1201828b4f886480ba956936/ast_serialize-0.13.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:3e129d44b0cffbea24a7f40117ce49e30993c3d119e584d4817ebd6f87246456", size = 897306, upload-time = "2026-10-17T00:02:52.479Z" }, + { url = "https://files.pythonhosted.org/packages/50/3f/2ad51e9823a790e7a70fd781e9595ab3abeace27f3aefcaa1939457efd72/ast_serialize-0.13.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:523d45bab16ddaf1399c04c4fe81bbf180f81e33c2f7bcbe76be56b0f7311ab4", size = 1236082, upload-time = "2026-10-17T00:02:54.82Z" }, + { url = "https://files.pythonhosted.org/packages/c0/5c/8e959db5cc2237f9d4c560a8e2253c98f9746883fb4774651e4cda458901/ast_serialize-0.13.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:638333fb8ae72c62753f9e794fda20290883ddc65b0d85e26e08f8bb227567ad", size = 1221242, upload-time = "2026-10-17T00:02:56.395Z" }, + { url = "https://files.pythonhosted.org/packages/d6/05/ab1c56c3b7adcac88764bba609502d6b4989e449b2dd60eb478e1d20e369/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc34c670747d85d4232834665e247009413eaebe47c69eaf60a5c439aaecaed9", size = 1282105, upload-time = "2026-10-17T00:02:58.275Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f4/41981fcd2ddffe0760e5326be9dcbd2eda9478a8430a637e5123bd8fcb26/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8c8ebd419811c435812d242fb2a173e57dfede198c9ac9a0a584666ebc9b9331", size = 1287905, upload-time = "2026-10-17T00:03:00.068Z" }, + { url = "https://files.pythonhosted.org/packages/6a/43/b783addc5681153707121991b7b47892f7e9c8ba58a0d03b71a3b023ba47/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8899c038b1c15ae8b98e379057fb3ef398ceb0b21d1737a8156dd5a0923b7e51", size = 1541806, upload-time = "2026-10-17T00:03:01.503Z" }, + { url = "https://files.pythonhosted.org/packages/47/10/969db095038ee15270f65ad20d1129c5e4da605bf7e93b70ae640f1c8a9f/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e14c4de98f965647a74e6eaf37e20207124e45451ccd36e84a6f6ebab45e0372", size = 1309946, upload-time = "2026-10-17T00:03:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/06/dd/7c623d2fb0b868948952d1787e629a43b866f7d4caefd5445e20aefacec4/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f3e8a3eff69af640db985360fef35073fb46956dfc3e5c614cf00920d2746dd", size = 1302773, upload-time = "2026-10-17T00:03:04.378Z" }, + { url = "https://files.pythonhosted.org/packages/d4/c2/4d49469110a19a44fc6269f4fd59679bcf84df068f185f89b73bee4da19b/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:75d7750fe98f6f35a96c43aba8d98a2e623e22c778a0c17ab9ede92eb10f5d82", size = 1299579, upload-time = "2026-10-17T00:03:05.88Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9a/2d381f1f262dda15affc8ad643fba1619eb07e7d9f8bd904deffef183006/ast_serialize-0.13.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43e8ddd63e80fc280c2f790157386f27f9b9f6ccb9c4f8e803b863cbfb085769", size = 1355969, upload-time = "2026-10-17T00:03:07.321Z" }, + { url = "https://files.pythonhosted.org/packages/9d/9c/2266794e6c25fa9ce51ce04a9bf4b7e330e3ac9351ba3fa14e487678acdb/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9221e09496fb942901993c1812ee87950cc0ecbb514c7c0f1f5fa6d78f701ed8", size = 1460227, upload-time = "2026-10-17T00:03:08.733Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d6/14deb28e5d8f328f626a0778ea2cf98fe908f2bbcccd804cb6f449ee1782/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7a5c38bdb93669c596f94fb7204dd6ebe4fadb19058ac9a2faf2948c2a175f96", size = 1562691, upload-time = "2026-10-17T00:03:10.25Z" }, + { url = "https://files.pythonhosted.org/packages/f5/19/7a28c0e5ba54f0d3e054f847a2a549b748cb179ab433321a8dc1b39e398d/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:acf4e9f055d75058ab01536da330648746f2f82b7697a85230547f2d5ebe895c", size = 1558669, upload-time = "2026-10-17T00:03:11.799Z" }, + { url = "https://files.pythonhosted.org/packages/18/9a/f9e2690eab58217f1d395a964a78efa7eaf813a1e38be8dfd9a8d45d0840/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:87427913c325adc811e81e7c9a757b061f47ef1e29114ac479aa0953a2687a98", size = 1665071, upload-time = "2026-10-17T00:03:13.284Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f3/ef5b159a0d7161b404f05e7ec21cf7735ee7edc10c000cd2cbbc7bb6b69a/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:ad88921f95b92bc144d9ad059a0e16f8c1ae69f5239f2d3c6536fd2774f73dfe", size = 1473364, upload-time = "2026-10-17T00:03:14.563Z" }, + { url = "https://files.pythonhosted.org/packages/1d/87/6e8ac4a865124a12cc4f915bf1bcfac3bba426cd5f58d3712a873ee61012/ast_serialize-0.13.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f8b17e2921e56621bcb46792bd3e0b82ca3b85de78d5936e7b8174ba1758b0aa", size = 1500566, upload-time = "2026-10-17T00:03:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/f5/1e/29155f1621433008013ad5c940a03d41a676be0680c41527040472d37583/ast_serialize-0.13.1-cp314-cp314t-win32.whl", hash = "sha256:2f9a9ce8b0994f757b941f7f7fe384a6666b4162b941ebc5c248081baea18e47", size = 1128674, upload-time = "2026-10-17T00:03:17.481Z" }, + { url = "https://files.pythonhosted.org/packages/bb/64/2506da65143ea138de5822e9173e0b72320a2f70e23bd723f044033175f1/ast_serialize-0.13.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5d99910e6b9a9b9360bcca2ae99aee36cc912ce058df2a57d1c33099b6cb2593", size = 1163306, upload-time = "2026-10-17T00:03:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8f/c7cb22638899b0cf15908809c270560e46395e65604bc59e4b623ab9f405/ast_serialize-0.13.1-cp314-cp314t-win_arm64.whl", hash = "sha256:a789cbe14d0a0e9ec9b32a456e5abc81ad8799479c3cfd88a29e3442262aebe6", size = 1133775, upload-time = "2026-10-17T00:03:20.377Z" }, + { url = "https://files.pythonhosted.org/packages/2b/e2/b0c140fe50fa39dd249e382f7733af6f816c90021fe7959a0b026f86237f/ast_serialize-0.13.1-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:23d768369b89d551077dc173da6892fad00443338417523b6732f4e8ba4a7ed5", size = 1236160, upload-time = "2026-10-17T00:03:21.945Z" }, + { url = "https://files.pythonhosted.org/packages/23/c4/a8a2ca21e3943555b48221f8fba546877a7ad39efaf6505506524967d7c4/ast_serialize-0.13.1-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:68f8829d114dab6c9db4a8bbb8d0e342958faee1b5e15a29022678df639960f8", size = 1222214, upload-time = "2026-10-17T00:03:23.462Z" }, + { url = "https://files.pythonhosted.org/packages/15/19/2f58f2ba64e4799b39de85c140aef09a03aa6ea53be75dd292a9d9b9d8ae/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e385b17d5bde892a6184003a9aff3518387908b73f8f5ea539313c07c8c0ede", size = 1282979, upload-time = "2026-10-17T00:03:24.695Z" }, + { url = "https://files.pythonhosted.org/packages/b1/af/ca6087fe84210c1ccd43e3182917bb70649fa1e69745a90c1d59ba08fea8/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:02686b281ec9b8e9614f6f0db27e12e4cac515f336c5d05b678bc9eb73ff2af5", size = 1288596, upload-time = "2026-10-17T00:03:26.04Z" }, + { url = "https://files.pythonhosted.org/packages/b0/2c/4ef964c763329a0347125e745ec0451ba4fbaceec9efe2ac729b7cd55f17/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a8ab2c2585b2e22e48cd4aff2136fce4eab4ced569b85f150ef9a5a5d6f34d49", size = 1545540, upload-time = "2026-10-17T00:03:27.52Z" }, + { url = "https://files.pythonhosted.org/packages/bb/85/aa961dbb75813cfa94de0ce9a16a9510c226b95f604c405421e727f44bb0/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c652d9f5d52d8b81eff3ff2c95c9dd0f281de45f6ad5d3fa95527550c396454a", size = 1309374, upload-time = "2026-10-17T00:03:29.091Z" }, + { url = "https://files.pythonhosted.org/packages/b4/e6/f2b67c327d9f9533e41791804403ccc2dc457821195a2e51f0b587a90b90/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:121e1d08d58b75c15491cc3003609cbad71ac2b0016043c36d30a6b2032b6605", size = 1303466, upload-time = "2026-10-17T00:03:30.451Z" }, + { url = "https://files.pythonhosted.org/packages/73/1d/33f18b81d1bb1f5d682943ea92fda8130a6593627f12f6201668d40824fd/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:30cf89b6a6844542d116c758725d13de5b97b9410a51ee577715f761270d0c1c", size = 1300576, upload-time = "2026-10-17T00:03:32.102Z" }, + { url = "https://files.pythonhosted.org/packages/25/cb/64914144925f1a4abdd9217e717b122255c38fc5ebe4342f92052f3bacbc/ast_serialize-0.13.1-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:732aaae9e498c973fc63fd7f308009b57c4afa6ff59d0b674d894e57d0bca01a", size = 1355960, upload-time = "2026-10-17T00:03:33.676Z" }, + { url = "https://files.pythonhosted.org/packages/ee/80/bd29379fab2bce19adc0bf9177002b11b23dfe8d8fecf409998b458b4624/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:d9c1a920b2aa0c1e016bed7721689717034ca0dba64ac3a3ff243b003d4e8dd8", size = 1459779, upload-time = "2026-10-17T00:03:35.09Z" }, + { url = "https://files.pythonhosted.org/packages/83/89/012dbe733de05ffdf8e40ca0f9fa5d417cb16ec93a74463c57c001914198/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:847cad4c22953823f5e76d0b126c6f9da670cfa45b1df41b9324e22189d77611", size = 1563399, upload-time = "2026-10-17T00:03:37.285Z" }, + { url = "https://files.pythonhosted.org/packages/69/37/ee4bcb0358bee38744eb0f18560446572d095fe3bf6b63d6f739d08a839c/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:aca07de3ca8e3d9f2b716c3c31f4f384fe4bd2d03e09912417cff7a3e08036f7", size = 1558808, upload-time = "2026-10-17T00:03:38.711Z" }, + { url = "https://files.pythonhosted.org/packages/54/1c/ff44c5355251d61ef22934c4430bb31a61de4781d25a992154aac9b8c82a/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:5db12d843d931b9d1f1519ce66b9cbfe72d8563b75c1204d255322254d1964dd", size = 1666324, upload-time = "2026-10-17T00:03:40.376Z" }, + { url = "https://files.pythonhosted.org/packages/c8/bb/6da4bfc8aa3403e6447607131bf3463acaf954d990c6c8b6787c1902ddfe/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:fb8021f4dc3296de18737e3b8b34994411da725374d74a0b9eb32a738f2a23d0", size = 1474089, upload-time = "2026-10-17T00:03:41.715Z" }, + { url = "https://files.pythonhosted.org/packages/21/97/897e33bef2eefd880e6350e375cc66bc5bcd00a139be480fd655f5b3f095/ast_serialize-0.13.1-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:0d764913374ba21df8cd618045b371ed104ce55c6276de7c59e554a4f5ba26fd", size = 1501542, upload-time = "2026-10-17T00:03:43.08Z" }, + { url = "https://files.pythonhosted.org/packages/8d/4c/821211ac83870c27e4412381b7c7838336456b40575bf840e4680b2018d9/ast_serialize-0.13.1-cp315-abi3.abi3t-win32.whl", hash = "sha256:2bad302ae2427454ef9dab71bd5c3ad67188860c39d36daee38f72031edb39c4", size = 1126118, upload-time = "2026-10-17T00:03:44.616Z" }, + { url = "https://files.pythonhosted.org/packages/1f/86/2b2d5e6c4fa352874a80618e742ffd32d249dee6c119f4233b51afc6b8f4/ast_serialize-0.13.1-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:032b09d86c785c3984e975378b7861f2a68ac44e22ba7b07b93a536b2524eeb8", size = 1163766, upload-time = "2026-10-17T00:03:45.986Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/d610cfcdabaa44fd62a54ca1fc57a7dba120643798b089a14914973562de/ast_serialize-0.13.1-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:929eda0c419e622ab49df3a4c45f81c57dcc10874bbad55d0470a4a03324dac3", size = 1134043, upload-time = "2026-10-17T00:03:48.453Z" }, + { url = "https://files.pythonhosted.org/packages/f0/89/0e30f9eddf7d44ccf9d25979030754fd0457af61fa4279acb8c293a04f97/ast_serialize-0.13.1-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:bf53a3ba1cbe45399fc1e4ccd684b8f2a11cca450f89706b79122f478534c6f5", size = 897546, upload-time = "2026-10-17T00:03:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/81/92/d705be88ff452b9ba0088b554a0cbd308fd01879da092b33fd2ad7d1aa60/ast_serialize-0.13.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:4bcf1d5f4eefecaa325a34ecf079b2741189e7d0821523f92cb343a389a67489", size = 1242416, upload-time = "2026-10-17T00:03:51.521Z" }, + { url = "https://files.pythonhosted.org/packages/60/c2/98370845a7f35cb8a25782e78284b48e01bf3128339881d48d613f304f8f/ast_serialize-0.13.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1b158f1c9e3a0d386913924c57995d6df78224868a4d0f215bdd41587c97b34a", size = 1229729, upload-time = "2026-10-17T00:03:52.825Z" }, + { url = "https://files.pythonhosted.org/packages/d8/8b/2d7ffed76addd9d3fcf37fb171400942735c8912264c763561a85407f6a8/ast_serialize-0.13.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77a6201b69ce42e919f2fd9a559138d5410b5d8afff7c3a6b1f500b0481ff7d8", size = 1292255, upload-time = "2026-10-17T00:03:54.148Z" }, + { url = "https://files.pythonhosted.org/packages/6b/2a/bb9e0e719d89cc950fadfcfcc9cb4bc7640419c4228c852f031ab8abf440/ast_serialize-0.13.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6eb0be64c9b8f1ed26637eef345abe75ea90351c92c85a8aec7838b3af971011", size = 1298501, upload-time = "2026-10-17T00:03:55.559Z" }, + { url = "https://files.pythonhosted.org/packages/7a/32/13ce07dce48226f5bd9cfd8f708ec101a8be05aa430cda4918ce31ce8c82/ast_serialize-0.13.1-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f2427bddfca208e84de700044fadf9aef2769a70bae740a1bdabb89ee9e1117", size = 1553758, upload-time = "2026-10-17T00:03:56.952Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fb/9b51d186db0c20ca28cbaaefd50f03b573c89327b9a2627820cbce22196b/ast_serialize-0.13.1-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88df960e67184a71874e5b2ee014f87d1254a0dbc1f903c7bd7b1ab7733ecaef", size = 1318258, upload-time = "2026-10-17T00:03:58.418Z" }, + { url = "https://files.pythonhosted.org/packages/c5/fa/da311559dd3dae9af5825cf21660775a929b970f72e3786af3f2cd667944/ast_serialize-0.13.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f99786e0d62082a5b121b3437f3167562133711df22128de1e9886f57f0b226", size = 1313260, upload-time = "2026-10-17T00:04:00.194Z" }, + { url = "https://files.pythonhosted.org/packages/2b/11/97a4b9e5b4776d72f47fcd9a09a497bbf68ec8c1ae074307a2c51e410d85/ast_serialize-0.13.1-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:a01b515b25a03184eff4acef25b981ae007916379682e908fc8785b342a0bf3e", size = 1309992, upload-time = "2026-10-17T00:04:01.575Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c4/62f85121a80cd451ef04f550cf0f7ea0ad9ff46cc4e9a5b9ae439b6527b1/ast_serialize-0.13.1-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:29422d7bcd9e7f39411571c36cec45ce8a7a9b3ad4b2e7aa64497c15a9e397f6", size = 1362715, upload-time = "2026-10-17T00:04:02.881Z" }, + { url = "https://files.pythonhosted.org/packages/c1/fa/c5f4a545f72a7681e3295bab6e3600ee0862ec4391bed961fe9f735b67e0/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6ffb0f548a7dc126067fdfb7b2325ab5dfcdeecc29faf5e0f0461c96d6ebfa3d", size = 1469931, upload-time = "2026-10-17T00:04:04.181Z" }, + { url = "https://files.pythonhosted.org/packages/38/a9/65aed4f8fa7a001a7cfdc03e68e844517ab041ba6fb614b7d78a1d6ac056/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:2876444a5f87a7c76f371af3c07c2fda563a08bc8af52568df812e8aba4bf568", size = 1570945, upload-time = "2026-10-17T00:04:05.778Z" }, + { url = "https://files.pythonhosted.org/packages/fb/8c/44b2da991cede79bfbf52089f33a5f0168945466b3d9cd38b5dcf08e7130/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:885a62ec6481ce58ab24c6357360d7a82023b62dd7e5e7ceccf59e9662a3272b", size = 1569978, upload-time = "2026-10-17T00:04:07.347Z" }, + { url = "https://files.pythonhosted.org/packages/46/e4/109b2a5beca700237767d5add920c2f8e1ba9bd5799b0faf143d0e9cf4ef/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:2015fd0165e463aac3b1aadc6a1fa1e5820c14ca950821f4fe02f8a3f6a388d6", size = 1676082, upload-time = "2026-10-17T00:04:08.7Z" }, + { url = "https://files.pythonhosted.org/packages/73/e0/7df70f6dbb11b5762da5416318f08316a1c51348e0d69ec79ca5f98f836a/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:eb41e1906502ac80d24ff343f45008552a3f5b5158607959fe6a07dbfc6eb037", size = 1483207, upload-time = "2026-10-17T00:04:10.162Z" }, + { url = "https://files.pythonhosted.org/packages/1f/bc/523aa1c08bd69b5654621beb3dc1a9035715a4a63762fb8d219d6fb3eb81/ast_serialize-0.13.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ca9acc67f796aa2554fa3037dccc022a4b08b38b2ad153edfac9c9bb7cb1f3c6", size = 1512140, upload-time = "2026-10-17T00:04:11.589Z" }, + { url = "https://files.pythonhosted.org/packages/66/83/6607060659998f00afdea0d8375c1e6e2b7d515ab905090d9a5a02f48851/ast_serialize-0.13.1-cp39-abi3-win32.whl", hash = "sha256:4f98da562e5901d69c7dd30885594c5438f859edf354e9ac7517d188fb026c3a", size = 1131889, upload-time = "2026-10-17T00:04:12.953Z" }, + { url = "https://files.pythonhosted.org/packages/9f/6f/bafd6490fe7b355627fabe95f4417e3d02da93102c8552550eb9a0774e1e/ast_serialize-0.13.1-cp39-abi3-win_amd64.whl", hash = "sha256:ddf7cbc9c8abf70102c4a088efe154bb86bfb825b0a10e556d2b15972cf99bd7", size = 1171153, upload-time = "2026-10-17T00:04:14.362Z" }, + { url = "https://files.pythonhosted.org/packages/11/25/4235f4b375a67a37ac595bf478a5407a744eaf7b64113c1c3a5432b8d954/ast_serialize-0.13.1-cp39-abi3-win_arm64.whl", hash = "sha256:48f11d96c9ec71119612f8e79a8c55c12f2b136716d68347433c561d27a25f43", size = 1139035, upload-time = "2026-10-17T00:04:15.767Z" }, +] + [[package]] name = "asttokens" version = "3.0.1" @@ -183,11 +250,14 @@ notebook = [ [package.dev-dependencies] dev = [ + { name = "mypy" }, { name = "nbformat" }, + { name = "pandas-stubs" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "ruff" }, + { name = "scipy-stubs" }, ] [package.metadata] @@ -211,11 +281,14 @@ provides-extras = ["dev", "notebook", "calms21"] [package.metadata.requires-dev] dev = [ + { name = "mypy", specifier = ">=2.4.0" }, { name = "nbformat", specifier = ">=5.10.4" }, + { name = "pandas-stubs", specifier = ">=3.0.5.260914,<3.1" }, { name = "pre-commit", specifier = ">=3.5" }, { name = "pytest", specifier = ">=7.4" }, { name = "pytest-cov", specifier = ">=4.1" }, { name = "ruff", specifier = ">=0.4" }, + { name = "scipy-stubs", specifier = ">=1.17.1.5,<1.18" }, ] [[package]] @@ -1002,9 +1075,12 @@ name = "ipython" version = "9.11.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and sys_platform == 'win32'", - "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.15' and sys_platform == 'win32'", + "python_full_version == '3.14.*' and sys_platform == 'win32'", + "python_full_version >= '3.15' and sys_platform == 'emscripten'", + "python_full_version == '3.14.*' and sys_platform == 'emscripten'", + "python_full_version >= '3.15' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.13.*' and sys_platform == 'win32'", "python_full_version == '3.12.*' and sys_platform == 'win32'", "python_full_version == '3.13.*' and sys_platform == 'emscripten'", @@ -1526,6 +1602,122 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/71/cf/e01dc4cc79779cd82d77888a88ae2fa424d93b445ad4f6c02bfc18335b70/libclang-18.1.1-py2.py3-none-win_arm64.whl", hash = "sha256:3f0e1f49f04d3cd198985fea0511576b0aee16f9ff0e0f0cad7f9c57ec3c20e8", size = 22361112, upload-time = "2024-03-17T16:42:59.565Z" }, ] +[[package]] +name = "librt" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/04/f5/9dc696772d241814bacac7880bac32f2930b5a6ebc1f85317b83161a011c/librt-0.16.0.tar.gz", hash = "sha256:ac38d6d8d66bf3d744148dbbc0b8e193e195a51e364ed55e224631f5721891fc", size = 219839, upload-time = "2026-09-29T00:55:32.891Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/0e/b04fd2d76b6a78a02286938251b2c119fb36ddece92c76cb4886ef7d44b8/librt-0.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fe4372c52d4849096c6cc1cda2817d293ec51440c890474ed59ef38d46556f18", size = 148058, upload-time = "2026-09-29T00:44:40.903Z" }, + { url = "https://files.pythonhosted.org/packages/a3/fc/e1ed24aadb32d38603c0c07f579d739eb4d90e64e90d525b024e4a999b4d/librt-0.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c72c5295a84bd249526da9bdca38f2e176d15c31c13bb0063c5053f4ca023421", size = 153073, upload-time = "2026-09-29T00:44:42.165Z" }, + { url = "https://files.pythonhosted.org/packages/be/38/133ab655409420e0c34a9a9836a7f4e5ae75fba758d7fe177099130d58ba/librt-0.16.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be56ba9c884143495b517f23fe794ae367d58cd89ea0fdd6d437e3c024a87f9f", size = 493052, upload-time = "2026-09-29T00:44:44.043Z" }, + { url = "https://files.pythonhosted.org/packages/94/42/aaa663bc9421f5bf7bbbe9d3316d7219749698ec7688ac3b519770a89c04/librt-0.16.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:ef46c1a29ffb8c72e882e22618ec618778eacd0578fb22c6e7cf9c11d15f357b", size = 485506, upload-time = "2026-09-29T00:44:45.479Z" }, + { url = "https://files.pythonhosted.org/packages/7b/59/83a6eb5087ba8b45b083859e2085256963e06d4c2531944cdd962670bfb5/librt-0.16.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d3c94211ee0c4f8d649ec06b7c115c0ec4eadb873a0e3154ca15cef3f814b071", size = 498708, upload-time = "2026-09-29T00:44:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/a0/e4/15a8d62b2bc12935080ec7277e2e0b9b233047781eb784a51d3e25af669a/librt-0.16.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:94aed6a8308818b91677957d1bd03188869cd7aeb23c5dba7912a6c0402f7602", size = 515981, upload-time = "2026-09-29T00:44:48.316Z" }, + { url = "https://files.pythonhosted.org/packages/46/4f/287cc282ac5dd815204c0779e79401bb0a568c3a1335fb319bf0838fb566/librt-0.16.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:349c0bcb87ebd07481b6ff781e25cdc699723dbe2212e57dabb27f7a13b7b87d", size = 508670, upload-time = "2026-09-29T00:44:49.726Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5b/ab7292add898ade0d2b5b75d32a923aae315861671aa5c39f5491309f7ae/librt-0.16.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:001bfd59a7d45b17e3e75f2a8c6405280b35e7b84471792778e718c4f368950e", size = 530393, upload-time = "2026-09-29T00:44:51.356Z" }, + { url = "https://files.pythonhosted.org/packages/7a/9e/8e6bcf027d4bcbc16a4e864b3d158a1d3e5f55ffa53af0238020cad89271/librt-0.16.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:28e038895b998d7a0c7798922ce8a1dc157675df5cf1c9ef0aca809ed804b7a1", size = 534302, upload-time = "2026-09-29T00:44:52.893Z" }, + { url = "https://files.pythonhosted.org/packages/96/a4/8c5ab4f3b3a8e905ff8305d76976a2e666f52fe049edd398aed2402beb36/librt-0.16.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:0dbe4096a7ecc00fa835d24510ad8545a4efef738dac96e0e63516783ccde905", size = 536150, upload-time = "2026-09-29T00:44:54.421Z" }, + { url = "https://files.pythonhosted.org/packages/ea/5e/7c4bc267e3cd4b8bd288de9f62d61fff203829c9cc649ea2aa12e9218cf0/librt-0.16.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5cd5b092441053364af968ea12084692cb9d4a22f3ce9524e377880bf028761e", size = 514316, upload-time = "2026-09-29T00:44:55.998Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/c8bc6b50f43c73819334233f5a3259c81216742cef56d2140c3dd377c3f0/librt-0.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3ddeb3c9dedb461bb457c6c7d9aa7fbf35329da313d1a7543d00c8d0f3473c96", size = 557731, upload-time = "2026-09-29T00:44:57.581Z" }, + { url = "https://files.pythonhosted.org/packages/93/fb/e241f91bc8de6c7ac1c6a20ea6dd381bc52312b44d711ffcd97140db1e57/librt-0.16.0-cp311-cp311-win32.whl", hash = "sha256:e05108e0849966f53a8d2d3112a7af881d0efaa479bc735bba91108f9f2350a7", size = 104922, upload-time = "2026-09-29T00:44:59.078Z" }, + { url = "https://files.pythonhosted.org/packages/d1/38/c892383d2bb1a6a80b2ec8ede84970a265a792c56614a06282bf3008517c/librt-0.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:5f49cff01bd608ef7d97104cb035c75455e79c2d70bf4a506cf773338ac1860d", size = 125093, upload-time = "2026-09-29T00:49:01.621Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/8bbde1ed0b21480163617648f638fdaec9374cf5e9c846ebcee34ec02a42/librt-0.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:1d28ae980ae2218f9c5b95d191e947296f918c9bf0b400d467a9430275bbe678", size = 115910, upload-time = "2026-09-29T00:52:40.131Z" }, + { url = "https://files.pythonhosted.org/packages/ad/76/bbdaeb87b7c47b5c7343e90222b9bfa4e4a8a83f647e02933ad0129225b1/librt-0.16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe52bf4641069e7978a14253b036cb9002def1926317e710f2e249f8a8c47742", size = 151028, upload-time = "2026-09-29T00:52:41.508Z" }, + { url = "https://files.pythonhosted.org/packages/fd/0c/ab8ed3dab0085931aec4a792c7eaac8dc6c5ff4691fda3a5360d9d8a9cd2/librt-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5bcc2c4726ced915b00de0c9856a4eeabfb3fddb93e10e0b8f735b7709358b6d", size = 155283, upload-time = "2026-09-29T00:52:42.873Z" }, + { url = "https://files.pythonhosted.org/packages/eb/36/494e79d460c80c1f030661e8287c9eca5e1ad652dc2b2180b6cd42abce0a/librt-0.16.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff7baa55f8e7c69851419e50a666015d02a74198716fd45c0125a2112e0a389f", size = 503185, upload-time = "2026-09-29T00:52:44.638Z" }, + { url = "https://files.pythonhosted.org/packages/9b/34/a8464038dd9db6e4381fa2b6eb73dc9a50888d77102c4c139304ac35cddc/librt-0.16.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:b95d5d92ab83d39e760a52091bb1baba664f3a2351e39b1e16801e5747c2f0e9", size = 496738, upload-time = "2026-09-29T00:52:46.441Z" }, + { url = "https://files.pythonhosted.org/packages/6a/53/e0e5e334ef0c6ed27039d323819368b9ef6712be87d55ee2bf9799398afd/librt-0.16.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b6d085d70bce51d43c5c7c36d63490770180d8779e71c49305c87b4213918de7", size = 513818, upload-time = "2026-09-29T00:52:48.068Z" }, + { url = "https://files.pythonhosted.org/packages/ad/f7/7ce72cbf19d0addd05090b339152fd0548a02562c2866a603e6e3b3da2df/librt-0.16.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36e53948e99bbe3ffea257124cfcae1cfb01831555c9a9c903c9f9a72db7fd07", size = 531952, upload-time = "2026-09-29T00:52:49.816Z" }, + { url = "https://files.pythonhosted.org/packages/83/22/0b1bcb6a8e723c8b4fd60dfc8ae8ec6461c54073fbc8685efeb8d900d407/librt-0.16.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:54d11f726aae9df5a6ffbbf0a03a52449bbac84a53ef03669cb41cdfd4ae41bf", size = 524465, upload-time = "2026-09-29T00:52:51.354Z" }, + { url = "https://files.pythonhosted.org/packages/64/2e/e9c23b8b9df1813da1be205deca9606beb7ddd033972246cd426d05374a0/librt-0.16.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4323193ac0cd025f85af531df8ba91bf24d1973b401697347a6282e8fd3fcf5e", size = 543191, upload-time = "2026-09-29T00:52:53.284Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e5/6a8b21b342c03ed7e230fa3afbfd2edc58e6e87ef1f0d11fa2b9a748c252/librt-0.16.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e42f8e098b9c5396fefa05fb1cc7e33b0e08fc51da106b5de4a45fd22aac6743", size = 546979, upload-time = "2026-09-29T00:52:54.932Z" }, + { url = "https://files.pythonhosted.org/packages/84/9e/b5129023eced1be01e01c22757f53be551d463b1bb7264f787927404c1d6/librt-0.16.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:39ec1d5a14e37baf1450a6cabf03fe552340808bf1ad9d71824ab90117716459", size = 555317, upload-time = "2026-09-29T00:52:56.869Z" }, + { url = "https://files.pythonhosted.org/packages/71/89/28bba5938c725fe91f06bf93f7fa6c6b150229df53a87b454b0d5c2a796e/librt-0.16.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d1aabe3925cbb4a08d15b7b20ba4011b53019da0c4173a25155139b7b1baed65", size = 535846, upload-time = "2026-09-29T00:52:58.475Z" }, + { url = "https://files.pythonhosted.org/packages/22/92/63773026614f888c5d4e370e395ce42ca604b89f70b3acdfedbf94851b80/librt-0.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:300c3ffdc459f4a779a8411ecb188e3ac0b1ff3a3a7b099642555dedae06c69b", size = 573700, upload-time = "2026-09-29T00:53:00.127Z" }, + { url = "https://files.pythonhosted.org/packages/66/8f/347d4677eefb57cd9f8e01d95a1dcee9a91b4e44e664173c32ff6f3752e5/librt-0.16.0-cp312-cp312-win32.whl", hash = "sha256:c17194318e4c0c0348b36f36c2ec7534436fe0a4c15582403162a4f08c80797a", size = 106158, upload-time = "2026-09-29T00:53:02.031Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2c/5193dc81127cd5ddfad031391b046bf32dda219b38463ab872407ca30646/librt-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:25a58a19ea8d83b68209f04912df765e9260635ef77646542ed4b4abe6bc7940", size = 126176, upload-time = "2026-09-29T00:53:03.445Z" }, + { url = "https://files.pythonhosted.org/packages/ff/3d/9668a400c8dd81d162eba38b33fa49fa6205f1a64493570a13fbb815c3ee/librt-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:f7be7cf555bc30ec12622e9447299cc4a9b8ff307548b634794353db0c2065dc", size = 116572, upload-time = "2026-09-29T00:53:04.815Z" }, + { url = "https://files.pythonhosted.org/packages/46/cd/ae5e0e9dba45d1399aa04a5395bcc0bead40d9fa06dc903634a7b4d7473d/librt-0.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c5e6144e68b577f157519f2ba88ca20e3ed61c29b00e5cdfa76cd2d45acf059a", size = 151036, upload-time = "2026-09-29T00:53:06.284Z" }, + { url = "https://files.pythonhosted.org/packages/41/5a/48a16e323c5f9447a94cce7b59babf60fa04e62c3365ecf060c77ed8b320/librt-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:33f41443a1f4e1f099331b3d8120e409fbff84b9760bc1cc9ea496f37ddaa5cc", size = 155211, upload-time = "2026-09-29T00:53:07.72Z" }, + { url = "https://files.pythonhosted.org/packages/3f/29/0f59299eb4251a409b2e690ad4b7d9f8a676db829d7817ec961f32b44f7e/librt-0.16.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7e510b7770bee609617a3374a96548eb114cae048023e3f049ee449e7ff2db32", size = 502587, upload-time = "2026-09-29T00:53:09.191Z" }, + { url = "https://files.pythonhosted.org/packages/de/ba/d6fb4ef8d1537c396079d72289f16be7cd35a366e5065c51253fea2760b6/librt-0.16.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:efc49c462d4516b8a58b00b490078fa64689fd1fe66970cc190131d7afb8027e", size = 496146, upload-time = "2026-09-29T00:53:11.016Z" }, + { url = "https://files.pythonhosted.org/packages/52/fc/8c50dd4d7cc97c0ee8f252c8a3104980f234391cf1519b554e8b9de08b60/librt-0.16.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:92caf82ebef5e12d21c72242b70d1e92536f1711cf2a727a4c276de4b4469087", size = 513336, upload-time = "2026-09-29T00:53:12.655Z" }, + { url = "https://files.pythonhosted.org/packages/a9/59/16c409c56f708eda2db9a0553662845d45e3871c77d70a240dae3f3bdc56/librt-0.16.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17bac7f7a16b328fff77e440287693eb017abde913595b5827ebccbc21ecd8a6", size = 531689, upload-time = "2026-09-29T00:53:14.39Z" }, + { url = "https://files.pythonhosted.org/packages/88/82/d34772a6c29d1446dcca6e64d74062efd508523ab351aa16625a4689d5cc/librt-0.16.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5b976054553670829985ed767feb78fb6bcede0175327c4844dd5c281c1be659", size = 524537, upload-time = "2026-09-29T00:53:16.064Z" }, + { url = "https://files.pythonhosted.org/packages/77/8f/24c5631313746131ccee53bc91fdc8374f9cf25e0082a1fee9c93bb98acc/librt-0.16.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0058f9d68721094105917254c72ac0569117bb7b13b9769cf45d26d89f9d21cd", size = 543231, upload-time = "2026-09-29T00:53:17.939Z" }, + { url = "https://files.pythonhosted.org/packages/f2/cb/5f8e0d41dbd8b499c2265e939c31acc9ba59845565bf99539ad1c06aebcf/librt-0.16.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:30b7beaf3f4487b7d8adef1f158b49067cb4d5a19fa7a3bf31a4e7a820e435c5", size = 546499, upload-time = "2026-09-29T00:53:19.666Z" }, + { url = "https://files.pythonhosted.org/packages/be/61/063052de441d1385f59cea4223f184bf9e5d125de1ae3239b490aa1e486e/librt-0.16.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:468df902df016a06eb0e40b0747dc8d14e47d7a38b18b63b1fb167d85cb94d63", size = 555261, upload-time = "2026-09-29T00:53:21.292Z" }, + { url = "https://files.pythonhosted.org/packages/14/11/a2ada0529372268d6401afa9d457a095b68cd7753532b6f7f33049a19b43/librt-0.16.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:aea7b1f2b125dad5de85f049136651bff256c883c65e6b9209b2da0a1ac3cdef", size = 536029, upload-time = "2026-09-29T00:53:23.07Z" }, + { url = "https://files.pythonhosted.org/packages/23/9d/5bb6d38853382986dca702fc7e06c8256d30f5fa0676d882773b744610dc/librt-0.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a8afb6557920860b7a3a596eb804cf37e09e7cf8a803db2478c202acc72d8c2e", size = 573748, upload-time = "2026-09-29T00:53:24.696Z" }, + { url = "https://files.pythonhosted.org/packages/99/f6/0025cde35ff7f607684dc775a2b2d732cce2561cec050a6ee1fb2e1fc6fe/librt-0.16.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:77c7a2b4fe2c1369e0d5aa1cade26740a7b14be32fbc9a5535d617d20065c39d", size = 79861, upload-time = "2026-09-29T00:53:26.089Z" }, + { url = "https://files.pythonhosted.org/packages/93/93/303b8592909bd583f83f02818ffbea3f7647ca1e22b1cd5465d04b8145fe/librt-0.16.0-cp313-cp313-win32.whl", hash = "sha256:02d89c813d5ff74b17df72d3a34819d132cd168e56b81bf755b809bd9e46b8c4", size = 106236, upload-time = "2026-09-29T00:53:27.43Z" }, + { url = "https://files.pythonhosted.org/packages/cf/24/80bbb463c60ed18e29cb26aba386ddb76609580e4e7160710e550587018c/librt-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:14ed6ebe3e4f85f326d7920011ad30ff49ed9334e62cf88caef9ba973d9e3a92", size = 126180, upload-time = "2026-09-29T00:53:28.7Z" }, + { url = "https://files.pythonhosted.org/packages/43/80/b1a6fbdd7da825cdd55c71aa81eb6cfa82c513c360774152eacb50b4a771/librt-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:83d4041a3d9b2fd053a8a4e1f22878b3e5833e2712956382d5c048d791454e91", size = 116557, upload-time = "2026-09-29T00:53:30.012Z" }, + { url = "https://files.pythonhosted.org/packages/1e/93/9e0cf7da129a93c3dc7f45bc3cd4a660f2aaa995aa8a6f95c2583ef41239/librt-0.16.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:931a0bb0fcac88f263e269e46eb30ba8e21402cd3c62ca40cb97034c0693fab1", size = 149844, upload-time = "2026-09-29T00:53:31.391Z" }, + { url = "https://files.pythonhosted.org/packages/8f/26/8a90d2a8f2b2e471bb486b7aec117b8ae622715ed6c39853aec48ea20073/librt-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1bc17e54e5305f8d40b7ca203671ff5a9e59c1d0f8ea0f625dcca53a3984de11", size = 154111, upload-time = "2026-09-29T00:53:32.718Z" }, + { url = "https://files.pythonhosted.org/packages/35/ce/67abb46258da4d3e42ff5b141db6f38c59357bef84c7979f183b22f924f1/librt-0.16.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:877698bf6bca5721d8be345f2fe09778e40ecadea8b58c73075f2b1a53666bf2", size = 494234, upload-time = "2026-09-29T00:53:34.466Z" }, + { url = "https://files.pythonhosted.org/packages/12/f9/ea7162414a16f8f1bbd3b493ad6d22b926c5471916e078350bdbca8c4e5f/librt-0.16.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5981c011b306781ce561e18e14230a14524a3d8109b97553666c942c18f31a96", size = 491173, upload-time = "2026-09-29T00:53:36.124Z" }, + { url = "https://files.pythonhosted.org/packages/b1/09/9b3e869060dd33f9989b80ba4fea306f6db8cecefcdfe6d7346ef0603f65/librt-0.16.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:afced3dfc17cd805ecf7a3d77996a71cf5f2c75aa66eb0c21a9930f4fc992f86", size = 505547, upload-time = "2026-09-29T00:53:37.686Z" }, + { url = "https://files.pythonhosted.org/packages/50/07/79007d2165f649ea93e08c0962d1d62c70af9cde77965255095bf9d96f9a/librt-0.16.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca8052401c55d7511dda6760719fda7618067e83535d7d0010096d216c34b667", size = 523057, upload-time = "2026-09-29T00:53:39.301Z" }, + { url = "https://files.pythonhosted.org/packages/61/0c/8fbaff66d0ba376d8864653f5acce1569bae27e89648671f26f7eca67ab8/librt-0.16.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1e511762a074005bb0aa569166779834e75e438370226930d0ce1866d4b6a33b", size = 515158, upload-time = "2026-09-29T00:53:41.012Z" }, + { url = "https://files.pythonhosted.org/packages/df/2e/23ff0dece76f07a4124413a57682efa0bbeb5765ac0122bc0955513f82ca/librt-0.16.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f1e8591bd8a5a628cd7f07954c6a1592359a878bf032957a8e9057a41d644311", size = 534183, upload-time = "2026-09-29T00:53:42.451Z" }, + { url = "https://files.pythonhosted.org/packages/26/c4/e11dea21d9a29486eba78887380374189d472734fa32cc50cb37ca44d3d0/librt-0.16.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4aaefb4ba6c07e1aeebb2795c8958148f1d6f9af3b555b53d23d766edb6d67a", size = 540663, upload-time = "2026-09-29T00:53:44.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/75/e873ae158a8b7f5359be33e7fd6c1fbe02d9a78a3e89a77de6b0e837f476/librt-0.16.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:d92db7a0f6aee44f1baee94750457e8d2d1c6ccea41842de6268d34e8dc7eddd", size = 545908, upload-time = "2026-09-29T00:53:45.812Z" }, + { url = "https://files.pythonhosted.org/packages/ba/36/8939d3f6a93e11bd9592e6fe28d2b44f1c2dc4bed6e22e72359a91e18ffb/librt-0.16.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:378dfaffb38e59c24a87cde5713cd865d51ff7383fa12947f3907f306ea1ca55", size = 523302, upload-time = "2026-09-29T00:53:47.596Z" }, + { url = "https://files.pythonhosted.org/packages/71/14/35309f44a077f0f42ade0e2e7cd88c0cea760c661af205c01ea90d3c0e1f/librt-0.16.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3e0c39bdc85370422e8b637be76eb1fd07d30967551b03e62267dd156f553152", size = 565911, upload-time = "2026-09-29T00:53:49.278Z" }, + { url = "https://files.pythonhosted.org/packages/78/0c/df6255b94967f3159ebc46f08d8e783c12da6ee269ddb74b2efed63c640c/librt-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:1b384b90ab79a7bc30b566895809a636e0666f21f3cf12b54823d025b7e83839", size = 79119, upload-time = "2026-09-29T00:53:50.938Z" }, + { url = "https://files.pythonhosted.org/packages/6b/44/d30d5a5461378c9d33f36736c6791c3b4c4ba4b1ffe0da7350aedcb2c9a0/librt-0.16.0-cp314-cp314-win32.whl", hash = "sha256:52327da75a94012e7f932f913d20d3876bed3c102be00e6c3e8600ff7bdd58a7", size = 100140, upload-time = "2026-09-29T00:53:52.205Z" }, + { url = "https://files.pythonhosted.org/packages/c2/98/769712f356a1e897df3581bb0c3100375d054a000de26099360ea65b5111/librt-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:3f0b8114c44b2ac06ff5dacd08e07e8e807ff4f46083f2a1602685122559be41", size = 120464, upload-time = "2026-09-29T00:53:53.495Z" }, + { url = "https://files.pythonhosted.org/packages/bf/d3/ae2abccc8bdc8b063c1613a77686e17b74e9d3d60cfe6f12c63fa2821b9f/librt-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:8caf96a4ef8fb27d0ac0d1ad8337d26a240acd4a02fe4345d0a8f264753e8f99", size = 110889, upload-time = "2026-09-29T00:53:54.817Z" }, + { url = "https://files.pythonhosted.org/packages/e4/26/0737d4be058dd6376eade7dd8b380d4b869b6394cee929393a5a431c45bb/librt-0.16.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:953107e2f68d0f3512c48f898b0dbf0ce5cc52bba0f318d847c985dc555ee4cc", size = 159838, upload-time = "2026-09-29T00:53:56.22Z" }, + { url = "https://files.pythonhosted.org/packages/01/96/9bc96531d7c620e9949af904470f02e3fa8f35129ab8e8f281c51eaa3788/librt-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ad37d5b9abd49c9a655dcda7ea52a8a752884062ef1ee71ae17c2f2a0f81fe6a", size = 162018, upload-time = "2026-09-29T00:53:57.532Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fa/b0289dcb186eb3f97221ba00da5f4bd3ba7fa5e99752d48aa8d336615334/librt-0.16.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c4aa329c17bd1aaea4f6e89335d8ccd494b3a5830b6654462273e50e11023f0", size = 703983, upload-time = "2026-09-29T00:53:59.024Z" }, + { url = "https://files.pythonhosted.org/packages/d9/ab/05ebbbde7530fc5eeb58fbd1581522a9f64f132fa703c4c595eed6e14760/librt-0.16.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0ead24d2562a49473dddd9efef8581f020007eb0054389c3ee3ffad38b1ca4c9", size = 683847, upload-time = "2026-09-29T00:54:00.671Z" }, + { url = "https://files.pythonhosted.org/packages/a3/75/f52aeecd4dbadbddf80725ba7de126d8bd5d0eb66247eae17a81ed90dd4d/librt-0.16.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:02118f56a9c36ddd07dfd9b919d9ecc117ba20a90987d56aa4c429fa34509188", size = 698045, upload-time = "2026-09-29T00:54:02.247Z" }, + { url = "https://files.pythonhosted.org/packages/e6/55/fa277a835cd6eb42380591ceb85f48c5b4d2b2d7e2cb9869e1e17d24d237/librt-0.16.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4e29522c62e28595ff7e324c6834ade51127707f0e255b18d1c1cf03d39c1048", size = 724640, upload-time = "2026-09-29T00:54:04.078Z" }, + { url = "https://files.pythonhosted.org/packages/25/e4/2cf64354f3fde8ebd591b3b48f96510bee24ac5f7d1e567adbbe210abdbd/librt-0.16.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3ff4b2367926b69c6215635902cccb04048e73094e9862900d27cb2c6bbff143", size = 731748, upload-time = "2026-09-29T00:54:05.741Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c9/c180af3e94e01aa529fa93d7733fec2abc47f222345400cf21d5481d5f8a/librt-0.16.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c6f1b27bf1632a7e016af9f145f82be95e1edd7721a646505c21059257cb5a04", size = 754960, upload-time = "2026-09-29T00:54:07.38Z" }, + { url = "https://files.pythonhosted.org/packages/95/d6/01073aa78c58f356b10d9c57b3fe9abb143df338142bcd316df417b89db0/librt-0.16.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:5696d7f52e7b37217cb3a8f92c744fe835942602fdd4c1a8bc4741d3bfdce15e", size = 747000, upload-time = "2026-09-29T00:54:09.06Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d8/1de3783908658d697a8cfc00582f61299ffba7796d7260c112e4700b1109/librt-0.16.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:6072e92dd876ff6ceeb6cf371e35e51f479349837391341f479b08df4564242b", size = 749569, upload-time = "2026-09-29T00:54:10.702Z" }, + { url = "https://files.pythonhosted.org/packages/b7/32/e817f66c96d6caa8bb8435ff93c4c220624d59efc506be59ab98fcd01d0c/librt-0.16.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:39ca4f2f2fe05de8e63493da592d84311adabe5bef52b193851981da9816b302", size = 729483, upload-time = "2026-09-29T00:54:12.25Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c9/23992ccd2b9d22798fdd0f61353183a47414e651da782ad83eb680f50833/librt-0.16.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9807485a908f00355820f18e91e045ffdcdc5adb68aaec40a1e2b88c5f7bba1", size = 776584, upload-time = "2026-09-29T00:54:13.837Z" }, + { url = "https://files.pythonhosted.org/packages/67/3b/e8af957f08e6e2e8e566b099e43d2748418aa565df6ee1d9d3331209fdfe/librt-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:94be5cb7bca4df6201f4183e9e4fa2086c655283d20b38cd84500a69057575a7", size = 104402, upload-time = "2026-09-29T00:54:15.568Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1c/946e6443d7cd32347a086043395e421e52fd603c9163d2ec970ceab8eed6/librt-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d46ca272b251d033dd4527b0dec5f261a28a52bd5fa0f99c117b0a1f8588cc2d", size = 126227, upload-time = "2026-09-29T00:54:17.165Z" }, + { url = "https://files.pythonhosted.org/packages/c2/a3/bc4f9959d3c62bcbf9e5fd3470a8bbd8bf33224ed4c25d7fba173201b8ac/librt-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b9d6d4b14e92d876f8026b54c20c445f36425214c1081dc76f74e40db386b82b", size = 116493, upload-time = "2026-09-29T00:54:18.567Z" }, + { url = "https://files.pythonhosted.org/packages/b6/4b/10fdb42dfab4c1533e1570e686b18e86ff4328b406361b39fb3016667638/librt-0.16.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:6fe436af2eaf630474f491af5d032cbe45f93fcff5c3b9fe4ab194a7255b20ff", size = 149829, upload-time = "2026-09-29T00:54:19.933Z" }, + { url = "https://files.pythonhosted.org/packages/8e/30/a90ca13f1d3d91af1680000a4038536907018fbd51764767287a05d28b8b/librt-0.16.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:8ff5d26c529336be9bd7ae04483235d77778ee7d6444a95353102b542601ce81", size = 154522, upload-time = "2026-09-29T00:54:21.306Z" }, + { url = "https://files.pythonhosted.org/packages/5c/dd/bcf364eacfa070bb1fc88d503111ae7177914197ba1fa717c748926e7930/librt-0.16.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:909d8e3c1faee44cb762b1c519ff8613dcc5ceae5c99987a00917b5a31fd1d6a", size = 497593, upload-time = "2026-09-29T00:54:22.798Z" }, + { url = "https://files.pythonhosted.org/packages/18/c1/2c4e81e347bdabfe8346bfc6dc37ae5e154d61c88e30848ec0606025a5e0/librt-0.16.0-cp315-cp315-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:6d4a64283ee61824b5790de882bc68e2d9d7a5143537cb7a966f7354f71646d4", size = 480459, upload-time = "2026-09-29T00:54:24.364Z" }, + { url = "https://files.pythonhosted.org/packages/ab/d7/fef2a3cb8400701be496f6e459876f650b3451f407e30cb243de571ae615/librt-0.16.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5810ba811297fdf37a1531a57667cb8ace0842013ca8606bf9eb7c24cf4be154", size = 507814, upload-time = "2026-09-29T00:54:25.997Z" }, + { url = "https://files.pythonhosted.org/packages/e1/6f/53762927a32e9dc9eb1d1c1f3528da281290c86d96929a8af652f671266e/librt-0.16.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e56aaf8c167548dc8e5d6f3bd0f48dcdd299a23c73be3f744aab79d99e9c7f5d", size = 525114, upload-time = "2026-09-29T00:54:27.747Z" }, + { url = "https://files.pythonhosted.org/packages/6c/67/0b9d031f303c4e8c691a9a8ef9d272f13b530c11cc563b819df621b6a348/librt-0.16.0-cp315-cp315-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f36c58e33b304b525c6c9c5076399c6ebf1109e17b9051a05a407b091b9215b", size = 520465, upload-time = "2026-09-29T00:54:29.331Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d5/2056a3a85864e882eb17a203a10ddb26fa748bc9718ec67e79059ab46cae/librt-0.16.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:242e00b3d4fa37c3d3c1ca5f5c9adb7d909ddb1eac9c41f2787320d00caa0af2", size = 537257, upload-time = "2026-09-29T00:54:30.915Z" }, + { url = "https://files.pythonhosted.org/packages/54/57/e0d79790c163cbc0909e209a6f62e30bb713b647f905a176cdc64848fd4d/librt-0.16.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:0253721561787b8df8443eb347b7a6461015354e5bdd37ee38a41fef220d2bb0", size = 527442, upload-time = "2026-09-29T00:54:32.478Z" }, + { url = "https://files.pythonhosted.org/packages/aa/50/1c0c95aba7af51f4752ea34fbf2eb79b36e7cae3a72536248e8c735de735/librt-0.16.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:3e483a8d69ede8067db70c0e83007423b6925de6fd53afed01d66160f2e9398c", size = 548139, upload-time = "2026-09-29T00:54:34.183Z" }, + { url = "https://files.pythonhosted.org/packages/98/91/a8a43dd5138d4f55f88846b8f0c85454a4fad69952cebfcff9948f831290/librt-0.16.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:69ba927445cfaaffb4081003ef5224c55a5c2ab67ef956f416ef744916e44121", size = 529708, upload-time = "2026-09-29T00:54:35.761Z" }, + { url = "https://files.pythonhosted.org/packages/2d/41/d5226881ab2b7c20d9d587b37bdd4a0ec8775a96d00ca87ac9f385587db4/librt-0.16.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:d6a365f2ab45a984d0e00eee0dd17f599ceab8cadab6ea07b6111c8132fc0e42", size = 567781, upload-time = "2026-09-29T00:54:37.409Z" }, + { url = "https://files.pythonhosted.org/packages/bb/bf/2345ba57a626e8c78c4ddcc724636a8df5a593fe46c1ee76bbf477e32b0d/librt-0.16.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:f01f3805f2dae4781c0c34b440e31740d082950bdaf89a6f601ad589a28af57a", size = 79177, upload-time = "2026-09-29T00:54:38.792Z" }, + { url = "https://files.pythonhosted.org/packages/1e/40/99e77936cc9207f629b077bf7cbc5e2f0827cddd7c29792f04ef881bd2c3/librt-0.16.0-cp315-cp315-win32.whl", hash = "sha256:b0e3e721c75d2e79a76d4422c79d7ba705fe1bbafec907037fe7a657a480a0e3", size = 100127, upload-time = "2026-09-29T00:54:40.251Z" }, + { url = "https://files.pythonhosted.org/packages/56/1e/801fe26bc622061b9dfd010e166d94142cb774b733217e6d98d1c0cf2638/librt-0.16.0-cp315-cp315-win_amd64.whl", hash = "sha256:bc02954b1295de798bbdb0b4e2d8a28c2117de8b5c73dcbeb27dc32572dfb971", size = 120480, upload-time = "2026-09-29T00:54:41.722Z" }, + { url = "https://files.pythonhosted.org/packages/bb/a9/d533983055bd36e112627384c2c038845d1df882540b8bcefb566475640b/librt-0.16.0-cp315-cp315-win_arm64.whl", hash = "sha256:c5db585d43449a5f54303d4b2774e45e1babd975cfe1630a3d708c0b80c3e560", size = 110915, upload-time = "2026-09-29T00:54:43.052Z" }, + { url = "https://files.pythonhosted.org/packages/55/fe/d62238fa9c653b0e0613290349467cb65711b5800e8e474f45e942a0ad95/librt-0.16.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f06c689cb14afd9b612727553a5ec5a40febf113ca41c4413a2b0b334285884b", size = 159714, upload-time = "2026-09-29T00:54:44.497Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ac/f31efe7818700be72ba4f9af8a80fa67c39808c8d26dacc55dc1f6f35172/librt-0.16.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:13b4e8aba90b0b1c82474e9844aa9ffe7ad3faa484350e1da64cb8188d903134", size = 162086, upload-time = "2026-09-29T00:54:45.968Z" }, + { url = "https://files.pythonhosted.org/packages/d9/16/4d7487bf86a9d7e8e18f37ba5538789233ea693637379b75701bd35ec9de/librt-0.16.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a269c46ae327d8e6f8c1f85f7516cb52c0fa48127565a1105a4f4a05ff2a0b4", size = 712727, upload-time = "2026-09-29T00:54:47.486Z" }, + { url = "https://files.pythonhosted.org/packages/77/74/50cd550ccc1a517b9ed62347625c01ddf8c4afad66490312677c011458f6/librt-0.16.0-cp315-cp315t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:a33e0dae1f8592146a4764d54ce842b278732d21a84e17c3bbe6b1bc158a2248", size = 681106, upload-time = "2026-09-29T00:54:49.126Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/7293f712975ee6fdd2251411fd9ef1c62bc99c7b83ecbe62dbc999b1fab7/librt-0.16.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:47ada6ea32636492c61aa8ad27ae3b9404bfe7a97e3ba946d1984236cc741da0", size = 705921, upload-time = "2026-09-29T00:54:50.905Z" }, + { url = "https://files.pythonhosted.org/packages/67/f7/8aab946f11d59d1bece9ffc65d994b200c789a8ca5a95c17e17e609f912a/librt-0.16.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c43bd6e642d8a248c114327f98dd25ac5a7cb5aa168ef02f0559b91874df16b8", size = 731786, upload-time = "2026-09-29T00:54:52.584Z" }, + { url = "https://files.pythonhosted.org/packages/43/76/1c42ab31e7cb8384ebf6d3af607213c495222474f4940443ae7639ab7685/librt-0.16.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c3d1bb7841a816ace6449bb26d3f9560dbfa20e71c568d23f0f62bf1e68f50b1", size = 744964, upload-time = "2026-09-29T00:54:54.217Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2e/4b19982d933d2dfced671e840b219e4c1cd3f507df6e6dabb51bbd4e3850/librt-0.16.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:3931f7a3db322e7f44e02a280e3949326ce9579ad388ee8d691dc7c76da9fb70", size = 765305, upload-time = "2026-09-29T00:54:55.815Z" }, + { url = "https://files.pythonhosted.org/packages/b8/1b/e872583de2dcb3ac7746e7a2321aeeb168274f3952a87dc66e05ddb29faf/librt-0.16.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:f4462528b6000afe8f16907b5c7c2553abf1df005ba5140e6eb394541c3624c3", size = 744959, upload-time = "2026-09-29T00:54:57.532Z" }, + { url = "https://files.pythonhosted.org/packages/10/de/a18c6bcfb297af2674233e90b3c3661c3a0af0f1434a0c966d2ef708a835/librt-0.16.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:80039ba9b6a7d5f1a0175a4cca6bbefead87bd854c80abad1cb30afe47a830db", size = 758138, upload-time = "2026-09-29T00:54:59.484Z" }, + { url = "https://files.pythonhosted.org/packages/22/1c/0df1d732539c297bb1a093e1fe3204d2faf76a9022cf4e1d1c2fce059790/librt-0.16.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:7a1d272724b581bb6bc769dfdafed6da2ecc9886ba2450311de55a4ac2e1e9cd", size = 744268, upload-time = "2026-09-29T00:55:01.179Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f7/ccaf31331f20c91a5bd9bd48ffc3f743f9c81bb5720cc7b2a720ca204f0a/librt-0.16.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:fbe4fb8c5445f7496d7f7f6bb0807875d09d47e6771ffa175fb2df2895fb86ba", size = 785783, upload-time = "2026-09-29T00:55:02.938Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ea/7421d9e6db894cd6cd788b604b3394a163ded6b942052b6f96c23ccf08e1/librt-0.16.0-cp315-cp315t-win32.whl", hash = "sha256:375bfe6b572a8f6cfc398709356046173bf27e64c4c5edaf5f7062f051fb4bf9", size = 104355, upload-time = "2026-09-29T00:55:04.533Z" }, + { url = "https://files.pythonhosted.org/packages/85/6d/7c31a506eb847bc58aeb2402d58e17605e821f68ab5df5e66fe274a6df41/librt-0.16.0-cp315-cp315t-win_amd64.whl", hash = "sha256:bd3150023d3dc2bc70f3784e59ffa1140d56ddba3d8125b3d6f9f85221279bfc", size = 126134, upload-time = "2026-09-29T00:55:06.017Z" }, + { url = "https://files.pythonhosted.org/packages/36/69/7a5d10ac409c4da0355e054a14371871da9b5557fcc42772cd00181c6cce/librt-0.16.0-cp315-cp315t-win_arm64.whl", hash = "sha256:8ceafb70f2a4f0826f11031942e59c0728fd98da112dc346d4352bde1e486866", size = 116391, upload-time = "2026-09-29T00:55:07.484Z" }, +] + [[package]] name = "markdown-it-py" version = "4.0.0" @@ -1747,6 +1939,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ad/3f/3d42e9a78fe5edf792a83c074b13b9b770092a4fbf3462872f4303135f09/ml_dtypes-0.5.4-cp314-cp314t-win_arm64.whl", hash = "sha256:11942cbf2cf92157db91e5022633c0d9474d4dfd813a909383bd23ce828a4b7d", size = 168825, upload-time = "2025-11-17T22:32:23.766Z" }, ] +[[package]] +name = "mypy" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ast-serialize" }, + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/4e/64300736cf0a0373a27b94a91b664ee7382e36f77b0621bae6381da3e180/mypy-2.4.0.tar.gz", hash = "sha256:77bdaebd452f43fcfc4cc3ba94352a3ea537cd01e3f2d0879f48673d2ec00d6e", size = 4064425, upload-time = "2026-10-01T20:40:39.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/67/e53965e67d23d48e78dd603aa7525a8d77e7818c23954cc217ccde5cd5c0/mypy-2.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d20e6c7c35fcbf2a0ebdd0eaeacfbc243009dfd33ab7822d54e213912e6dbbd", size = 14315745, upload-time = "2026-10-01T20:39:25.03Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5d/a588ccd57625822d8484d7aff4e9c70f88d042a91c2ec84324a2bc14b3f3/mypy-2.4.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3011537be6cf1de4511c0255a324362a812b58184bbe61e15f59c8b31033bd74", size = 14465652, upload-time = "2026-10-01T20:38:57.19Z" }, + { url = "https://files.pythonhosted.org/packages/05/af/b4978a5566cb829ca563dc08e3ad8d944f116078786ad54ba3dff2bcb64d/mypy-2.4.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a96b07a49b7b1d025ce59c1b3acbcf24bead9a83da4523c4a6bde1bb94e7a0e1", size = 15419180, upload-time = "2026-10-01T20:40:27.808Z" }, + { url = "https://files.pythonhosted.org/packages/b2/8c/52c71f31c54d0a749d832cc73b933c6e38c52d9b270d5503e031c73723dc/mypy-2.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7c4f8f8d1d1c0e2832d8ee7113dd08f6df6c7aad9e863fcbed9f25832be0b8c4", size = 15669212, upload-time = "2026-10-01T20:39:20.443Z" }, + { url = "https://files.pythonhosted.org/packages/e8/55/ba63121494404fd7d3787ec176ccd02820d6ec125ac4e7c4c0b79f2cc6c7/mypy-2.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:ba05652540bf12828e52abae807b024b09ca144ff4f75e2450a81d69c376425b", size = 11299572, upload-time = "2026-10-01T20:40:03.627Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9c/385c81b5c0d3a2917f077c4da660d0493f9521ccfd4b058e33b860f72d02/mypy-2.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:6306086b87cf7f8a29aa618d9fd9bffb56c59247166b9660fdb54d86d7714ecd", size = 10869751, upload-time = "2026-10-01T20:40:25.483Z" }, + { url = "https://files.pythonhosted.org/packages/68/ed/e5d7cf4017e74a1c1e1c4058ce8f614fc1e3e7606564f47166e22bfc9f95/mypy-2.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e05ff2925d8b37ad26c80c1b9dc43ae5d455da2df1e23c24c095a6425917c57e", size = 14378593, upload-time = "2026-10-01T20:38:48.222Z" }, + { url = "https://files.pythonhosted.org/packages/30/7d/12d994886a922f0f1997becc9c6625198d61eeb5962558a886af7dd38d54/mypy-2.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:29243242cf72582b65f9582ad9e56e8cb281566ed3519f4cd70bb8b9f2977e90", size = 14464697, upload-time = "2026-10-01T20:39:11.059Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9e/bcc9af755425ad17790bf11d73c2ea7592914cf309ee1340997670f9d57a/mypy-2.4.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29eb0b9427a6b11b992e452f6cceb8af724f4dceb47e779d0b35e405e996ea5e", size = 15552257, upload-time = "2026-10-01T20:39:56.269Z" }, + { url = "https://files.pythonhosted.org/packages/af/0c/3343fc4525d6f00d75ad17a93a9f052d5163641cb8911840d4a12cb59ff2/mypy-2.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e3ebe2f72a2a1156065a9851570ffbf50c0a93cdccadef9c6e05c508a4fd10b1", size = 15800443, upload-time = "2026-10-01T20:38:54.815Z" }, + { url = "https://files.pythonhosted.org/packages/8f/0e/69aac6b8159da7c53e6115a2be1bf48e85503402cadbf506dd708e8b2ad7/mypy-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:236e0d68f6941992b0811128e652590f590db444ab29ad8f1324765b9298b946", size = 11381574, upload-time = "2026-10-01T20:39:49.406Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d3/d32ce4feb5993eec09d2024bef16cedc9b93701b1446b86092f08b24491b/mypy-2.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:82d0f94c8587ccb472622ee7795280aaa38a06640d5f45b3f16909d6dd86a989", size = 10944729, upload-time = "2026-10-01T20:40:15.203Z" }, + { url = "https://files.pythonhosted.org/packages/44/f2/eb15183c97c69d7cbfac990a6efd33a19ecfd97dab9e714c742fa78a784f/mypy-2.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7da85fbcff6dac1abcc636707bed38b45598131fb7a605d9719c70b5cc733af8", size = 14213395, upload-time = "2026-10-01T20:40:21.178Z" }, + { url = "https://files.pythonhosted.org/packages/8c/b5/ba91b6ff65e4d6b6ff53b2b3b3ac5f1babf0c7c27d0b43a0196b1c967926/mypy-2.4.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4209da39d85cf240f762af622d8180fcdfcb4727d021f44ade62d613a1a43324", size = 14373009, upload-time = "2026-10-01T20:39:58.86Z" }, + { url = "https://files.pythonhosted.org/packages/d5/c4/484275efc935c0003e55e4e8a33e4b8e99528ee956c12256ece4708f903f/mypy-2.4.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6be721bd4bd57576193653b75b4af3461c9d0bf7dd8b528f782e9be210dc75bb", size = 15465557, upload-time = "2026-10-01T20:39:01.794Z" }, + { url = "https://files.pythonhosted.org/packages/50/30/66eb6fdd0875e3c9025a02f0bb0ea2e524b74274b658c37fde0068c4939d/mypy-2.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e1fde197ae65be856a034a91b70ed747a16562ca69577785f06c661548424bf1", size = 15728673, upload-time = "2026-10-01T20:38:43.724Z" }, + { url = "https://files.pythonhosted.org/packages/58/bc/2aa98fd7f49c42dba8e9c065886fadffdd00f3c654cff3f2a7103a797ac8/mypy-2.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:295ecf2e57542cd836ca537486951289678c8c7d1ee6ad74ebe29b2168a003cf", size = 11389803, upload-time = "2026-10-01T20:39:37.547Z" }, + { url = "https://files.pythonhosted.org/packages/49/41/17b60df2d946792ef6af43b89351f5c9ddabc69053206b2304945572a744/mypy-2.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:bc378bdad4e9f12b5bd96466083d1e71acf00594ec9c7b2bdb5e02816f77f303", size = 10951060, upload-time = "2026-10-01T20:39:04.015Z" }, + { url = "https://files.pythonhosted.org/packages/e0/66/924be0b653372ed31ad5c48e26044cc00840e591943a56e61621cf05b60e/mypy-2.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:058165f564ccf559c68c70fec2091fca5891110480210c22594635e3f6683437", size = 14234230, upload-time = "2026-10-01T20:39:44.535Z" }, + { url = "https://files.pythonhosted.org/packages/56/39/c4f176880a4177123576de6cec6309feea8f42fca2bf2f6584e88054f656/mypy-2.4.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fa247e02b505a45a2775f69df38d360d197e3790bc60f717595db9eda358b6e", size = 14392944, upload-time = "2026-10-01T20:40:05.78Z" }, + { url = "https://files.pythonhosted.org/packages/bc/1c/26e16977e25ef2494a74f8ffc872a76ec2c3aa43c3156057cbdf88f352c5/mypy-2.4.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20e9a5cd875837520c43db98dea0b6d0c2197833d95c30127d8f570fb9b1f00b", size = 15441463, upload-time = "2026-10-01T20:40:32.412Z" }, + { url = "https://files.pythonhosted.org/packages/31/9c/9e4b049f0ecefbfd6817ca2a16eea55dd74a07950268edc6cffd29ccfe98/mypy-2.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9f03a7828cca2b0adcd6662aee8f2711ff8830e1027641fdea3ab0b787483566", size = 15705406, upload-time = "2026-10-01T20:40:34.844Z" }, + { url = "https://files.pythonhosted.org/packages/4a/5e/e861b5f6c5ef9ee6cd24683aed1edecf82a26dbd536049f9b7850b586267/mypy-2.4.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:9279488933040b638c0ab739084c0ca100efeea6db581bf5d7628d8e89de53fe", size = 7897523, upload-time = "2026-10-01T20:39:39.861Z" }, + { url = "https://files.pythonhosted.org/packages/3a/87/61ffee58b25a956532a6006fae84918a2de849ed25f397492b2e815fe7b3/mypy-2.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:2106b55105ba5ea9be4f53a24517fc5fa927ff1585edc9bc1a975abb72caef89", size = 11576345, upload-time = "2026-10-01T20:39:41.553Z" }, + { url = "https://files.pythonhosted.org/packages/a1/88/a331c20698971c2ce8d1c30f317fd61b5be13a85b1f053e12dfa22ac6568/mypy-2.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:528c8744b8b5e3ecb8774f86af38d2376216816e9908317ad055f3c9c2d74799", size = 11172062, upload-time = "2026-10-01T20:38:59.534Z" }, + { url = "https://files.pythonhosted.org/packages/47/c0/4f7daa73270dced8e86c6f4a911c68082d03a84e6c1ec9bd916028d66131/mypy-2.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0bb95cf34899e4619c61ab0a8667804e139e580b30d5df12af2102dfe44d0c97", size = 15711208, upload-time = "2026-10-01T20:39:30.597Z" }, + { url = "https://files.pythonhosted.org/packages/2f/05/f1afa303c678be24cf7a266d38fb24b3de4599a024c2f9ba0d5905a3efa3/mypy-2.4.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86d616fe84c6eab8026f8c50ab5bcb90db780d2ccd233d971e34e92bede9b359", size = 15935442, upload-time = "2026-10-01T20:39:22.781Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d6/6a1a45459b63716e0d035f4892a926d3054f0a8cbfa02551d228a9946083/mypy-2.4.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3f86fd1313dd69d013e265f1fdcd12ea7a9d606f9875b2a3db946cd334555f3", size = 17240044, upload-time = "2026-10-01T20:39:32.95Z" }, + { url = "https://files.pythonhosted.org/packages/87/85/ae33bee66c13f98d421964d87bf0888be941063bc75c1204a4cf142cf1cc/mypy-2.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:720434d48542ecfe84d32d287b727569d3fc8f5769acd39051130e490a5c295c", size = 17545856, upload-time = "2026-10-01T20:39:08.827Z" }, + { url = "https://files.pythonhosted.org/packages/df/de/eeff209b65c3e267818d79333bb9f058da6e4e73761461bd94748a2eb628/mypy-2.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a6e851b82c0661f69f1630fc16172c68787a6a9cf0991e7c6437d60976cdcd76", size = 12369342, upload-time = "2026-10-01T20:40:10.516Z" }, + { url = "https://files.pythonhosted.org/packages/2b/43/e62d8d5c1dd737aa248968302ff7d6eabf997c3301773ed8bcb64932ca77/mypy-2.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:3bd0e340f0ebe65c548210f53be3fd8192e83964760caf0c28bef368e68b0d37", size = 11849099, upload-time = "2026-10-01T20:39:51.834Z" }, + { url = "https://files.pythonhosted.org/packages/2e/5f/335b8980055118dc131355155883fb676bb2161a0d97e08658e479c776fb/mypy-2.4.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:afa89837d9be67e0cadfa33bca3bb7efdda98c3b07e74dc3b635ebfb1c8a926a", size = 14241302, upload-time = "2026-10-01T20:39:13.227Z" }, + { url = "https://files.pythonhosted.org/packages/18/37/1482fdc49332b145828912b15f16eee0c4ca70a8bce0f6514ece79b14680/mypy-2.4.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb443e81057896132d3642d6be219e6efd158691ac7883e3ba8fcb469865f05d", size = 14408200, upload-time = "2026-10-01T20:38:52.576Z" }, + { url = "https://files.pythonhosted.org/packages/04/09/dce2e8f6c1b31053c430ef6963f6f7a38ccb49b90c5e01e37ed0129b7d5a/mypy-2.4.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7f38f57d344f8b6accb40e01c3d83cfc590498231724d16c07ffb7940f157818", size = 15444409, upload-time = "2026-10-01T20:39:18.1Z" }, + { url = "https://files.pythonhosted.org/packages/43/c5/91b68306da4cd280cb15be35dbb5ffc343cabd67c4b121b6625bf2d13177/mypy-2.4.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e76172710bd4e5eeae061abfd68347e5264632e02778be61784671ae3a2132f5", size = 15731186, upload-time = "2026-10-01T20:40:23.426Z" }, + { url = "https://files.pythonhosted.org/packages/64/71/2d0340182a8f27352fb1e25531355951108b506097c433937e9eec452ffd/mypy-2.4.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:f83353e47ab520bf6fd4df8f5897d9fe081211f2fbc4b7d37736a3e3c166cbcf", size = 7897838, upload-time = "2026-10-01T20:38:50.779Z" }, + { url = "https://files.pythonhosted.org/packages/3e/08/32703c117e134c02efa2a82705bb40cee91eaedea10b6530e20eba651b1f/mypy-2.4.0-cp315-cp315-win_amd64.whl", hash = "sha256:970b221ed5842213d98e3c480c08f795ace4b1f81fb21e1b126bd0476bce1c34", size = 11575700, upload-time = "2026-10-01T20:39:53.982Z" }, + { url = "https://files.pythonhosted.org/packages/ab/08/08bb269feafdaad031046ee2d771528a64467e6a180f962afc28c4a3ccd8/mypy-2.4.0-cp315-cp315-win_arm64.whl", hash = "sha256:502b94b0b331f7dafe32fd6b151797ddbb4f32385b362e722c783a025e5954a3", size = 11172908, upload-time = "2026-10-01T20:40:12.91Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3f/c5c92626006ca92c7686adfbece47fc6a0daa0e54753952a9ad13ce0561c/mypy-2.4.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:c9de622fd397495695d0598ddc789222bfcfec9d7c9ec3a1e385c855e3bc5e01", size = 15705970, upload-time = "2026-10-01T20:40:17.79Z" }, + { url = "https://files.pythonhosted.org/packages/10/f3/863365f7997a76a5a1dd42d7902ab05afa4124e8419089cbca1ce2554db1/mypy-2.4.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f459f0b4f0596d9d51fe7716b404b35287b99e77da98a7af90a65dd5fd61141", size = 15942234, upload-time = "2026-10-01T20:40:37.243Z" }, + { url = "https://files.pythonhosted.org/packages/f6/30/2f45b1f425a2c95dbe1a3f4d076bfd42b76e9615dba5906230703b14e4eb/mypy-2.4.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f9b028548b3af480e2b1ed8df14ccaac86f99c9f600d1580770ab7ba3dcd40f0", size = 17218364, upload-time = "2026-10-01T20:39:06.497Z" }, + { url = "https://files.pythonhosted.org/packages/43/8b/5b2bbfc69e84800b78fa2dba16d995b003f93b573558e412c5490d6e6c37/mypy-2.4.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:cb734b2668c1f40d07ce093bbeb4407e9527c67901627b0e1679825be3f09975", size = 17557934, upload-time = "2026-10-01T20:38:46.077Z" }, + { url = "https://files.pythonhosted.org/packages/df/c0/1a5dc601c22041a7bbfe1ff71cf097fbbd4fb49930867c6789baf5102106/mypy-2.4.0-cp315-cp315t-win_amd64.whl", hash = "sha256:172e30b8fea631fe310f0c665477f52d9ea40bb4e99e0c81dc30118563b13710", size = 12350955, upload-time = "2026-10-01T20:40:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/1c/bc/697e9e26fc2a86c094ad67ee1e419971b7f01ded66ee66231b09e9f3e12e/mypy-2.4.0-cp315-cp315t-win_arm64.whl", hash = "sha256:5786ef987b3767e51aaa53f20aec104c0252b42ecda7aef8e8b4cbae279b05c5", size = 11842738, upload-time = "2026-10-01T20:40:30.026Z" }, + { url = "https://files.pythonhosted.org/packages/81/12/46ae8670c98a3cd0286ca5645c2f918f8f6be65edfed81b916010619f668/mypy-2.4.0-py3-none-any.whl", hash = "sha256:d01c5d26a352acc6d5cf3128225477e1e8465e8d3029d4c345807fbf7f3cf093", size = 2800734, upload-time = "2026-10-01T20:39:26.837Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + [[package]] name = "namex" version = "0.1.0" @@ -1937,6 +2198,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/a7/b35835e278c18b85206834b3aa3abe68e77a98769c59233d1f6300284781/numpy-2.4.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:4b42639cdde6d24e732ff823a3fa5b701d8acad89c4142bc1d0bd6dc85200ba5", size = 12504685, upload-time = "2026-03-09T07:58:50.525Z" }, ] +[[package]] +name = "numpy-typing-compat" +version = "20251206.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/5f/29fd5f29b0a5d96e2def96ecba3112fc330ecd16e8c97c2b332563c5e201/numpy_typing_compat-20251206.2.4.tar.gz", hash = "sha256:59882d23aaff054a2536da80564012cdce33487657be4d79c5925bb8705fcabc", size = 5011, upload-time = "2025-12-06T20:02:04.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/7c/5c2892e6bc0628a2ccf4e938e1e2db22794657ccb374672d66e20d73839e/numpy_typing_compat-20251206.2.4-py3-none-any.whl", hash = "sha256:a82e723bd20efaa4cf2886709d4264c144f1f2b609bda83d1545113b7e47a5b5", size = 6300, upload-time = "2025-12-06T20:01:57.578Z" }, +] + [[package]] name = "opt-einsum" version = "3.4.0" @@ -2034,6 +2307,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/a7/9eb8b63ea378274517233ba0f2f0091a4cf38d92f91c9b1e6e646af31a8b/optree-0.19.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:7bcb4e3e0b2b6db57e0882f0ba6b03a07d1489887e7e75d75c42b0e2be9f0019", size = 335393, upload-time = "2026-02-23T01:56:36.659Z" }, ] +[[package]] +name = "optype" +version = "0.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/86/e6f1f6f3487492dfcf3b7a2d4e2534d27af6ac05b364b276706906c34865/optype-0.17.1.tar.gz", hash = "sha256:07bfa32b795dea28fba8605a6288d36370d072f25183fb9c29b5a90f4b6f5638", size = 53572, upload-time = "2026-05-17T22:13:28.725Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/d4/c6a2b043e33f0dd012486dcebe0593585588d400175d22aad42049c88321/optype-0.17.1-py3-none-any.whl", hash = "sha256:82f2508ca31cb21e53a41648482d890fe1f5c6cb153720551af41161555adaf1", size = 65954, upload-time = "2026-05-17T22:13:27.549Z" }, +] + +[package.optional-dependencies] +numpy = [ + { name = "numpy" }, + { name = "numpy-typing-compat" }, +] + [[package]] name = "overrides" version = "7.7.0" @@ -2112,6 +2403,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/b0/34937815889fa982613775e4b97fddd13250f11012d769949c5465af2150/pandas-3.0.1-cp314-cp314t-win_arm64.whl", hash = "sha256:108dd1790337a494aa80e38def654ca3f0968cf4f362c85f44c15e471667102d", size = 9452085, upload-time = "2026-02-17T22:20:14.331Z" }, ] +[[package]] +name = "pandas-stubs" +version = "3.0.5.260914" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/93/8948ae6c1e1e3d6833596fd266f7be2d27c1451b8be094975ad42c5e842e/pandas_stubs-3.0.5.260914.tar.gz", hash = "sha256:3f6fc1f147f68fd89c007105e7c94a948acb4ecd7eb20dc1c02e153c4ed5c250", size = 117622, upload-time = "2026-09-14T16:42:35.065Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/cb/5ad79e02a556cc23fed5816de0109fa8af660c66cfa5f4af74c3e8d4cd26/pandas_stubs-3.0.5.260914-py3-none-any.whl", hash = "sha256:39a1300c5c5c55fdf609e3476805decce5d5015539a4dcb683449f8feaeee2fb", size = 177344, upload-time = "2026-09-14T16:42:33.771Z" }, +] + [[package]] name = "pandocfilters" version = "1.5.1" @@ -2130,6 +2433,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl", hash = "sha256:2c549f800b70a5c4952197248825584cb00f033b29c692671d3bf08bf380baff", size = 106894, upload-time = "2026-02-09T15:45:21.391Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + [[package]] name = "pexpect" version = "4.9.0" @@ -2895,6 +3207,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/39/338d9219c4e87f3e708f18857ecd24d22a0c3094752393319553096b98af/scipy-1.17.1-cp314-cp314t-win_arm64.whl", hash = "sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21", size = 25489165, upload-time = "2026-02-23T00:22:29.563Z" }, ] +[[package]] +name = "scipy-stubs" +version = "1.17.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "optype", extra = ["numpy"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/30/7a2e621918d1317ab972f797161131f2635648ad5d92baf0695dd009e4f9/scipy_stubs-1.17.1.5.tar.gz", hash = "sha256:284b1dd1dd46107a614971d170030d310cd88b2ac6b483f85285ee0ff87720bd", size = 399933, upload-time = "2026-05-25T21:34:33.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/26/d4bc2ba3427a623f79a6c10c8f427c7a55b56eb8b3eddc369319d97f741b/scipy_stubs-1.17.1.5-py3-none-any.whl", hash = "sha256:58ebf054a86c000c72e8982e121c4ead0d3d9ba7a6c38aa5fa71b07f96a427fd", size = 607388, upload-time = "2026-05-25T21:34:32.073Z" }, +] + [[package]] name = "seaborn" version = "0.13.2" From f2570dafb20910fa1524220b1fd04df5af33b7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Thu, 29 Oct 2026 23:29:03 +0100 Subject: [PATCH 13/25] build: trim runtime dependencies, complete metadata and slim the sdist matplotlib and seaborn move from the runtime dependencies to the notebook extra: the package never imports them, the Python tutorial does. pip install banos now pulls numpy, pandas and scipy only. The dev extra is removed; the dev dependency group, which uv sync installs by default, is kept. SETUP.md runs uv sync instead of uv sync --extra dev. Metadata: authors and maintainers, and the license as an SPDX expression with its file (license = "MIT", license-files). hatchling 1.27 is the first version that writes them as License-Expression and License-File, so the build backend is bounded to >=1.27,<2. The MIT license classifier is removed: the expression replaces it (PEP 639). The sdist lists what it includes: src, tests with their fixtures, the bundled recordings the tests read, demo/quickstart.py, README, LICENSE and CHANGELOG. The Keras model, notebooks, Matlab code, HTML report, images and repository files are left out (about 62 kB instead of 1.2 MB). The wheel keeps the same files; only its metadata changes. The lock changes only in the banos entry; no pinned version moves. --- SETUP.md | 2 +- pyproject.toml | 32 +++++++++++++++++++++----------- uv.lock | 20 +++++--------------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/SETUP.md b/SETUP.md index ef68420..f0333d9 100644 --- a/SETUP.md +++ b/SETUP.md @@ -15,7 +15,7 @@ pip install uv # Install the package in editable mode with dev dependencies -uv sync --extra dev +uv sync # Verify everything works uv run pytest -q diff --git a/pyproject.toml b/pyproject.toml index a9654df..317677b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,13 +3,15 @@ name = "banos" version = "1.0.0" description = "Behavior ANnOtation Score: ethological metrics for evaluating behavior annotation quality" readme = "README.md" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] requires-python = ">=3.11" +authors = [{ name = "Benoît J. T. Girard" }] +maintainers = [{ name = "Benoît J. T. Girard" }] keywords = ["behavior", "annotation", "ethology", "video", "scoring", "metrics"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -21,20 +23,14 @@ dependencies = [ "numpy>=1.24", "pandas>=2.0", "scipy>=1.10", - "matplotlib>=3.7", - "seaborn>=0.12", ] [project.optional-dependencies] -dev = [ - "pytest>=7.4", - "pytest-cov>=4.1", - "pre-commit>=3.5", - "ruff>=0.4", -] notebook = [ "jupyter>=1.0", "nbformat>=5.9", + "matplotlib>=3.7", + "seaborn>=0.12", ] calms21 = [ "tensorflow>=2.12", @@ -48,12 +44,26 @@ Repository = "https://github.com/BelloneLab/BANOS" Issues = "https://github.com/BelloneLab/BANOS/issues" [build-system] -requires = ["hatchling"] +requires = ["hatchling>=1.27,<2"] # 1.27 writes the SPDX license (License-Expression) build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["src/banos"] +[tool.hatch.build.targets.sdist] +# The package, its tests and the data they read; not the MATLAB code, +# notebooks, Keras model, reports or images of the repository. +include = [ + "/src", + "/tests", + "/data/dataset_human_vs_human", + "/demo/quickstart.py", + "/README.md", + "/LICENSE", + "/CHANGELOG.md", +] +exclude = ["/tests/fixtures/matlab"] + [tool.ruff] line-length = 100 diff --git a/uv.lock b/uv.lock index 015893d..412dab0 100644 --- a/uv.lock +++ b/uv.lock @@ -224,11 +224,9 @@ name = "banos" version = "1.0.0" source = { editable = "." } dependencies = [ - { name = "matplotlib" }, { name = "numpy" }, { name = "pandas" }, { name = "scipy" }, - { name = "seaborn" }, ] [package.optional-dependencies] @@ -237,15 +235,11 @@ calms21 = [ { name = "tensorflow" }, { name = "tqdm" }, ] -dev = [ - { name = "pre-commit" }, - { name = "pytest" }, - { name = "pytest-cov" }, - { name = "ruff" }, -] notebook = [ { name = "jupyter" }, + { name = "matplotlib" }, { name = "nbformat" }, + { name = "seaborn" }, ] [package.dev-dependencies] @@ -263,21 +257,17 @@ dev = [ [package.metadata] requires-dist = [ { name = "jupyter", marker = "extra == 'notebook'", specifier = ">=1.0" }, - { name = "matplotlib", specifier = ">=3.7" }, + { name = "matplotlib", marker = "extra == 'notebook'", specifier = ">=3.7" }, { name = "nbformat", marker = "extra == 'notebook'", specifier = ">=5.9" }, { name = "numpy", specifier = ">=1.24" }, { name = "pandas", specifier = ">=2.0" }, - { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.4" }, - { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=4.1" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.4" }, { name = "scikit-learn", marker = "extra == 'calms21'", specifier = ">=1.3" }, { name = "scipy", specifier = ">=1.10" }, - { name = "seaborn", specifier = ">=0.12" }, + { name = "seaborn", marker = "extra == 'notebook'", specifier = ">=0.12" }, { name = "tensorflow", marker = "extra == 'calms21'", specifier = ">=2.12" }, { name = "tqdm", marker = "extra == 'calms21'", specifier = ">=4.65" }, ] -provides-extras = ["dev", "notebook", "calms21"] +provides-extras = ["notebook", "calms21"] [package.metadata.requires-dev] dev = [ From c7831c7e98c8fc4dfef29dabba7e08a035e40840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:19:23 +0100 Subject: [PATCH 14/25] docs: describe the metrics, options and errors as computed The README now defines bouts, overlap, greedy matching and each metric as the code computes them: IC is 1 - switches / (bout length - 1) over the found real bouts, SO and TP average over every overlapping pair, and F1 is 0 when no bout is matched. It says when each value is NaN and that NaN values are left out of the means, and that optimal matching gives the same counts as greedy on bouts read from 0/1 columns. New sections cover the aggregation and missed_bouts options with an example each, the input format and return shapes, the errors and warnings, the pooled chain of the lower-level API and the MATLAB options. The per-file mean is shown to mix detection quality with the share of recordings without the behavior. Installation lists numpy, pandas and scipy and the notebook and calms21 extras. The citation section gives the published article first and the arXiv preprint second. Links point to absolute GitHub URLs, so that they work on the PyPI page. --- README.md | 336 ++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 273 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 8b02c77..3781eb8 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![PyPI](https://img.shields.io/pypi/v/banos)](https://pypi.org/project/banos/) [![PyPI Downloads](https://static.pepy.tech/badge/banos)](https://pepy.tech/project/banos) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/BelloneLab/BANOS/blob/main/LICENSE) [![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/) [![CI](https://github.com/BelloneLab/BANOS/actions/workflows/ci.yml/badge.svg)](https://github.com/BelloneLab/BANOS/actions/workflows/ci.yml) [![MATLAB File Exchange](https://www.mathworks.com/matlabcentral/images/matlab-file-exchange.svg)](https://www.mathworks.com/matlabcentral/fileexchange/157916-banos) @@ -26,7 +26,7 @@ temporal qualities: does the predicted segment overlap well with the ground-trut bout? Does the prediction start and end at the right time? Is the prediction internally consistent, or does it flicker on and off inside the bout? -BANOS addresses these questions with four complementary metrics. +BANOS addresses these questions with four complementary metrics, computed for each behavior. ## The 4 BANOS Metrics @@ -34,17 +34,65 @@ BANOS addresses these questions with four complementary metrics.

-1. **Detection Accuracy (DA)** — Precision, Recall, and F1 score over matched - behavioral segments (TP/FP/FN at the bout level). - -2. **Segment Overlap (SO)** — Temporal Intersection over Union (tIoU) between each - matched predicted segment and its ground-truth counterpart. - -3. **Temporal Precision (TP)** — Penalizes start/end time errors: - `1 / (1 + |start deviation| + |end deviation|)`. - -4. **Intra-bout Continuity (IC)** — Measures label stability inside each - ground-truth bout: `1 - (label switches / bout length)`. +Annotations are binary: one column per behavior, one row per frame, 1 when the behavior is +present. A **bout** is a run of consecutive frames with value 1. Real bouts come from the +ground truth, predicted bouts from the annotation being evaluated. A predicted bout +**overlaps** a real bout when they share at least one frame. + +1. **Detection Accuracy (DA)** — precision, recall and F1 over bouts. Predicted bouts are + taken in time order, and each is matched to the first real bout it overlaps that is not + matched yet (greedy matching). TP is the number of matched pairs, FP the number of + predicted bouts left unmatched, FN the number of real bouts left unmatched (TP here is + a count, not the Temporal Precision metric). + Precision = TP / (TP + FP), recall = TP / (TP + FN), + F1 = 2 · precision · recall / (precision + recall). F1 is 0 when no bout is matched + (TP = 0) but some bouts were predicted or missed (FP + FN > 0). + +2. **Segment Overlap (SO)** — the temporal intersection over union (shared frames / frames + in the union) of every overlapping (predicted, real) bout pair, averaged over all such + pairs. + +3. **Temporal Precision (TP)** — `1 / (1 + |start difference| + |end difference|)`, in + frames, for every overlapping (predicted, real) bout pair, averaged over all such pairs. + It falls quickly: a bout predicted 2 frames late at both ends scores 1 / 5 = 0.2. + +4. **Intra-bout Continuity (IC)** — label stability inside the real bouts that are found, + that is, with at least one predicted frame inside. For each found real bout longer than + one frame: `1 - switches / (bout length - 1)`, where the bout length is in frames and + the switches are the changes of the predicted value between consecutive frames of the + bout (a bout has bout length - 1 such frame pairs). IC is the mean over these bouts. + One-frame real bouts have no frame pair and are left out. + +SO and TP average over pairs, not over real bouts: a real bout overlapped by two predicted +bouts gives two terms, and a predicted bout that overlaps two real bouts gives one term with +each. A real bout with no predicted frame inside it (a missed bout) gives no SO, TP or IC +term by default (see the `missed_bouts` option below). + +**Undefined values.** For a behavior with real bouts in a recording (or, with +`aggregation="pooled"`, in at least one recording), recall and F1 are always defined, and +a metric is NaN when it is undefined: + +- precision, when nothing is predicted (TP + FP = 0); +- SO and TP, when no predicted bout overlaps a real bout; +- IC, when no real bout longer than one frame is found. + +With `missed_bouts="zero"`, each missed bout adds a term equal to 0: SO and TP are then +always defined, and IC is NaN only when no real bout is longer than one frame. Per +recording, a behavior without real bouts follows the absent-behavior rule (see +`aggregation` below). Pooled, a behavior with no real bout in any recording has an +undefined recall, SO, TP and IC, and an undefined precision and F1 too when nothing is +predicted either (TP = FP = FN = 0). + +NaN values are left out of every mean, over recordings or behaviors: a NaN never counts as +0 or 1, and a mean with no defined value is NaN. + +**Matching.** `matching="optimal"` replaces greedy matching with the Hungarian algorithm +(`scipy.optimize.linear_sum_assignment`), which finds the largest possible number of +matched pairs. On bouts read from 0/1 columns, as in `banos.score()` and the pipeline +functions, greedy matching already finds that number, so both options give the same counts +and the same metrics. They can differ only when `banos.core.calculate_tp_fp_fn` is given +bouts directly and the bouts of one side overlap one another or are not in time order; +`banos.score()` always passes disjoint bouts in time order, so it is unaffected. ## Installation @@ -52,8 +100,17 @@ BANOS addresses these questions with four complementary metrics. pip install banos ``` -Requires Python 3.11+. Core dependency: `pandas`. Optional: `scipy` (for -`matching='optimal'`). +Requires Python 3.11+. Dependencies: `numpy` (>= 1.24), `pandas` (>= 2.0) and +`scipy` (>= 1.10). + +Two optional extras install what the tutorial notebooks need (the notebooks are in the +[GitHub repository](https://github.com/BelloneLab/BANOS/tree/main/demo), not in the +package): + +- `pip install "banos[notebook]"`: jupyter, nbformat, matplotlib and seaborn, for + `demo/tutorial_python.ipynb`; +- `pip install "banos[calms21]"`: tensorflow, scikit-learn and tqdm, for + `demo/tutorial_calms21_task1.ipynb` (together with the `notebook` extra). ## Quick Start @@ -61,76 +118,182 @@ Requires Python 3.11+. Core dependency: `pandas`. Optional: `scipy` (for import banos import pandas as pd -# Load annotations as DataFrames (rows=frames, columns=behaviors, values=0/1) +# One recording: one row per frame, one 0/1 column per behavior pred = pd.read_csv("machine_annotations.csv") gt = pd.read_csv("human_annotations.csv") -# Score a single recording metrics = banos.score(pred, gt) -# {'behavior1': {'precision': 0.9, 'recall': 0.85, 'f1_score': 0.87, -# 'so': 0.76, 'tp': 0.82, 'ic': 0.91}, ...} +# {'attack': {'precision': ..., 'recall': ..., 'f1_score': ..., 'so': ..., 'tp': ..., 'ic': ...}, ...} -# Score multiple recordings +# Several recordings: a dict of (prediction, ground truth) pairs group_metrics, overall_metrics = banos.score({ - 'recording_1': (pred_1, gt_1), - 'recording_2': (pred_2, gt_2), + "recording_1": (pred_1, gt_1), + "recording_2": (pred_2, gt_2), }) ``` -`banos.score()` returns: +**Input.** For each recording, the prediction and the ground truth are DataFrames with the +same number of rows (frames). Rows are read by position: the index is ignored. Columns are +matched by name and hold 0 and 1 (integers, floats or booleans). NaN values are replaced by +0, and columns that cannot be scored are dropped, each with a warning (see Errors and +warnings below). -- **Single recording**: `dict[behavior → dict[metric → float]]` -- **Multiple recordings**: `(per-recording metrics, overall averaged metrics)` +**Returns.** `banos.score()` returns, with either aggregation: -Absent-behavior scoring (v0.2.0): when a behavior has no ground-truth bouts, -BANOS checks whether the machine also predicted nothing — if so, all metrics = **1.0** -(correct absence); if the machine predicted bouts that don't exist in GT, all metrics = **0.0** -(false detection). This is excluded from NaN-averaging correctly. +- for one recording (a DataFrame pair): `{behavior: {metric: value}}`, with the metrics + `precision`, `recall`, `f1_score` (DA), `so`, `tp` and `ic`; +- for several recordings (a dict): a tuple `(group_metrics, overall_metrics)`, where + `group_metrics` is `{behavior: {metric: value}}` over all recordings and + `overall_metrics` is `{metric: value}`. -## Examples & Tutorials +## Options -| Resource | Description | -|----------|-------------| -| [`demo/quickstart.py`](demo/quickstart.py) | Minimal Python example — score annotations in two lines | -| [`demo/quickstart.m`](demo/quickstart.m) | Minimal MATLAB example | -| [`demo/tutorial_python.ipynb`](demo/tutorial_python.ipynb) | Full Python tutorial: classic BANOS usage + frame F1 vs BANOS comparison | -| [`demo/tutorial_matlab.m`](demo/tutorial_matlab.m) | Full MATLAB tutorial mirroring the Python notebook | -| [`demo/tutorial_calms21_task1.ipynb`](demo/tutorial_calms21_task1.ipynb) | Human vs machine: CalMS21 Task 1 baseline, F1 + BANOS (Python only) | +`banos.score(data, gt=None, *, matching="greedy", missed_bouts="exclude", aggregation="per_file")` + +### Combining recordings: `aggregation` + +**`aggregation="per_file"` (default).** Each recording is scored on its own, then the +values are averaged: `group_metrics` is the mean of each behavior over the recordings, +`overall_metrics` the mean of every (recording, behavior) value, NaN left out. Each +recording weighs the same, whatever its number of bouts. -## Advanced Usage +*Absent-behavior rule:* when the ground truth of a recording has no bout of a behavior, +that behavior scores 1.0 on all six metrics if nothing is predicted (correct absence), and +0.0 on all six if at least one bout is predicted (false detection). -### Optimal Matching +The per-file mean therefore mixes detection quality with the share of recordings without +the behavior. A model that predicts nothing scores 1.0 on every recording without the +behavior and F1 = 0 on the others: on the bundled data, its per-file F1 is 0.5 for attack +(absent from 5 of the 10 recordings) and 0.4 for mount (absent from 4), while it finds no +bout at all. -By default, predicted segments are matched to ground-truth bouts greedily (by -overlap). For a globally optimal assignment, use the Hungarian algorithm: +Use it to know how good the annotation of each recording is: look at the value of each +recording (`calculate_banos_for_each_file`, see Lower-level API) and at their mean, with +the share of recordings without each behavior in mind. + +**`aggregation="pooled"`.** Each behavior is scored once over all recordings: TP, FP and +FN are summed over the recordings and give precision, recall and F1, and the SO, TP and IC +terms of all recordings are averaged together. There is no absent-behavior rule: a +recording without the behavior only adds its predicted bouts to FP. A behavior with no +real and no predicted bout in any recording is NaN on all six metrics. `overall_metrics` is +the mean over the behaviors, NaN left out. Every bout weighs the same, and predicting +nothing gives F1 = 0: this is the right choice to compare or rank models and methods on +the same recordings. ```python -metrics = banos.score(pred, gt, matching='optimal') +import pandas as pd +import banos + +gt = pd.DataFrame({"attack": [0, 1, 1, 1, 0, 0, 1, 1, 1, 0]}) # two real bouts +silent = pd.DataFrame({"attack": [0] * 10}) # a model that predicts nothing + +data = { + "video_1": (silent, gt), # both real bouts missed + "video_2": (silent, silent), # no attack in this video: correct absence +} + +group, overall = banos.score(data) # per_file +# group["attack"]: precision 1.0, recall 0.5, f1_score 0.5, so 1.0, tp 1.0, ic 1.0 + +group, overall = banos.score(data, aggregation="pooled") +# group["attack"]: precision nan, recall 0.0, f1_score 0.0, so nan, tp nan, ic nan ``` -Requires `scipy` (`pip install scipy`). +Per file, `video_2` scores 1.0 everywhere, while `video_1` has recall and F1 0 and its +precision, SO, TP and IC undefined: every mean but recall and F1 is 1.0. Pooled, the model +has found 0 of the 2 bouts. + +### Missed bouts: `missed_bouts` -### Lower-level API +**`missed_bouts="exclude"` (default).** A real bout with no predicted frame inside it gives +no SO, TP or IC term. SO, TP and IC then describe the quality of the bouts that were found, +and are read together with recall, the share of real bouts found. + +**`missed_bouts="zero"`.** Each missed real bout adds a term equal to 0 to SO, to TP and, +if it is longer than one frame, to IC, so these metrics also drop when bouts are missed. + +Precision, recall, F1 and the absent-behavior rule do not depend on this option. ```python -from banos import preprocess_data, calculate_banos_for_each_file, aggregate_metrics +gt = pd.DataFrame({"attack": [0, 1, 1, 1, 0, 0, 1, 1, 1, 0]}) # two real bouts +pred = pd.DataFrame({"attack": [0, 1, 1, 1, 0, 0, 0, 0, 0, 0]}) # finds the first only + +banos.score(pred, gt)["attack"] +# precision 1.0, recall 0.5, f1_score 0.667, so 1.0, tp 1.0, ic 1.0 + +banos.score(pred, gt, missed_bouts="zero")["attack"] +# precision 1.0, recall 0.5, f1_score 0.667, so 0.5, tp 0.5, ic 0.5 +``` + +The two options combine: `banos.score(data, missed_bouts="zero", aggregation="pooled")`. -data_dict = { - 'file1': (pred_df_1, gt_df_1), - 'file2': (pred_df_2, gt_df_2), +### Errors and warnings + +`banos.score()` raises `ValueError`: + +- for an unknown `matching`, `missed_bouts` or `aggregation` value, before reading the data; +- when `data` is a DataFrame and `gt` is missing; +- when the prediction and the ground truth of a recording have different numbers of + frames, or a recording has no frame. + +It emits a `UserWarning` naming the recording and the column: + +- for each column whose NaN values are replaced by 0; +- for each column dropped because it holds values other than 0 and 1 (probabilities, for + example); +- for each column dropped because it is missing, or was dropped, on the other side. + +A single DataFrame pair is named `'single'` in these messages. Valid input gives no +warning. `preprocess_data()` raises and warns in the same way. + +## Lower-level API + +`banos.score()` chains three functions; each step can be called on its own, for example +to keep the results of each recording. + +```python +from banos import ( + preprocess_data, + calculate_banos_for_each_file, aggregate_metrics, # per_file + calculate_components_for_each_file, pool_components, # pooled +) + +data = { + "recording_1": (pred_1, gt_1), + "recording_2": (pred_2, gt_2), } +clean, dropped = preprocess_data(data) # dropped: the columns removed from each recording -preprocessed, dropped = preprocess_data(data_dict) -per_file_metrics = calculate_banos_for_each_file(preprocessed) -group_metrics, overall = aggregate_metrics(per_file_metrics) +# Per file: the metrics of each recording, then their means +file_metrics = calculate_banos_for_each_file(clean) # {recording: {behavior: {metric: value}}} +group_metrics, overall_metrics = aggregate_metrics(file_metrics) + +# Pooled: the bout counts and terms of each recording, then one score per behavior +file_components = calculate_components_for_each_file(clean) # {recording: {behavior: components}} +group_metrics, overall_metrics = pool_components(file_components) ``` +`calculate_banos_for_each_file` and `calculate_components_for_each_file` take the +`matching` and `missed_bouts` options of `banos.score()`. The components of a behavior in a +recording are what its metrics are computed from: +`{"n_tp": ..., "n_fp": ..., "n_fn": ..., "so_terms": [...], "tp_terms": [...], "ic_terms": [...]}`, +the bout counts and the lists of terms that SO, TP and IC average. For a single recording +that has already been preprocessed, `compute_behavior_metrics(pred, gt)` and +`compute_behavior_components(pred, gt)` return its per-behavior metrics and components. + +`banos.core` holds the building blocks: `identify_bouts`, `calculate_tp_fp_fn`, +`calculate_precision_recall_f1`, `calculate_tiou`, `calculate_so` and +`calculate_so_terms`, `calculate_tp_metric` and `calculate_tp_terms`, `calculate_ic` and +`calculate_ic_terms`, `count_switches_within_bout`, and the preprocessing functions +`is_logical_column`, `drop_non_logical_columns`, `match_headers` and `preprocess_data`. +Import the metric functions from `banos.core`. + ## MATLAB A MATLAB implementation is provided in `matlab/BANOS/`. Usage mirrors the Python API. ```matlab -% Add toolbox to path +% Add the toolbox to the path for this session run('matlab/BANOS/setup.m'); % Score a single recording (pred/gt = cell arrays: row 1 = headers, rows 2+ = binary data) @@ -140,33 +303,79 @@ metrics = BANOS_score(pred, gt); dataStruct.Recording_1 = {pred1, gt1}; dataStruct.Recording_2 = {pred2, gt2}; [groupMetrics, overallMetrics] = BANOS_score(dataStruct); + +% Options as name-value pairs (names and values are case-insensitive) +metrics = BANOS_score(pred, gt, 'missed_bouts', 'zero'); +[groupMetrics, overallMetrics] = BANOS_score(dataStruct, 'aggregation', 'pooled'); ``` -See [`demo/tutorial_matlab.m`](demo/tutorial_matlab.m) for a full walkthrough. +The MATLAB code computes the same metrics with the same options and rules; the parity +tests compare the Python results with golden values written by the MATLAB code, within +1e-6. MATLAB uses greedy matching only (`'matching', 'optimal'` raises an error). A +single recording is named `single_recording` in messages, and errors and warnings carry +identifiers: `BANOS:lengthMismatch`, `BANOS:noFrame`, `BANOS:nanReplaced`, +`BANOS:columnDropped`, `BANOS:unknownOption`, `BANOS:missingOptionValue` and +`BANOS:invalidOption`. `BANOScalculateBANOSForEachFile` returns the components of each +recording as a second output, and `BANOSpoolComponents` pools them. + +See [`demo/tutorial_matlab.m`](https://github.com/BelloneLab/BANOS/blob/main/demo/tutorial_matlab.m) +for a full walkthrough. Also available on [MATLAB File Exchange](https://www.mathworks.com/matlabcentral/fileexchange/157916-banos). +## Examples & Tutorials + +| Resource | Description | +|----------|-------------| +| [`demo/quickstart.py`](https://github.com/BelloneLab/BANOS/blob/main/demo/quickstart.py) | Minimal Python example — score annotations in two lines | +| [`demo/quickstart.m`](https://github.com/BelloneLab/BANOS/blob/main/demo/quickstart.m) | Minimal MATLAB example | +| [`demo/tutorial_python.ipynb`](https://github.com/BelloneLab/BANOS/blob/main/demo/tutorial_python.ipynb) | Full Python tutorial: classic BANOS usage + frame F1 vs BANOS comparison (`notebook` extra) | +| [`demo/tutorial_matlab.m`](https://github.com/BelloneLab/BANOS/blob/main/demo/tutorial_matlab.m) | Full MATLAB tutorial mirroring the Python notebook | +| [`demo/tutorial_calms21_task1.ipynb`](https://github.com/BelloneLab/BANOS/blob/main/demo/tutorial_calms21_task1.ipynb) | Human vs machine: CalMS21 Task 1 baseline, F1 + BANOS (Python only; `calms21` and `notebook` extras) | + +The demos run from a clone of the repository. The quickstarts and the two full tutorials +score the example recordings in `data/dataset_human_vs_human/`; the CalMS21 notebook +downloads its test data on first run. + ## Contributing BANOS is functionally complete by design. We welcome **bug reports** and -**documentation improvements**. See [CONTRIBUTING.md](CONTRIBUTING.md) for details on -how to report issues, submit fixes, and the project's maintenance lifecycle. +**documentation improvements**. See +[CONTRIBUTING.md](https://github.com/BelloneLab/BANOS/blob/main/CONTRIBUTING.md) for +details on how to report issues, submit fixes, and the project's maintenance lifecycle. -To report a security vulnerability, see [SECURITY.md](SECURITY.md) — please do not use -public GitHub Issues for security reports. +To report a security vulnerability, see +[SECURITY.md](https://github.com/BelloneLab/BANOS/blob/main/SECURITY.md) — please do not +use public GitHub Issues for security reports. ## Citation -If you use BANOS, please cite: +If you use BANOS, please cite the article: + +> Chindemi G, Bellone C, Girard B. *From eye to AI: Studying rodent social behavior in the +> era of machine learning.* Neuroscience & Biobehavioral Reviews 188 (2026) 106766. +> https://doi.org/10.1016/j.neubiorev.2026.106766 + +The preprint of this article is: -> Chindemi G., Bellone C., Girard B. *From eye to AI: studying rodent social behavior -> in the era of machine learning.* arXiv:2508.04255 (2025). +> Chindemi G, Bellone C, Girard B. *From eye to AI: studying rodent social behavior in the +> era of machine Learning.* arXiv:2508.04255 (2025). > https://doi.org/10.48550/arXiv.2508.04255 ```bibtex +@article{chindemi2026eye, + title = {From eye to {AI}: Studying rodent social behavior in the era of machine learning}, + author = {Chindemi, Giuseppe and Bellone, Camilla and Girard, Benoit}, + journal = {Neuroscience \& Biobehavioral Reviews}, + volume = {188}, + pages = {106766}, + year = {2026}, + doi = {10.1016/j.neubiorev.2026.106766} +} + @article{chindemi2025eye, - title = {From eye to {AI}: studying rodent social behavior in the era of machine learning}, + title = {From eye to {AI}: studying rodent social behavior in the era of machine {L}earning}, author = {Chindemi, Giuseppe and Bellone, Camilla and Girard, Benoit}, journal = {arXiv preprint arXiv:2508.04255}, year = {2025}, @@ -176,8 +385,9 @@ If you use BANOS, please cite: ## Changelog -See [CHANGELOG.md](CHANGELOG.md) for release history. +See [CHANGELOG.md](https://github.com/BelloneLab/BANOS/blob/main/CHANGELOG.md) for release +history. ## License -MIT — see [LICENSE](LICENSE). +MIT — see [LICENSE](https://github.com/BelloneLab/BANOS/blob/main/LICENSE). From 732af1515590a265d0e16d98cb8b1e552c4baa38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:19:51 +0100 Subject: [PATCH 15/25] docs: add a citation file for the software and the article CITATION.cff (format 1.2.0) describes the software, BANOS 1.1.0 under the MIT license, and gives the article in Neuroscience & Biobehavioral Reviews as preferred citation, with the arXiv preprint as a reference. It has no release date yet. The README points to it. --- CITATION.cff | 47 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 4 ++++ 2 files changed, 51 insertions(+) create mode 100644 CITATION.cff diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..45b0802 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,47 @@ +cff-version: 1.2.0 +message: "If you use BANOS, please cite the article given as preferred citation." +title: BANOS +abstract: "Behavior ANnOtation Score: ethological metrics for evaluating behavior annotation quality." +type: software +version: 1.1.0 +license: MIT +repository-code: "https://github.com/BelloneLab/BANOS" +authors: + - given-names: "Benoît J. T." + family-names: Girard +keywords: + - behavior + - annotation + - ethology + - video + - scoring + - metrics +preferred-citation: + type: article + title: "From eye to AI: Studying rodent social behavior in the era of machine learning" + authors: + - given-names: Giuseppe + family-names: Chindemi + - given-names: Camilla + family-names: Bellone + - given-names: Benoit + family-names: Girard + journal: "Neuroscience & Biobehavioral Reviews" + volume: 188 + start: 106766 + year: 2026 + doi: 10.1016/j.neubiorev.2026.106766 +references: + - type: article + title: "From eye to AI: studying rodent social behavior in the era of machine Learning" + authors: + - given-names: Giuseppe + family-names: Chindemi + - given-names: Camilla + family-names: Bellone + - given-names: Benoit + family-names: Girard + journal: "arXiv preprint arXiv:2508.04255" + year: 2025 + doi: 10.48550/arXiv.2508.04255 + url: "https://arxiv.org/abs/2508.04255" diff --git a/README.md b/README.md index 3781eb8..cd6dd52 100644 --- a/README.md +++ b/README.md @@ -383,6 +383,10 @@ The preprint of this article is: } ``` +The same references are in +[CITATION.cff](https://github.com/BelloneLab/BANOS/blob/main/CITATION.cff), which GitHub +offers as "Cite this repository". + ## Changelog See [CHANGELOG.md](https://github.com/BelloneLab/BANOS/blob/main/CHANGELOG.md) for release From c0774618c2bb241f96328581e5e397228504d862 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:21:43 +0100 Subject: [PATCH 16/25] docs: add the 1.1.0 changelog entry and complete the 0.2.0 one The 1.1.0 entry, not yet dated, lists the numerical changes (F1 0 instead of NaN when no real bout is matched, IC over the found real bouts, IC read by position, the 41 golden values that move and the figures of the bundled data), the added options and functions, the changed errors, warnings, packaging and CI, and the fixes. The 0.2.0 entry gains the two changes it did not list: for a recording with real bouts and no match, F1, SO and TP, and precision when nothing was predicted, went from 0 to NaN; and a one-frame real bout, which scored 0 in IC, is left out of IC. Earlier dates and versions are unchanged. --- CHANGELOG.md | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5b9cd7..5ab64d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,131 @@ All notable changes to this project will be documented in this file. +## [1.1.0] — Unreleased + +Two metrics are corrected (detection F1 and IC), two options are added (`missed_bouts` and +`aggregation`), and invalid input now raises an error or a warning. With the default +options and valid input, precision, recall, SO and TP are unchanged. + +### Numerical changes +- **Detection F1 is 0, not NaN, when no real bout is matched.** F1 is now 0 when TP = 0 + and FP + FN > 0 (a recording with real bouts and nothing predicted, or only false + bouts), the value of 2 TP / (2 TP + FP + FN). It stays NaN only when TP = FP = FN = 0; + every other value is unchanged. Such a recording now stays in the means instead of + leaving them. Precision and recall are unchanged (NaN on a zero denominator). This + brings back the 0 of v0.1.5, which v0.2.0 had turned into NaN. +- **IC averages over the found real bouts only.** A real bout with no predicted frame + inside it holds no switch and scored IC 1.0: missing a bout raised IC, and a model that + predicted nothing scored IC 1.0. IC is now the mean of `1 - switches / (bout length - 1)` + over the real bouts with at least one predicted frame, one-frame bouts still left out, + and NaN when no such bout is found. +- **IC reads the prediction by position.** IC read the prediction by index label, so + DataFrames indexed other than 0..n-1 (frames numbered from 1, reversed, float or string + labels, concatenated recordings) got a wrong IC or an exception. Input with the default + index is unchanged. +- **Golden values.** The fixtures move to `golden_python_v110.json` and + `golden_matlab_v110.json`; the v0.2.0 files are kept as archives. 41 of the 300 + per-recording values change: 2 F1 values (Recording_8 mount F1 from NaN to 0, + Recording_8 overall F1 from 0.6062 to 0.4546) and 39 IC values, in every recording + (Recording_8 mount IC from 1.0 to NaN: both its real bouts are missed). The other 259 + are unchanged. `python tests/fixtures/compare_fixtures.py --mode 3` shows them. +- **Bundled data**, the 10 recordings scored together with the default options + (`banos.score(data)`, four behaviors), 1.0.0 → 1.1.0: F1 mount 0.8780 → 0.7902, overall + 0.7746 → 0.7553; IC attack 0.9956 → 0.9949, investigation 0.9918 → 0.9875, mount + 0.9969 → 0.9942, other 0.9945 → 0.9927, overall 0.9947 → 0.9923. Precision, recall, SO + and TP are unchanged. +- **A model that predicts nothing**, on the same data, per file: F1 attack 1.0 → 0.5, + mount 1.0 → 0.4, investigation and other NaN → 0, overall 1.0 → 0.225; IC investigation + and other 1.0 → NaN. The 0.5 and 0.4 that remain are the share of recordings without + the behavior (absent-behavior rule, unchanged). Pooled, its F1 is 0 for every behavior. +- The MATLAB code gives the same numbers (parity tests within 1e-6). + +### Added +- `missed_bouts` option of `banos.score()`: `"exclude"` (default) leaves a missed real + bout out of SO, TP and IC; `"zero"` adds a term equal to 0 to SO, to TP and, for a bout + longer than one frame, to IC. Precision, recall and F1 do not depend on it. +- `aggregation` option of `banos.score()`: `"per_file"` (default) scores each recording, + then averages; `"pooled"` sums the TP, FP and FN counts of all recordings and averages + all their SO, TP and IC terms, for each behavior, without the absent-behavior rule, and + `overall_metrics` is the mean over the behaviors. The result has the same form in both. +- Functions `calculate_so_terms`, `calculate_tp_terms` and `calculate_ic_terms` + (`banos.core`), `compute_behavior_components`, `calculate_components_for_each_file` and + `pool_components` (`banos`). `calculate_so`, `calculate_tp_metric`, `calculate_ic`, + `compute_behavior_metrics` and `calculate_banos_for_each_file` take the keyword + `missed_bouts`. +- MATLAB: the `'missed_bouts'` and `'aggregation'` options of `BANOS_score`; + `BANOSparseOptions` and `BANOSpoolComponents`; the terms as second output of + `BANOScalculateSO`, `BANOScalculateTP` and `BANOScalculateIC`, and the components as + second output of `BANOScomputeBehaviorMetrics` and `BANOScalculateBANOSForEachFile`. +- Golden values of the 10 recordings scored together, for each combination of the two + options (`all_recordings` in the v110 fixtures), checked by the regression and parity + tests. +- `py.typed`: type checkers read the annotations of the package. +- `CITATION.cff`, with the article to cite. +- Mode 3 of `tests/fixtures/compare_fixtures.py` (v0.2.0 against v1.1.0), and a test that + runs the script and compares its output with `reports/fixture_comparison.html`. + +### Changed +- **Input errors.** `ValueError` when the prediction and the ground truth of a recording + have different numbers of frames (a shorter prediction raised `KeyError` when a real + bout longer than one frame reached past its last frame, and was scored otherwise; a + longer one was scored) or when a recording has no frame (it scored 1.0 on all six + metrics). +- **Warnings.** A `UserWarning` names the recording and the column for each column + dropped (values other than 0 and 1, or missing or invalid on the other side) and for + each column whose NaN values are replaced by 0; both were silent. The warnings point at + the line that called `banos.score()` or `preprocess_data()`. Valid input gives no + warning. +- **Options checked on entry.** `banos.score()` and the pipeline functions raise + `ValueError` for an unknown `matching`, `missed_bouts` or `aggregation` value even when + nothing would be scored; an unknown `matching` used to pass unnoticed when no behavior + had real bouts. +- **MATLAB.** `BANOSpreprocessData` raises and warns in the same way (identifiers + `BANOS:lengthMismatch`, `BANOS:noFrame`, `BANOS:nanReplaced`, `BANOS:columnDropped`). + Options are stricter: an unknown option name, which was ignored, and an unknown + `'matching'` value, which was treated as greedy, raise an error (`BANOS:unknownOption`, + `BANOS:missingOptionValue`, `BANOS:invalidOption`). +- **Imports.** `calculate_so`, `calculate_tp_metric` and `calculate_ic` are no longer + importable from `banos.pipeline`, which only used them; import them from `banos.core`. +- **Packaging.** matplotlib and seaborn move from the runtime dependencies to the + `notebook` extra: `pip install banos` installs numpy, pandas and scipy only. The `dev` + extra is removed; the `dev` dependency group, which `uv sync` installs, stays. Authors + and maintainers are set, the license is an SPDX expression (`License-Expression: MIT`, + with `License-File`) and the license classifier is removed. The build backend is + `hatchling>=1.27,<2`. The sdist holds the package, its tests with the data they read, + `demo/quickstart.py`, README, LICENSE and CHANGELOG (about 70 kB instead of 1.2 MB). +- **CI.** The test matrix runs the parity tests; a `Type check (mypy)` job runs mypy on + `src/`; checkouts do not keep credentials. The publish workflow checks that the tag is + `v` followed by the version in `pyproject.toml` and runs the whole CI before building + and publishing. +- **Development tools.** Pre-commit runs ruff 0.15.7, the version of the lock and CI. CI + also lints `demo/quickstart.py`. mypy, pandas-stubs and scipy-stubs join the dev group; + pytest-html leaves it. +- **Tests.** `tests/test_comparison_report.py`, which was always skipped, is removed: + `test_parity_group_metrics` checks the same values, and now also fails when Python and + MATLAB report different behaviors. + +### Fixed +- The `score()` docstring and the `BANOS_score` help said that a metric is NaN when the + ground truth has no bout; per file, such a behavior scores 1.0 or 0.0. +- README: the IC formula (the denominator is the bout length minus 1), SO and TP (averaged + over every overlapping pair, not over matched pairs), the return value for several + recordings, the dependencies, and links that did not work on the PyPI page. +- Tutorials: the text said that the prediction shifted by 15 frames still detects both + bouts (DA 1.0); it shares no frame with them, so DA is 0 (it printed NaN before this + release). +- `compare_fixtures.py` swapped the `VALUE->NULL` and `NULL->VALUE` tags and misdescribed + the v0.1.5 to v0.2.0 changes in its legend; it no longer fails on a Windows console. +- `test_preprocess_data_does_not_mutate_caller_dict` compared a dict's id with itself and + could not fail; it now compares the caller's data before and after. `test_score_version` + compares `banos.__version__` with the version in `pyproject.toml`. +- Type errors reported by mypy: `is_logical_column` returns a Python `bool`, and + `drop_non_logical_columns` is annotated with the column labels it returns. +- `SECURITY.md` said that BANOS reads CSV files (it reads no file: it takes DataFrames or + MATLAB cell arrays) and that `uv.lock` pins the dependencies users get (it pins the + development and CI environment). `SETUP.md` said to allow all actions (the repository + allows selected actions, pinned by SHA). + ## [1.0.0] — 2025-02-27 First stable release. Metric computation and the golden fixtures are unchanged from 0.3.0. @@ -36,10 +161,17 @@ No change to metric computation: `src/banos/` and the golden fixtures are unchan - **Absent-behavior scoring**: when GT has no bouts, metrics are no longer `NaN`/`0`: - Machine also has no bouts → all metrics = **1.0** (correct absence, rewarded) - Machine predicts bouts that don't exist in GT → all metrics = **0.0** (false detection, penalized) +- **Undefined metrics are NaN instead of 0** (missing from this entry until 1.1.0): for a + behavior with real bouts in a recording and no predicted bout matching any of them (so + none overlapping them), F1, SO and TP became NaN instead of 0, and so did precision when + nothing was predicted. The averages leave NaN out, so such a recording no longer + counted in these means. 1.1.0 scores this F1 as 0 again. - Python minimum version raised to 3.9 ### Bug Fixes - Fixed IC boundary: last transition within GT bout was not counted (`range(gt_end-1)` → `range(gt_end)`) +- **One-frame real bouts are left out of IC** (missing from this entry until 1.1.0): such a + bout has no frame pair to compare; v0.1.5 gave it an IC of 0, whatever was predicted. - Fixed variable shadowing of `tp` count and `tp` metric in `compute_behavior_metrics` - Fixed Matlab `BANOSmatchHeaders`: now matches by column name, not column index - Fixed `aggregate_metrics`: NaN values now excluded from averages (nanmean behavior) From ea78c3d7eb9e0d68aabfce2538fe6e1d65003af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:23:47 +0100 Subject: [PATCH 17/25] docs: update the release, security, setup and contributing guides RELEASE.md: the version is bumped on a branch and merged by pull request, then the merged commit is tagged; the publish workflow checks the tag against pyproject.toml and runs the whole CI, parity tests included, before publishing. The fixture table names the v1.1.0 files as current and the v0.2.0 and v0.1.5 files as archives, and the regenerate commands add the comparison report and the v0.1.5 command, which reproduces that file identically with Python 3.12 or later. SECURITY.md: BANOS reads no file, and uv.lock pins the development and CI environment, not the versions users install. The runtime closure is numpy, pandas and scipy. SETUP.md: the repository allows selected actions pinned by SHA, not all actions; the required check names, the protected tags and the pypi environment are listed, with the CI, notebook and build commands. The CalMS21 section matches the notebook: it downloads the zip, and loads the model from demo/models. CONTRIBUTING.md: there is no issue template; the CI checks to run before a pull request are listed. --- CONTRIBUTING.md | 18 ++++++++--- RELEASE.md | 85 +++++++++++++++++++++++++++++++++---------------- SECURITY.md | 19 +++++++---- SETUP.md | 66 +++++++++++++++++++++++++++++++------- 4 files changed, 139 insertions(+), 49 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9846aca..76ad4dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,10 +25,9 @@ Contact: open a [GitHub Issue](https://github.com/BelloneLab/BANOS/issues). ## How to Report a Bug -Open a [GitHub Issue](https://github.com/BelloneLab/BANOS/issues) using the bug report -template. Please include: +Open a [GitHub Issue](https://github.com/BelloneLab/BANOS/issues). Please include: -- BANOS version (`pip show banos` or check `pyproject.toml`) +- BANOS version (`pip show banos`, or `banos.__version__`) - Python version (or MATLAB version, if applicable) - A minimal reproducible example (small CSV + code that demonstrates the wrong output) @@ -41,8 +40,19 @@ template. Please include: Code style: - Python: follow existing style enforced by `ruff` (lint + format) -- Use type hints consistent with the existing codebase +- Use type hints consistent with the existing codebase; CI runs `mypy` on `src/` - Tests are required for bug fixes (see `tests/`) +- A fix that changes a metric changes both implementations, Python (`src/banos/`) and + MATLAB (`matlab/BANOS/`), and regenerates the golden fixtures (see + [`RELEASE.md`](RELEASE.md)); the parity tests compare the two + +Before opening the PR, run the checks of CI (see [`SETUP.md`](SETUP.md)): +```bash +uv run ruff check src/ tests/ demo/quickstart.py +uv run ruff format --check src/ tests/ demo/quickstart.py +uv run mypy src/ +uv run pytest +``` ## What We Accept diff --git a/RELEASE.md b/RELEASE.md index 94ef1c6..d6c95be 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -4,46 +4,59 @@ Before tagging a release, verify: -- [ ] All CI tests pass on `main` (`pytest` green, coverage >= 80%) -- [ ] `uv run pytest tests/test_regression.py` passes (verifies golden fixtures) -- [ ] Version bumped consistently in `pyproject.toml` and `src/banos/__init__.py` -- [ ] `CHANGELOG.md` has an entry for this version with a summary of changes +- [ ] CI is green on `main`: `Lint (ruff)`, `Type check (mypy)` and `Test (Python 3.11)` + to `Test (Python 3.14)`, which run the whole suite, regression and parity tests + included, with coverage >= 80% +- [ ] The version is the same in `pyproject.toml`, `src/banos/__init__.py`, `uv.lock` + (run `uv lock`), `CITATION.cff` and the `Version` line of `matlab/BANOS/Contents.m` + (`test_score_version` compares the first two) +- [ ] `CHANGELOG.md` has an entry for this version, dated on release day; the same date + is set in `CITATION.cff` (`date-released`) and in `matlab/BANOS/Contents.m` - [ ] README examples are up to date --- ## Python Package — PyPI -Publishing is fully automated via GitHub Actions on version tag push. +Publishing is automated by GitHub Actions when a version tag is pushed. `main` only +accepts changes through a pull request, so the version is bumped on a branch. ```bash -# 1. Bump version in two places -# pyproject.toml: version = "X.Y.Z" -# src/banos/__init__.py: __version__ = "X.Y.Z" +# 1. On a branch, bump the version +# pyproject.toml: version = "X.Y.Z" +# src/banos/__init__.py: __version__ = "X.Y.Z" +# CITATION.cff: version: X.Y.Z and date-released: YYYY-MM-DD +# matlab/BANOS/Contents.m: % Version X.Y.Z DD-Mon-YYYY +uv lock -# 2. Update CHANGELOG.md +# 2. Date the CHANGELOG.md entry -# 3. Commit -git commit -am "chore: bump version to vX.Y.Z" +# 3. Commit, open a pull request, and merge it once CI is green +git commit -am "chore: release vX.Y.Z" -# 4. Tag -git tag vX.Y.Z +# 4. Tag the merged commit on main +git fetch origin +git tag -a vX.Y.Z origin/main -m "BANOS X.Y.Z" -# 5. Push tag — this triggers the publish workflow automatically +# 5. Push the tag — this triggers the publish workflow git push origin vX.Y.Z ``` The `publish.yml` workflow: -1. Runs the full test suite (excluding parity tests) -2. Builds the package with `uv build` -3. Publishes to PyPI using OIDC trusted publishing (no API token required) +1. Checks that the tag is `v` followed by the version in `pyproject.toml`, and stops + otherwise +2. Runs the whole CI workflow (`ci.yml`): lint, type check, and the test matrix on + Python 3.11 to 3.14, parity tests included +3. Builds the package with `uv build` and publishes it to PyPI using OIDC trusted + publishing (no API token required). This job runs in the `pypi` environment, which + waits for the maintainer's approval The `README.md` is automatically bundled because `pyproject.toml` specifies `readme = "README.md"` — no manual copying needed. **Note:** OIDC trusted publishing requires a one-time setup in your PyPI account: -go to the project page → Publishing → Add a new pending publisher, and configure -it for this GitHub repository and the `publish.yml` workflow. +go to the project page → Publishing → Add a new publisher, and configure it for this +GitHub repository, the `publish.yml` workflow and the `pypi` environment. --- @@ -52,8 +65,10 @@ it for this GitHub repository and the `publish.yml` workflow. MATLAB File Exchange does not have a public API for automated uploads. Updates are done manually when a new version is released. -1. Download the release ZIP from the GitHub releases page: - `https://github.com/BelloneLab/BANOS/releases` +1. Download the source ZIP of the tag: + `https://github.com/BelloneLab/BANOS/archive/refs/tags/vX.Y.Z.zip` + (also linked from the tag on the GitHub releases page, + `https://github.com/BelloneLab/BANOS/releases`) 2. Log in to MATLAB File Exchange: `https://www.mathworks.com/matlabcentral/fileexchange/` @@ -72,12 +87,28 @@ Updates are done manually when a new version is released. | File | Represents | |------|-----------| -| `tests/fixtures/golden_python_v020.json` | **Current Python (v0.2.0)** — score 1.0 for correct absence, 0.0 for false detection, corrected IC counting. Canonical reference. | -| `tests/fixtures/golden_matlab_v020.json` | **Current Matlab (v0.2.0)** — same semantics as Python. Used for parity tests. | -| `tests/fixtures/golden_python_v015.json` | **Original Python (v0.1.5)** — returns 0 for absent behaviors, IC misses last frame pair. Kept for historical comparison only. | - -To regenerate fixtures after a deliberate algorithm change: +| `tests/fixtures/golden_python_v110.json` | **Current Python (v1.1.0)** — detection F1 0 when no real bout is matched, IC averaged over the found real bouts. Each `Recording_N` entry scores one recording alone with the default options; `all_recordings` scores the 10 recordings together for each combination of `missed_bouts` and `aggregation`. Canonical reference, used by the regression tests. | +| `tests/fixtures/golden_matlab_v110.json` | **Current MATLAB (v1.1.0)** — the same entries, computed by the MATLAB code. Used by the parity tests. | +| `tests/fixtures/golden_python_v020.json` | **Python v0.2.0** (archive) — 1.0 for correct absence, 0.0 for false detection, NaN for undefined metrics (F1 included), IC counting the last frame pair. | +| `tests/fixtures/golden_matlab_v020.json` | **MATLAB v0.2.0** (archive) — same semantics as Python v0.2.0. | +| `tests/fixtures/golden_python_v015.json` | **Original Python (v0.1.5)** (archive) — 0 for absent behaviors and undefined metrics, IC misses the last frame pair. Kept for historical comparison only. | + +To regenerate the current fixtures after a deliberate algorithm change, from the project +root: ```bash -python tests/fixtures/python/generate_golden_fixtures.py +uv run python tests/fixtures/python/generate_golden_fixtures.py matlab -batch "run('tests/fixtures/matlab/generate_matlab_fixtures.m')" ``` + +Then regenerate the comparison report (`reports/fixture_comparison.html`), which +`tests/test_compare_fixtures.py` checks against the fixtures: +```bash +uv run python tests/fixtures/compare_fixtures.py --html +``` + +The archives are not regenerated. The v0.1.5 file can be reproduced with the BANOS 0.1.5 +package from PyPI and Python 3.12 or later (with Python 3.11, about a hundred values +differ in their last digit): +```bash +uv run --isolated --no-project --python 3.12 --with "BANOS==0.1.5" python tests/fixtures/python/generate_archive_fixtures.py +``` diff --git a/SECURITY.md b/SECURITY.md index a988ae6..e8ee213 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -32,22 +32,26 @@ BANOS is a pure-Python / MATLAB computation library. It: - Has **no network access** - Has **no authentication or authorization layer** - Has **no persistent data storage** -- Reads only local CSV files passed explicitly by the user +- Reads **no file**: it takes pandas DataFrames (Python) or cell arrays (MATLAB) that the + caller has already loaded. Only the demos and tutorials read files, the CSV files of + the repository and, for the CalMS21 tutorial, a dataset it downloads The attack surface is minimal. Security reports are most relevant to dependency vulnerabilities (e.g., malicious `pandas` or `scipy` releases) or unsafe handling of -user-supplied CSV input. +user-supplied data. ## Dependency Policy BANOS accompanies a peer-reviewed publication and is maintained in security-only mode. Dependency handling follows two rules. -**Numerical results must not change.** Runtime dependencies are pinned in `uv.lock`. -Every update is validated by the golden regression tests +**Numerical results must not change.** `uv.lock` pins the development and CI +environment. Every update of the lock is validated by the golden regression tests (`tests/test_regression.py`, tolerance `1e-10`) across the whole supported Python matrix before it is merged. An update that moves a metric is rejected — the golden -fixtures are never regenerated to accommodate a dependency bump. +fixtures are never regenerated to accommodate a dependency bump. Users who install +`banos` are not bound by the lock: they get any version of its dependencies within the +bounds declared in `pyproject.toml`. **Updates are deliberate, never automatic.** Dependabot alerts are enabled for monitoring, but automatic security updates and version updates are disabled. Each @@ -61,5 +65,6 @@ publishing, and unlike Python packages they cannot affect a computed result. One caveat when reading alerts: Dependabot reports every entry of `uv.lock` as `runtime` scope. In practice the BANOS runtime closure is limited to `numpy`, -`pandas`, `scipy`, `matplotlib` and `seaborn`. The `notebook` and `calms21` extras -are optional tutorial environments and are not needed to import or use BANOS. +`pandas` and `scipy`, and the packages they depend on. The `notebook` and `calms21` +extras are optional tutorial environments, and the `dev` dependency group is the +development environment; neither is needed to import or use BANOS. diff --git a/SETUP.md b/SETUP.md index f0333d9..572dd42 100644 --- a/SETUP.md +++ b/SETUP.md @@ -2,8 +2,18 @@ ## GitHub Repository Settings -- **Branch protection on `main`**: require PR review + CI passing before merge -- **Actions permissions**: allow all actions (needed for CI and PyPI publish) +- **Branch protection on `main`**: require PR review + CI passing before merge. The + required checks are the CI job names `Lint (ruff)` and `Test (Python 3.11)` to + `Test (Python 3.14)`, so these jobs must keep their names. `Type check (mypy)` runs + too, and blocks a merge only once it is added to the required checks +- **Tags `v*`**: only the maintainer may create them, since a pushed tag starts the + publish workflow +- **Actions permissions**: allow only selected actions — actions created by GitHub, plus + `actions/checkout`, `astral-sh/setup-uv` and `pypa/gh-action-pypi-publish` — and + require actions to be pinned to a full commit SHA. The workflows pin each action by + SHA, with its version in a comment; Dependabot updates them +- **Environment `pypi`**: requires the maintainer's approval before the publishing job + runs - **Secrets**: none required (OIDC replaces API tokens for PyPI) --- @@ -14,23 +24,54 @@ # Install uv (package manager) pip install uv -# Install the package in editable mode with dev dependencies +# Install the package in editable mode with the dev dependency group uv sync -# Verify everything works +# Verify everything works (the parity tests need no MATLAB) uv run pytest -q ``` +The checks run by CI: +```bash +uv run ruff check src/ tests/ demo/quickstart.py +uv run ruff format --check src/ tests/ demo/quickstart.py +uv run mypy src/ +uv run pytest tests/ -v +``` + +`uv build` builds the sdist and the wheel into `dist/`. + ### Optional: pre-commit hooks ```bash uv run pre-commit install ``` +### Python tutorial + +`demo/tutorial_python.ipynb` needs the `notebook` extra: +```bash +uv run --extra notebook jupyter notebook demo/tutorial_python.ipynb +``` + +The notebook is generated by `scripts/create_tutorial_notebook.py`. To regenerate it and +store its outputs, from the project root: +```bash +uv run python scripts/create_tutorial_notebook.py +uv run --extra notebook jupyter nbconvert --to notebook --execute --inplace \ + --ExecutePreprocessor.record_timing=False demo/tutorial_python.ipynb +``` + --- ## Human-vs-Machine Tutorial — CalMS21 Task 1 -The `demo/tutorial_calms21_task1.ipynb` notebook requires: +The `demo/tutorial_calms21_task1.ipynb` notebook runs with the `calms21` and `notebook` +extras: +```bash +uv run --extra calms21 --extra notebook jupyter notebook demo/tutorial_calms21_task1.ipynb +``` + +It requires: ### 1. Model weights (already in repo) @@ -74,13 +115,14 @@ cp results/task1_baseline/task1_seed_42_model.h5 ../../demo/models/ ### 2. CalMS21 test data (downloaded at runtime) -The test JSON (~600 MB) is **not** included in the repo. The notebook downloads -it automatically on first run from CaltechDATA to `.cache/calms21/` (gitignored). +The test JSON (~600 MB) is **not** included in the repo. On first run, the notebook +downloads the CalMS21 Task 1 zip (~457 MB) from CaltechDATA to `.cache/calms21/` +(gitignored) and extracts the test JSON from it. -If the automatic download fails, download manually from: -https://data.caltech.edu/records/s0vdx-0k302 +If the automatic download fails, download `task1_classic_classification.zip` manually +from: https://data.caltech.edu/records/s0vdx-0k302 -Place the file at: `.cache/calms21/calms21_task1_test.json` +Place the file at: `.cache/calms21/task1_classic_classification.zip` ### 3. Optional — publish model to GitHub Releases @@ -93,4 +135,6 @@ gh release create calms21-assets-v1 demo/models/task1_seed_42_model.h5 \ --notes "Pre-trained Conv-1D baseline model (seed 42) for tutorial_calms21_task1.ipynb" ``` -Then update `MODEL_URL` in the notebook's setup cell to the release asset URL. +The notebook loads the model from `demo/models/` (`MODEL_PATH` in its setup cell); it +has no download step for the model, so a copy downloaded from the release must be placed +there, or `MODEL_PATH` changed. From aeb4a54b9071a51ed0157710538a5a679dd9deb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:24:30 +0100 Subject: [PATCH 18/25] docs: make help texts and docstrings match the code score() no longer calls optimal matching "scientifically optimal": it gives the largest number of matched pairs, which greedy matching already reaches on bouts read from 0/1 columns. The docstring of test_matching.py says when the two can differ: only on bouts given directly, when those of one side overlap one another or are not in time order. score() and BANOS_score list F1 among the values that can be NaN when pooled, and BANOS_score names the missing-value error. compute_behavior_metrics and BANOScomputeBehaviorMetrics state the absent-behavior rule. score() and preprocess_data say that a warning is emitted per column, not per NaN value. BANOScalculateIC says that IC is averaged over the found bouts and that one-frame bouts are left out, and BANOSidentifyBouts describes its [start, end] vectors. generate_archive_fixtures.py says that it writes the v0.1.5 file identically with Python 3.12 or later only. No code changes. --- matlab/BANOS/BANOS_score.m | 10 ++++++---- matlab/BANOS/BANOScalculateIC.m | 9 +++++---- matlab/BANOS/BANOScomputeBehaviorMetrics.m | 3 +++ matlab/BANOS/BANOSidentifyBouts.m | 3 ++- src/banos/__init__.py | 15 ++++++++++----- src/banos/core/preprocessing.py | 6 +++--- src/banos/pipeline.py | 3 +++ .../fixtures/python/generate_archive_fixtures.py | 9 ++++++--- tests/test_matching.py | 7 +++++-- 9 files changed, 43 insertions(+), 22 deletions(-) diff --git a/matlab/BANOS/BANOS_score.m b/matlab/BANOS/BANOS_score.m index eda0e37..1f7fb78 100644 --- a/matlab/BANOS/BANOS_score.m +++ b/matlab/BANOS/BANOS_score.m @@ -31,8 +31,9 @@ % 'pooled' scores each behavior once on the bout counts % and terms of all recordings (see BANOSpoolComponents). % The outputs have the same form in both modes. -% Option names and values are case-insensitive. An unknown name or value -% raises an error (BANOS:unknownOption, BANOS:invalidOption). +% Option names and values are case-insensitive. An unknown name or value, +% or a name without a value, raises an error (BANOS:unknownOption, +% BANOS:invalidOption, BANOS:missingOptionValue). % % NaN VALUES: % With 'per_file', a behavior with no ground-truth bouts in a recording @@ -41,8 +42,9 @@ % With 'pooled', this 1.0/0.0 rule does not apply and overallMetrics % averages the pooled behaviors with nanmean. % A metric is NaN when it is undefined: precision when nothing is -% predicted, recall when there is no GT bout (pooled), SO, TP and IC when -% they have no term (no overlap, or no found GT bout). +% predicted, recall when there is no GT bout (pooled), F1 when there is no +% bout at all (pooled), SO, TP and IC when they have no term (no overlap, +% or no found GT bout). % % EXAMPLE — single recording: % pred = [{'event1','event2'}; num2cell(predData)]; diff --git a/matlab/BANOS/BANOScalculateIC.m b/matlab/BANOS/BANOScalculateIC.m index c6042d9..55016e1 100644 --- a/matlab/BANOS/BANOScalculateIC.m +++ b/matlab/BANOS/BANOScalculateIC.m @@ -1,9 +1,10 @@ function [ic, icTerms] = BANOScalculateIC(predSequence, gtBouts, varargin) - % Calculates the Intra-bout Continuity (IC) for each ground truth bout. + % Calculates the Intra-bout Continuity (IC) over the found ground truth bouts. % - % IC = 1 - (switches / boutLength) averaged over the found GT bouts. + % IC = 1 - (switches / boutLength) averaged over the found GT bouts, where + % boutLength = end - start is the number of frame pairs of the bout. % A GT bout is found when at least one of its frames is predicted (value 1). - % Zero-length bouts (boutLength = 0) are always excluded from the average. + % One-frame bouts (boutLength = 0) are always excluded from the average. % Returns NaN if there is no term (matches Python behaviour — NaN % propagates through nanmean). % @@ -17,7 +18,7 @@ % Returns: % ic - The average of the terms, or NaN if undefined. % icTerms - Row vector of the terms, one per found GT bout longer than - % zero, and 0 for each missed one with 'zero'. + % one frame, and 0 for each missed one with 'zero'. options = BANOSparseOptions(varargin, {'missed_bouts'}); icTerms = []; diff --git a/matlab/BANOS/BANOScomputeBehaviorMetrics.m b/matlab/BANOS/BANOScomputeBehaviorMetrics.m index 57b89b3..3e30a48 100644 --- a/matlab/BANOS/BANOScomputeBehaviorMetrics.m +++ b/matlab/BANOS/BANOScomputeBehaviorMetrics.m @@ -1,6 +1,9 @@ function [behaviorMetrics, behaviorComponents] = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix, varargin) % Computes metrics for each behavior in the prediction and ground truth matrices. % + % A behavior with no GT bout scores 1.0 on all six metrics if nothing is + % predicted (correct absence) and 0.0 otherwise (false detection). + % % Name-value option: % 'missed_bouts' - how SO, TP and IC count a GT bout that no predicted % bout overlaps: 'exclude' (default) or 'zero' diff --git a/matlab/BANOS/BANOSidentifyBouts.m b/matlab/BANOS/BANOSidentifyBouts.m index ce659bf..0ad54fc 100644 --- a/matlab/BANOS/BANOSidentifyBouts.m +++ b/matlab/BANOS/BANOSidentifyBouts.m @@ -5,7 +5,8 @@ % sequence - A binary vector representing predictions or ground truth. % % Returns: - % bouts - A cell array of tuples, each tuple contains start and end indices of a bout. + % bouts - A cell array with one [start, end] row vector per bout (1-based + % indices, both included). bouts = {}; start = NaN; diff --git a/src/banos/__init__.py b/src/banos/__init__.py index e5e2c22..7f35875 100644 --- a/src/banos/__init__.py +++ b/src/banos/__init__.py @@ -61,8 +61,11 @@ def score( Ground truth DataFrame. Required when ``data`` is a single DataFrame. matching : {'greedy', 'optimal'} Bout matching algorithm. - 'greedy': first-match wins (fast, default, backward compatible) - 'optimal': Hungarian algorithm (scientifically optimal, requires scipy) + 'greedy': each predicted bout, in time order, takes the first unmatched + GT bout it overlaps (default) + 'optimal': Hungarian algorithm (scipy), the largest number of matched + pairs. On bouts read from 0/1 columns, as here, greedy matching already + reaches it, so both give the same results. missed_bouts : {'exclude', 'zero'} How SO, TP and IC count a GT bout with no predicted frame inside it. 'exclude': left out of their averages (default) @@ -92,15 +95,17 @@ def score( With ``aggregation='pooled'``, this 1.0/0.0 rule does not apply and ``overall_metrics`` averages the pooled behaviors, NaN excluded. A metric is NaN when it is undefined: precision when nothing is predicted, - recall when there is no GT bout (pooled), SO, TP and IC when they have no - term (no overlap, or no found GT bout). + recall when there is no GT bout (pooled), F1 when there is no bout at all + (pooled), SO, TP and IC when they have no term (no overlap, or no found GT + bout). An unknown ``matching``, ``missed_bouts`` or ``aggregation`` value raises ValueError before the data is read. The input goes through ``preprocess_data``: a ValueError is raised when the prediction and the ground truth of a file differ in length or a file has no - frame, and a UserWarning names each column dropped and each NaN replaced by 0. + frame, and a UserWarning, naming the file and the column, is emitted for each + column dropped and for each column whose NaN values are replaced by 0. """ _check_options(matching, missed_bouts) if aggregation not in ("per_file", "pooled"): diff --git a/src/banos/core/preprocessing.py b/src/banos/core/preprocessing.py index fbd32ea..8a67a8f 100644 --- a/src/banos/core/preprocessing.py +++ b/src/banos/core/preprocessing.py @@ -52,9 +52,9 @@ def preprocess_data(data_dict: dict[str, tuple[pd.DataFrame, pd.DataFrame]]) -> """Preprocess prediction and ground truth DataFrames. Fills NaN with 0, drops non-binary columns, aligns headers. - Each NaN replacement and each dropped column (values other than 0 and 1, - or a column present on one side only) emits a UserWarning naming the file - and the column. + A UserWarning naming the file and the column is emitted for each column + whose NaN values are replaced by 0 (with their count) and for each dropped + column (values other than 0 and 1, or a column present on one side only). Parameters ---------- diff --git a/src/banos/pipeline.py b/src/banos/pipeline.py index d01eaf0..662973c 100644 --- a/src/banos/pipeline.py +++ b/src/banos/pipeline.py @@ -100,6 +100,9 @@ def compute_behavior_metrics( ) -> dict: """Compute all BANOS metrics for each behavior column. + A behavior with no GT bout scores 1.0 on all six metrics if nothing is + predicted (correct absence) and 0.0 otherwise (false detection). + ``missed_bouts`` sets how SO, TP and IC count a GT bout that no predicted bout overlaps: left out ('exclude', default) or a term equal to 0 ('zero'). diff --git a/tests/fixtures/python/generate_archive_fixtures.py b/tests/fixtures/python/generate_archive_fixtures.py index 79d419e..c600ed5 100644 --- a/tests/fixtures/python/generate_archive_fixtures.py +++ b/tests/fixtures/python/generate_archive_fixtures.py @@ -9,9 +9,12 @@ It was written by this script, which imported the 0.1.5 module from a copy at archive/python/BANOS/BANOS.py. That folder is not part of this repository, so the import below only works where the BANOS 0.1.5 package from PyPI (module -BANOS.BANOS) is installed. Run that way from the project root, the script -writes the committed file again, unchanged: - uv run --isolated --no-project --with "BANOS==0.1.5" python tests/fixtures/python/generate_archive_fixtures.py +BANOS.BANOS) is installed. Run that way from the project root with Python 3.12 +or later, the script writes the committed file again, unchanged: + uv run --isolated --no-project --python 3.12 --with "BANOS==0.1.5" python tests/fixtures/python/generate_archive_fixtures.py +With Python 3.11, about a hundred values differ in their last digit: the 0.1.5 +code averages with the built-in sum(), which adds floats with compensation +from Python 3.12 on. """ import json diff --git a/tests/test_matching.py b/tests/test_matching.py index 9c662f6..0e5dcbb 100644 --- a/tests/test_matching.py +++ b/tests/test_matching.py @@ -11,8 +11,11 @@ that overlaps. Fast (O(n*m)) but can make suboptimal early commitments. Optimal matching: Hungarian algorithm (scipy.optimize.linear_sum_assignment) -finds the assignment with the most overlapping (matched) pairs. Slower (O(n^3)) -but scientifically preferable when bouts are densely overlapping. +finds the assignment with the most overlapping (matched) pairs. Slower (O(n^3)). +On bouts read from 0/1 columns, as in score(), greedy matching already finds +the most pairs, so both give the same counts. They can differ only when +calculate_tp_fp_fn is given bouts directly and the bouts of one side overlap +one another, as below, or are not in time order. The test_optimal_beats_greedy test documents the exact scenario where greedy fails: pred(0,10) greedily takes gt(0,3) — the first overlap it finds — leaving From 6fa260c039323d04ccb23566e8e2e13d6c1ba343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:26:35 +0100 Subject: [PATCH 19/25] docs: correct the tutorials and store the current notebook outputs The tutorials said that the prediction shifted by 15 frames still detects both real bouts (DA 1.0) and that a small shift gives a TP near 1.0. That prediction shares no frame with the real bouts, so DA is 0 and SO and TP are undefined; the 2-frame shift gets TP 0.2 and SO 0.667. The interpretation now says so, and the comparison with CalMS21 mentions the pooled aggregation. The MATLAB tutorial computes the frame F1 of this example as the Python one does (0, not NaN, when undefined), names the behaviors it uses, and states the frame F1 it prints (0.815). create_tutorial_notebook.py writes the imports in the order ruff expects and the wording of the committed notebook, which is generated again from it and executed: the printed metrics are those of this code (overall F1 0.7486, IC 0.9921; shifted prediction DA 0.000). The CalMS21 notebook is not run here, as it downloads its data: a note after its title says that its stored outputs come from BANOS 0.2.0 and that F1 and IC differ with 1.1.0. Its comparison table no longer calls SO a measure of matched bouts. --- demo/tutorial_calms21_task1.ipynb | 14 +++- demo/tutorial_matlab.m | 65 ++++++++++++----- demo/tutorial_python.ipynb | 104 ++++++++++++++-------------- scripts/create_tutorial_notebook.py | 45 ++++++------ 4 files changed, 139 insertions(+), 89 deletions(-) diff --git a/demo/tutorial_calms21_task1.ipynb b/demo/tutorial_calms21_task1.ipynb index c8df4f4..11c7443 100644 --- a/demo/tutorial_calms21_task1.ipynb +++ b/demo/tutorial_calms21_task1.ipynb @@ -41,6 +41,18 @@ "the repo at `demo/models/task1_seed_42_model.h5`." ] }, + { + "cell_type": "markdown", + "id": "a1000000-note", + "metadata": {}, + "source": [ + "> **Note on the stored outputs.** The outputs stored in this notebook were produced\n", + "> with an earlier version of BANOS (0.2.0, as the setup cell prints). With BANOS 1.1.0,\n", + "> the detection F1 (DA) and IC values differ (see `CHANGELOG.md`, 1.1.0, Numerical\n", + "> changes); precision, recall, SO and TP do not. Run the notebook to get the values of\n", + "> 1.1.0." + ] + }, { "cell_type": "markdown", "id": "a1000001", @@ -672,7 +684,7 @@ " (\"BANOS DA (F1)\", banos_da,\n", " \"Bout-level detection accuracy\"),\n", " (\"BANOS SO\", banos_so,\n", - " \"Temporal overlap quality of matched bouts\"),\n", + " \"Temporal quality of overlapping bouts\"),\n", " (\"BANOS TP\", banos_tp,\n", " \"Boundary precision (start/end frame accuracy)\"),\n", " (\"BANOS IC\", banos_ic,\n", diff --git a/demo/tutorial_matlab.m b/demo/tutorial_matlab.m index 59a9b10..e115b66 100644 --- a/demo/tutorial_matlab.m +++ b/demo/tutorial_matlab.m @@ -14,7 +14,7 @@ clear; clc; -% Detect project root (works whether run from root or example/) +% Detect project root from the location of this file (demo/) thisDir = fileparts(mfilename('fullpath')); if exist(fullfile(thisDir, 'matlab'), 'dir') projectRoot = thisDir; @@ -40,9 +40,9 @@ %% 1.1 Setup — already done above (addpath, BEHAVIORS, datasetDir) %% 1.2 Single Recording ------------------------------------------------------- -% Recording_1, behaviors event1/event2/event3 (dropping 'other') +% Recording_1, behaviors attack/investigation/mount (dropping 'other') % -% v0.2.0 absent-behavior rule: +% Absent-behavior rule (since v0.2.0; per-file scoring, the default): % - GT has no bouts AND machine predicts nothing -> all metrics = 1.0 (correct absence) % - GT has no bouts BUT machine predicts bouts -> all metrics = 0.0 (false detection) @@ -50,7 +50,7 @@ predTable = readtable(fullfile(recDir, 'humanAnnotation_2.csv')); gtTable = readtable(fullfile(recDir, 'humanAnnotation_1.csv')); -% Keep only event1/event2/event3 +% Keep only attack/investigation/mount predTable = predTable(:, BEHAVIORS); gtTable = gtTable(:, BEHAVIORS); @@ -125,7 +125,7 @@ fprintf(' SO : Temporal IoU -- how well do detected bouts overlap GT?\n'); fprintf(' TP : Boundary precision -- are bout start/end frames accurate?\n'); fprintf(' IC : Intra-bout continuity -- is each detected bout clean?\n\n'); -fprintf('Absent-behavior rule (v0.2.0):\n'); +fprintf('Absent-behavior rule (since v0.2.0; per-file scoring, the default):\n'); fprintf(' GT has no bouts AND machine predicts nothing -> all metrics = 1.0 (correct absence)\n'); fprintf(' GT has no bouts BUT machine predicts bouts -> all metrics = 0.0 (false detection)\n\n'); @@ -171,7 +171,7 @@ end framef1_overall = mean(perRecF1, 'omitnan'); -fprintf('Frame-based F1 (our approach): %.3f\n', framef1_overall); % Expected: ~0.791 +fprintf('Frame-based F1 (our approach): %.3f\n', framef1_overall); % Expected: 0.815 %% 2b. BANOS metrics: same recordings, same behaviors ------------------------ @@ -217,8 +217,13 @@ fprintf(' 2. Explicit absent-behavior scoring: we reward correct absence (score 1)\n'); fprintf(' and penalize false detection (score 0). CalMS21 implicitly handles\n'); fprintf(' this via pooling.\n\n'); -fprintf('We do NOT align with CalMS21 pooled approach because BANOS quantifies\n'); -fprintf('annotation quality per recording, not globally only.\n\n'); +fprintf('By default, we do NOT align with CalMS21 pooled approach because BANOS\n'); +fprintf('quantifies annotation quality per recording, not globally only.\n'); +fprintf('To compare or rank models on a whole dataset,\n'); +fprintf('BANOS_score(data, ''aggregation'', ''pooled'') scores each behavior once on\n'); +fprintf('the bout counts and terms of all recordings, without the absent-behavior\n'); +fprintf('rule. The per-file mean, by contrast, mixes detection quality with the\n'); +fprintf('share of recordings without the behavior (see the README).\n\n'); fprintf('For publications: state both the frame F1 methodology and BANOS metrics.\n\n'); %% 2e. Thought experiment -- the shifted annotator --------------------------- @@ -255,7 +260,7 @@ fprintf('%s\n', repmat('-', 1, 52)); for pi = 1:3 p = preds{pi}; - ff1 = local_frame_f1(p, gt_s); + ff1 = local_frame_f1_raw(p, gt_s); pred_cell = [{'beh'}; num2cell(p)]; b_m = BANOS_score(pred_cell, gt_cell); b = b_m.beh; @@ -264,18 +269,25 @@ end fprintf('\nInterpretation:\n'); -fprintf(' Frame F1 collapses to 0 for a 15-frame shift -- same as completely missing.\n'); -fprintf(' BANOS DA = 1.0 for all predictions (both bouts are still detected).\n'); -fprintf(' BANOS TP quantifies the temporal shift: large shift -> low TP.\n'); -fprintf(' BANOS SO measures fractional temporal IoU of matched bouts.\n\n'); -fprintf('Frame F1 says "shifted 15f is equally wrong as missing entirely".\n'); -fprintf('BANOS tells you HOW the system is wrong -- and by how much.\n\n'); +fprintf(' Frame F1 collapses to 0 for a 15-frame shift (B); a 2-frame shift (C)\n'); +fprintf(' gives 0.80.\n'); +fprintf(' BANOS DA counts bouts: a predicted bout detects a GT bout when they share\n'); +fprintf(' at least one frame. C detects both GT bouts: DA = 1.0. B shares no frame\n'); +fprintf(' with them: both GT bouts are missed and both predicted bouts are false,\n'); +fprintf(' so DA = 0, and SO and TP are undefined (NaN), having no overlapping pair\n'); +fprintf(' (with ''missed_bouts'', ''zero'' they would be 0).\n'); +fprintf(' BANOS TP quantifies the boundary error, 1 / (1 + |start diff| + |end diff|):\n'); +fprintf(' C, 2 frames late at both ends, gets 1 / 5 = 0.2.\n'); +fprintf(' BANOS SO measures the temporal IoU of overlapping bouts: for C, 8 shared\n'); +fprintf(' frames out of 12, 0.667.\n\n'); +fprintf('Frame F1 gives C a single 0.80.\n'); +fprintf('BANOS tells you HOW the system is wrong -- every bout found (DA), but late (SO, TP).\n\n'); fprintf('Tutorial complete!\n'); %% ========================================================================= -%% Local helper: frame-level F1 with explicit absent-behavior scoring +%% Local helpers: frame-level F1, with and without absent-behavior scoring %% ========================================================================= function f1 = local_frame_f1(pred_col, gt_col) @@ -310,3 +322,24 @@ f1 = NaN; end end + +function f1 = local_frame_f1_raw(pred_col, gt_col) + % Frame-level F1 without absent-behavior scoring, as frame_f1_raw in the + % Python tutorial: precision, recall and F1 are 0 when undefined. + tp_f = sum(pred_col == 1 & gt_col == 1); + fp_f = sum(pred_col == 1 & gt_col == 0); + fn_f = sum(pred_col == 0 & gt_col == 1); + prec = 0; + rec = 0; + if (tp_f + fp_f) > 0 + prec = tp_f / (tp_f + fp_f); + end + if (tp_f + fn_f) > 0 + rec = tp_f / (tp_f + fn_f); + end + if (prec + rec) > 0 + f1 = 2 * prec * rec / (prec + rec); + else + f1 = 0; + end +end diff --git a/demo/tutorial_python.ipynb b/demo/tutorial_python.ipynb index 53532c1..f08d7f9 100644 --- a/demo/tutorial_python.ipynb +++ b/demo/tutorial_python.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "7766bc43", + "id": "bc5c61b1", "metadata": {}, "source": [ "# BANOS Tutorial — Python\n", @@ -18,7 +18,7 @@ }, { "cell_type": "markdown", - "id": "85d2fa80", + "id": "2b5469e0", "metadata": {}, "source": [ "---\n", @@ -27,7 +27,7 @@ }, { "cell_type": "markdown", - "id": "9ff378d4", + "id": "087abc03", "metadata": {}, "source": [ "### 1.1 Setup" @@ -36,14 +36,14 @@ { "cell_type": "code", "execution_count": 1, - "id": "c2ce5a40", + "id": "e11e85f3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "banos version: 0.2.0\n" + "banos version: 1.0.0\n" ] } ], @@ -66,14 +66,14 @@ }, { "cell_type": "markdown", - "id": "67656986", + "id": "65aef272", "metadata": {}, "source": [ "### 1.2 Single Recording\n", "\n", "Recording_1, behaviors `attack`/`investigation`/`mount` (dropping `other`).\n", "\n", - "> **Note (v0.2.0 absent-behavior rule):** \n", + "> **Note (absent-behavior rule, since v0.2.0; per-file scoring, the default):** \n", "> - GT has no bouts AND machine predicts nothing → all metrics = **1.0** (correct absence) \n", "> - GT has no bouts BUT machine predicts bouts → all metrics = **0.0** (false detection)" ] @@ -81,7 +81,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "eb84e32f", + "id": "d671d64c", "metadata": {}, "outputs": [ { @@ -110,7 +110,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "f59929aa", + "id": "f68e54f7", "metadata": {}, "outputs": [ { @@ -118,17 +118,17 @@ "output_type": "stream", "text": [ "Per-behavior metrics — Recording 1\n", - " precision recall da (F1) so tp ic\n", + " precision recall DA (F1) so tp ic\n", "attack 1.0 1.0000 1.0000 1.0000 1.0000 1.0000\n", - "investigation 0.8 0.7347 0.7660 0.5444 0.0846 0.9829\n", - "mount 1.0 0.5333 0.6957 0.7333 0.0785 0.9909\n" + "investigation 0.8 0.7347 0.7660 0.5444 0.0846 0.9806\n", + "mount 1.0 0.5333 0.6957 0.7333 0.0785 0.9829\n" ] } ], "source": [ "metrics = banos.score(pred, gt)\n", "\n", - "metrics_df = pd.DataFrame(metrics).T.rename(columns={\"f1_score\": \"da (F1)\"})\n", + "metrics_df = pd.DataFrame(metrics).T.rename(columns={\"f1_score\": \"DA (F1)\"})\n", "print(\"Per-behavior metrics — Recording 1\")\n", "print(metrics_df.round(4).to_string())" ] @@ -136,12 +136,12 @@ { "cell_type": "code", "execution_count": 4, - "id": "f202751f", + "id": "8d710a0b", "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAADwCAYAAADM38HGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbe5JREFUeJzt3Qd0FGUXBuA3vfeQ0HvvvYkUAakqiDRBiorSBUQFf+lNVLoIAtKRXkSQLiAl0qtAqAFCT0JII33/c7+4m91kAwkGWHbf55xRdnZ2djKzyc6de787VhqNRgMiIiIiIiIzYP2yN4CIiIiIiCi7MMAhIiIiIiKzwQCHiIiIiIjMBgMcIiIiIiIyGwxwiIiIiIjIbDDAISIiIiIis8EAh4iIiIiIzAYDHCIiIiIiMhsMcIiIiIiIyGwwwCEiegZWVlbo27fvC993CxcuVO8dFBQESzVy5Ei1D8j0yHGR46PFzysRvQwMcIjoudOe5GgnR0dHFC9eXAUI9+7deyEnw9rJ2toauXLlQsuWLfH3338/1/d+1Wn3neyzmzdvpns+IiICTk5O/ynYGz9+PDZs2ABLVLBgQYPPpouLC6pXr47Fixe/7E0zC3fu3MGQIUPQoEEDuLm5qX28Z8+el71ZRPQCMMAhohdm9OjRWLJkCX788UfUrl0bs2bNQq1atRATE/Pc31veS95bgi05GT979izq1q2LkydP4lXywQcf4PHjxyhQoMALe08HBwcsX7483fx169b953U/S4DzzTffqH1gDipWrKg+lzJJQPno0SN07doVc+fOhTl4GZ9XrcDAQEycOBG3bt1CuXLlXvj7E9HLY/sS35uILEyzZs1QtWpV9e+PP/4YPj4+mDx5Mn777Td07NjxP61bgiRnZ+cMn3/vvffg6+ure9yqVSuULVsWq1evVieZrwobGxs1ZZen7TfRvHlzFeB8+eWXBvN//fVXtGjRAmvXrsWLEB0drbIctra2ajIHefLkQefOnXWPu3XrhsKFC2PKlCno0aMHTI32GLysz2tWVKlSBaGhofD29saaNWvQtm3bl7IdRPTiMYNDRC/NG2+8of5/7do13bylS5eqExMpfZITkw4dOqQrj6pfv74KTo4dO6ayMHKC/vXXX2fpvXPmzKn+n/ZEOS4uDiNGjEDRokVV5iJfvnzqxF7mGyPZB9kWWbZMmTLYunWrwfPXr19H7969UaJECfUzSVAnJ1r6Y2iOHj2qymcWLVqUbv3btm1Tz23atOmJYxp++ukn9f6yHblz50afPn0QHh6eLfvt/fffV5muCxcu6ObdvXsXf/75p3rOmMzsR/k55IRZfm5tmZac4OuXx507d069h5eXF+rUqWPwXFry2ZESL/m5ZHn5Gbdv326wn5s0aaICXTkWhQoVwocffghTkiNHDpQsWRJXrlwxmJ+cnIypU6eqYywlnv7+/vj000/x8OHDdOvYsmUL6tWrp8qy3N3dUa1aNRWM6pPAXvt7JvtDgizJdOiTY+Hq6qq2RYJcWV+nTp3Uc3IcBw4cqLZX5r/99tsIDg5Oty3GPq9Smiclovv371fHS34eCeqMleadPn1a/SyynXnz5sXYsWOxYMGCTI1Dk+2SvyFEZHnM4xIYEb2StCdxctIvxo0bh2HDhqFdu3Yqw/PgwQPMmDFDnaieOHECnp6eutfKlVnJCEkAJCdncsL3JGFhYboTRTmRGzNmjDqxkvfSkufkRE1OvD755BOUKlUKZ86cUVfTL168mK6USpaTMi0JYORkavr06WjTpg1u3Lih+5mOHDmCgwcPqu2UEzQ5KZNyOQk25ORdTsYlqyUneKtWrVLlSfpWrlypTtblxDwjcsI/atQoNGrUCL169VKlOfIe8t4HDhyAnZ3dM+83Iftftl1OkqXMULtdcvIrGZy0MrsfpSxLjrOc5MpyokiRIgbrkmCwWLFiqpRNo9FkuI3y88t+kNJH2UZ7e3scOnRIBWFvvvkm7t+/r/4vJ+QyLkM+S3IssqPMLjslJiaqQEGOuT4JZiRY6N69O/r3768uCkipp/xe6B9jWUaCNgmEhg4dqn5OWUYCb20wql2PBD4TJkxQ4+CmTZum1pP290y2Rz57Elz+8MMPumyfHDcJKGWdss9lPxv7LGTk8uXLKqv60Ucfqc/8/PnzVUAlQZdsu5DfUxk/I8GM/CySOZo3b54KmImInkhDRPScLViwQM5MNTt37tQ8ePBAc/PmTc2KFSs0Pj4+GicnJ01wcLAmKChIY2Njoxk3bpzBa8+cOaOxtbU1mF+vXj21vtmzZz/1vUeMGKGWTTt5enpqtm7darDskiVLNNbW1pp9+/YZzJf3kdccOHBAN08e29vbay5fvqybd+rUKTV/xowZunkxMTHptikgIEAtt3jxYt28oUOHauzs7DRhYWG6eXFxcWo7P/zww3T78tq1a+rx/fv31Xa8+eabmqSkJN1yP/74o1pu/vz5z7Tf9PedHLPBgwdrihYtqnuuWrVqmu7du+v2RZ8+fZ5pP7q4uGi6du2a4Xt37Ngxw+e0Ll26pN6vdevWBvtAJCcnq/+vX79evebIkSMaU1GgQAF13GT/yiSf9Q8++CDd/pT9KPOWLVtm8Hr5/OrPDw8P17i5uWlq1Kihefz4sdH9EB8fr/Hz89OULVvWYJlNmzapdQ0fPlw3T46LzBsyZIjBuk6ePKnm9+7d22D++++/r+bL8cno86r9uWXeX3/9pZsnn2MHBwfN559/rpvXr18/jZWVlebEiRO6eaGhoRpvb+9063ya1atXq9fs3r07068holcXS9SI6IWRDINcQZdyJckgSAZg/fr1ahyCXEmXK/+SUQkJCdFNUkomV/B3795tsC65iitXoTNLxons2LFDlSxJiYt0cZNsi2RX9Mt2JNsgJUL626AtpUu7DfLz6Gccypcvr0qCrl69qpsnpTVaCQkJKoMiZVtylfz48eO659q3b6+e188oyLZKmZk8l5GdO3ciPj4eAwYMUN3OtGT8hmzL5s2b/9N+05Ir9XLVXbJC2v9nVJ6W1f34JD179nzqMpIRks/O8OHDDfaB0JayabMSUuon+9lUyDGW3wmZZCC8ZLXk+Hz//fcG+9PDwwONGzc22J+S7ZDfIe3+lM93ZGSkylBJdtLYfpAyPclmSdZRfxnJvsjxSvt5EZIV1PfHH3+o/0smSZ98BjOrdOnSeP3113WP5eeXMk793x3JOkkTEv0xclJypi2TIyLKCEvUiOiFmTlzpgosZNyLlEbJCY32hPTSpUuqBEmCGWP0y6yEBEVShqQl3af0O2vJc/r191Jmpd9kQMpj5L369eunxqRot+H8+fPqZMsYOTHUlz9//nTLSGmR/rgI2SYpA5KgSkpu9MusZJu1KlSooE4wpfRLynaE/Fu2WRsYGCNjfITsS33y80vZm/b5jPZbZlWqVEltn5SpSbAggWdG25XV/fgkMk4mM6WO8jmSk+aMyDgOCWillE1K5aREUBpNSJD2pJKntJ+rrJCgRD/ANaZGjRpqXElSUpLq7Cf/ls+P/jGS/Snb4efn98T9qS35lHFWWf28CDm+UlaoT35XpTwx7Tpkf6ctJzS2zoxk5ndH3kcCnLTkAgER0ZMwwCGiF0bGWmi7qKUlV+DlKrMMkDbWdUmuVOtLe+L42WefGQzSlxPaJ93zQtYnJ5fSwU3bGUq2Qa6iS2c3YyTzpC+j7lD6QYwEUBLcyNVtOVmTk175OSWDJe+nTzI1Mg5Jrs7LmJ6NGzeq7nLZ2THsaSfcTyLBgIztkW2TbU2bLdHK6n58XturT/a5dNKSex/9/vvvqnmDjFWZNGmSmpf285XR5yor5LhrmyZkRAJYyQQKGesiQYYMwJcxMYMGDdLtTwluli1bZnQdGQWS2UGCv4yO83+Rmd8dIqJnxQCHiEyCXA2Wkxu5Yi9ZnqySDl367XbTDtI2RgZQi6ioKBXgyDacOnUKDRs2NNql61nISbUMopYTaa3Y2Nh0Hc6EBA2SYZByOslwyY00JRB6Eu39RaSxgGRstKRsTQaia0+es4MEOFIGJjdQlFKqjGRlP2bHfpb3kyBAmjY8reV3zZo11SSBpGSjpNxpxYoVatB8Zj5XWaEdLJ8VUiomwbk0VZDGAtrPpZQivvbaa08M+LQZFckEZZTl0P+8pM3AybzM3K9GlpH9LRkj/ayNvD47yftIOWRaxuYREenjGBwiMgnvvvuuuqorJ/hpr+LKYxm78iRSniQn89pJxic8rauajL+RUitt6Y+M/5EyMmM3WZQyJcn0ZJX8TGl/HukMJyVJacm4Fcl8SGmaTLly5VKldU8iP6uUM0kHN/33+eWXX1RZU1Y6Wz2NnEBLq2IpuZNsXEaysh/lBN5YsJcVUmomWQbpnpY2K6bdJ1L6lPY4aIOhjFqAG/tcZWWS4/csvvrqK/V51+4/2Z/yeZHOf8aCdO3+ky5xkl2T4yNBtLH9IBlU+bzPnj3b4OeWzKmUFWbm8yJd+IR85vTJZyM7SUYrICDA4Ga88nubUSaLiEiLGRwiMgly8izjD6QdrLTvlZNWOVmTLIQ0IpA2woMHD/5PmRQpQ5ITvdu3b6sAQE565URPm0WQu65Lq2YZ2C4Dt+WKuZxYyv1fZL6UNWVUYpcRKTeSbIeUpsnJspywydV4bRtpY1kcyZLIAHAZi/O08iApT5J9JoFh06ZNVXtmuZIu98WRNsDPmn3IiJRsPU1W9qMEorI/pJxN7t8jGTwpHcwKyVb873//UwGADFyXYFlKq6QRgqxTTvilzEz2SevWrdVnTQbjSwAhjRjkHi+mRAIIGUcj+0TuZyQZHcnmyM8hJ/sSyMiYNBmbIw0IpJxNxpTJzyLjiyQbJcdee/8gyabJDV1lH8jrJk6cqBoZyHqlBFLbJlruTyP3tnkaCQzldbI/JYiWNtG7du3K9syKZM+kFbU0V5BST22baBm/I4FOZrJ/8jdF/PPPP+r/8ruoHWf0zTffZOv2EpEJedlt3IjI/GlbxWamRe/atWs1derUUe2DZSpZsqRqmRsYGGjQ7rhMmTKZem9jbaJlvbVq1dKsWrUq3fLSRnfixIlq/dK21svLS1OlShXNqFGjNI8ePdItl7aVr34LXP22xw8fPlTtlH19fTWurq6aJk2aaC5cuJBuOf2Wx9rt3L9/f7rnjbXd1baFln0lrab9/f01vXr1Uu+tLyv7TX/fSQvjJzG2LzK7H2Vf1K1bV7ULl/Vo98mT3jttm2gtaYldqVIl3fvJz7tjxw713PHjx1XL6fz586vnpVVyy5YtNUePHtW8LPIZaNGihdHnFi5cqH5GOd5ac+bMUftQ9pW0gy5Xrpzmyy+/1Ny+fdvgtRs3btTUrl1bLefu7q6pXr26Zvny5QbLrFy5UrevpO1yp06dVLt2fXIs5HfFGGkx3b9/f9XqXZZ56623VPv3zLaJNvZzy/GSSZ+0iH799dfVdubNm1czYcIEzfTp09U67969q3kaYy3itRMRmS8r+c/LDrKIiIiIMkMadvz8889q7FxGzQqIyLJxDA4RERGZpLQtumVskpSZ1alTh8ENEWWIY3CIiIjIJElrdblnkTTgkLFCMnZOugsOGzbsZW8aEZkwBjhERERkkqQBhDQImTNnjmoqULlyZRXkPK27IBFZNo7BISIiIiIis8ExOEREREREZDYY4BARERERkdlggENERERERGaDAQ4REREREZkNdlEjxapXTe4JE6GZ9TePhwnh8TA9PCamhcfDtPB4mN7xoBePGRwiIiIiIjIbDHCIiIiIiMhsMMAhIiIiIiKzwQCHiIiIiIjMBgMcIiIiIiIyGwxwiIiIiIjIbDDAISIiIiIis8EAh4iIiIiIzAYDHCIiIiIiMhsMcIiIiIiIKEv++usvvPXWW8idOzesrKywYcOGp75mz549qFy5MhwcHFC0aFEsXLgw3TIzZ85EwYIF4ejoiBo1auDw4cNZ2zAGOERERERElFXR0dGoUKGCCkgy49q1a2jRogUaNGiAkydPYsCAAfj444+xbds23TIrV67EoEGDMGLECBw/flytv0mTJrh//36Wts02yz8NERERERFZtGbNmqkps2bPno1ChQph0qRJ6nGpUqWwf/9+TJkyRQUxYvLkyejRowe6d++ue83mzZsxf/58DBkyJNPvxRI1IiIiIiJ6rgICAtCoUSODeRLYyHwRHx+PY8eOGSxjbW2tHmuXySxmcIiIiIiICHFxcWrSJ+NlZPqv7t69C39/f4N58jgiIgKPHz/Gw4cPkZSUZHSZCxcuZOm9GOAQEREREVkI6z61MnxueI4mGDVqlME8GQ8zcuRIvEoY4BARERERWQhr24xHqAwdOlQN8teXHdkbkTNnTty7d89gnjx2d3eHk5MTbGxs1GRsGXltVnAMDhERERGRhbC2sc5wkmBGAg79KbsCnFq1amHXrl0G83bs2KHmC3t7e1SpUsVgmeTkZPVYu0ymf8Zs2WJ6aaRP+NSpU3kEiIiIiChTGZyMpqyIiopS7Z5l0raBln/fuHFDlw3q0qWLbvmePXvi6tWr+PLLL9WYmp9++gmrVq3CwIEDdctI9mju3LlYtGgRzp8/j169eql21NquapnFACebBQUFqZsdaQ+2Vrdu3dCqVavsfjuz8HrRitjY6wfcmvA7NLP+xjsV6j71NfWKVcaxoYsQO/0vXBq1Gl1rtki3TO96bXBt7Ho8nr4Xf3/5C6oVKG3wvIOtPX7sMBgh329D5JQ/seaTCfBz84al4/EwLTwepoXHw/TwmJgWHg/LCXCOHj2KSpUqqUkbnMi/hw8frh7fuXNHF+wIaREtLZ8layP3t5F20fPmzdO1iBbt27fHDz/8oNZRsWJFdT69devWdI0HnvozZmlpoufAxcEJp25dQp8VP2Rq+YI+ubC5zyTsvngMFcd3wdQ/V2Je56F4s1QN3TLtqjTC5DafYdTmeag8vitOBV/Ctv5TkcPNS7fMlLYD8Fa5Omg772vUm9ILuT18se7Tby3+GPN4mBYeD9PC42F6eExMC4+H5QQ49evXh0ajSTctXLhQPS//37NnT7rXnDhxQnVqu3LlikoApNW3b19cv35dLXPo0CHUqFEj6z9jll9BKpKsU6cOPD094ePjg5YtW6qDpI1OhUSwksmRAymdJyTV9ttvv6l5MmkP+FdffYXixYvD2dkZhQsXxrBhw5CQkGCwl3///XdUq1YNjo6O8PX1RevWrTM8ChIJy3alrXE0ZVv/CcCwjT9jw6m9mVq+5+vv4lrobQxeOx0X7gZh5t41WHNiNwY27KBbZlDDjph74DcsDNiM83eD0HP5RMTEx+LDWi3V8+6OLvio9lsYtGYadgcew/Ebgei+eCxeK1IeNQqVgSXj8TAtPB6mhcfD9PCYmBYej1d7DI65MJ+f5AWSWkBJw0lqTgIJuQmRBB0yEOrw4cNqmZ07d6rU3Lp16zB48GC0a9cOTZs2VfNkql27tlrOzc1NRbjnzp3DtGnTVN2h3NFVS1J5su7mzZuriFfer3r16ka367vvvlN3ed2+fTsaNmwIc1WrcFnsvHDEYN62c3+jVuFy6t92Nraokr+EwTJyRUEea5epUqAk7G3tDJYJvHcd10PvoFahlGWIx+NVxN8P08LjYXp4TEwLj8eLZ2Nvk+FkLtgm+hm0adPG4PH8+fORI0cOFaTI/4VkdvRb2kn7O0m1pW1z98033xg0DJBgaMWKFWoAlhg3bhw6dOhg0JNc6hbTkkzQkiVLsHfvXpQpY94ZiJzuPrgXEWYwTx57OLnC0c4BXs5usLWxNbLMQ5T0L6hbR1xCPB49jjJcJjJMPUc8Hq8q/n6YFh4P08NjYlp4PF486yyWor2KGOA8g0uXLqnBT1IXGBISojI3QgZSlS5tOJD9aVauXInp06erEjfpRpGYmKha8mnJ4KoePXo8cR0ySEuySpJRkjK3Z7lDLRERERGZP2sLCHDM/yd8Dt566y2EhYWpcjIJcmQS8fHxWVpPQEAAOnXqpMrPNm3apErQ/ve//xmsRzI/T/P6668jKSlJtdp7mgkTJsDDw8NgknmvkrsRofB3N+x2Jo8lGxObEIeQqHAkJiUaWcZLvVa7Dgc7e5X1MVjGzVu3DPF4vIr4+2FaeDxMD4+JaeHxePGsOQaH0goNDUVgYKAqLZNxLqVKlcLDhw91z8tNioQEHPpkftp5Bw8eRIECBVRQU7VqVRQrVkx1jdBXvnz5pzYMkDE5W7Zswfjx41VrvSeRnuSPHj0ymGTeqyTg6lk0LFHVYF7jUtURcPWM+ndCUiKO3QhEwxLVdM9LYwd5rF3m2PULiE9MQMOSqcsU98+PAj65EHAtZRni8XgV8ffDtPB4mB4eE9PC4/HqdlEzZSxRyyIvLy81vmbOnDnIlSuXKkuTgf1afn5+Kusindby5s2rOp9JlkTG12zbtk0FR/J6mScBjbxextxIlzRpKLB+/XqD9xsxYoQKpIoUKaLG4kgJ2x9//KHG3OiTpgUyv1mzZrC1tcWAAQOMbr+Uo5laSZq0lCyaI6/ucSGf3KiQtxjCoiNw8+E9jH+nF/J45kDXRaPV87P3rUPf+u9hYuu+mH/wd7xRoiraVW6IFjM/161j8q7lWNR1GI7eOI/DQecw4I32cHFwxIKAzer5iNho/HLwd0xu0x9h0Y/U4xntPsfBK6dx6No/sGQ8HqaFx8O08HiYHh4T08LjYfpsbK1g7hjgZJF0TJOApH///ihbtixKlCihxtBIO2i1Q21t1ePRo0ercTpSPiYtoWUcjfxfMjUy1mb37t14++231d1bpd+3jItp0aKFahMtbaW1ZL2rV6/GmDFj8O2336rxOXXrGr8RprSuliBJSt5sbGzQr18/vAqq5i+FPYN+Mrg/jZAWz90Xj0EuD1/k905tzhAUekcFM1PafobPGrRDcPh9fLx0ArafTykVFKuO7UQOV0+MbtlDDWA8GXwJTWcMxP3I1MYDA1dPRbImGWs/maBu+rnt3CH0XvEdLB2Ph2nh8TAtPB6mh8fEtPB4mD57M2oHnRErjfTPJYtn1aumxe8DU6GZ9TePhwnh8TA9PCamhcfDtPB4mN7xMDUlF76X4XMXuq2BOWAGh4iIiIjIQthbm38GhwEOEREREZGFsLfhGBwiIiIiIjIT9hYwBocZHCIiIiIiC2HPDA4REREREZkLe2ZwiIiIiIjIXNgzg0NERERERObCydYG5o5jcIiIiIiILIQ9MzhERERERGQu7DkGh4iIiIiIzIU9MzhERERERGQu7KzN/0af5n+nHyIiIiIiUuysM56yaubMmShYsCAcHR1Ro0YNHD58OMNl69evDysrq3RTixYtdMt069Yt3fNNmzbN8naxyQARERERkYWwy6YStZUrV2LQoEGYPXu2Cm6mTp2KJk2aIDAwEH5+fumWX7duHeLj43WPQ0NDUaFCBbRt29ZgOQloFixYoHvs4OCQ5W1jBoeIiIiIyELYZVMGZ/LkyejRowe6d++O0qVLq0DH2dkZ8+fPN7q8t7c3cubMqZt27Nihlk8b4EhAo7+cl5dXln9GBjhERERERBY0BscugymzJBNz7NgxNGrUSDfP2tpaPQ4ICMjUOn755Rd06NABLi4uBvP37NmjMkAlSpRAr169VKYnq1iiRkRERERkIeyekN6Ii4tTU9qMStoysZCQECQlJcHf399gvjy+cOHCU7dBxuqcPXtWBTlpy9PeffddFCpUCFeuXMHXX3+NZs2aqaDJxibzNyhlBoeIiIiIyELYPSGDM2HCBHh4eBhMMi+7SWBTrlw5VK9e3WC+ZHTefvtt9VyrVq2wadMmHDlyRGV1soIBDhERERGRhbCzyXgaOnQoHj16ZDDJvLR8fX1VRuXevXsG8+WxjJt5kujoaKxYsQIfffTRU7e1cOHC6r0uX778fAOchIQEFClSBOfPn8/qS4mIiIiIyEQzOA4ODnB3dzeYjHUxs7e3R5UqVbBr1y7dvOTkZPW4Vq1aT3z/1atXqzK4zp07P3Vbg4OD1RicXLlyPd8Ax87ODrGxsVl9GRERERERmUkXtUGDBmHu3LlYtGiRSnxIQwDJzkhXNdGlSxej2R8pT5PyMx8fH4P5UVFR+OKLL/D3338jKChIBUvvvPMOihYtqtpPP/cmA3369MHEiRMxb9482NqyTwERERER0avAMZvug9O+fXs8ePAAw4cPx927d1GxYkVs3bpV13jgxo0bqrOaPrlHzv79+7F9+/Z065OSt9OnT6uAKTw8HLlz58abb76JMWPGZPleOFYajUaT1R+odevWKqpydXVVg4DStneTG/kQEREREZFp+TWwT4bPvV9iJszBM6VfPD090aZNm+zfGnpp/hfwCfe+iRhXaw4exq182ZtB//JyaI8TpUpyf5iQSucvwG5gnZe9GfSvhCn78emfH3N/mIif35iHKSd7vuzNoH8NrDjb5PaFbZqsijl6pgBnwYIF2b8lRERERET0XNlaZ/5+Mq+q/zSARurupJZOyN1Gc+TIkV3bRURERERE2czWAjI4z/QTSoeEDz/8ULVsq1u3rppkIJD0s46Jicn+rSQiIiIiov/M1somw8miAxxpC7d37178/vvvqsuBTL/99pua9/nnn2f/VhIRERERUbZkcGwzmCy6RG3t2rVYs2YN6tevr5vXvHlzODk5oV27dpg1a1Z2biMREREREWUDW47BMU7K0LQ9rvX5+fmxRI2IiIiIyETZmlGmJiPP9BPWqlULI0aMQGxsrG7e48ePMWrUKPUcERERERGZHlsLGIPzTCVq06ZNQ5MmTZA3b15UqFBBzTt16hQcHR2xbdu27N5GIiIiIiLKBrYsUTOubNmyuHTpEpYtW4YLFy6oeR07dkSnTp3UOBwiIiIiIjI9thZQovbM98FxdnZGjx49sndriIiIiIjoubFlBifVxo0b0axZM9jZ2al/P8nbb7/NjyURERERkYmxtWIGR6dVq1a4e/eu6pQm/86IlZUVkpKSXswRIiIiIiKiTLOzsTP7vZXpErXk5GSj/yYiIiIioleDjdUzj1B5ZTxTjurmzZvZvyVERERERPTcAxybDCaLDnAKFiyIevXqYe7cuXj48GH2bxUREREREWU7GwY4xh09ehTVq1fH6NGjkStXLjUmZ82aNYiLi+PHkIiIiIjIRNlY22Y4WXQGp1KlSvj+++9x48YNbNmyBTly5MAnn3wCf39/fPjhh9m/lURERERE9J/ZMIPzZNIxrUGDBqpUbefOnShUqBAWLVqU6R1cv359DBgwAKbsRW6jlP5NnTr1hbwXEREREVkeGyu7DCdz8Z9yUcHBwfj111/VdPbsWdSqVQszZ87M9OvXrVun7qtjCvbs2aOCNRlT5Onp+Vy3ceHChSpoCg8PN5h/5MgRuLi4wBJd3nkLF7fcROyjeHjkd0WlzkXhXdjd6LJ7JpxESOCjdPNzlvdGnUHldI8jbkfjzKpreBAYDk2SBu55XFCrb2k4+zjqlgm9/Ahn1wYh7EoErKyt4JnfFa8PLgcbextYujUrDmHpwgMIC4lC0eL++HxoC5Qpl9fospt+O4Gxw9YbzLO3t8VfR4cbXX7imI1Yv/ooBnzRFB0+qJ3u+fj4RHzUaQ4uBd7F4lW9ULxkLlg63/ffh9+HH8HO1xePL1xA8LixiDlzJsPlPZs0Qa7+n8E+Tx7EXb+O25N+QMRff+met3Z2Ru5Bn8OjYUPYenoiLjgYD5YuQejKlbplii5aDLfq1Q3WG7JiBW6OGglL1+u1dzHojY7I6eaN07evYMC6KThy43yGN9X7qtEH+KBaM+Tx8MXF+zcxdNMsbL9wSLeMtZU1hjf9EO9XeRM53XxwOyIEiw//gfE7Ui8aDmvyIdpVaoh8nn6IT0rE8eBADN88B4dvnIOlq5+nARrnbwIPew8ER93EiovLERR5LcPlG+ZthLp56sPb0RtRCVE4fv8Y1l9di8TkxEyv09baFm2LtkNV/+qwtbLFubB/8GvgMkQmRMDSnd0WjJO/38Tj8Hj4FHDBa92Lw7+o8e/030adwJ1zhudDIn8lbzQfUkH9+8jqa7hy8D6iQmNhbWuNHIXcUL1DIfgX89Atv7RvAKIexBqso0bHwqjUqkC2/3yvOhszaiaQkWf6CX/++WcV1Bw4cAAlS5ZEp06d8Ntvv6FAgax9iLy9vWHqXuQ2SqmfJbp56D5Or7iCyl2Lw7uwGy5tv4V9P5xBk2+rwdHdPt3ytfuVQXKiRvc4LjoBO4cdRd5qqfsv6v5j7Bl3EgXr5kTp1gVg52SLiFvRsLazNghu9k06g5It8qNi56KwtrZC+M0oSU3C0u3YegbTvt+Kr4a9pYKaFUsDMKDnYqzc2B/ePq5GX+Pi6oBVG/unzshgP+7ZdQ5nTwcjh59bhu//4+Tt8M3hpgIcAjybNUOer4bg5siRiDl9Cjm6dEWRufNwvnkzJIaFpT8WFSuh4A+TcHvKZDzaswfeLVui0IwfEfheG8ReuqSWkfW51aiB619+ifhbt+D22mvIN3w4Eu7fR8Tu3bp1haxahTszpuseJz9+bPGHpG3FN/B9q77os/oHHL5+Dv3rtcPmTyejzISOeBCV/kRtdPNPVODSc9VEBN6/gTdLVMea7uNRd3pPnLyVcjy+aNgJn9ZuhQ+Xj8O5O9dQJX9JzOvwNSJio/HjvjVqmUsPbuKzdVNwLfQ2nOwc8Fm9dvij52SUHNcBIdHp39dSVPWrhveKtcOvgUtx7dFVNMzXCP0rDsCIv79BZEJkuuWr+VdH6yJtsOjCAlx9dAV+zv7oVkrK6zVYfXlVptfZrmgHlPMthzlnZ+Nx4mN0LP4+epbrje+PfwtLdvngPRxcfBl1Py4Bv2LuOPPHTWwefwodp9SAk0f67/Qmn5dFcmLq7UdiIxOx+ssjKFzTTzfPM5cz6nQvBnd/JyTGJ+P05pvYPO4UOk6vCSe984Rq7QqhVMPUC2J2juZ/Iv8sbLJxrI0kNmTYitwrs0KFCpgxY4Yap5/RBf7u3bsbzHNwcEBsbGpgqtFoMGLECFUdJomA1157DbNmzUKxYsWe/xicsWPHokaNGjh27JjK3AwdOjTLwU3a8i8pzxo/frwaw+Pm5ob8+fNjzpw5umVr166Nr776yuD1Dx48UNmVv/69KilNDgYPHow8efKoTIhso2RmtK5fv4633noLXl5e6vkyZcrgjz/+QFBQkMreCHlOSu+6deuWbhvFnTt30KJFCzg5OamSPAn00paWTZ48GeXKlVPvkS9fPvTu3RtRUVHqOdkeObiPHj1S7yPTyJEpV0PTrkfGOL3zzjtwdXWFu7s72rVrh3v37umel9dVrFgRS5YsUa/18PBAhw4dEBmZ/g+6Kbu4LRiF6uVCwddzqixL5a7FYGNvjaC/jJ/c2rvawdHTXjfdP/tQZVzyVk8NcM6uuaYyOuXbF4FXATe4+jkhdyVfg4Dp1K9XULRRHpRsmR8eeVzglssZ+ar7wUYvCLJUyxcfxDttqqBlq8ooVMRPBTqOTnbYtOF4hq+Rz7KPr1vqZCQQun8vApMm/IFRE96Dja3xLNnBfRdxKOAy+n/eJFt/pleZX9duCF29GmHr1yH2yhXcHDkCybGx8Hm3jdHlc3T5ABH79+P+/PmIu3oVd6ZPx+Pz55Dj/U66ZVwqVUTobxsQdeQw4m/fQujqVXgcGAiX8uUN1pUc+xiJISG6KTk6GpZuQP0O+CXgdyw6/AfO3wtC79XfIyY+Ft1qtDS6fKeqTTBx5xJsPf+3Ck5+PrgBW84HYGD9DrplahUsi9/P7seWcwG4/vAu1p3agx2Bh1EtfyndMiuO78CfF4+qdZy7ew2DN8yAh5MryuUuAkvWKF9j7L+9DwfvHMCdmDtYFrgU8cnxqJ27jtHli3gUxZVHl3Hk3mGExobifNg59e+C7oUyvU5HGye8lrsOVl9ahcCHF3Aj8joWnl+Aop5FUci9MCyZBB+lGuZGyQa54J3XRQU6tvbWuLD7jtHlHV3t4OzpoJuCT4fB1sEaRfQCnGJ1/JG3vLcKcLzzuaB2l6KIf5yE0Osp51Zado42BuuSx/T8xuCsXLkSgwYNUgHJ8ePHVYDTpEkT3L9/P8PXyPmsnEtrJzk31/fdd99h+vTpmD17Ng4dOqTOpWWd+kFQZjzTmZyceMsGyA+SnSZNmoSqVavixIkTKijo1asXAgMD1XOSJVqxYoWK7PR3bO7cufH666+rx3379kVAQIBa7vTp02jbti2aNm2KS/9esezTp48KgiQgOnPmDCZOnKiCBwlC1q5dq5aR95MdPm3aNKPb2KVLF9y+fVsFKvIaCcLSHkhra2t1cP755x81JunPP//El19+qQvUJIjRP8ASlKUlN1OV4CYsLAx79+7Fjh07cPXqVbRv395guStXrmDDhg3YtGmTmmTZb799da4eyVWb8KBI+JX20s2TUjH/Ml4IvZK5NP+1fXeRr4YfbB1S/pBpkjW4ezoMrjmdse+H0/i930HsGn0ct46F6F4TGxGPsKuRKuD5c+wJ/N7/YErp28X0pW+WJiEhEYHn76BazSIGn+lqNYrgzKngDF/3OCYerZpMwtuNf8AX/X/F1cv3032mR329Fp27vYbCRVO/uPSFhkZhwqiNGDm+DRwcTaN89WWzsrODc5kyiAw4mDpTo0FkQACcK1Y0+hqXChUNl5eSzf0H4KK3fPSJk/Bo8Abs/FKOhWv1GnAoWBARBw4YvM6r5VsodzAAJTduRK6Bg2DlmFriaYnsbGxROW9x7Lp4VDdPvpf+vHQUNQuUMfoaB1s7xCYadhmNTYhD7cKpwWRA0Fk0KF4FxXLkU4/L5y6K1wqXV0FRRtvxca13EP44EqdvX4alsrGyQX63AipI0dJAgwth51E4g0BDght5TUG3lIDG19EXZX3K4WzomUyvs4B7AVWidv5h6jL3Yu6qgKmwh+UGnEmJyXhwNQp5yxl+p+ct5417lzL3nS6BUNHafhkGJ/Ie53bdhr2zLXwKGF5IO/HbDSz4aB9Wf3UEJzfeQHISb0z/PAMcuaDfo0cPdeG+dOnSKihxdnbG/Pnzn3gxNGfOnLpJGpTp/y2Vc+RvvvlGnQOXL18eixcvVufdcq6bFc+Uo5KNEzExMSrYiY+PN3heNuhZNG/eXAU2QrI1U6ZMwe7du1GiRAmVvZBMyv79+3UBjWRPOnbsqLZHtmPBggXq/xL0CAkctm7dquZLdkiea9OmjcquiMKFC6crRfPz8zMYg6PvwoULqpmCjJWRQEzMmzcvXdpMP+MjmRXJePXs2RM//fQT7O3tVaZFe4AzsmvXLhWEXbt2TQVgQg6yZJ3k/atVq6Y7aZSUn2S9xAcffKBeO27cOLwK4iIToEkGHD0MT2Yd3O0QcSfmqa8PuxqBiOBoVP2weOo6IxKQGJuEwM03UKZNIZRrWxh3z4Qh4Md/UO+rCshR0hPR91OuBJzbEITyHYrAI78Lrh+4h7++O4XGY6vCLaczLFX4wxgkJSXD28dwPJiXjwuCrj0w+poCBX3wv1Gt1FidqKhYLFt4AD26zMXydX3hlzOlRnrJ/P2wsbVGu041ja5D/rCN+WYdWrerilJl8uD2Ld5jS9h4esHK1hYJoaEG+ysxNASOhVKvOOuz9fVFYkj65WW+VvDYMcg3egzK7v0LmoQEtf9vDh+G6KOpJ+4PN21C/O3bqmzNqURx5P58MBwLFcS1/nqliBbG18UDtja2uB9pWBp4LzIMJfyMVzJsv3AYn9XvgH1XTuFK6C28UawKWpWvBxvr1GuM3+1aCndHF5wdsgxJmmTYWFlj2B9zsPz4DoN1NS9dG8u6jISznSPuRISi2ayBCI223AszrnausLG2QWS84clzRHwEcjob/46VbI2rnRu+qPIVrP4t19kbvAdbrv+R6XW627sjITlBlaalXcbD3vhYE0sQGyHf6Zp0pWhOHnYIv/307O+9yxEIuxmNej1Lpnvu+rEQ7Jh2DonxSXD2tEfL/1UwKE8r1zQPfAu5qYzQ3YuPcGj5VcSEx6F2l6yVNlkCm2wYgyPn/lLJJVVc+hdDGzVqpJINGZGKJqn6kvPXypUrq/NzObcVcs4rpW6yDi05Z5aKLFmnVCll1jP9hFIaJiVcEjwYk5SU9CyrNQiMtAGANjsi41PefPNNLFu2TAU4shPkh5XxQEKCAXnf4sVTT3SFZGx8fHzUv/v376+yQtu3b1c7T4KdrARjkt2xtbVVB0SraNGiqqxNnwRBEyZMUAFRREQEEhMTVWpNAkKJbDPj/PnzKrDRBjdComMJvuQ5bYAjAZQ2uBFyX6InpQZlf6S9X5HUP76qrv11Fx55XQwaEmizfLkr+6J4k5RB8Z4FXBF6OQJXd99WAY52mUINUkrjhJSyPTgXjqB9d1VQRJlXrkJ+NWmVr5AfHVrNwPo1R/Fp34a4cO42Vi77G4tW9tRdIElr1a+HEBMTj64f1eWufwFydP4ALhUq4EqvXqpEzbVqNeQdljIGR7JDQsrWtGIvXUTCgwcotnAR7PPlQ/zNmzxOmTRo/TTMbv8lzg5dpv72XAm9rcrbulVvYTCup2Plxvhg6ShVflYhTzFMatUfdyJCsORI6nftnsvHUfWH7vB18cRHNd/Cr11H47Wpnxgd+0PGFfcsgWYFmquGANcirsLPyQ/ti3dA8/iW+CNoE3fbS3Thzzvwzu9itCFB7jJeaPtdVRVEnf/zDnZM/QfvjquiC6YqtEz9DpLMjlxQ+2tuIGp0LMLS8yyMwcnoPDHtuWJISIg679bPwAh5LOe/xkjCQrI7cu4tQzV++OEHVdkkFU958+ZVwY12HWnXqX3uuZaoSYZCNkxq42QsigQ6UoolmYyNGzfiWaXtViYnQhLhaUmZmtxQNCEhQWVvJBOjzcZIRGhjY6OiyZMnT+omCQa05WYff/yxKvOSLIcERJKFkcFQ2UnG87Rs2VIdPClhk+3RdpZLm+nKDk/bZ2lJ4CXRsP4k814WBzc7WFkDsY8SDOZLFsbRyGBEfYlxSapBgTQSSLdOGyu45zYMJt1yOyMmNOWX1skzZd3uuV0yXMZSeXo5w8bGGmGhhlfbHoZGq7E1mWFrZ6M6nwXfSLnKffJYEB6GRaNVk8l4rdJINd29HY7pk7ahVdPJapljh6/i7KmbqFt1tHq+bcuU39vuHX/G6P+tg6VKCn8ITWIi7P69UKNl6+OLhJDUskt9MlbG1jf98jJfWDk4INeAAbg18VtE7NmN2IsXEfLrMjzc8gf8umd8L7OY06fV/x3yW25XopDoR0hMSoSfm2EDGn83b9yNCM3gNeF4b/7X8PiqMYqMeQ9lJ7yPqLjHuBp2W7fMt2/1xve7lmHViV04e+cqlh3dhml7V+HLhh8YrEvG+lwJuYVD1//BJyu/RWJyErpnMPbHEkgHtKTkJLilyZpIhuVRvPHM1tuF38GhuwE4cGcfbkffwsmQE9hwZT2aFWgGK1hlap2SqbGztoOTrZORZSy3i5qju3ynW+HxI8PzncePEtSYmCdJiE3ClYP31NgdY6RkzSOnM/yLe6B+z5Lqe14CnYz4FXVHcpIGkWk6qxFglZSY4fQ8zxOl27IM9ZDx4/Xq1VOdiiWBoU1WZKdnCnBkTInU3UmAIOkoSTV17txZjct5nifLUo8nmRAJqCTAkYBH/+ajEklK9kKyKvqTfimYZESkXEx26ueff666NAgpHXta9kkiT8nGyBghrcuXL6vW0loS0EiAIeOJatasqTJKUjuoT97raVmuUqVK4ebNm2rSOnfunOooIZmcZyWpRAlO9Sf99OKLJu0ePQu64f651H0o6W157FPkyWn+4MMPkJyQjPy1/dOt06uQGyLTlLhF3Y2Bs2/K+AH5vzQoSL/MYzj7vroZrexgZ2eLEqVy4cihq7p58pmWx+UqGG8TnZaUuF25dA8+OVLqo5u9VRFL1/RWLZ+1k3RR69TtNUyb1UUtM2hIcyxZnbrM5Jmd1fwx37VFz34NYamkfCzmn3/gVrNW6kwrK7jVrImYkyeNvib61EnD5SV4r10b0f8uLyVv1vI3L+3FEKlX1yubSsupZErZSMKDjLPE5i5BtWe+iDeKVzG8J1yxKvj7+j9PfG1cYjxuPwpRbaNbl6+H38/s0z3nbO+IZKnX1SMn2dI++knkeQfbJ18MMmdJmiQ1wL+UV2ozBglSSnqVxNWI1L9h+uytHdSYGn3JSM7SOq9HXFctpUvqLePv7A8fRx/Vmc1SSdYkR2FX3Dpj+J1+6+xD+Bd78nf6lb/vIylRg+L/VlU8lUajxuNkJCQoUjXzdHLneM50pB16BlNmzxN9fX1VYkG/+ZWQx08agpH2Ir2cv8u5tNC+7r+s8z+VqEVHR6uxKkLKs6RkTU7kJZsiXRSeF+mk0KpVKwwbNkxlZmT8jZa8vwQ8EhlKcCE7TLZLxqNINkU6n0nmqVmzZmpZCUpkfI8EEkKCNPmSkoH6MhZIMlPSgECftMSW0rZPPvlEtayTAyNBkiyrLbuRgEoyTJIZko5t0kpbBl3pk7IyyTjJtkmjBilbS1u6Ju8j+1N+JhlwJYGVjE+SiFc7/udZGEszvmxSRnZk7gUVlGjbRCfGJetKxw7PuQAnL/t0ZWPX9t1RZWgOrun/eJVolg9//3QOviU84VfKU43BuXMyFPWGpAyyluMly/yzIUjd+0amoP131bifmn2fPYA0Fx271MaYb9ajVOncKF0uL1YuDUDs43i0aJVSninNAnL4u6P3Z43V419m70bZ8vmQN783IiNSxuDcvROOd95NOQn08HRWkz7poiad1goUShkXkjOX4dg3J+eUk7a8+bx143gs1f1FC1FgwreIOXsW0WdOw69LV1g7OSF0fUpmq8C33yL+3n3cmZKSDXuweAmKLV4Mv27d8WjvHng1b6EaFdwckXJfIumEFnn4MHJ/8QWSY+NSStSqVYf3O++orI6QMjSvli0RsfcvJIWHw7FEceQdMhSRR46ojI8lm7pnBea//z8cu3kBR66fV22iXeydsOjQZvX8gve/wa1HD/DN5pSrktXzl0ZuD1+cun1Z/X94kw/VxcEf/vxVt87N/xzAkMZdcCP8nmoTXTFvcQyo3x4LD/2hC4CGNuqCTf8cUGVrUqLWq8676r46a0+ltvW2RDtv7lBtnoMiryMo4ppq6Wxv44CDt1MaZshz4XHh2HA15ffldOgp1SXtRuQNXIu4pkrU3i7UCqdDTusCn6etMzbpMQ7c3o+2xdojOiEasUmx6FC8o2pgIGVvlqx8i3zY/dMF5CjiBr8i7jj9RzAS4pJQon5KZubPH8/BxdsBNd4vkq65QMGqvnB0s0uX2Tm+PggFq/jC2csBsZEJ6j470WHxuk5rMubm/qUIVcZm72SjHkur6mKv5zR6jmDxklLv95SWg2PmzhPlYn2VKlXUuaycm2svhspjafqVGXKxXyqq5LxbSHdiCWRkHZLlETLUQyrGZIjJcw9wJJMh41HkRF1O0CW1JP+WE3kZA/I8yQm/7Ii6deuqVtL6pJmADOiXoOPWrVsqupQsipSMaXekdFKTG5RKFzPpsCaNDIS0lh41ahSGDBmiukFIoCSD99OSgf4fffSRen85CJKxktpBx387C8n+kOyWdGiTiFeWk2VkfVpSbyhZJOmIFhoaqtrraVtFa8kJuNxbqF+/fmod8mUo25vdJXWmQDqgSbOBc+uDdDf6rPN5OV2JWkxobLpbqkjmJfRihLoppzF5qviqdtOBm2/i5LLLcMvphFp9y8C3eOqJcrEmeZGUkIxTy68gPipBvW/dL8qrltKWrnHTcqrZwNyf/kRoSBSKlciJKbM+0LV+vnv3kSpD0JKgZsKo39Sybu5OKFk6F+Ys7qFaTNN/F75lC2y9vJGrfz/Y+ubA4/PnceWTHkj8t/GAXa7c6iqpVvTJEwj6YjByfTYAuQYORNz1IFzr11d3DxwR9Pkg5B44CAW+/x62Hh6qmcDtqVPVjTy1mSO3WrV1wVT83TsI37Edd2fNsvhDuvrkn8jh6okRTT9GTndvnLp1GS1//hz3o1KuWufz8jfIxjjY2WNU8x4o7JNblaZJZ7Ruy8bgUWxqi1u5v82oZj0wo83n8HP1Ujf6nHtwI8ZuX6CeT0pORgn/Aupmob6uHgiNjsDRG+fRYEYfNWbHkh29f0Q1BpDSMykRC468iemnpupuuOnt6GOQsVHjbDQavFO4NTwdPBGVEInTIaew4er6TK9TrLq8Ahokq3vfSEe1c6H/4NeLS2Hpitb2V+Nkjqy6hpjwePgWdEWLoeVVYwARKWXget8fIvx2DO5eeIQW/0vfnVeSmOG3YrBt71kV3EgAJIHTOyMrqZbR2szR5YP3cXRNkPped/dzRPnm+VChZeo4ZtKjd0Pb/0JaRHft2lVdeJd738gFeUmCaO91I+e+cn6tre4aPXq0Oi+XZIBUJMn9c6RNtAwh0Z77SjJCzuVl2IsEPJLUkOZh2iAqs6w0+n2XM2np0qUqoyCNBqQkS068pZ2xRHMSFKRtZWzOJFiSsjdpLNCw4atbRvO/gE9e9ibQv8bVmoOHcal3k6eXy8uhPU6USt/Rh16eSucvwG6g8Xuc0IuXMGU/Pv0z5QSFXr6f35iHKSd7vuzNoH8NrGhYxWMSIldn/Jxb2yyt6scff9Td6FOyLnKbFOl6pr2XpCRAtAmDgQMHqiEisqxUgEkGSIIZqbpKe6NPuQ2LBEF16tRRXYjTNhF7LgFOWtIdTDomSEZFsibmTMYfSXmZlI/JPWzk/jaSLbp48WK6Af+vEgY4poMBjmlhgGN6GOCYFgY4poUBjmkxyQDn0fKMn/NIHf7xKvvvjbClNtjZ2aB1sjmT8TVff/216sYm7Zml3ExaV7/KwQ0RERERWQZNkmHnWn3Gb+RgIQGOjGWRdJMMApKuZWnbEkuWw1w1adJETUREREREljoGx+wCnM8++0wFONKZrGzZshneuI+IiIiIiExIMgMco1asWIFVq1bp2roREREREdGr3SbaojM40i1NWrwREREREdErJNn8A5wn3yI5A3KfmWnTpqlWbkRERERE9AplcJIymCwtg/Puu++maySwZcsWlClTJl0HMelxTUREREREJibJfAKZ/xzgeHik3gFetG7d+nlsDxERERERPS+JSWa/bzMd4CxYsOD5bgkRERERET1f8RnfB8eix+CIxMRE7Ny5Ez///DMiIyPVvNu3byMqKio7t4+IiIiIiLIzg5OYwWTJXdSuX7+Opk2b4saNG4iLi0Pjxo3h5uaGiRMnqsezZ8/O/i0lIiIiIqL/JtF8AplszeDIjT6rVq2Khw8fwsnJyWBczq5du7Jz+4iIiIiIKLskJmY8WXIGZ9++fTh48KC6H46+ggUL4tatW9m1bURERERElI00SRlncKwsOcBJTk5GkpGdExwcrErViIiIiIjIBCWyRM2oN998E1OnTtU9trKyUs0FRowYgebNm7+4A0RERERERJmXyBI1oyZNmoQmTZqgdOnSiI2Nxfvvv49Lly7Bx8cHy5cv50eMiIiIiMgUJZp/BueZStTy5s2LU6dOYcWKFTh9+rTK3nz00Ufo1KmTQdMBIiIiIiIyIYnmH+A8Uxe10NBQ2NraonPnzujXrx98fX0RGBiIo0ePZv8WEhERERFR9kg0/xK1LAU4Z86cUZ3S/Pz8ULJkSZw8eRLVqlXDlClTMGfOHDRo0AAbNmx4fltLRERERETPLtH8b/RppdFoNJlduFmzZipzM2TIECxZsgSbNm1SY3Hmzp2rnpdszrFjx/D3338/z20mIiIiIqJnkLSya4bP2bRfZHkBjpSi/fnnnyhfvrwad+Pu7o4jR46gSpUq6vkLFy6gZs2aCA8Pf57bTM/Biot9uV9NRIfiP2L6qV4vezPoX/0rzMKxEiW5P0xIlcALsOlX+2VvBv0racZBWPWqyf1hIjSz/kZkwvqXvRn0Lze71ia3LxKXfpDhc7adl2RpXTNnzsT333+Pu3fvokKFCpgxYwaqV69udFlJiCxevBhnz55VjyV+GD9+vMHy3bp1w6JFhkGWJFO2bt36/ErUwsLCkDNnTvVvV1dXuLi4wMvLS/e8/DsyMjJLG0BERERERC9IfFLGUxasXLkSgwYNUreJOX78uApwJBi5f/++0eX37NmDjh07Yvfu3QgICEC+fPnUrWdu3bplsFzTpk1x584d3fQsHZqz3GRA7nnzpMdERERERGSaNHFJGU5ZMXnyZPTo0QPdu3dXt46ZPXs2nJ2dMX/+fKPLL1u2DL1790bFihXVWP558+YhOTkZu3btMljOwcFBJVS0k34y5bm1iZbUkbyxkHvg9OzZU2VyRFxcXJY3gIiIiIiIXgxNQsaBjJzLpz2fl/N+7bm/Vnx8vBp3P3ToUN08a2trNGrUSGVnMiMmJgYJCQnw9vZOl+mRhmYS2LzxxhsYO3asutfmc8vgdO3aVb2hh4eHmqRNdO7cuXWP5bkuXbpkaQOIiIiIiOjF0CQkZzhNmDBBd16vnWReWiEhIUhKSoK/v7/BfHks43Ey46uvvlJxhARF+uVpMk5HsjoTJ07E3r17VZMzea/nlsFZsGBBllZOREREREQmJD45w6ckIyPjavSlzd5kh2+//RYrVqxQ2RpHR0fd/A4dOuj+Xa5cOdXYrEiRImq5hg0bPr8SNSIiIiIiMr8SNUcj5WgZdVa2sbHBvXv3DObLY21Dsoz88MMPKsDZuXOnCmCepHDhwuq9Ll++nKUAJ8tNBoiIiIiIyPxK1DLL3t5etXnWbxCgbRhQq1atDF/33XffYcyYMartc9WqVZ/6PsHBwQgNDUWuXLmQFczgEBERERFZCE0WApknkVI2GZ8vgYrcy2bq1KmIjo5WXdWEjMvPkyePbgyPjKkZPnw4fv31VxQsWFA3VkduPSOT3GNz1KhRaNOmjcoCXblyBV9++SWKFi2q2k9nBQMcIiIiIiJLkZC1AfsZad++PR48eKCCFglWpP2zZGa0jQdu3LihOqtpzZo1S3Vfe++99wzWI/fRGTlypCp5O336tLrRZ3h4uGpAIPfJkYxPVscBMcAhIiIiIrIQmmzK4Ii+ffuqyRhpDKAvKCjoietycnLCtm3bsmW7GOAQEREREVmI5GwMcEwVAxwiIiIiIguRxACHiIiIiIjMRXJi9ozBMWXM4BARERERWYhkZnCIiIiIiMhcJDPAISIiIiIic5EYxxI1IiIiIiIyE8nM4BARERERkblIzqYbfZoyNhkwE3IzpQYNGuDhw4fw9PTEq+bQ5ps4uC4IUQ/j4V/IFc0/LYm8xT0yXP5xVAJ2LbmM8wH38TgyAZ5+TmjaoziKV82hno+LScSfy66o56MfxSNXYTc061ECedKs88HNKOxYeAlBZ8ORnJSMHPlc0X5oebU+S3dmazBO/H4DMeHx8CngirofFod/UXejy64feRy3z4Wnm1+gkg9aDq2g/n141VVcOngfUaGxsLG1Ro7CbqjRoTByFks9JpsnnkZIUCQeRyTAwcUWect5oXanonDxztodjM1Rjvffh/9HH8Euhy8eX7iAG2PGIubMmQyX92zaBHk++wz2efIgLug6gn/4ARF//aV7vkrgBaOvC/7uO9z7Zb76t0PBgsj75RdwrVwZVnZ2eBwYiFvTpiPq0CFYul6vv4vBDTshp7s3Tt26jM/WTMaR6+eNLmtrbYMhb3ZBl+rNkcfTF4H3b2Dobz9h2/nU/ejq4IzRLXqgVYV68HP1wongixi4diqO3khdZ+sK9fDpa61ROX8J+Lh4oPK3XXHq1iVYuteLVsQXjTujSv4SyO2ZA61mf4nfTqV+1o2pV6wyJr/3GcrkKoSbD+9h7JaFWPT3ZoNletdro9arjnHwZfRbOQlHrp/TPe9ga49J7/VHhyqN4WBrp45n7+Xf435kGCzdquUBWLJgL0JDolCsRC588fXbKFsun9Flf99wFKO+WWMwz97eFgePj9U9/nnmDmzfehr37obDzs4GpUrnRe/+b6Js+fy6ZQb2XYSLF27jYVg03NydUL1mUfQf1Aw5/Ix/b1mypETeB4fouTu77y62zQvEW31KqQDk7403sGT4cfSb/RpcPe3TLZ+YkIzFw47DxdMe7YdUgJuPAx7dfwxHVzvdMr/NOIf716Pw7qCycPN2wOk9d7Bo2HH0/akW3H0c1TJhd2Lwy1dHUblxbjR4vwgcnG1x/0YUbO1tLP6oXzp4D/sXX0L9HiXgX8wDpzbfxO/jTuL9qTXh7JH+mDQbXM7gD2ZsZAJWfnEERWr56eZ55nZWQZK7vxMS45NS1jn2JDrPqAUn95R15injiSqtC8DZyx7RYfE4uOQStk4+gzZjq1r0MfFq1gx5hw7BjREjEX3qFPy6dkWxX+bhn6bNkBiW/mTKpVIlFJ40CbcmT8aj3Xvg/VZLFJn5I86/2waxl1JOiE+9VsfgNR5166LAuLF4uG27bl7R2bMRdz0IF7t2RXJsHPy6dkHR2bNwtvGbSAwJgaVqV7khJrXuj94rv8eh6//gs/rtsaX3FJQa0xEPoh6mW35My0/RqVoTfLr8W1y4dx1NStXA2o+/RZ0pn+Jk8EW1zNz3h6BMrsLoung0bj96gE7VmmJ732koO+593H6Usq9d7J2w/+oprD6xC3PeH/rCf25T5eLgpAK9+Qd/x/qeE5+6fEGfXNjcZxJm71uPTgtGoGGJqpjXeSjuPArB9n+DznZVGmFym8/Qc/lEHLr2Dwa80QHb+k9FiZHt8SAy5RhPaTsALcrWRtt5X+PR4yj82H4w1n36Ler88Aks2fYtpzDlu00YOrw1ypbPh+VLDqDfp79g7e+D4e3javQ1Lq4OWLtpsO6xVZrnCxTMgS+/fht58nojLi4Rvy7ehz6f/IINf3wBL++UdVatXhgf9mgA3xxuuH8vAtN+2IyvBi7F/GW9n+vP+ypKtoASNeuXvQGvmvr166Nfv34YMGAAvLy84O/vj7lz5yI6Ohrdu3eHm5sbihYtii1btuhes3fvXlSvXh0ODg7IlSsXhgwZgsTERN3zBQsWxNSpUw3ep2LFihg5cqTusZWVFebNm4fWrVvD2dkZxYoVw8aNG9VzQUFBKnsjZJtk2W7duuFVcXDDdVRpkheVGuWBX35XtOxdCnYONjix45bR5U/svKUyOB3/VwH5S3vCy98JBct5I2chN/V8QlwSzh+8jze7F0PBsl7wye2sAhjvXE448kewbj2SASpWxRdvdi+OXEXc4Z3LGSVr+BkNqizNyU03UaZhbpRqkBveeV1UoGNrb43zu28bXV6CSxdPB90UfPohbB2sUbRmaoBTvE5O5CvvDQ9/J/jkc0WdLsUQ/zgJIdejdMtUbJkfOYt7wD2HE3KV8EDlVgVw91KERVxtehL/7t0Qsmo1QtetQ+yVK7gxYgSSY2Ph06aN0eX9unyAR/v2q0xM7NWruD1tOmLOnYNf5066ZSRA0Z88G76ByEOHEB+c8jti4+UJx0IFcXfOXDwOvIi469dxa9Jk2Dg7w6lYMViyAQ06YF7ARiw8tBnn7wah18rvEBMfh+61WhpdvnP1JpiwfRG2nAvAtdDbmL1/PbacO4hBb3RUzzva2ePdCvUx5LefsO/KSVwJuYXRW37B5QfB6FnnXd16lh7ZirFbF2Bn4JEX9rO+Crb+E4BhG3/GhlN7M7V8z9ffVcdh8NrpuHA3CDP3rsGaE7sxsGEH3TKDGnbE3AO/YWFAyjGWQCcmPhYf/nuM3R1d8FHttzBozTTsDjyG4zcC0X3xWLxWpDxqFCoDS7Zs8X60eq863m5dFYWL+GPo8FZwdLTHxvVHM3yNnLf4+rrpJh/flO9zraYtKqJGrWLIm88HRYr6Y+CXLREdFYdLF+/qlunU5XWUq5AfuXJ7oUKlAuj6cX2cOX0TiRZQjvUsAU5yBpO5YIDzDBYtWgRfX18cPnxYBTu9evVC27ZtUbt2bRw/fhxvvvkmPvjgA8TExODWrVto3rw5qlWrhlOnTmHWrFn45ZdfMHZsauo1s0aNGoV27drh9OnTap2dOnVCWFgY8uXLh7Vr16plAgMDcefOHUybNg2vAsnG3LkcicIVvHXzrK2tULiiN24GPjL6msBDD5CvpAc2z76A7z7Yi5l9DuKvVdeQnKRRz8v/k5M16oRcn529DW78W0Ylz188GgKfPM5YPPw4vuu8B3M+P6RK2iydBBMPrkYib7nUY2JlbaUe370Ykal1nPvzNorV9oedo02G7/HPztuwd7aFbwHjV/RioxJwcd895CruoUraLJWUhjmXKYOIgwdTZ2o0iDwYANdKFY2+xrViRUQG6C0PIGL/AbhUNL68rY8PPOrVQ8ialL8jIulhuAqOvFu9A2snJ8DGBjnat0dCSAhi/vkHlsrOxhZV8pXArsDUkzWNRoNdgUdQq2BZo6+RUqa4hHiDeY8T4vFa4fLq37bWtrC1sUVsQlyaZeLUCTNlr1qFy2LnBcMgcdu5v1GrcLnUY5y/hMEycozlsXaZKgVKwt7WzmCZwHvXcT30DmoVSlnGEiUkJOLCuVuoUbOobp61tbUqFzt96nqGr3scE4+Wjb9Fi4YTMKjfIly5fO+J77F+9WG4ujmieIlcRpd59CgGWzedRPmK+WFrx6oMSwxwOAbnGVSoUAHffPON+vfQoUPx7bffqoCnR48eat7w4cNVICOByO+//64CkB9//FFdoShZsiRu376Nr776Si0nv/iZJVmZjh1TrviNHz8e06dPV0FW06ZN4e2dcjLq5+f3So3BiYmIV8GGq5dh1kSyKCHB0UZf8/DuY1w7/RDl6udE5xGVVKnZplkXkJSUjAYdU0rNJADau+IafPO6wNXTAWf+uoubgeEqSyNkXI5kD/avuYY3OhdF427FcPlYCFZOOIVu46qojJClio1IgCZZA+c0mSx5/PB2zFNff+9yBMJuRuONXqXSPRd0LATbpv6jStSkxPDtbyrqytO0Di69jDPbgpEYlwz/Yu5oOSRlDI+lspWsrK0tEkNDDeYnhIbAsXAh46/x9UVCiOHyiaEhsPP1Nbq8T+tWSIqORvj21PI0cbFbdxT5aSYqHj8mVwWQEBaGSx/3QFJE5gJdc+Tr4qmCkXsRhqWB9yLDUMK/gNHXSNmTlDj99W92pmHxqmo8jY1Vyt//qLgYHLx6Bv9r2h3n715X6+pYpTFqFSqrsjiUvXK6+6Q/fhFh8HByhaOdA7yc3Ywf44iHKOlfULcOCVqlNM1gmcgw9ZylCn8Yo76L05aiyeOgaw+MvkbKz4aNbqPG6kRFxmLpwr/wYeefsGrDIPjnTB2juW/PeXz9xXLExiaoMrSZcz6Cp5eLwbqmT96CVcsPIvZxgsrmTJnZ9Tn9pK+2xMSUC8LmzHIvi/4H5cunXlGzsbGBj48PypVLvWIjZWvi/v37OH/+PGrVqqWCG63XXnsNUVFRCP63FORZ3tfFxQXu7u7qPbIiLi4OERERBpPMe5VoNICLhz3e7lMauYu6o+zrOVG3XSEc3ZK6P2XsjVxxm9RtH8a8uwuHfr+BcnVz6o6DnMALKUmr3aqAakLwettCKF7NF0e28oTiv5DsjU9+F6MNCfKU8UL776uhzZgqyF/RB9umnEXMI8Mr25Xezo/2E6ur4EeyeTt/PKeOJT0/vm3aIOz3TdDEGx6L/COGq8AqsFMnnG/bDuE7d6oxOLY5Upp5UOYMWDtVBSrnvlmO2Cl7Mb3tICz8ezOS9T7XXZeMhhWsEDxuIx5P2YO+9dtixbGdBssQmaPyFQug5TtVUKJkblSpVhjfT/0AXl6uWLfasJlJ1epF8Ova/pi/tBdqvVYcQwf/irBQwwCzS/e6WLa6P36c85H6/hgxdBW/P4yQURIZTeaCGZxnYGeXOphdyEmz/jztSXRycuZSfZLFSXsCl5CQkKn3zex7aE2YMEGVuukbMWIESr6Pl8LZ3V79EZLuafqiwuPh6mW8c5Zke6RkydomNWjMkddFrUNK3mztrFWm5sNvqyE+Nkl1VJNGA6smnoZXTqfU97WxQo78hld/fPO54sa59IOELYmju50qSZPuafrkcdqsTloJsUm4fOAeqrcvbPR5KVnzzOkM5IQaa7O0fwDO/3kbVVqnXBUVktGRSZoSeOVxxqJeB3HvUoRa3hIlPnwITWKiKiPTZ+cjWRrjA/1lTI2dr+Hythks71qlChwLF8bVAQMN5rvVrAmP+vVxslp1JEenZFNvjhoN99q14dOqFe7NnQtLFBIdjsSkRPi7G2Z5/d28013x170mKhzvzh2iStV8XNxV04AJb/fG1dDUcYZXQ27hjel94GzvqMZ33I0IxfLuo9VYEcpesm/THT93b5WNkTLBkKgk48fY3Uu9VrsOBzt7lfXRz+LI50C7jCXy9HKGjY11usBDHvv4Gi9HTktKykqUyo2bNwz3o5OzPfLl90W+/FDZmdbNv8dv646ge48Geu/voibJChUq7IcWjSbgzKkbKoiiVIkWMCyJGZznrFSpUggICDAIYA4cOKCaEeTNm1c9zpEjhxo3oyVZlWvXrmXpfeztU048k5Ke/KmVkrpHjx4ZTDLvZZFgJFdRN1w9nXpiICVr106FIV8J4ye00lhAytJkOa2Q2zFw87ZX69Nn72ijghtpSnDlRChK1ki58izL5SnmjpBgw5Kr0FvR8Mxh2S2itS2cg8+mBnqS8ZLHOYs/ud3m5b/vIylRgxKv58zUe8nvRdITan61vzZPWsbcaRIS1JgX91q1UmdaWcGtVk1EnThp9DVRJ0/Crabe8jIounZtRJ9Mv7zPe+8h+uxZ1QJanxp3ozYgTQZBo4FVFkprzU1CUiKO3QzEG8WrGFxseqN4VQQEnX3ia+MS41VwI22j361YHxvP7Eu3jAxklxNkTyc3vFmyBjaeTr8M/TcBV8+qzmn6GpeqjoCrZ1KP8Y1ANCxRzeAYy2PtMseuX0B8YgIalkxdprh/fhTwyYWAaxm3bzd3dna2KFk6Dw4fuqybJxdijxy6jPIVMhdkSInb5Ut3VRnak8g5QHx8xikH7XnXk5axVInM4NB/1bt3b9UhTZoR9O3bVzUBkIzJoEGDdONv3njjDSxcuBBvvfWWGj8jY3Ok9C0rChQooP4Ab9q0STUgcHJygqtr+qsl0slNJlMiJWLrp/yDPEXdkae4OwJ+u6EyL5Ua5VbPr5t8VrWCbtw1pXNTtWb5cHjTTWyZG4gaLfMh7HYM9q2+pv6tdfl4iDov883jooKh7QsuqvE42nWK194tiNXfnUaBsp4oVM5bvebi4RB0G5964mKpKrbMh10zz8OvsBv8irrj1B83kRiXhFL1U/aflI3JvWlqvV/E4HWSjSlUzReObnbpMjtH1wWhUFVf1QJa2kif2XpLtYLWtpK+e+kR7l+JRK6SHuoeOBH3HuPQymuqrbSlZm+07i1YiIITv1WBSMzp06pNtAQg0lVNyHPx9+7j9uTJ6vH9xUtQYsli+HXvjkd798C7eQs4ly2D68OHG6zX2sUFXk2bIHhi+ta6USdPqLE2Bb/9FndmzkRyXBx827VV99V5tGcPLNnU3SuwoPM3OHbjAg5fP6faRLs4OGLh35vU8ws/GIZb4Q/wv99nq8fVC5RGHs8cOBl8Sf1/eLOPYG1lhe93LtOtU4IZSf7LPXKK+ubFxFZ9VEvpBf+uU8jYkPxeOZHbI2UsVQn/lHuASEAkYz8suU100RwpFwxFIZ/cqJC3GMKiI9Q9bsa/00vt966LRqvnZ+9bh77138PE1n1Va+k3SlRVrb9bzPxct47Ju5ZjUddh6j5Eh4POYcAbKcd4QUDKvXIiYqPxy8HfMblNf4RFP1KPZ7T7HAevnFZtpS1Zpy51MPJ/q1G6TF6UKZsPvy7dj8eP4/FWq5Tv1uFDV8LPzwN9BzZVj+fO2oly5fMjb34fNQZn8YK/cPf2Q7RqU03XgGD+nD9Rt0FpFfSEP4xW99l5cD8CjZqklO6fPX0D/5wNRsXKBeHu7oTgm6GYNWOH6rrG7E165lSKlhGWqD1nefLkwR9//IEvvvhCNSeQZgAfffSRrkmBkAyKZGxatmwJDw8PjBkzJssZHHkfKT2TFtTSrrpLly4qaHoVyBgaGfQvN+aMehiHnIXd8MGoyroStUcPYtUXv5ZHDkd8MLoyts67iFn9/lbBT8238qNOm9Qyp9joROxcfBkRIbFwcrND6dr+aPhBEYNuXKVq+amW1BIcbZkTCN88zuomnwXKeMHSSQc0udnmoVVXVWmab0E3tPy6gq5ELTLE8JiIh7ejcefCIzV2Ji0ZSx1+OwZbJ51RN2aVAMiviDtaj6qsWkYLaQ1+9dB9dUNQaTAg7yXjdJoMLAibNJk5S/NwyxbYensjd/9+sMuRA4/Pn1eD/bWNB+xz5daNKxPRJ07g6uDByDNgAPIMGoi4oCBc6dNXdw8cLe8WLdSFkbBNhjc41HZRk/fIPWAAii9aBCs7Wzy+dBlX+vRJl+2xNKuO74KvqydGtuiBnG7eOHnrEpr/NAj3/70/Sj4vfyRrUrOO0gZ6dItPUNg3N6LiHqt20XK/G/3SJg8nF4x7qxfyeuZAWEwE1p3ag29+/xmJyalZ+bfLvY75nVO/O5Z3H6P+P+qPX1RbaUtVNX8p7Bn0k+6x3J9GSIvn7ovHIJeHL/J7p2aVg0LvqGBmStvP8FmDdggOv4+Pl07Q3QNHrDq2EzlcPTG6ZQ/VNECC06YzBhrcxHPg6qnqOK/9ZIIqP9x27hB6r/gOlu7NZhXw8GE0Zv+4A6EhkSheMjdmzP5Q1/r57p1wVZquFRHxGGNHrlPLSnAiGaBflvZSLaaFlJNLg4JNG5eq4MbD0xmly+bF3EWfqpbRQtpQ7955FnNm7lTBlARCMk7no0/fUDcNJcsLcKw0HL1LAFZc7Mv9YCI6FP8R00/1etmbQf/qX2EWjpUoyf1hQqoEXoBNv9ovezPoX0kzDsKqV03uDxOhmfU3IhPWv+zNoH+52bU2uX2x069Ehs81um8eF7AY1hIRERERWYhEC8jgWHbdBxERERGRBUnMxiYDM2fORMGCBeHo6IgaNWqo+zM+yerVq9U9IWV5ucWKDOPQJ4VlMhY9V65cajx5o0aNcClNeXVmMMAhIiIiIrIQ2RXgrFy5UjXNkuZZx48fV2PNmzRpkuE9Gg8ePKhuWC9j0U+cOIFWrVqp6ezZ1A6U3333nbqR/ezZs3Ho0CF130dZZ2xsbJa2jQEOEREREZEF3QcnMYMpKyZPnowePXqo5lalS5dWQYmzszPmz59vdPlp06ahadOmqvGW3EZFmmpVrlwZP/74oy57I52HpRHXO++8o25wv3jxYty+fRsbNmzI0rYxwCEiIiIishCJT8jgxMXFqfsx6k8yL634+HgcO3ZMlZBpye1P5LHc/9EYma+/vJDsjHZ56SB89+5dg2Wku7CUvmW0zoywyQARERERkYXokJRxp7SRI0eq247okxI0ma8vJCRE3Vze3z+lVbeWPL5w4YLRdUvwYmx5ma99Xjsvo2UyiwEOERERERFB7s0o42r0mdoN4jODAQ4REREREUGCmcwENL6+vrCxscG9e/cM5svjnDlTb6yrT+Y/aXnt/2WedFHTX6ZixfQ3EX8SjsEhIiIiIqJMs7e3R5UqVbBr1y7dvOTkZPW4Vq1aRl8j8/WXFzt27NAtX6hQIRXk6C8jY4Ckm1pG68wIMzhERERERJQlUsrWtWtXVK1aFdWrV1cd0KKjo1VXNdGlSxfkyZMHEyZMUI8/++wz1KtXD5MmTUKLFi2wYsUKHD16FHPmzFHPW1lZYcCAARg7diyKFSumAp5hw4Yhd+7cqp10VjDAISIiIiKiLGnfvj0ePHigbswpTQCkjGzr1q26JgE3btxQndW0ateujV9//VW1gf76669VECPtn8uWLatb5ssvv1RB0ieffILw8HDUqVNHrVNuDJoVDHCIiIiIiCjL+vbtqyZj9uzZk25e27Zt1ZQRyeKMHj1aTf8Fx+AQEREREZHZYIBDRERERERmgwEOERERERGZDQY4RERERERkNhjgEBERERGR2WCAQ0REREREZoMBDhERERERmQ0GOEREREREZDYY4BARERERkdmw0mg0mpe9EUT/VVxcHCZMmIChQ4fCwcGBO/Ql4/EwPTwmpoXHw7TweJgWHg/6rxjgkFmIiIiAh4cHHj16BHd395e9ORaPx8P08JiYFh4P08LjYVp4POi/YokaERERERGZDQY4RERERERkNhjgEBERERGR2WCAQ2ZBGguMGDGCDQZMBI+H6eExMS08HqaFx8O08HjQf8UmA0REREREZDaYwSEiIiIiIrPBAIeIiIiIiMwGAxwiIiKyaFZWVtiwYUO2L0svlv6xCQoKUo9PnjzJw2CBGOCQWfnggw8wfvz4bFtffHw8ChYsiKNHj2bbOs3FgwcP0KtXL+TPn18NCM2ZMyeaNGmCAwcO6JY5ePAgmjdvDi8vLzg6OqJcuXKYPHkykpKSXuq2myP5In/SNHLkSN0Xvnby8fHBm2++iRMnTrzszTdr3bp1Q6tWrXSP7969i379+qFw4cLqdydfvnx46623sGvXrpe6naa0v7SfUXt7exQtWhSjR49GYmLic3vPO3fuoFmzZtm+rKUeNzs7OxQqVAhffvklYmNjX/amkQVigEMv5Q+fv78/GjdujPnz5yM5Odnoa+Rk2cbGBkeOHMnUe5w6dQp//PEH+vfvr5tXv359oyd72i/KdevWqRM8OdEzdqVHvlwHDx6Mr7766j/9/OaoTZs26sR40aJFuHjxIjZu3Kj2d2hoqHp+/fr1qFevHvLmzYvdu3fjwoUL+OyzzzB27Fh06NABGo3mZf8IZkVOurTT1KlT4e7ubjBPPsdaO3fuVPO2bduGqKgodbIWHh7+UrffUkiQWaVKFfz555/4/vvvcebMGWzduhUNGjRAnz59XvbmmYymTZuqz+ilS5fw+eefqwBd9pexi1DZQS7QSLCZ3cta6nG7evUqpkyZgp9//ll1OCV64TREL0DXrl01TZs21dy5c0cTHBysOXbsmGbcuHEaV1dXTbNmzTQJCQkGy1+/fl09179/f03Pnj0z9R4fffSR5tNPPzWYV69ePU2PHj3U++pPWosXL9aMGjVKM3fuXDnb1pw4cSLdesPCwjT29vaas2fPPvPPb24ePnyo9teePXuMPh8VFaXx8fHRvPvuu+me27hxo3rtihUrXsCWWqYFCxZoPDw80s2/du1aus/5gQMH1LytW7e+4K20rL9/77zzjvq3/L3LkyeP+h0x9ntFhvtLq3HjxpqaNWvqnhs7dqwmV65cmoIFC6rnb9y4oWnbtq363Ht5eWnefvtt9XnX98svv2hKly6t/p7nzJlT06dPH91z8juwfv169e+4uDj1nCzj4OCgyZ8/v2b8+PFGlxWnT5/WNGjQQOPo6Kjx9vZW3zmRkZHpjv/333+v1inL9O7dWxMfH2/2x02+AypVqqT+nZSUpPajHDPZV+XLl9esXr3aYHn5nm3RooXGzc1NnQPUqVNHc/nyZfXc4cOHNY0aNVLfLe7u7pq6deuqcwl9+sfG2N87shzM4NALoy1jypMnDypXroyvv/4av/32G7Zs2YKFCxcaLLtgwQK0bNlSlUAtX74cjx8/fuK6peRpzZo1qswjLWdnZ/W++pN+Sdvw4cPRqFGjDNct5VWvvfYaVqxY8Uw/tzlydXVVk9Q6x8XFpXt++/btKpOjnzXQkmNUvHhxdVzp5XNycsrWK+GUsbCwMJWtkUyNi4tLuuc9PT25+57wOdV+RqWULzAwEDt27MCmTZuQkJCgMv5ubm7Yt2+fKpOVv0+STdC+ZtasWWq/f/LJJyprJhlnKX0zZvr06er5VatWqfdZtmyZKlU2Jjo6Wr23fE9ItcHq1atVhrRv374Gy0kW+8qVK+r/kvWW77y033vm5uzZs6pMWSohxIQJE7B48WLMnj0b//zzDwYOHIjOnTtj79696vlbt26hbt266lxBMpzHjh3Dhx9+qKu4iIyMRNeuXbF//378/fffKFasmCqBlvlEadmmm0P0Ar3xxhuoUKGCKhX7+OOP1Ty5CCMBzsyZM1GyZEn1JSTBiwQjGTl9+jQePXqEqlWrPpftrF69uvripBS2trbqy7lHjx7qy0oCVilHk9Kz8uXLq5I1UapUKaO7TI6rdhl6eaQsbcyYMepkUD7j9HxdvnxZ/X2Tzz9ljuwvCWiknFLGLcnYPwkO582bpztxXrp0qSp1lnlSaizkO0QCxj179qgyZCmNlVI3KZPVqlatmtH3vHHjhjp5rlOnjlpfgQIFMty+X3/9VY0xkRN3bdD6448/qgs5EydOVOXYQgIgmS9l13L8W7RooX4u+RtqTiTglL8nEpTIxS9ra2v1c8u/ZXysBH+1atVSy8oYNAlWpIxNvj/kO9/Dw0NdTJRSdiEXw/TPF/TNmTNHHWMJkOSCKJE+ZnDopZM/9lKXriV/AGNiYtRVMSFXeH755ZcnruP69evqi8PPzy/dcz/99JMu4yCTfMllVe7cudV7kOEYnNu3b6srnXKlVE4kJNDRvyrJcTamqXbt2up3QU66ZOzaypUrdSdi9Pzw9yHrJ8rSnETGiLVv316NwxHSrEQb3Aj5DEvwKBkc7d95b29vFXhI1uT+/fvqb1XDhg0zPWZUxmOWKFFCjemUjHRGzp8/ry7S6WfkJOMvAZdkf7TKlCmjvqO0cuXKpbbL3MhYMtl3hw4dUtmW7t27q+8KOT7yvS5jb/W/jyUwlGMk5HWvv/66LrhJ6969eyoglOBTAiEZZyhjCCUgJUqLGRwyiS997VU3IY0H5MtMsgSiY8eO+OKLL9QfwSJFihhdh5SwSVpbfz1anTp1wv/+97//VAYi5RHyx5kMycmHfGHJNGzYMJWFkwGlMshd++UvJ9NpyfzSpUtzd74kEtDI/pfmGiyLenHkxEz+RknDDXr6ibKUlUkgIxeYtN8HIm15n5zkSuMGKSVLK0eOHCqLkBVyoebatWuqfFouuLVr106VMUslwbNKe9Iun4OMGuy8yuTYaEv/5Ltcgj+5QFm2bFk1b/PmzapMXZ+2YYO2XDYjEjBJ6fO0adNUVk1eJ9kglteSMczg0EsnJ7vSTlJboy7dtyTrIl9oMskfQ0l3yx/LjPj6+qoAxNgfOrnSI39wtZMsm1WyXfJFSU8mJ81Sky4lIXIFddKkSemWkYyPdEaSwJVeDmlLLBcLGNy8WPI7IZlpKcWR35O02Mku/YmytKHXD24yCkjkb4pk8PX/1sskf/8lsyNjaLLShluyA3Khbe7cueqCwNq1a9X3QFpShisZJP3jKWOAJKiSDJAlk30gY22/+eYb9d0gAYlkW9IeI/l7JKS8WUrBZUyVMbJfJaMm424kIybrCwkJecE/Fb0qGODQSyUDCWXAp6SwhVyBk7bC8oUh6WrtJCfKUvqU0f1TKlasqP5/7ty55zZYslKlSs9l3a8iuYom9dBS+y7jn+Rqpwyu/e677/DOO++okxOpq5YmEjKoV5aRMkS5kiflH++99566KkpkaSS4kb9jMuZJTprlxFwu8sjAdu3YBMoaydLLhSv52yMnyPL3SEpm5WQ4ODhYLSPlbfI9IvtZ9vnx48cxY8YMo+uTe3VJExTJtMlYQfnbJs1pjF0QkPeWTLZkF+R7QpoIyFghGTPKsk+gbdu2qjRPvg+k6Yw0FpAmC1KRoT0G8lhIY4aIiAg1llPuPSfHacmSJbpSP8mAymP5fZESONn3T8v6kOViiRq9MDLIUG5wJ1/uUksr3YSkq4oMDuzSpYtaRk6A5eRXm87Wkis8Q4cOVa+RwZlpSXZFruLJgEVtsJMZckVOrihJfbbQ/iFN221NvjRlMDalkNrpGjVqqPscyBeVXHGTYyT10XLFTshxlC/7cePGqbpqqYeXLygpFxwwYIDRckIicycDq+XETn4vZDyg3DNE/n5JiZWUZFHWSafMv/76S92v7N1331VdtSTzL2NuJBMjJACRv0HyN0tOtCUgkr9RxkjGRy7WyAm2nJxLMwK5x5qxUjd5b2mAIM0LZDl5LBfsJEiilIY0ErjI/pTAUz7r8r0v98mRgFHbUVVIyaxc9JSSdGk6IPtevs9lTJP2/EAumMlr5PtGmhYY69RJJKykVzR3BT1vctVee5VG/uDJ4GapzX3//ffVF498cUhLSOmCdvjwYaPdbSQtLVfKpOOaMXJyIAMWAwICdPPkxpPyB1I7JiQtyQrJIMi0ZByJdkCrrE/eW4IgXi0iIiIiMm0McMhsSKMBqXmWeunsLPWQOmwJxrRXmYiIiIjIdHEMDpkNya5IBic7Bx1K0wJpSSp1w0RERERk+pjBISIiIiIis8EMDhERERERmQ0GOEREREREZDYY4BARERERkdlggENERERERGaDAQ4REREREZkNBjhERGSWrKyssGHDhpe9GURE9IIxwCEiouemW7duKtDo2bNnuuf69OmjnpNlMmPPnj1q+fDw8Ewtf+fOHTRr1izL20xERK82BjhERPRc5cuXDytWrMDjx49182JjY/Hrr78if/782f5+coNekTNnTjg4OGT7+omIyLQxwCEioueqcuXKKshZt26dbp78W4KbSpUq6eYlJydjwoQJKFSoEJycnFChQgWsWbNGPRcUFIQGDRqof3t5eRlkfurXr4++fftiwIAB8PX1RZMmTYyWqAUHB6Njx47w9vaGi4sLqlatikOHDvHoExGZGduXvQFERGT+PvzwQyxYsACdOnVSj+fPn4/u3bursjMtCW6WLl2K2bNno1ixYvjrr7/QuXNn5MiRA3Xq1MHatWvRpk0bBAYGwt3dXQVBWosWLUKvXr1w4MABo+8fFRWFevXqIU+ePNi4caPK7hw/flwFVUREZF4Y4BAR0XMngcrQoUNx/fp19VgCESlb0wY4cXFxGD9+PHbu3IlatWqpeYULF8b+/fvx888/q+BEMi/Cz88Pnp6eBuuXgOi7777L8P2lHO7Bgwc4cuSIbj1FixZ9bj8vERG9PAxwiIjouZMsTIsWLbBw4UJoNBr1bykn07p8+TJiYmLQuHHjdONp9MvYMlKlSpUnPn/y5Em1Hm1wQ0RE5osBDhERvbAyNRkrI2bOnJmuhExs3rxZlZHpy0yjABlT8yT65WxERGTeGOAQEdEL0bRpU5WRkcH/2kYAWqVLl1aBzI0bN1Q5mjH29vbq/0lJSVl+7/Lly2PevHkICwtjFoeIyMyxixoREb0QNjY2OH/+PM6dO6f+rc/NzQ2DBw/GwIEDVcOAK1euqCYAM2bMUI9FgQIFVHC0adMmNZ5Gm/XJDOmeJo0FWrVqpcb/XL16VTUtCAgIyPafk4iIXi4GOERE9MJI9zOZjBkzZgyGDRumuqmVKlVKZXykZE3aRgspXRs1ahSGDBkCf39/XblbZkj2Z/v27apBQfPmzVGuXDl8++236QItIiJ69VlpZLQnERERERGRGWAGh4iIiIiIzAYDHCIiIiIiMhsMcIiIiIiIyGwwwCEiIiIiIrPBAIeIiIiIiMwGAxwiIiIiIjIbDHCIiIiIiMhsMMAhIiIiIiKzwQCHiIiIiIjMBgMcIiIiIiIyGwxwiIiIiIjIbDDAISIiIiIimIv/A9zGph+3IgL6AAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAADwCAYAAADM38HGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbglJREFUeJzt3Qd4U2UXB/B/996lZe+99xLZyFRBZG8UPxBEwAUqG0FUtggCskGWgAiyl4zKngJlllUKtKUTuvM9560JSZtCiwVC8v89T6C5ubm5uTdt7rnnvOdaaTQaDYiIiIiIiMyA9cteASIiIiIiouzCAIeIiIiIiMwGAxwiIiIiIjIbDHCIiIiIiMhsMMAhIiIiIiKzwQCHiIiIiIjMBgMcIiIiIiIyGwxwiIiIiIjIbDDAISIiIiIis8EAh4joGVhZWWHAgAEvfNstXLhQvXZQUBAs1ahRo9Q2INMj+0X2jxY/r0T0MjDAIaLnTnuQo705OjqiePHiKkC4e/fuCzkY1t6sra2RK1cutGrVCn///fdzfe1XnXbbyTa7efNmusejoqLg5OT0n4K98ePHY/369bBEBQsWNPhsuri4oHr16li8ePHLXjWzcOfOHQwdOhQNGjSAm5ub2sZ79ux52atFRC8AAxwiemHGjBmDJUuW4Mcff0Tt2rUxa9Ys1KpVCw8fPnzury2vJa8twZYcjJ89exZ169bFyZMn8Srp1q0bHj16hAIFCryw13RwcMCvv/6abvratWv/87KfJcD5+uuv1TYwBxUrVlSfS7lJQBkZGYkePXpg7ty5MAcv4/OqFRgYiIkTJ+L27dsoV67cC399Inp5bF/iaxORhWnevDmqVq2qfn7//ffh4+ODyZMn4/fff0enTp3+07IlSHJ2ds7w8XfffRe+vr66+61bt0bZsmWxevVqdZD5qrCxsVG37PK07SZatGihApzPP//cYPry5cvRsmVL/Pbbb3gRYmNjVZbD1tZW3cxBnjx50LVrV939nj17onDhwpgyZQr69OkDU6PdBy/r85oVVapUQVhYGLy9vbFmzRq0a9fupawHEb14zOAQ0UvTsGFD9f+1a9d005YuXaoOTKT0SQ5MOnbsmK48qn79+io4OXbsmMrCyAH6l19+maXXzpkzp/o/7YFyfHw8Ro4ciaJFi6rMRb58+dSBvUw3RrIPsi4yb5kyZbBlyxaDx69fv44PP/wQJUqUUO9Jgjo50NIfQ3P06FFVPrNo0aJ0y9+6dat6bOPGjU8c0/DTTz+p15f1yJ07N/r374+IiIhs2W6dO3dWma4LFy7opoWEhGDXrl3qMWMysx3lfcgBs7xvbZmWHODrl8edO3dOvYaXlxfq1Klj8Fha8tmREi95XzK/vMdt27YZbOemTZuqQFf2RaFChdC7d2+Ykhw5cqBkyZK4cuWKwfSUlBRMnTpV7WMp8fT398f//vc/PHjwIN0yNm/ejHr16qmyLHd3d1SrVk0Fo/oksNf+nsn2kCBLMh36ZF+4urqqdZEgV5bXpUsX9Zjsx8GDB6v1lelvvfUWbt26lW5djH1epTRPSkT379+v9pe8HwnqjJXmnT59Wr0XWc+8efNi3LhxWLBgQabGocl6yd8QIrI85nEKjIheSdqDODnoF9988w2GDx+O9u3bqwzP/fv3MWPGDHWgeuLECXh6euqeK2dmJSMkAZAcnMkB35OEh4frDhTlQG7s2LHqwEpeS0sekwM1OfD64IMPUKpUKZw5c0adTb948WK6UiqZT8q0JICRg6np06ejbdu2uHHjhu49HTlyBAcPHlTrKQdoclAm5XISbMjBuxyMS1ZLDvBWrVqlypP0rVy5Uh2sy4F5RuSAf/To0WjcuDH69eunSnPkNeS1Dxw4ADs7u2febkK2v6y7HCRLmaF2veTgVzI4aWV2O0pZluxnOciV+USRIkUMliXBYLFixVQpm0ajyXAd5f3LdpDSR1lHe3t7HDp0SAVhb7zxBu7du6f+lwNyGZchnyXZF9lRZpedkpKSVKAg+1yfBDMSLPTq1QsDBw5UJwWk1FN+L/T3scwjQZsEQsOGDVPvU+aRwFsbjGqXI4HPhAkT1Di4adOmqeWk/T2T9ZHPngSXP/zwgy7bJ/tNAkpZpmxz2c7GPgsZuXz5ssqqvvfee+ozP3/+fBVQSdAl6y7k91TGz0gwI+9FMkfz5s1TATMR0RNpiIieswULFsiRqWbHjh2a+/fva27evKlZsWKFxsfHR+Pk5KS5deuWJigoSGNjY6P55ptvDJ575swZja2trcH0evXqqeXNnj37qa89cuRINW/am6enp2bLli0G8y5ZskRjbW2t2bdvn8F0eR15zoEDB3TT5L69vb3m8uXLummnTp1S02fMmKGb9vDhw3TrFBAQoOZbvHixbtqwYcM0dnZ2mvDwcN20+Ph4tZ69e/dOty2vXbum7t+7d0+txxtvvKFJTk7Wzffjjz+q+ebPn/9M201/28k++/TTTzVFixbVPVatWjVNr169dNuif//+z7QdXVxcND169MjwtTt16pThY1qXLl1Sr9emTRuDbSBSUlLU/+vWrVPPOXLkiMZUFChQQO032b5yk896t27d0m1P2Y4ybdmyZQbPl8+v/vSIiAiNm5ubpkaNGppHjx4Z3Q4JCQkaPz8/TdmyZQ3m2bhxo1rWiBEjdNNkv8i0oUOHGizr5MmTavqHH35oML1z585quuyfjD6v2vct0/766y/dNPkcOzg4aD755BPdtI8++khjZWWlOXHihG5aWFiYxtvbO90yn2b16tXqObt37870c4jo1cUSNSJ6YSTDIGfQpVxJMgiSAVi3bp0ahyBn0uXMv2RUQkNDdTcpJZMz+Lt37zZYlpzFlbPQmSXjRLZv365KlqTERbq4SbZFsiv6ZTuSbZASIf110JbSpV0HeT/6GYfy5curkqCrV6/qpklpjVZiYqLKoEjZlpwlP378uO6xDh06qMf1MwqyrlJmJo9lZMeOHUhISMCgQYNUtzMtGb8h67Jp06b/tN205Ey9nHWXrJD2/4zK07K6HZ+kb9++T51HMkLy2RkxYoTBNhDaUjZtVkJK/WQ7mwrZx/I7ITcZCC9ZLdk/33//vcH29PDwQJMmTQy2p2Q75HdIuz3l8x0dHa0yVJKdNLYdpExPslmSddSfR7Ivsr/Sfl6EZAX1/fnnn+p/ySTpk89gZpUuXRqvv/667r68fynj1P/dkayTNCHRHyMnJWfaMjkiooywRI2IXpiZM2eqwELGvUhplBzQaA9IL126pEqQJJgxRr/MSkhQJGVIWtJ9Sr+zljymX38vZVb6TQakPEZe66OPPlJjUrTrcP78eXWwZYwcGOrLnz9/unmktEh/XISsk5QBSVAlJTf6ZVayzloVKlRQB5hS+iVlO0J+lnXWBgbGyBgfIdtSn7x/KXvTPp7RdsusSpUqqfWTMjUJFiTwzGi9srodn0TGyWSm1FE+R3LQnBEZxyEBrZSySamclAhKowkJ0p5U8pT2c5UVEpToB7jG1KhRQ40rSU5OVp395Gf5/OjvI9mesh5+fn5P3J7akk8ZZ5XVz4uQ/Stlhfrkd1XKE9MuQ7Z32nJCY8vMSGZ+d+R1JMBJS04QEBE9CQMcInphZKyFtotaWnIGXs4yywBpY12X5Ey1vrQHjh9//LHBIH05oH3SNS9keXJwKR3ctJ2hZB3kLLp0djNGMk/6MuoOpR/ESAAlwY2c3ZaDNTnolfcpGSx5PX2SqZFxSHJ2Xsb0bNiwQXWXy86OYU874H4SCQZkbI+sm6xr2myJVla34/NaX32yzaWTllz76I8//lDNG2SsyqRJk9S0tJ+vjD5XWSH7Xds0ISMSwEomUMhYFwkyZAC+jIkZMmSIbntKcLNs2TKjy8gokMwOEvxltJ//i8z87hARPSsGOERkEuRssBzcyBl7yfJklXTo0m+3m3aQtjEygFrExMSoAEfW4dSpU2jUqJHRLl3PQg6qZRC1HEhrxcXFpetwJiRokAyDlNNJhksupCmB0JNory8ijQUkY6MlZWsyEF178JwdJMCRMjC5gKKUUmUkK9sxO7azvJ4EAdK04Wktv2vWrKluEkhKNkrKnVasWKEGzWfmc5UV2sHyWSGlYhKcS1MFaSyg/VxKKeJrr732xIBPm1GRTFBGWQ79z0vaDJxMy8z1amQe2d6SMdLP2sjzs5O8jpRDpmVsGhGRPo7BISKT8M4776izunKAn/YsrtyXsStPIuVJcjCvvcn4hKd1VZPxN1JqpS39kfE/UkZm7CKLUqYkmZ6skveU9v1IZzgpSUpLxq1I5kNK0+SWK1cuVVr3JPJepZxJOrjpv84vv/yiypqy0tnqaeQAWloVS8mdZOMykpXtKAfwxoK9rJBSM8kySPe0tFkx7TaR0qe0+0EbDGXUAtzY5yorN9l/z+KLL75Qn3ft9pPtKZ8X6fxnLEjXbj/pEifZNdk/EkQb2w6SQZXP++zZsw3et2ROpawwM58X6cIn5DOnTz4b2UkyWgEBAQYX45Xf24wyWUREWszgEJFJkINnGX8g7WClfa8ctMrBmmQhpBGBtBH+9NNP/1MmRcqQ5EAvODhYBQBy0CsHetosglx1XVo1y8B2GbgtZ8zlwFKu/yLTpawpoxK7jEi5kWQ7pDRNDpblgE3OxmvbSBvL4kiWRAaAy1icp5UHSXmSbDMJDJs1a6baM8uZdLkujrQBftbsQ0akZOtpsrIdJRCV7SHlbHL9HsngSelgVki24quvvlIBgAxcl2BZSqukEYIsUw74pcxMtkmbNm3UZ00G40sAIY0Y5BovpkQCCBlHI9tErmckGR3J5sj7kIN9CWRkTJqMzZEGBFLOJmPK5L3I+CLJRsm+114/SLJpckFX2QbyvIkTJ6pGBrJcKYHUtomW69PItW2eRgJDeZ5sTwmipU30zp07sz2zItkzaUUtzRWk1FPbJlrG70igk5nsn/xNEf/884/6X34XteOMvv7662xdXyIyIS+7jRsRmT9tq9jMtOj97bffNHXq1FHtg+VWsmRJ1TI3MDDQoN1xmTJlMvXaxtpEy3Jr1aqlWbVqVbr5pY3uxIkT1fKlba2Xl5emSpUqmtGjR2siIyN186Vt5avfAle/7fGDBw9UO2VfX1+Nq6urpmnTppoLFy6km0+/5bF2Pffv35/ucWNtd7VtoWVbSatpf39/Tb9+/dRr68vKdtPfdtLC+EmMbYvMbkfZFnXr1lXtwmU52m3ypNdO2yZaS1piV6pUSfd68n63b9+uHjt+/LhqOZ0/f371uLRKbtWqlebo0aOal0U+Ay1btjT62MKFC9V7lP2tNWfOHLUNZVtJO+hy5cppPv/8c01wcLDBczds2KCpXbu2ms/d3V1TvXp1za+//mowz8qVK3XbStoud+nSRbVr1yf7Qn5XjJEW0wMHDlSt3mWeN998U7V/z2ybaGPvW/aX3PRJi+jXX39drWfevHk1EyZM0EyfPl0tMyQkRPM0xlrEa29EZL6s5J+XHWQRERERZYY07Pj555/V2LmMmhUQkWXjGBwiIiIySWlbdMvYJCkzq1OnDoMbIsoQx+AQERGRSZLW6nLNImnAIWOFZOycdBccPnz4y141IjJhDHCIiIjIJEkDCGkQMmfOHNVUoHLlyirIeVp3QSKybByDQ0REREREZoNjcIiIiIiIyGwwwCEiIiIiIrPBAIeIiIiIiMwGAxwiIiIiIjIb7KJGilW/mtwSJkIz62/uDxPC/WF6uE9MC/eHaeH+ML39QS8eMzhERERERGQ2GOAQEREREZHZYIBDRERERERmgwEOERERERGZDQY4RERERERkNhjgEBERERGR2WCAQ0REREREZoMBDhERERERmQ0GOEREREREZDYY4BARERERUZb89ddfePPNN5E7d25YWVlh/fr1T33Onj17ULlyZTg4OKBo0aJYuHBhunlmzpyJggULwtHRETVq1MDhw4eztmIMcIiIiIiIKKtiY2NRoUIFFZBkxrVr19CyZUs0aNAAJ0+exKBBg/D+++9j69atunlWrlyJIUOGYOTIkTh+/LhaftOmTXHv3r0srZttlt8NERERERFZtObNm6tbZs2ePRuFChXCpEmT1P1SpUph//79mDJligpixOTJk9GnTx/06tVL95xNmzZh/vz5GDp0aKZfiyVqRERERET0XAUEBKBx48YG0ySwkekiISEBx44dM5jH2tpa3dfOk1nM4BAREREREeLj49VNn4yXkdt/FRISAn9/f4Npcj8qKgqPHj3CgwcPkJycbHSeCxcuZOm1GOAQEREREVkI6/61MnxsRI6mGD16tME0GQ8zatQovEoY4BARERERWQhr24xHqAwbNkwN8teXHdkbkTNnTty9e9dgmtx3d3eHk5MTbGxs1M3YPPLcrOAYHCIiIiIiC2FtY53hTYIZCTj0b9kV4NSqVQs7d+40mLZ9+3Y1Xdjb26NKlSoG86SkpKj72nky/R6zZY3ppZE+4VOnTuUeICIiIqJMZXAyumVFTEyMavcsN20baPn5xo0bumxQ9+7ddfP37dsXV69exeeff67G1Pz0009YtWoVBg8erJtHskdz587FokWLcP78efTr10+1o9Z2VcssBjjZLCgoSF3sSLuztXr27InWrVtn98uZhdeLVsSGfj/g9oQ/oJn1N96uUPepz6lXrDKODVuEuOl/4dLo1ehRs2W6eT6s1xbXxq3Do+l78ffnv6BagdIGjzvY2uPHjp8i9PutiJ6yC2s+mAA/N29YOu4P08L9YVq4P0wP94lp4f6wnADn6NGjqFSpkrppgxP5ecSIEer+nTt3dMGOkBbR0vJZsjZyfRtpFz1v3jxdi2jRoUMH/PDDD2oZFStWVMfTW7ZsSdd44KnvMUtzEz0HLg5OOHX7Evqv+CFT8xf0yYVN/Sdh98VjqDi+O6buWol5XYfhjVI1dPO0r9IYk9t+jNGb5qHy+B44desStg6cihxuXrp5prQbhDfL1UG7eV+i3pR+yO3hi7X/+9bi9zH3h2nh/jAt3B+mh/vEtHB/WE6AU79+fWg0mnS3hQsXqsfl/z179qR7zokTJ1SntitXrqgEQFoDBgzA9evX1TyHDh1CjRo1sv4es/wMUpFknTp14OnpCR8fH7Rq1UrtJG10KiSClUyO7EjpPCGptt9//11Nk5t2h3/xxRcoXrw4nJ2dUbhwYQwfPhyJiYkGW/mPP/5AtWrV4OjoCF9fX7Rp0ybDvSCRsKxX2hpHU7blnwAM3/Az1p/am6n5+77+Dq6FBePT36bjQkgQZu5dgzUndmNwo466eYY06oS5B37HwoBNOB8ShL6/TsTDhDj0rtVKPe7u6IL3ar+JIWumYXfgMRy/EYhei8fhtSLlUaNQGVgy7g/Twv1hWrg/TA/3iWnh/ni1x+CYC/N5Jy+Q1AJKGk5ScxJIyEWIJOiQgVCHDx9W8+zYsUOl5tauXYtPP/0U7du3R7NmzdQ0udWuXVvN5+bmpiLcc+fOYdq0aaruUK7oqiWpPFl2ixYtVMQrr1e9enWj6/Xdd9+pq7xu27YNjRo1grmqVbgsdlw4YjBt67m/UatwOfWznY0tquQvYTCPnFGQ+9p5qhQoCXtbO4N5Au9ex/WwO6hVKHUe4v54FfH3w7Rwf5ge7hPTwv3x4tnY22R4MxdsE/0M2rZta3B//vz5yJEjhwpS5H8hmR39lnbS/k5SbWnb3H399dcGDQMkGFqxYoUagCW++eYbdOzY0aAnudQtpiWZoCVLlmDv3r0oU8a8MxA53X1wNyrcYJrc93ByhaOdA7yc3WBrY2tkngco6V9Qt4z4xAREPooxnCc6XD1G3B+vKv5+mBbuD9PDfWJauD9ePOsslqK9ihjgPINLly6pwU9SFxgaGqoyN0IGUpUubTiQ/WlWrlyJ6dOnqxI36UaRlJSkWvJpyeCqPn36PHEZMkhLskqSUZIyt2e5Qi0RERERmT9rCwhwzP8dPgdvvvkmwsPDVTmZBDlyEwkJCVlaTkBAALp06aLKzzZu3KhK0L766iuD5Ujm52lef/11JCcnq1Z7TzNhwgR4eHgY3GTaqyQkKgz+7obdzuS+ZGPiEuMRGhOBpOQkI/N4qedql+FgZ6+yPgbzuHnr5iHuj1cRfz9MC/eH6eE+MS3cHy+eNcfgUFphYWEIDAxUpWUyzqVUqVJ48OCB7nG5SJGQgEOfTE877eDBgyhQoIAKaqpWrYpixYqprhH6ypcv/9SGATImZ/PmzRg/frxqrfck0pM8MjLS4CbTXiUBV8+iUYmqBtOalKqOgKtn1M+JyUk4diMQjUpU0z0ujR3kvnaeY9cvICEpEY1KPp6nuH9+FPDJhYBrqfMQ98eriL8fpoX7w/Rwn5gW7o9Xt4uaKWOJWhZ5eXmp8TVz5sxBrly5VFmaDOzX8vPzU1kX6bSWN29e1flMsiQyvmbr1q0qOJLnyzQJaOT5MuZGuqRJQ4F169YZvN7IkSNVIFWkSBE1FkdK2P7880815kafNC2Q6c2bN4etrS0GDRpkdP2lHM3UStKkpWTRHHl19wv55EaFvMUQHhuFmw/uYvzb/ZDHMwd6LBqjHp+9by0G1H8XE9sMwPyDf6BhiapoX7kRWs78RLeMyTt/xaIew3H0xnkcDjqHQQ07wMXBEQsCNqnHo+Ji8cvBPzC57UCEx0aq+zPaf4KDV07j0LV/YMm4P0wL94dp4f4wPdwnpoX7w/TZ2FrB3DHAySLpmCYBycCBA1G2bFmUKFFCjaGRdtBqg9raqvtjxoxR43SkfExaQss4GvlfMjUy1mb37t1466231NVbpd+3jItp2bKlahMtbaW1ZLmrV6/G2LFj8e2336rxOXXrGr8QprSuliBJSt5sbGzw0Ucf4VVQNX8p7Bnyk8H1aYS0eO61eCxyefgiv/fj5gxBYXdUMDOl3cf4uEF73Iq4h/eXTsC286mlgmLVsR3I4eqJMa36qAGMJ29dQrMZg3Ev+nHjgcGrpyJFk4LfPpigLvq59dwhfLjiO1g67g/Twv1hWrg/TA/3iWnh/jB99mbUDjojVhrpn0sWz6pfTYvfBqZCM+tv7g8Twv1herhPTAv3h2nh/jC9/WFqSi58N8PHLvRcA3PADA4RERERkYWwtzb/DA4DHCIiIiIiC2FvwzE4RERERERkJuwtYAwOMzhERERERBbCnhkcIiIiIiIyF/bM4BARERERkbmwZwaHiIiIiIjMhZOtDcwdx+AQEREREVkIe2ZwiIiIiIjIXNhzDA4REREREZkLe2ZwiIiIiIjIXNhZm/+FPs3/Sj9ERERERKTYWWd8y6qZM2eiYMGCcHR0RI0aNXD48OEM561fvz6srKzS3Vq2bKmbp2fPnukeb9asWZbXi00GiIiIiIgshF02laitXLkSQ4YMwezZs1VwM3XqVDRt2hSBgYHw8/NLN//atWuRkJCgux8WFoYKFSqgXbt2BvNJQLNgwQLdfQcHhyyvGzM4REREREQWwi6bMjiTJ09Gnz590KtXL5QuXVoFOs7Ozpg/f77R+b29vZEzZ07dbfv27Wr+tAGOBDT683l5eWX5PTLAISIiIiKyoDE4dhncMksyMceOHUPjxo1106ytrdX9gICATC3jl19+QceOHeHi4mIwfc+ePSoDVKJECfTr109lerKKJWpERERERBbC7gnpjfj4eHVLm1FJWyYWGhqK5ORk+Pv7G0yX+xcuXHjqOshYnbNnz6ogJ2152jvvvINChQrhypUr+PLLL9G8eXMVNNnYZP4CpczgEBERERFZCLsnZHAmTJgADw8Pg5tMy24S2JQrVw7Vq1c3mC4Znbfeeks91rp1a2zcuBFHjhxRWZ2sYIBDRERERGQh7Gwyvg0bNgyRkZEGN5mWlq+vr8qo3L1712C63JdxM08SGxuLFStW4L333nvquhYuXFi91uXLl59vgJOYmIgiRYrg/PnzWX0qERERERGZaAbHwcEB7u7uBjdjXczs7e1RpUoV7Ny5UzctJSVF3a9Vq9YTX3/16tWqDK5r165PXddbt26pMTi5cuV6vgGOnZ0d4uLisvo0IiIiIiIyky5qQ4YMwdy5c7Fo0SKV+JCGAJKdka5qonv37kazP1KeJuVnPj4+BtNjYmLw2Wef4e+//0ZQUJAKlt5++20ULVpUtZ9+7k0G+vfvj4kTJ2LevHmwtWWfAiIiIiKiV4FjNl0Hp0OHDrh//z5GjBiBkJAQVKxYEVu2bNE1Hrhx44bqrKZPrpGzf/9+bNu2Ld3ypOTt9OnTKmCKiIhA7ty58cYbb2Ds2LFZvhaOlUaj0WT1DbVp00ZFVa6urmoQUNr2bnIhHyIiIiIiMi3LA/tn+FjnEjNhDp4p/eLp6Ym2bdtm/9rQS/NVwAfc+ibim1pz8CB+5cteDfqXl0MHnChVktvDhFQ6fwF2g+u87NWgfyVO2Y//7Xqf28NE/NxwHqac7PuyV4P+NbjibJPbFrZpsirm6JkCnAULFmT/mhARERER0XNla53568m8qv7TABqpu5NaOiFXG82RI0d2rRcREREREWUzWwvI4DzTO5QOCb1791Yt2+rWratuMhBI+lk/fPgw+9eSiIiIiIj+M1srmwxvFh3gSFu4vXv34o8//lBdDuT2+++/q2mffPJJ9q8lERERERFlSwbHNoObRZeo/fbbb1izZg3q16+vm9aiRQs4OTmhffv2mDVrVnauIxERERERZQNbjsExTsrQtD2u9fn5+bFEjYiIiIjIRNmaUaYmI8/0DmvVqoWRI0ciLi5ON+3Ro0cYPXq0eoyIiIiIiEyPrQWMwXmmErVp06ahadOmyJs3LypUqKCmnTp1Co6Ojti6dWt2ryMREREREWUDW5aoGVe2bFlcunQJy5Ytw4ULF9S0Tp06oUuXLmocDhERERERmR5bCyhRe+br4Dg7O6NPnz7ZuzZERERERPTc2DKD89iGDRvQvHlz2NnZqZ+f5K233uLHkoiIiIjIxNhaMYOj07p1a4SEhKhOafJzRqysrJCcnPxi9hAREREREWWanY2d2W+tTJeopaSkGP2ZiIiIiIheDTZWzzxC5ZXxTDmqmzdvZv+aEBERERHRcw9wbDK4WXSAU7BgQdSrVw9z587FgwcPsn+tiIiIiIgo29kwwDHu6NGjqF69OsaMGYNcuXKpMTlr1qxBfHw8P4ZERERERCbKxto2w5tFZ3AqVaqE77//Hjdu3MDmzZuRI0cOfPDBB/D390fv3r2zfy2JiIiIiOg/s2EG58mkY1qDBg1UqdqOHTtQqFAhLFq0KNMbuH79+hg0aBBM2YtcRyn9mzp16gt5LSIiIiKyPDZWdhnezMV/ykXdunULy5cvV7ezZ8+iVq1amDlzZqafv3btWnVdHVOwZ88eFazJmCJPT8/nuo4LFy5UQVNERITB9CNHjsDFxQWW6PKO27i4+SbiIhPgkd8VlboWhXdhd6Pz7plwEqGBkemm5yzvjTpDyunuRwXH4syqa7gfGAFNsgbueVxQa0BpOPs46uYJuxyJs78FIfxKFKysreCZ3xWvf1oONvY2sHRrVhzC0oUHEB4ag6LF/fHJsJYoUy6v0Xk3/n4C44avM5hmb2+Lv46OMDr/xLEbsG71UQz6rBk6dqud7vGEhCS812UOLgWGYPGqfiheMhcsnW/nzvDr/R7sfH3x6MIF3PpmHB6eOZPh/J5NmyLXwI9hnycP4q9fR/CkHxD111+6x62dnZF7yCfwaNQItp6eiL91C/eXLkHYypW6eYouWgy36tUNlhu6YgVujh4FS9fvtXcwpGEn5HTzxungKxi0dgqO3Dif4UX1vmjcDd2qNUceD19cvHcTwzbOwrYLh3TzWFtZY0Sz3uhc5Q3kdPNBcFQoFh/+E+O3Pz5p2LpcXXzwWmtUzlsCPi4eqPp9T5wKvvxC3q+pq5+nAZrkbwoPew/cirmJFRd/RVD0tQznb5S3MermqQ9vR2/EJMbg+L1jWHf1NySlJGV6mbbWtmhXtD2q+leHrZUtzoX/g+WByxCdGAVLd3brLZz84yYeRSTAp4ALXutVHP5FjX+n/z76BO6cMzweEvkreaPF0Arq5yOrr+HKwXuICYuDta01chRyQ/WOheBfzEM3/9IBAYi5H2ewjBqdCqNS6wLZ/v5edTZm1EwgI8/0Dn/++WcV1Bw4cAAlS5ZEly5d8Pvvv6NAgax9iLy9vWHqXuQ6SqmfJbp56B5Or7iCyj2Kw7uwGy5tu419P5xB02+rwdHdPt38tT8qg5Qkje5+fGwidgw/irzVHm+/mHuPsOebkyhYNydKtykAOydbRN2OhbWdtUFws2/SGZRsmR8VuxaFtbUVIm7GSGoSlm77ljOY9v0WfDH8TRXUrFgagEF9F2PlhoHw9nE1+hwXVwes2jDw8YQMtuOenedw9vQt5PBzy/D1f5y8Db453FSAQ4Bn8+bI88VQ3Bw1Cg9Pn0KO7j1QZO48nG/RHEnh4en3RcVKKPjDJARPmYzIPXvg3aoVCs34EYHvtkXcpUtqHlmeW40auP7550i4fRtur72GfCNGIPHePUTt3q1bVuiqVbgzY7rufsqjRxa/S9pVbIjvWw9A/9U/4PD1cxhYrz02/W8yykzohPsx6Q/UxrT4QAUufVdNROC9G3ijRHWs6TUedaf3xcnbqfvjs0Zd8L/ardH7129w7s41VMlfEvM6fomouFj8uG9N6n51cMKBq6ex5sQu/NxxqMXvB62qftXwbrH2WB64FNcir6JRvsYYWHEQRv79NaITo9Ntp2r+1dGmSFssurAAVyOvwM/ZHz1LSXm9Bqsvr8r0MtsX7YhyvuUw5+xsPEp6hE7FO6NvuQ/x/fFvLXrfXD54FwcXX0bd90vAr5g7zvx5E5vGn0KnKTXg5JH+O73pJ2WRkvT48iNx0UlY/fkRFK7pp5vmmcsZdXoVg7u/E5ISUnB6001s+uYUOk2vCSe944Rq7QuhVKPHJ8TsHM3/QP5Z2GTjWBtJbMiwFblWZoUKFTBjxgw1Tj+jE/y9evUymObg4IC4uMeBqUajwciRI1V1mCQCXnvtNcyaNQvFihV7/mNwxo0bhxo1auDYsWMqczNs2LAsBzdpy7+kPGv8+PFqDI+bmxvy58+POXPm6OatXbs2vvjiC4Pn379/X2VX/vr3rKQ0Ofj000+RJ08elQmRdZTMjNb169fx5ptvwsvLSz1epkwZ/PnnnwgKClLZGyGPSeldz549062juHPnDlq2bAknJydVkieBXtrSssmTJ6NcuXLqNfLly4cPP/wQMTEx6jFZH9m5kZGR6nXkNmpU6tnQtMuRMU5vv/02XF1d4e7ujvbt2+Pu3bu6x+V5FStWxJIlS9RzPTw80LFjR0RHp/+Dbsoubr2FQvVyoeDrOVWWpXKPYrCxt0bQX8YPbu1d7eDoaa+73Tv7QGVc8lZ/HOCcXXNNZXTKdygCrwJucPVzQu5KvgYB06nlV1C0cR6UbJUfHnlc4JbLGfmq+8FGLwiyVL8uPoi321ZBq9aVUaiInwp0HJ3ssHH98QyfI59lH1+3xzcjgdC9u1GYNOFPjJ7wLmxsjWfJDu67iEMBlzHwk6bZ+p5eZX49eiJs9WqEr1uLuCtXcHPUSKTExcHnnbZG58/RvRui9u/HvfnzEX/1Ku5Mn45H588hR+cuunlcKlVE2O/rEXPkMBKCbyNs9So8CgyES/nyBstKiXuEpNBQ3S0lNhaWblD9jvgl4A8sOvwnzt8Nwoerv8fDhDj0rNHK6PxdqjbFxB1LsOX837gWFoyfD67H5vMBGFy/o26eWgXL4o+z+7H5XACuPwjB2lN7sD3wMKrlL6WbZ9nRrfhm20LsvHj0hbzPV0XjfE2wP3gfDt45gDsP72BZ4FIkpCSgdu46Rucv4lEUVyIv48jdwwiLC8P58HPq54LuhTK9TEcbJ7yWuw5WX1qFwAcXcCP6OhaeX4CinkVRyL0wLJkEH6Ua5UbJBrngnddFBTq29ta4sPuO0fkdXe3g7Omgu906HQ5bB2sU0QtwitXxR97y3irA8c7ngtrdiyLhUTLCrqceW2nZOdoYLEvu0/Mbg7Ny5UoMGTJEBSTHjx9XAU7Tpk1x7969DJ8jx7NyLK29ybG5vu+++w7Tp0/H7NmzcejQIXUsLcvUD4Iy45mO5OTAW1ZA3kh2mjRpEqpWrYoTJ06ooKBfv34IDAxUj0mWaMWKFSqy09+wuXPnxuuvv67uDxgwAAEBAWq+06dPo127dmjWrBku/XvGsn///ioIkoDozJkzmDhxogoeJAj57bff1DzyerLBp02bZnQdu3fvjuDgYBWoyHMkCEu7I62trdXO+eeff9SYpF27duHzzz/XBWoSxOjvYAnK0pKLqUpwEx4ejr1792L79u24evUqOnToYDDflStXsH79emzcuFHdZN5vv311zh7JWZuIoGj4lfbSTZNSMf8yXgi7krk0/7V9IchXww+2Dql/yDQpGoScDodrTmfs++E0/vjoIHaOOY7bx0J1z4mLSkD41WgV8OwadwJ/DDyYWvp2MX3pm6VJTExC4Pk7qFaziMFnulqNIjhz6laGz3v0MAGtm07CW01+wGcDl+Pq5XvpPtOjv/wNXXu+hsJFH39x6QsLi8GE0RswanxbODiaRvnqy2ZlZwfnMmUQHXDw8USNBtEBAXCuWNHoc1wqVDScX0o29x+Ai978sSdOwqNBQ9j5pe4L1+o14FCwIKIOHDB4nlerN1HuYABKbtiAXIOHwMrxcYmnJbKzsUXlvMUNggz5Xtp16ShqFihj9DkOtnaISzLsMhqXGI/ahR8HkwFBZ9GgeBUUy5FP3S+fuyheK1xeBUWUMRsrG+R3K6CCFN3+gAYXws+jcAaBhgQ38pyCbqkBja+jL8r6lMPZsDOZXmYB9wKqRO38g8fz3H0YogKmwh6P/3ZamuSkFNy/GoO85Qy/0/OW88bdS5n7TpdAqGhtvwyDE3mNczuDYe9sC58ChifSTvx+Awve24fVXxzByQ03kJLMC9M/zwBHTuj36dNHnbgvXbq0CkqcnZ0xf/78J54MzZkzp+4mDcr0/5bKMfLXX3+tjoHLly+PxYsXq+NuOdbNimfKUcnKiYcPH6pgJyEhweBxWaFn0aJFCxXYCMnWTJkyBbt370aJEiVU9kIyKfv379cFNJI96dSpk1ofWY8FCxao/yXoERI4bNmyRU2X7JA81rZtW5VdEYULF05Xiubn52cwBkffhQsXVDMFGSsjgZiYN29eurSZfsZHMiuS8erbty9++ukn2Nvbq0yLdgdnZOfOnSoIu3btmgrAhOxkyTrJ61erVk130CgpP8l6iW7duqnnfvPNN3gVxEcnQpMCOHoYHsw6uNsh6s7Dpz4//GoUom7Fomrv4o+XGZWIpLhkBG66gTJtC6Fcu8IIOROOgB//Qb0vKiBHSU/E3ks9E3BufRDKdywCj/wuuH7gLv767hSajKsKt5zOsFQRDx4iOTkF3j6G48G8fFwQdO2+0ecUKOiDr0a3VmN1YmLisGzhAfTpPhe/rh0Av5ypNdJL5u+Hja012nepaXQZ8odt7Ndr0aZ9VZQqkwfBt3mNLWHj6QUrW1skhoUZbK+ksFA4Fnp8xlmfra8vkkLTzy/TtW6NG4t8Y8ai7N6/oElMVNv/5ojhiD36+MD9wcaNSAgOVmVrTiWKI/cnn8KxUEFcG6hXimhhfF08YGtji3vRhqWBd6PDUcLPeCXDtguH8XH9jth35RSuhN1Gw2JV0Lp8PdhYPz7H+N3OpXB3dMHZocuQrEmBjZU1hv85B78e3/7c39OrzNXOFTbWNohOMDx4jkqIQk5n49+xkq1xtXPDZ1W+gNW/5Tp7b+3B5ut/ZnqZ7vbuSExJVKVpaefxsDc+1sQSxEXJd7omXSmak4cdIoKfnv29ezkK4TdjUa9vyXSPXT8Wiu3TziEpIRnOnvZo9VUFg/K0cs3ywLeQm8oIhVyMxKFfr+JhRDxqd89aaZMlsMmGMThy7C+VXFLFpX8ytHHjxirZkBGpaJKqLzl+rVy5sjo+l2NbIce8Uuomy9CSY2apyJJlSpVSZj3TO5TSMCnhkuDBmOTk5GdZrEFgpA0AtNkRGZ/yxhtvYNmyZSrAkY0gb1bGAwkJBuR1ixd/fKArJGPj4+Ojfh44cKDKCm3btk1tPAl2shKMSXbH1tZW7RCtokWLqrI2fRIETZgwQQVEUVFRSEpKUqk1CQglss2M8+fPq8BGG9wIiY4l+JLHtAGOBFDa4EbIdYmelBqU7ZH2ekVS//iquvZXCDzyuhg0JNBm+XJX9kXxpqmD4j0LuCLschSu7g5WAY52nkINUkvjhJSy3T8XgaB9ISoooswrVyG/ummVr5AfHVvPwLo1R/G/AY1w4VwwVi77G4tW9tWdIElr1fJDePgwAT3eq8tN/wLk6NoNLhUq4Eq/fqpEzbVqNeQdnjoGR7JDQsrWtOIuXUTi/fsotnAR7PPlQ8LNm9xPmTRk3TTM7vA5zg5bpv72XAkLVuVtPau3NBjX06lyE3RbOhrnQq6hQp5imNR6IO5EhWLJEePftfRsinuWQPMCLVRDgGtRV+Hn5IcOxTuiRUIr/Bm0kZv1Jbqw6w6887sYbUiQu4wX2n1XVQVR53fdwfap/+Cdb6rogqkKrR5/B0lmR06o/TU3EDU6FWHpeRbG4GR0nJj2WDE0NFQdd+tnYITcl+NfYyRhIdkdOfaWoRo//PCDqmySiqe8efOq4Ea7jLTL1D72XEvUJEMhKya1cTIWRQIdKcWSTMaGDRvwrNJ2K5MDIYnwtKRMTS4ompiYqLI3konRZmMkIrSxsVHR5MmTJ3U3CQa05Wbvv/++KvOSLIcERJKFkcFQ2UnG87Rq1UrtPClhk/XRdpZLm+nKDk/bZmlJ4CXRsP5Npr0sDm52sLIG4iITDaZLFsbRyGBEfUnxyapBgTQSSLdMGyu45zYMJt1yO+NhWOovrZNn6rLdc7tkOI+l8vRyho2NNcLDDM+2PQiLVWNrMsPWzkZ1Prt1I/Us98ljQXgQHovWTSfjtUqj1C0kOALTJ21F62aT1TzHDl/F2VM3UbfqGPV4u1apv7e9Ov2MMV+thaVKjngATVIS7P49UaNl6+OLxNDHZZf6ZKyMrW/6+WW6sHJwQK5Bg3B74reI2rMbcRcvInT5MjzY/Cf8emV8LbOHp0+r/x3yW25XotDYSCQlJ8HPzbABjb+bN0KiwjJ4TgTenf8lPL5ogiJj30XZCZ0RE/8IV8ODdfN8++aH+H7nMqw6sRNn71xV422m7V2Fzxt1e+7v6VUmHdCSU5LhliZrIhmWyATjJcdvFX4bh0ICcODOPgTH3sbJ0BNYf2UdmhdoDitYZWqZkqmxs7aDk62TkXkst4uao7t8p1vhUaTh8c6jyEQ1JuZJEuOSceXgXTV2xxgpWfPI6Qz/4h6o37ek+p6XQCcjfkXdkZKsQXSazmoEWCUnZXh7nseJ0m1ZhnrI+PF69eqpTsWSwNAmK7LTMwU4MqZE6u4kQJB0lKSaunbtqsblPM+DZanHk0yIBFQS4EjAo3/xUYkkJXshWRX9m34pmGREpFxMNuonn3yiujQIKR17WvZJIk/JxsgYIa3Lly+r1tJaEtBIgCHjiWrWrKkySlI7qE9e62lZrlKlSuHmzZvqpnXu3DnVUUIyOc9KUokSnOrf9NOLL5q0e/Qs6IZ75x5vQ0lvy32fIk9O8986fB8piSnIX9s/3TK9CrkhOk2JW0zIQzj7po4fkP+lQUH6eR7B2ffVzWhlBzs7W5QolQtHDl3VTZPPtNwvV8F4m+i0pMTtyqW78MmRWh/d/M2KWLrmQ9XyWXuTLmpder6GabO6q3mGDG2BJasfzzN5Zlc1fex37dD3o0awVFI+9vCff+BWs9bjiVZWcKtZEw9PnjT6nNhTJw3nl+C9dm3E/ju/lLxZy9+8tCdDpF5dr2wqLaeSqWUjifczzhKbu8TkJBy/dRENi1cxvCZcsSr4+/o/T3xufFICgiNDVdvoNuXr4Y8z+3SPOds7IkXqdfXIQba0j6aMJWuS1QD/Ul6PmzFIkFLSqySuRj3+G6bP3tpBjanRl4KULC3zetR11VK6pN48/s7+8HH0UZ3ZLJVkTXIUdsXtM4bf6bfPPoB/sSd/p1/5+x6SkzQo/m9VxVNpNGo8TkZCg6JVM08nd47nTEfaoWdwy+xxoq+vr0os6De/EnL/SUMw0p6kl+N3OZYW2uf9l2X+pxK12NhYNVZFSHmWlKzJgbxkU6SLwvMinRRat26N4cOHq8yMjL/RkteXgEciQwkuZIPJesl4FMmmSOczyTw1b95czStBiYzvkUBCSJAmX1IyUF/GAklmShoQ6JOW2FLa9sEHH6iWdbJjJEiSebVlNxJQSYZJMkPSsU1aacugK31SViYZJ1k3adQgZWtpS9fkdWR7ynuSAVcSWMn4JIl4teN/noWxNOPLJmVkR+ZeUEGJtk10UnyKrnTs8JwLcPKyT1c2dm3fHVWG5uCa/o9Xieb58PdP5+BbwhN+pTzVGJw7J8NQb2jqIGvZXzLPP+uD1LVv5Ba0P0SN+6k54NkDSHPRqXttjP16HUqVzo3S5fJi5dIAxD1KQMvWqeWZ0iwgh787Pvy4ibr/y+zdKFs+H/Lm90Z0VOoYnJA7EXj7ndSDQA9PZ3XTJ13UpNNagUKp40Jy5jIc++bknHrSIW8+b904Hkt1b9FCFJjwLR6ePYvYM6fh170HrJ2cELYuNbNV4NtvkXD3Hu5MSc2G3V+8BMUWL4Zfz16I3LsHXi1aqkYFN0emXpdIOqFFHz6M3J99hpS4+NQStWrV4f322yqrI6QMzatVK0Tt/QvJERFwLFEceYcOQ/SRIyrjY8mm7lmB+Z2/wrGbF3Dk+nnVJtrF3gmLDm1Sjy/o/DVuR97H15tSz0pWz18auT181TVr5P8RTXurk4M/7FquW+amfw5gaJPuuBFxV7WJrpi3OAbV74CFh1LHhQgvZzfk9/RHLo/U35nifqklOSHR4WoMkKXacXO7avMcFH0dQVHXVEtnexsHHAxObZghj0XER2D91dTfl9Nhp1SXtBvRN3At6poqUXurUGucDj2tC3yetsy45Ec4ELwf7Yp1QGxiLOKS49CxeCfVwEDK3ixZ+Zb5sPunC8hRxA1+Rdxx+s9bSIxPRon6qZmZXT+eg4u3A2p0LpKuuUDBqr5wdLNLl9k5vi4IBav4wtnLAXHRieo6O7HhCbpOazLm5t6lKFXGZu9ko+5Lq+pir+c0eoxg8ZIfX+8pLQfHzB0nysn6KlWqqGNZOTbXngyV+9L0KzPkZL9UVMlxt5DuxBLIyDIkyyNkqIdUjMkQk+ce4EgmQ8ajyIG6HKBLakl+lgN5GQPyPMkBv2yIunXrqlbS+qSZgAzol6Dj9u3bKrqULIqUjGk3pHRSkwuUShcz6bAmjQyEtJYePXo0hg4dqrpBSKAkg/fTkoH+7733nnp92QmSsZLaQcd/OwvJ9pDslnRok4hX5pN5ZHlaUm8oWSTpiBYWFqba62lbRWvJAbhcW+ijjz5Sy5AvQ1nf7C6pMwXSAU2aDZxbF6S70GedT8rpStQehsWlu6SKZF7CLkapi3Iak6eKr2o3HbjpJk4uuwy3nE6oNaAMfIs/PlAu1jQvkhNTcOrXK0iISVSvW/ez8qqltKVr0qycajYw96ddCAuNQbESOTFlVjdd6+eQkEhVhqAlQc2E0b+red3cnVCydC7MWdxHtZim/y5i82bYenkj18CPYOubA4/On8eVD/og6d/GA3a5cquzpFqxJ08g6LNPkevjQcg1eDDirwfh2kcDdNfAEUGfDEHuwUNQ4PvvYevhoZoJBE+dqi7kqc0cudWqrQumEkLuIGL7NoTMmmXxu3T1yV3I4eqJkc3eR053b5y6fRmtfv4E92JSz1rn8/I3yMY42NljdIs+KOyTW5WmSWe0nsvGIjLucYvbj9dOwejmfTCj7Sfwc/VSF/qce3ADxm1boJvnzTJ18Evnr3T3l/cYo/4fs2U+xm7NuGuRuTt674hqDCClZ1Iidiv6Jqafmqq74Ka3o49BxkaNs9Fo8HbhNvB08ERMYjROh57C+qvrMr1MseryCmiQoq59Ix3VzoX9g+UXl8LSFa3tr8bJHFl1DQ8jEuBb0BUth5VXjQFEtJSB631/iIjghwi5EImWX6XvzitJzIjbD7F171kV3EgAJIHT26MqqZbR2szR5YP3cHRNkPped/dzRPkW+VCh1eNxzKRH74K2/4W0iO7Ro4c68S7XvpET8pIE0V7rRo595fhaW901ZswYdVwuyQCpSJLr50ibaBlCoj32lWSEHMvLsBcJeCSpIc3DtEFUZllp9PsuZ9LSpUtVRkEaDUhJlhx4SztjieYkKEjbyticSbAkZW/SWKBRo1e3jOargA9e9irQv76pNQcP4h9fTZ5eLi+HDjhRKn1HH3p5Kp2/ALvBxq9xQi9e4pT9+N+u1AMUevl+bjgPU072fdmrQf8aXNGwisckRK/O+DG3dlla1I8//qi70KdkXeQyKdL1THstSUmAaBMGgwcPVkNEZF6pAJMMkAQzUnWV9kKfchkWCYLq1KmjuhCnbSL2XAKctKQ7mHRMkIyKZE3MmYw/kvIyKR+Ta9jI9W0kW3Tx4sV0A/5fJQxwTAcDHNPCAMf0MMAxLQxwTAsDHNNikgFO5K8ZP+bxePjHq+y/N8KWwZHOzgatk82ZjK/58ssvVTc2ac8s5WbSuvpVDm6IiIiIyDJokg071+ozfiEHCwlwZCyLpJtkEJB0LUvblliyHOaqadOm6kZEREREZKljcMwuwPn4449VgCOdycqWLZvhhfuIiIiIiMiEpDDAMWrFihVYtWqVrq0bERERERG92m2iLTqDI93SpMUbERERERG9QlLMP8B5pksky3Vmpk2bplq5ERERERHRK5TBSc7gZmkZnHfeeSddI4HNmzejTJky6TqISY9rIiIiIiIyMcnmE8j85wDHw+PxFeBFmzZtnsf6EBERERHR85KUbPbbNtMBzoIFC57vmhARERER0fOVkPF1cCx6DI5ISkrCjh078PPPPyM6OlpNCw4ORkxMTHauHxERERERZWcGJymDmyV3Ubt+/TqaNWuGGzduID4+Hk2aNIGbmxsmTpyo7s+ePTv715SIiIiIiP6bJPMJZLI1gyMX+qxatSoePHgAJycng3E5O3fuzM71IyIiIiKi7JKUlPHNkjM4+/btw8GDB9X1cPQVLFgQt2/fzq51IyIiIiKibKRJzjiDY2XJAU5KSgqSjWycW7duqVI1IiIiIiIyQUksUTPqjTfewNSpU3X3raysVHOBkSNHokWLFi9uBxERERERUeYlsUTNqEmTJqFp06YoXbo04uLi0LlzZ1y6dAk+Pj749ddf+REjIiIiIjJFSeafwXmmErW8efPi1KlTWLFiBU6fPq2yN++99x66dOli0HSAiIiIiIhMSJL5BzjP1EUtLCwMtra26Nq1Kz766CP4+voiMDAQR48ezf41JCIiIiKi7JFk/iVqWQpwzpw5ozql+fn5oWTJkjh58iSqVauGKVOmYM6cOWjQoAHWr1///NaWiIiIiIieXZL5X+jTSqPRaDI7c/PmzVXmZujQoViyZAk2btyoxuLMnTtXPS7ZnGPHjuHvv/9+nutMRERERETPIHlljwwfs+mwyPICHClF27VrF8qXL6/G3bi7u+PIkSOoUqWKevzChQuoWbMmIiIinuc603Ow4uIAblcT0bH4j5h+qt/LXg3618AKs3CsREluDxNSJfAC7AbXedmrQf9KnLIfVv1qcnuYCM2svxGduO5lrwb9y82ujclti6Sl3TJ8zLbrkiwta+bMmfj+++8REhKCChUqYMaMGahevbrReSUhsnjxYpw9e1bdl/hh/PjxBvP37NkTixYZBlmSTNmyZcvzK1ELDw9Hzpw51c+urq5wcXGBl5eX7nH5OTo6OksrQEREREREL0hCcsa3LFi5ciWGDBmiLhNz/PhxFeBIMHLv3j2j8+/ZswedOnXC7t27ERAQgHz58qlLz9y+fdtgvmbNmuHOnTu627N0aM5ykwG55s2T7hMRERERkWnSxCdneMuKyZMno0+fPujVq5e6dMzs2bPh7OyM+fPnG51/2bJl+PDDD1GxYkU1ln/evHlISUnBzp07DeZzcHBQCRXtTT+Z8tzaREvqSF5YyDVw+vbtqzI5Ij4+PssrQEREREREL4YmMeNARo7l0x7Py3G/9thfKyEhQY27HzZsmG6atbU1GjdurLIzmfHw4UMkJibC29s7XaZHGppJYNOwYUOMGzdOXWvzuWVwevTooV7Qw8ND3aRNdO7cuXX35bHu3btnaQWIiIiIiOjF0CSmZHibMGGC7rhee5NpaYWGhiI5ORn+/v4G0+W+jMfJjC+++ELFERIU6ZenyTgdyepMnDgRe/fuVU3O5LWeWwZnwYIFWVo4ERERERGZkISUDB+SjIyMq9GXNnuTHb799lusWLFCZWscHR110zt27Kj7uVy5cqqxWZEiRdR8jRo1en4lakREREREZH4lao5GytEy6qxsY2ODu3fvGkyX+9qGZBn54YcfVICzY8cOFcA8SeHChdVrXb58OUsBTpabDBARERERkfmVqGWWvb29avOs3yBA2zCgVq1aGT7vu+++w9ixY1Xb56pVqz71dW7duoWwsDDkypULWcEMDhERERGRhdBkIZB5Eillk/H5EqjItWymTp2K2NhY1VVNyLj8PHny6MbwyJiaESNGYPny5ShYsKBurI5cekZuco3N0aNHo23btioLdOXKFXz++ecoWrSoaj+dFQxwiIiIiIgsRWLWBuxnpEOHDrh//74KWiRYkfbPkpnRNh64ceOG6qymNWvWLNV97d133zVYjlxHZ9SoUark7fTp0+pCnxEREaoBgVwnRzI+WR0HxACHiIiIiMhCaLIpgyMGDBigbsZIYwB9QUFBT1yWk5MTtm7dmi3rxQCHiIiIiMhCpGRjgGOqGOAQEREREVmIZAY4RERERERkLlKSsmcMjiljBoeIiIiIyEKkMINDRERERETmIoUBDhERERERmYukeJaoERERERGRmUhhBoeIiIiIiMxFSjZd6NOUscmAmZCLKTVo0AAPHjyAp6cnXjWHNt3EwbVBiHmQAP9Crmjxv5LIW9wjw/kfxSRi55LLOB9wD4+iE+Hp54RmfYqjeNUc6vH4h0nYteyKejw2MgG5CruheZ8SyJNmmfdvxmD7wksIOhuBlOQU5Mjnig7DyqvlWbozW27hxB838DAiAT4FXFG3d3H4F3U3Ou+6UccRfC4i3fQClXzQalgF9fPhVVdx6eA9xITFwcbWGjkKu6FGx8LIWezxPtk08TRCg6LxKCoRDi62yFvOC7W7FIWLd9auYGyOcnTuDP/33oNdDl88unABN8aOw8MzZzKc37NZU+T5+GPY58mD+KDruPXDD4j66y/d41UCLxh93q3vvsPdX+arnx0KFkTezz+Da+XKsLKzw6PAQNyeNh0xhw7B0vV77R0MadgJOd28cTr4CgatnYIjN84bndfW2gZfNO6GbtWaI4+HLy7eu4lhG2dh24XH29HayhojmvVG5ypvIKebD4KjQrH48J8Yv32Rbp7hTXujfaVGyOfph4TkJBy/FYgRm+bg8I1zsGSvF62Iz5p0RZX8JZDbMwdaz/4cv596/Fk3pl6xypj87scok6sQbj64i3GbF2LR35sM5vmwXlu13Jzu3jh16zI+WjkJR64/3tYOtvaY9O5AdKzSBA62dth6/hA+/PV73IsOh6Vb9WsAlizYi7DQGBQrkQufffkWypbLZ3TeP9Yfxeiv1xhMs7e3xcHj43T3f565Hdu2nMbdkAjY2dmgVOm8+HDgGyhbPr9unsEDFuHihWA8CI+Fm7sTqtcsioFDmiOHn/HvLUuWnMTr4BA9d2f3hWDrvEC82b+UCkD+3nADS0Ycx0ezX4Orp326+ZMSU7B4+HG4eNqjw9AKcPNxQOS9R3B0tdPN8/uMc7h3PQbvDCkLN28HnN5zB4uGH8eAn2rB3cdRzRN+5yF++eIoKjfJjQadi8DB2Rb3bsTA1t7G4vf6pYN3sX/xJdTvUwL+xTxwatNN/PHNSXSeWhPOHun3SfNPyxn8wYyLTsTKz46gSC0/3TTP3M4qSHL3d0JSQnLqMsedRNcZteDknrrMPGU8UaVNATh72SM2PAEHl1zClsln0HZcVYveJ17NmyPvsKG4MXIUYk+dgl+PHij2yzz806w5ksLTH0y5VKqEwpMm4fbkyYjcvQfeb7ZCkZk/4vw7bRF36ZKa59RrdQye41G3Lgp8Mw4Ptm7TTSs6ezbirwfhYo8eSImLh1+P7ig6exbONnkDSaGhsFTtKjbE960HoP/qH3D4+jkMrNcem/43GWUmdML9mPSB/pgWH6jApe+qiQi8dwNvlKiONb3Go+70vjh5O3V/fNaoC/5XuzV6//oNzt25hir5S2Jexy8RFReLH/elHvxdun8TH6+dgmthwXCyc8DH9drjz76TUfKbjgiNTf+6lsLFwQmnbl/C/IN/YF3fiU+dv6BPLmzqPwmz961DlwUj0ahEVczrOgx3IkOx7Xxq0Nm+SmNMbvsx+v46EYeu/YNBDTti68CpKDGqA+5HP1DzTGk3CC3L1ka7eV8i8lEMfuzwKdb+71vU+eEDWLJtm09hyncbMWxEG5Qtnw+/LjmAj/73C37741N4+7gafY6LqwN+2/ip7r5VmscLFMyBz798C3nyeiM+PgnLF+9D/w9+wfo/P4OXd+oyq1YvjN59GsA3hxvu3Y3CtB824YvBSzF/2YfP9f2+ilIsoETN+mWvwKumfv36+OijjzBo0CB4eXnB398fc+fORWxsLHr16gU3NzcULVoUmzdv1j1n7969qF69OhwcHJArVy4MHToUSUlJuscLFiyIqVOnGrxOxYoVMWrUKN19KysrzJs3D23atIGzszOKFSuGDRs2qMeCgoJU9kbIOsm8PXv2xKvi4PrrqNI0Lyo1zgO//K5o9WEp2DnY4MT220bnP7HjtsrgdPqqAvKX9oSXvxMKlvNGzkJu6vHE+GScP3gPb/QqhoJlveCT21kFMN65nHDkz1u65UgGqFgVX7zRqzhyFXGHdy5nlKzhZzSosjQnN95EmUa5UapBbnjndVGBjq29Nc7vDjY6vwSXLp4Outut0w9g62CNojUfBzjF6+REvvLe8PB3gk8+V9TpXgwJj5IRej1GN0/FVvmRs7gH3HM4IVcJD1RuXQAhl6Is4mzTk/j36onQVasRtnYt4q5cwY2RI5ESFweftm2Nzu/XvRsi9+1XmZi4q1cRPG06Hp47B7+uXXTzSICif/Ns1BDRhw4h4Vbq74iNlyccCxVEyJy5eBR4EfHXr+P2pMmwcXaGU7FisGSD6nfELwF/YNHhP3H+bhA+XP09HibEoWeNVkbn71K1KSbuWIIt5/9WwcnPB9dj8/kADK7fUTdPrYJl8cfZ/dh8LgDXH4Rg7ak92B54GNXyl9LNs+L4duy6eFQt41zINXy6fgY8nFxRLncRWLIt/wRg+Iafsf7U3kzN3/f1d9Q2/PS36bgQEoSZe9dgzYndGNzo8f4Y0qgT5h74HQsDNuF8SJAKdGQf966Vuo/dHV3wXu03MWTNNOwOPIbjNwLRa/E4vFakPGoUKgNLtmzxfrR+tzrealMVhYv4Y9iI1nB0tMeGdUczfI4ct/j6uuluPr6p3+dazVpWRI1axZA3nw+KFPXH4M9bITYmHpcuhujm6dL9dZSrkB+5cnuhQqUC6PF+fZw5fRNJFlCO9SwBTkoGN3PBAOcZLFq0CL6+vjh8+LAKdvr164d27dqhdu3aOH78ON544w1069YNDx8+xO3bt9GiRQtUq1YNp06dwqxZs/DLL79g3LjHqdfMGj16NNq3b4/Tp0+rZXbp0gXh4eHIly8ffvvtNzVPYGAg7ty5g2nTpuFVINmYO5ejUbiCt26atbUVClf0xs3ASKPPCTx0H/lKemDT7Av4rttezOx/EH+tuoaUZI16XP5PSdGoA3J9dvY2uPFvGZU8fvFoKHzyOGPxiOP4rusezPnkkCpps3QSTNy/Go285R7vEytrK3U/5GJUppZxblcwitX2h52jTYav8c+OYNg728K3gPEzenExibi47y5yFfdQJW2WSkrDnMuUQdTBg48najSIPhgA10oVjT7HtWJFRAfozQ8gav8BuFQ0Pr+tjw886tVD6JrUvyMi+UGECo68W78NaycnwMYGOTp0QGJoKB7+8w8slZ2NLSrnLY6dFx8frGk0Guy6dBQ1Cxg/sJXypbikeINpcYnxqF24vO5+QNBZNCheBcVypJbxlM9dFK8VLq+CoozW4/1abyPiUTROB1/OpndnGWoVLosdF44YTNt67m/UKlxOt22l3E1/HtnHcl87T5UCJWFva2cwT+Dd67gedge1CqXOY4kSE5Nw4dxt1KhZVDfN2tpalYudPnU9w+c9epiAVk2+RctGEzDko0W4cvnuE19j3erDcHVzRPESuYzOExn5EFs2nkT5ivlha8eqDEsMcDgG5xlUqFABX3/9tfp52LBh+Pbbb1XA06dPHzVtxIgRKpCRQOSPP/5QAciPP/6ozlCULFkSwcHB+OKLL9R88oufWZKV6dSpk/p5/PjxmD59ugqymjVrBm/v1INRPz+/V2oMzsOoBBVsuHoZZk0kixJ6K9bocx6EPMK10w9Qrn5OdB1ZSZWabZx1AcnJKWjQKbXUTAKgvSuuwTevC1w9HXDmrxDcDIxQWRoh43Ike7B/zTU07FoUTXoWw+VjoVg54RR6flNFZYQsVVxUIjQpGjinyWTJ/QfBD5/6/LuXoxB+MxYN+z0+86wVdCwUW6f+o0rUpMTwra8r6srTtA4uvYwzW28hKT4F/sXc0Wpo6hgeS2UrWVlbWySFhRlMTwwLhWPhQsaf4+uLxFDD+ZPCQmHn62t0fp82rZEcG4uIbY/L08TFnr1Q5KeZqHj8mJwVQGJ4OC693wfJUZkLdM2Rr4sHbG1s042zuBsdjhJ+BYw+Z9uFw/i4fkfsu3IKV8Juo2GxKmhdvh5s9P7+f7dzqcoKnB26DMmaFNhYWWP4n3Pw6/HtBstqUbo2lnUfBWc7R9yJCkPzWYMRFmv8ZBAZl9PdB3ej0uy/qHCVDXO0c4CXs5vax+nneYCS/gV1y4hPTFClaWk/B/KYpYp48FB9F6ctRZP7QdfuG32OlJ8NH9NWjdWJiY7D0oV/oXfXn7Bq/RD453w8RnPfnvP48rNfEReXqMrQZs55D55eLgbLmj55M1b9ehBxjxJVNmfKzB7P6Z2+2pKSUk8ImzPLPS36H5Qv//ism42NDXx8fFCu3OMzNlK2Ju7du4fz58+jVq1aKrjReu211xATE4Nb/5aCPMvruri4wN3dXb1GVsTHxyMqKsrgJtNeJRoN4OJhj7f6l0buou4o+3pO1G1fCEc3P96eMvZGzrhN6rkPY9/ZiUN/3EC5ujl1+0EO4IWUpNVuXUA1IXi9XSEUr+aLI1uytl8offbGJ7+L0YYEecp4ocP31dB2bBXkr+iDrVPO4mFkgsE8ld7Kjw4Tq6vgR7J5O348p/YlPT++bdsi/I+N0CQY7ov8I0eowCqwSxecb9ceETt2qDE4tjlSm3lQ5gxZNw2X79/E2WHL8PD73ZjWdogqb5OTO/rjejpVboJuS0ej+qTeaizOkAad0K1aM4Nl7bl8HFV/6IW60/upJgXLe4xBDtdX56QWUVrlKxZAq7eroETJ3KhSrTC+n9oNXl6uWLvasJlJ1epFsPy3gZi/tB9qvVYcwz5djvAwwwCze6+6WLZ6IH6c8576/hg5bBW/P4yQURIZ3cwFA5xnYGf3eDC7kINm/Wnag+iUlMyl+iSLk/YALjExMVOvm9nX0JowYQI8PDwMbjLtZXF2t1d/hKR7mr6YiAS4ehnvnCXZHikts7Z5HDTmyOuiliElb0IyNb2/rYavVjfEkAWv44PJNZCcpIFXTqfHr2tjhRz5Dc/++OZzReT9OFgyR3c7VZIm3dP0yf20WZ20EuOScfnAXZRqmNvo41Ky5pnTWY2zkQyP7IPzuwzH9UhGRxoSyHidNwaVwfUTYbh7yXIzBkkPHkCTlKTKyPTZ+UiWxvhAfxlTY+drOL9tBvO7VqkCx8KFEbp6tcF0t5o14VG/Pq4OHoLY4yfw6Nw53Bw9JnXsT+vWsFShsZFISk6Cn5thltffzRshUWEZPCcC787/Eh5fNEGRse+i7ITOiIl/hKvhjz/73775Ib7fuQyrTuzE2TtXsezoVkzbuwqfN+pmsCwZB3Il9DYOXf8HH6z8FkkpyeiVwdgfMk72k797mv3n7q2yMVI6GBoTofZx+nm8dPtY/news1dZn8x+DiyBp5czbGys0wUect/H13g5clpSUlaiVG7cvGG4HZ2c7ZEvv6/KzIwY+656nd/XGpYaSkZHMkI1axfD+O8748C+QJw5dSMb3pl5SUrO+GYuGOA8Z6VKlUJAQIBBAHPgwAHVjCBv3rzqfo4cOdS4GS3Jqly7di1Lr2Nvn3rgmZz85E+nlNRFRkYa3GTay2JrZ41cRd1w9fTjUgA5q3ntVDjylTDeJloaC0hZmv7Zz9Dgh3DztlfL02fvaKO6qElTgisnwlCyRuqZZ5kvTzF3hN4yLLkKux0LzxyW3SJa28L51tnUTkHajJfcz1n8ye02L/99TwWSJV7PmanXkt+L5CfU/Gp/bZ40j7nTJCaqMS/utWo9nmhlBbdaNRFz4qTR58ScPAm3mnrzy6Do2rURezL9/D7vvovYs2dVC2h9atyNWoE02TONBlZZKK01N4mqPfNFNCxexeBkU4NiVfD39SePTYpPSkBwZKhqG92mfD38cWaf7jFne0ekaAw/58kpyap99JPI49KumDIv4OpZ1TlNX5NS1RFw9YxuHx+7EYhGJaoZ7GO5r53n2PULSEhKRKOSj+cp7p8fBXxyIeBaxu3bzZ2dnS1Kls6Dw4cejwuTE7FHDl1G+QrGSzjTkhK3y5dCVBnak8gxQEJCxikH7XHXk+axVEkWkMHhGJzn7MMPP1Qd0qQZwYABA1QTgJEjR2LIkCG68TcNGzbEwoUL8eabb6rxMzI2R0rfsqJAgQLqD/DGjRtVAwInJye4uqY/WyKd3ORmSqREbN2Uf5CnqDvyFHdHwO83kBCXjEqNU7MAayefVa2gm/RI7dxUrXk+HN54E5vnBqJGq3wID36IfauvqZ+1Lh8PVcdlvnlcVDC0bcFFNR5Hu0zx2jsFsfq70yhQ1hOFynmr51w8HIqe4x8fuFiqiq3yYefM8/Ar7Aa/ou449edNJMUno1T91O0nZWNybZpanQ27N0k2plA1Xzi62aXL7BxdG4RCVX1VC2hpI31my23VClrbSjrkUiTuXYlGrpIe6ho4UXcf4dDKa6qttGR8LNndBQtRcOK3KhB5ePq0ahMtAYh0VRPyWMLdewiePFndv7d4CUosWQy/Xr0QuXcPvFu0hHPZMrg+YoTBcq1dXODVrCluTUzfWjfm5Ak11qbgt9/izsyZSImPh2/7duq6OpF79sCSTd2zAvM7f4VjNy/gyPXzqk20i70TFh1KvY7Kgs5f43bkfXy96Wd1v3r+0sjt4YtTwZfV/yOa9lZ//3/YtVy3zE3/HMDQJt1xI+KuahNdMW9xDKrfAQsP/akLgIY17o6N/xzAnahQ+Lp4ol+dd9R1dX47tRuWTNpEF82ResJQFPLJjQp5iyE8Nkpd42b82/2QxzMHeiwaox6fvW8tBtR/FxPbDFCtpRuWqIr2lRuh5cxPdMuYvPNXLOoxHEdvnMfhoHMY1LADXBwcsSAgdR9L++5fDv6ByW0HIjw2Ut2f0f4THLxyWrWVtmRdutfBqK9Wo3SZvChTNh+WL92PR48S8Gbr1O/WEcNWws/PAwMGp5Zfzp21A+XK50fe/D5qDM7iBX8hJPgBWretpmtAMH/OLtRtUFoFPREPYtV1du7fi0Ljpqml+2dP38A/Z2+hYuWCcHd3wq2bYZg1Y7vquiYlcGTInAKZjDDAec7y5MmDP//8E5999plqTiDNAN577z1dkwIhGRTJ2LRq1UqVjI0dOzbLGRx5HemyJi2opV119+7dVdD0KpAxNDLoXy7MGfMgHjkLu6Hb6Mq6EjUpGdMbwgSPHI7oNqYytsy7iFkf/a2Cn5pv5kedtqmDP0VcbBJ2LL6MqNA4OLnZoXRtfzTqVsSgG1epWn6qJbUER5vnBMI3j7O6yGeBMl6wdNIBTS62eWjVVVWa5lvQDa2+rKArUYsONdwn4kFwLO5ciFRjZ9KSk9ARwQ+xZdIZdWFWCYD8irijzejKqmW0kNbgVw/dUxcElQYD8loyTqfp4IKwSZOZszQPNm+Grbc3cg/8CHY5cuDR+fNqsL+28YB9rty6cWUi9sQJXP30U+QZNAh5hgxGfFAQrvQfoLsGjpZ3y5bqxEj4RsMLHGq7qMlr5B40CMUXLYKVnS0eXbqMK/37p8v2WJrVJ3epcS8jm72fehHI25fR6udPcC8mNeuZz8vfIBsjpUyjW/RBYZ/cqjRNOqP1XDYWkXGPy3jk+jajm/fBjLafwM/VS13oc+7BDRi3bYF6PDklBSX8C6iLhfq6eiAsNkodfDeY0V+1jLZkVfOXwp4hP+nuy/VphLR47rV4LHJ5+CK/9+OsclDYHRXMTGn3MT5u0B63Iu7h/aUTdNfAEauO7VD7eEyrPqppwMlbl9BsxmCD5hKDV09V+/m3DyaoLNrWc4fw4YrvYOneaF4BDx7EYvaP2xEWGo3iJXNjxuzeutbPIXciVGm6VlTUI4wbtVbNK8GJZIB+WdpPtZgWUsosDQo2bliqghsPT2eULpsXcxf9T7WMFtKGeveOs5gzc4cKpiQQknE67/2vobpoKFlegGOl4ehdkusrXBzA7WAiOhb/EdNP9XvZq0H/GlhhFo6VKMntYUKqBF6A3WDDC5XSy5M4ZT+s+tXkLjARmll/Izpx3cteDfqXm10bk9sWO/xKZPhY43vmcQKLYS0RERERkYVIsoAMjmXXfRARERERWZCkbGwyMHPmTBQsWBCOjo6oUaOGuj7jk6xevVpdE1Lml0usyDAOfVJYJmPRc+XKpcaTN27cGJfSlFdnBgMcIiIiIiILkV0BzsqVK1XTLGmedfz4cTXWvGnTphleo/HgwYPqgvUyFv3EiRNo3bq1up09e1Y3z3fffacuZD979mwcOnRIXfdRlhkXl7VLeDDAISIiIiKyEEnZdB2cyZMno0+fPqq5VenSpVVQ4uzsjPnz5xudf9q0aWjWrJlqvCWXUZGmWpUrV8aPP/6oy95I52FpxPX222+rC9wvXrwYwcHBWL9+fZbWjQEOEREREZGFSHpCBic+Pl5dj1H/JtPSSkhIwLFjx1QJmZa0v5f7cv1HY2S6/vxCsjPa+aWDcEhIiME80l1YSt8yWmZG2GSAiIiIiMhCdEzOuFPaqFGj1GVH9EkJmkzXFxoaqi4u7++f2qpbS+5fuHDB6LIleDE2v0zXPq6dltE8mcUAh4iIiIiIINdmlHE1+kztAvGZwQCHiIiIiIggwUxmAhpfX1/Y2Njg7t27BtPlfs6cjy+sq0+mP2l+7f8yTbqo6c9TsWL6i4g/CcfgEBERERFRptnb26NKlSrYuXOnblpKSoq6X6tWLaPPken684vt27fr5i9UqJAKcvTnkTFA0k0to2VmhBkcIiIiIiLKEill69GjB6pWrYrq1aurDmixsbGqq5ro3r078uTJgwkTJqj7H3/8MerVq4dJkyahZcuWWLFiBY4ePYo5c+aox62srDBo0CCMGzcOxYoVUwHP8OHDkTt3btVOOisY4BARERERUZZ06NAB9+/fVxfmlCYAUka2ZcsWXZOAGzduqM5qWrVr18by5ctVG+gvv/xSBTHS/rls2bK6eT7//HMVJH3wwQeIiIhAnTp11DLlwqBZwQCHiIiIiIiybMCAAepmzJ49e9JNa9eunbplRLI4Y8aMUbf/gmNwiIiIiIjIbDDAISIiIiIis8EAh4iIiIiIzAYDHCIiIiIiMhsMcIiIiIiIyGwwwCEiIiIiIrPBAIeIiIiIiMwGAxwiIiIiIjIbDHCIiIiIiMhsWGk0Gs3LXgmi/yo+Ph4TJkzAsGHD4ODgwA36knF/mB7uE9PC/WFauD9MC/cH/VcMcMgsREVFwcPDA5GRkXB3d3/Zq2PxuD9MD/eJaeH+MC3cH6aF+4P+K5aoERERERGR2WCAQ0REREREZoMBDhERERERmQ0GOGQWpLHAyJEj2WDARHB/mB7uE9PC/WFauD9MC/cH/VdsMkBERERERGaDGRwiIiIiIjIbDHCIiIiIiMhsMMAhIiIii2ZlZYX169dn+7z0Yunvm6CgIHX/5MmT3A0WiAEOmZVu3bph/Pjx2ba8hIQEFCxYEEePHs22ZZqL+/fvo1+/fsifP78aEJozZ040bdoUBw4c0M1z8OBBtGjRAl5eXnB0dES5cuUwefJkJCcnv9R1N0fyRf6k26hRo3Rf+Nqbj48P3njjDZw4ceJlr75Z69mzJ1q3bq27HxISgo8++giFCxdWvzv58uXDm2++iZ07d77U9TSl7aX9jNrb26No0aIYM2YMkpKSnttr3rlzB82bN8/2eS11v9nZ2aFQoUL4/PPPERcX97JXjSwQAxx6KX/4/P390aRJE8yfPx8pKSlGnyMHyzY2Njhy5EimXuPUqVP4888/MXDgQN20+vXrGz3Y035Rrl27Vh3gyYGesTM98uX66aef4osvvvhP798ctW3bVh0YL1q0CBcvXsSGDRvU9g4LC1OPr1u3DvXq1UPevHmxe/duXLhwAR9//DHGjRuHjh07QqPRvOy3YFbkoEt7mzp1Ktzd3Q2myedYa8eOHWra1q1bERMTow7WIiIiXur6WwoJMqtUqYJdu3bh+++/x5kzZ7BlyxY0aNAA/fv3f9mrZzKaNWumPqOXLl3CJ598ogJ02V7GTkJlBzlBI8Fmds9rqfvt6tWrmDJlCn7++WfV4ZTohdMQvQA9evTQNGvWTHPnzh3NrVu3NMeOHdN88803GldXV03z5s01iYmJBvNfv35dPTZw4EBN3759M/Ua7733nuZ///ufwbR69epp+vTpo15X/6a1ePFizejRozVz586Vo23NiRMn0i03PDxcY29vrzl79uwzv39z8+DBA7W99uzZY/TxmJgYjY+Pj+add95J99iGDRvUc1esWPEC1tQyLViwQOPh4ZFu+rVr19J9zg8cOKCmbdmy5QWvpWX9/Xv77bfVz/L3Lk+ePOp3xNjvFRluL60mTZpoatasqXts3Lhxmly5cmkKFiyoHr9x44amXbt26nPv5eWleeutt9TnXd8vv/yiKV26tPp7njNnTk3//v11j8nvwLp169TP8fHx6jGZx8HBQZM/f37N+PHjjc4rTp8+rWnQoIHG0dFR4+3trb5zoqOj0+3/77//Xi1T5vnwww81CQkJZr/f5DugUqVK6ufk5GS1HWWfybYqX768ZvXq1Qbzy/dsy5YtNW5ubuoYoE6dOprLly+rxw4fPqxp3Lix+m5xd3fX1K1bVx1L6NPfN8b+3pHlYAaHXhhtGVOePHlQuXJlfPnll/j999+xefNmLFy40GDeBQsWoFWrVqoE6tdff8WjR4+euGwpeVqzZo0q80jL2dlZva7+Tb+kbcSIEWjcuHGGy5byqtdeew0rVqx4pvdtjlxdXdVNap3j4+PTPb5t2zaVydHPGmjJPipevLjar/TyOTk5ZeuZcMpYeHi4ytZIpsbFxSXd456entx8T/icaj+jUsoXGBiI7du3Y+PGjUhMTFQZfzc3N+zbt0+VycrfJ8kmaJ8za9Ystd0/+OADlTWTjLOUvhkzffp09fiqVavU6yxbtkyVKhsTGxurXlu+J6TaYPXq1SpDOmDAAIP5JIt95coV9b9kveU7L+33nrk5e/asKlOWSggxYcIELF68GLNnz8Y///yDwYMHo2vXrti7d696/Pbt26hbt646VpAM57Fjx9C7d29dxUV0dDR69OiB/fv34++//0axYsVUCbRMJ0rLNt0UoheoYcOGqFChgioVe//999U0OQkjAc7MmTNRsmRJ9SUkwYsEIxk5ffo0IiMjUbVq1eeyntWrV1dfnJTK1tZWfTn36dNHfVlJwCrlaFJ6Vr58eVWyJkqVKmV0k8l+1c5DL4+UpY0dO1YdDMpnnJ6vy5cvq79v8vmnzJHtJQGNlFPKuCUZ+yfB4bx583QHzkuXLlWlzjJNSo2FfIdIwLhnzx5VhiylsVLqJmWyWtWqVTP6mjdu3FAHz3Xq1FHLK1CgQIbrt3z5cjXGRA7ctUHrjz/+qE7kTJw4UZVjCwmAZLqUXcv+b9mypXpf8jfUnEjAKX9PJCiRk1/W1tbqfcvPMj5Wgr9atWqpeWUMmgQrUsYm3x/yne/h4aFOJkopu5CTYfrHC/rmzJmj9rEESHJClEgfMzj00skfe6lL15I/gA8fPlRnxYSc4fnll1+euIzr16+rLw4/P790j/3000+6jIPc5Esuq3Lnzq1egwzH4AQHB6sznXKmVA4kJNDRPyvJcTamqXbt2up3QQ66ZOzaypUrdQdi9Pzw9yHrB8rSnETGiHXo0EGNwxHSrEQb3Aj5DEvwKBkc7d95b29vFXhI1uTevXvqb1WjRo0yPWZUxmOWKFFCjemUjHRGzp8/r07S6WfkJOMvAZdkf7TKlCmjvqO0cuXKpdbL3MhYMtl2hw4dUtmWXr16qe8K2T/yvS5jb/W/jyUwlH0k5Hmvv/66LrhJ6+7duyoglOBTAiEZZyhjCCUgJUqLGRwyiS997Vk3IY0H5MtMsgSiU6dO+Oyzz9QfwSJFihhdhpSwSVpbfzlaXbp0wVdfffWfykCkPEL+OJMhOfiQLyy5DR8+XGXhZECpDHLXfvnLwXRaMr106dLcnC+JBDSy/aW5BsuiXhw5MJO/UdJwg55+oCxlZRLIyAkm7feBSFveJwe50rhBSsnSypEjh8oiZIWcqLl27Zoqn5YTbu3bt1dlzFJJ8KzSHrTL5yCjBjuvMtk32tI/+S6X4E9OUJYtW1ZN27RpkypT16dt2KAtl82IBExS+jxt2jSVVZPnSTaI5bVkDDM49NLJwa60k9TWqEv3Lcm6yBea3OSPoaS75Y9lRnx9fVUAYuwPnZzpkT+42pvMm1WyXvJFSU8mB81Sky4lIXIGddKkSenmkYyPdEaSwJVeDmlLLCcLGNy8WPI7IZlpKcWR35O02Mku/YGytKHXD24yCkjkb4pk8PX/1stN/v5LZkfG0GSlDbdkB+RE29y5c9UJgd9++019D6QlZbiSQdLfnzIGSIIqyQBZMtkGMtb266+/Vt8NEpBItiXtPpK/R0LKm6UUXMZUGSPbVTJqMu5GMmKyvNDQ0Bf8ruhVwQCHXioZSCgDPiWFLeQMnLQVli8MSVdrb3KgLKVPGV0/pWLFiur/c+fOPbfBkpUqVXouy34VyVk0qYeW2ncZ/yRnO2Vw7XfffYe3335bHZxIXbU0kZBBvTKPlCHKmTwp/3j33XfVWVEiSyPBjfwdkzFPctAsB+ZykkcGtmvHJlDWSJZeTlzJ3x45QJa/R1IyKwfDt27dUvNIeZt8j8h2lm1+/PhxzJgxw+jy5Fpd0gRFMm0yVlD+tklzGmMnBOS1JZMt2QX5npAmAjJWSMaMsuwTaNeunSrNk+8DaTojjQWkyYJUZGj3gdwX0pghKipKjeWUa8/JflqyZImu1E8yoHJffl+kBE62/dOyPmS5WKJGL4wMMpQL3MmXu9TSSjch6aoigwO7d++u5pEDYDn41aazteQMz7Bhw9RzZHBmWpJdkbN4MmBRG+xkhpyRkzNKUp8ttH9I03Zbky9NGYxNqaR2ukaNGuo6B/JFJWfcZB9JfbScsROyH+XL/ptvvlF11VIPL19QUi44aNAgo+WEROZOBlbLgZ38Xsh4QLlmiPz9khIrKcmirJNOmX/99Ze6Xtk777yjumpJ5l/G3EgmRkgAIn+D5G+WHGhLQCR/o4yRjI+crJEDbDk4l2YEco01Y6Vu8trSAEGaF8h8cl9O2EmQRKkNaSRwke0pgad81uV7X66TIwGjtqOqkJJZOekpJenSdEC2vXyfy5gm7fGBnDCT58j3jTQtMNapk0hYSa9obgp63uSsvfYsjfzBk8HNUpvbuXNn9cUjXxzSElK6oB0+fNhodxtJS8uZMum4ZowcHMiAxYCAAN00ufCk/IHUjglJS7JCMggyLRlHoh3QKsuT15YgiGeLiIiIiEwbAxwyG9JoQGqepV46O0s9pA5bgjHtWSYiIiIiMl0cg0NmQ7IrksHJzkGH0rRAWpJK3TARERERmT5mcIiIiIiIyGwwg0NERERERGaDAQ4REREREZkNBjhERERERGQ2GOAQEREREZHZYIBDRERERERmgwEOERGZJSsrK6xfv/5lrwYREb1gDHCIiOi56dmzpwo0+vbtm+6x/v37q8dknszYs2ePmj8iIiJT89+5cwfNmzfP8joTEdGrjQEOERE9V/ny5cOKFSvw6NEj3bS4uDgsX74c+fPnz/bXkwv0ipw5c8LBwSHbl09ERKaNAQ4RET1XlStXVkHO2rVrddPkZwluKlWqpJuWkpKCCRMmoFChQnByckKFChWwZs0a9VhQUBAaNGigfvby8jLI/NSvXx8DBgzAoEGD4Ovri6ZNmxotUbt16xY6deoEb29vuLi4oGrVqjh06BD3PhGRmbF92StARETmr3fv3liwYAG6dOmi7s+fPx+9evVSZWdaEtwsXboUs2fPRrFixfDXX3+ha9euyJEjB+rUqYPffvsNbdu2RWBgINzd3VUQpLVo0SL069cPBw4cMPr6MTExqFevHvLkyYMNGzao7M7x48dVUEVEROaFAQ4RET13EqgMGzYM169fV/clEJGyNW2AEx8fj/Hjx2PHjh2oVauWmla4cGHs378fP//8swpOJPMi/Pz84OnpabB8CYi+++67DF9fyuHu37+PI0eO6JZTtGjR5/Z+iYjo5WGAQ0REz51kYVq2bImFCxdCo9Gon6WcTOvy5ct4+PAhmjRpkm48jX4ZW0aqVKnyxMdPnjyplqMNboiIyHwxwCEiohdWpiZjZcTMmTPTlZCJTZs2qTIyfZlpFCBjap5Ev5yNiIjMGwMcIiJ6IZo1a6YyMjL4X9sIQKt06dIqkLlx44YqRzPG3t5e/Z+cnJzl1y5fvjzmzZuH8PBwZnGIiMwcu6gREdELYWNjg/Pnz+PcuXPqZ31ubm749NNPMXjwYNUw4MqVK6oJwIwZM9R9UaBAARUcbdy4UY2n0WZ9MkO6p0ljgdatW6vxP1evXlVNCwICArL9fRIR0cvFAIeIiF4Y6X4mN2PGjh2L4cOHq25qpUqVUhkfKVmTttFCStdGjx6NoUOHwt/fX1fulhmS/dm2bZtqUNCiRQuUK1cO3377bbpAi4iIXn1WGhntSUREREREZAaYwSEiIiIiIrPBAIeIiIiIiMwGAxwiIiIiIjIbDHCIiIiIiMhsMMAhIiIiIiKzwQCHiIiIiIjMBgMcIiIiIiIyGwxwiIiIiIjIbDDAISIiIiIis8EAh4iIiIiIzAYDHCIiIiIiMhsMcIiIiIiICObi/6GDqoy7NoNmAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -168,7 +168,7 @@ }, { "cell_type": "markdown", - "id": "22b2f9d6", + "id": "9dc6aaff", "metadata": {}, "source": [ "### 1.3 Multi-Recording Batch\n", @@ -179,7 +179,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "40a13d44", + "id": "00028acb", "metadata": {}, "outputs": [ { @@ -189,10 +189,10 @@ "Overall metrics — 10 recordings (nanmean across behaviors and recordings)\n", " precision 0.9355\n", " recall 0.6759\n", - " f1_score 0.7745\n", + " f1_score 0.7486\n", " so 0.6980\n", " tp 0.3381\n", - " ic 0.9948\n" + " ic 0.9921\n" ] } ], @@ -215,12 +215,12 @@ { "cell_type": "code", "execution_count": 6, - "id": "45419804", + "id": "0780cda5", "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAADwCAYAAADM38HGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAhg5JREFUeJzt3QV8E+cbB/Bf3V0pVoq7S4HhDsN1OAyGDduGDYehw4a7u7u7FJeixSmlQN29zf/zvOXSJE1Ly79ASZ7vPhnN5XK53F3u7rn3eZ/TkclkMjDGGGOMMcaYBtD93jPAGGOMMcYYY1mFAxzGGGOMMcaYxuAAhzHGGGOMMaYxOMBhjDHGGGOMaQwOcBhjjDHGGGMagwMcxhhjjDHGmMbgAIcxxhhjjDGmMTjAYYwxxhhjjGkMDnAYY4wxxhhjGoMDHMbYd9ejRw+4urp+79lg31GtWrXEQ/L69Wvo6Ohg3bp12Xq9DBgwAPXr1//es8G+Ito3NWvWLMumJ23b//77L761iRMnis/OSvHx8cidOzeWLFmSpdNl7P/BAQ5jarx69QqDBg1CoUKFYGpqKh7FihXDwIED4enp+UMHEnRws7S0RHR0dKrXnz17Jl7/0oNvVFSUOICeO3cO2mj79u3o0qULChYsKJah4gm7qtjYWIwcORIuLi4wMTFB5cqVcfLkyUytR+lhZGQkttXx48cjJiYmC78R+9x+YtWqVRgzZgwvKKa1DAwMMHz4cPzzzz+8/2HZBgc4jKk4dOgQSpQogY0bN6JevXqYN28eFixYgMaNG+PIkSMoU6YM3rx588MuN319fRGIHDx4MNVrmzdvhrGx8RdPm6Y7adKkTAc4K1euhJeXF350S5cuxf79+8XVTBsbm88GKXPnzkXnzp3F9qWnp4cmTZrg0qVLGfosCmpoG6UHTYeuMk+ZMgW9e/eGJsibN68Iwrt27YrsitZbvnz5ULt27e89K4xlyNixY9Ve3Pp/9ezZEwEBAdiyZQuvCZYt6H/vGWAsO3nx4gU6duwoTq5Onz6NHDlyKL0+c+ZM0Qyvq5v+tYHIyEiYmZkhO6IT42rVqmHr1q1o37690mt0cGratCl27979TeZFWk50BVATULCRM2dOsX1QkJyW69evY9u2bZg9ezb+/PNPMaxbt27iPSNGjMCVK1cyFKhSa5FiqlTVqlXFeqWAx8nJCdlJZn8T1DL1/wTbXxul5dAFgX79+n3vWWH/p6SkJMTFxWXr7S2r0H6DHllBJpOJFhtqgba2tkaDBg1ESmmvXr2yZPqM/T+4BYcxBbNmzRInYmvXrk0V3BA6MAwePFhcoVe8Em9ubi6CI7oCb2FhIa7KE5rWH3/8IcanwKJw4cIi9YsODBnpa0DDKeVLNX/6yZMnIjihVDM7OzsMGTIkU6kBv/zyC44ePYqQkBD5sBs3bogUNXpNHRp36NCh8u9SoEABEfDRyYH0PRwcHMTf1IojpU9J85/eclLXB4emS1fIS5YsKU48aNqNGjXCzZs35eNQSlf16tXFwZWmTcv3e6YL0bL5XPBLdu3aJVps+vbtKx9G35FaXzw8PPD27dtMfzYta1oWtG29fPlS6TVa1z/99JMIMGi5UxD78OHDVNOQtita1nTSQsvz77//Vhrnzp07ojWTtj1a5nXr1sXVq1eVxqFtmebn/PnzIvBydHRErly55K+vWLEC+fPnF59RqVIlXLx4MdW8qPtdSNvQu3fv0LJlS/E3zSsFiYmJiUrvDwwMFK0/NJ+0fXTv3h337t1LNc0PHz6Iq880f7Rd0+++RYsW4vPTQy1tdMWaWnkVUeslfcaOHTtEyg5Nl9YtLafnz58rjUvfu127dsiTJ4/4bNp+hg0bluoKe0a/t2LfjsWLF8PNzU2k19KJJ21TtG1QKx/NEy17+p5BQUGpvltGthdK1aX5os+g7+fs7CxObGm5K5L2WfTdaXxaF1ZWVmKZU4vv51CaJwX+t27dEgE8zTe1mi1btkxt2ueECRPEvklannTBgIYrovmhFGQKUIsXLy7GPXbs2Gfn5cSJE6IFn74vpSzv2bMn0/tJVdJvgcatWLGi2A9ndjnT/kT6valavny5eO3Bgwdp9sFJSEgQ24U0H7Qvpv2o6nKT+iIdP34cFSpUEOuCpi+hvmj0u1C3TTH2rXELDmMq6Wl0QKL+EJlBB4iGDRuKE0w6uaCTCjqZaN68Oc6ePStOXOnASAeGv/76S5yoUOrbl6KTUDrYTJ8+XZxc/vfffwgODsaGDRsy9P7WrVuLK890gJautlHrTZEiRVCuXLlU49OJSM2aNcV8//bbb+KEjFoZRo8ejffv32P+/PnihItStPr3749WrVqJzyClSpVKdzmlhZYZnYjSyfSvv/4q3ksnhPR96eBKJ1x0sKXpT548WRyY6STq8uXLyO4oSKA+M3TyrYhO9sndu3eVguiMkk7KFdPjqFWJTu5pudOJFq1LWk+0Dmg+pMCSTqTopJZa0yjwouEUjFIqI52oE1rmNA7NN5040rh0gkMnoXRypfq7oeCGtgvqG0TBPlm9erXYhuhklU4EKRij34mtrW2GvjOd0NN3oc+ibejUqVOYM2eOODmjbY/QyeTPP/8sWspoGG3XlDpIy0FVmzZtxPf6/fffxXf28/MTgbO3t3e6hS9o+6cTxbJly6p9fcaMGSLYpSAkNDRUXDyhgP7atWvycXbu3CnWB80jXaig+V24cCF8fHzEa5n93hI6cacWCfpOdLJJn037jDp16ogAjPp+0W+FPovmb82aNZneXmgZ0bqjQIVOumkZ0sk6/Uu/UdWTaPp8Ckxon3X79m3Rd4kCX/qMz6F9G10UoWl06tRJBI/0nQ0NDeX7L1rntB3RCTZtv0WLFsX9+/fFfvbp06fYt2+f0jTPnDkjpkOBjr29/WeLnNDFnw4dOoj9Ji0fughGwSkFRlKRiYzsJxXRPjc8PFyMS8uL1hPtN2m5Sq3aGVnOFIBS0Evfhz5ftV8gBXHptSjT/nX9+vVo27atuCBH2yitp8ePH2Pv3r1K41IqMa0Dmuc+ffqIiyCS8uXLi+MefeesLMrA2BeRMcaE0NBQalaRtWzZMtUSCQ4Olvn7+8sfUVFR8te6d+8u3jdq1Cil9+zbt08Mnzp1qtLwtm3bynR0dGTPnz8Xz1+9eiXGW7t2barPpeETJkyQP6e/aVjz5s2VxhswYIAYfu/evXTXJs2rmZmZfD7q1q0r/k5MTJQ5OzvLJk2aJJ+f2bNny983ZcoU8b6nT58qTY++s56enszb21s8p2WjOs+fW07Sa3nz5pU/P3PmjBh38ODBqcZNSkoS/86bN0+MQ5+ZHRUvXlxWs2bNNF+rU6dOquEPHz4U32nZsmUZWo/S9kjb0r///iu2qxIlSsiXUXh4uMza2lrWp08fpfd/+PBBZmVlpTS8Ro0aMgsLC9mbN2+UxpWmRei3YWhoKHvx4oV8mK+vr3gfvV9C2zJ9j+rVq8sSEhLkw+Pi4mSOjo6yMmXKyGJjY+XDV6xYIcZXXF7qfhfSNjR58mSleSxbtqysfPny8ue7d+8W482fP18+jLZxWuaK06Tfteq2nlFdunSR2dnZpRp+9uxZMc2iRYsqfccFCxaI4ffv35cPU9yPSKZPny7Wo+J6yOj3lpaZg4ODLCQkRD589OjRYnjp0qVl8fHx8uGdOnUS6zMmJibT24u6ed+6dav4nAsXLqTaZ/Xq1Utp3FatWqldfqpom6D3z5kzRz6MlittQ7Qt0TZFNm7cKNPV1ZVdvHhR6f30W6L3X758WT6MntO49HvLCNo30Xtou1I8XuTIkUOsg8zuJ6X1RN8/KChIPt7+/fvF8IMHD2Z6OdO6pOWh+Ht7//69+J6K2420PiR3794Vz3/99Velz/jzzz/FcNoXqy6HY8eOqV1OtC+g12fOnJnGkmTs2+EUNcY+CQsLE//SlTBVdIWarkRLD0r/UKV6FZUKElAaEqW0KaIrZHSMpTSQL0XV3BTRlVrpMzOKUtHoai6l6NDVTPo3rfQ0uppMV+6pZYDScqQHpefQleULFy5k+HNVl5M61AeIrkxSuokq6cowpboQujKfVvpHdkUpSNTipErqA5CRTsDUIiJtj9TqSFfiqW8VLQ9pGdHVX0qZoSuuiuuNtktqCaDWReLv7y/WIV0Np6vOiqRp0XqmFB1KkaJ0GQmldNF2Q1fOpd+QhK7w0mdJKL2QWkjoKjhdfZdQCg6lLWWUar8X2jYV0/LoqjpdAafPl1BriurvhlJsaD7od0CtBJlBKULpFZKgK+6K35HmkSjOJ32+4vqkdUMtW7R/oNaSzH5vCbUsKC5PqWWN+mwp9r+g4dTSQy0OmdleVOed0mNpvCpVqojn1EKTkXmnZai6zahD80wtBhJarvSctiVKXZP2UdRqQ611ivNOrVZEcd4JtXRQmllGUbVDapmWUCsm9Zuj9UT7zi/ZT1KLkOI29LltJL3lTNOi5aFY4IVS12jfSK+lRTpmUBU01eMUOXz4sNJwaoWj1j11pO9C88jY98Ypaox9QrnmJCIiItUyoTQcSiX4+PGjUsdu+Q9JX1+pjwGhSmt0UJSmK6GDsPT6l6IyxIooTYVO4D7Xb0CR1A+GUhgoJYryv+lEWd00KD2DUpikPjaq6MCaEeqWkzqUGkXLjtKW0kIHbUpzofSKUaNGiT4OlN5BaRbp9YOhlB06qfsSND+KJ61fik5aVPPbidSPSvGkJi0UDEmV8CilidJbaD0ovpfWG5FO8lRJKXLSCVV6aSwUBFEKjmJKiuI2TSdS1M+D0mEUT4YUSdu86vZLwYhi0JQeqT+W6omVYoBCn0OBl2oKJG3fiijIpBQpOpmjogx04kipNXTiSulAn6PYl06VaqAonfwpzielwVH63oEDB1IFWJTWltnvndZnS8GOagqgNFyaRka3F+l3RH3tqFiG6u9fdd4/tzxUUzVV0b5AtUAFpXgS2l/ReqN5p5SqjO6jVLfNz6FtRzXtTnEeaHvJ7H4yI9tIRpcz9U+k9Un7c9oXEvqbUqOl+VSHfiu0v1T9bdD3oYtIqsep9Jab9HvI6vvsMPYlOMBh7BM6ONBJkdQZU5F0BTStAIJOlDLSuVydtA4Gqp2mv2Qa6aF5poCAcq/pBFexmIEqOnmlPHPqd6FOegfQrFpOquhEnq6I0pVZuspIV+3pgE4nZ9TSoNhyoIi+s7rOuBlBn5XevW0yirYz6aq5IsrTl07oPoe+n2IHd7qqSlev6co2nTATqWWL+lWoO2HPqmpKaclIoJZZaa3XL0X9gKi/DvXRoD5y48aNE/0PqFUzrf41hPrMpNfqk9Z8SieB9Pum3xSdwFKfGFp3dBJP2wW1aKm2Smbme6c17ufmKTPbC/WHob4W1KeQTqKp5ZveTyfa6lpUP/fZ/y/6TCpIQhUE1VEN7r7GtpnZ/WRGlklGlzPtW6l1lfrMUKVPuhhH/RGnTZuWpceQ9Jab9HugPk2MfW8c4DCmgDprUqsAdfaVOnx/KSo1TR2BqeVHsRWHKlVJrytetVOsaPa5Fh66Uqh4JY06DNPB7nMdZVVRahF1MKagg8pjp4VaiKhlS7VilKqsunJHn0cnm3Tyl14rDs03Xa2kB53Y0MGcqn5RIJLWvFLH7MymI0lKly6NrEAnKjSPlJ6jePVa6oBOr39J0EQVuOhqL3U+pqvatBwJdeZOb91JrSfqgnsJXZWmFhF19yuibZrWxeeKBEjbPG2/iq0EVHKZbpqZVcuXPoeWL7U4KbbiqFYxk9ByolYcetC80fKn7WTTpk1pfgYFJNSZn66iZya9TkId4KnzO11goBYjSUZv9vo1ZHR7od8PldGnbY1aoCRSC1BW8/X1TVVmnJYdkfZ5NO9UJY/2BV+jBYG2HQo8FKetbh4ysp/MqMwuZ2rVpu2J3kOtWTS/6aWnSb8VOnbQNKXsAkIBEh2TpN9sRtBvmChOh7HvhfvgMKaArrzRCRH1RaAd/P9ztZFSwOgq7aJFi5SGU1UfOkhSdTBCJ7h0xUs1P5uuwqVFtQ8QVUMi0jQzim5QSOVBaR7TS8mhq4hUvpiCDlV0EKQKZ0Q6mVQN1jKLKlvRsqYDe1rrQF0pUikwUJf+pVjph05AvuTxuZt3ZhSl0dG2QdWQJDTPVJmJWgu/pIKa1BeL1gFV8JJadWj7osCPggh1aWdS8FKjRg0R7FLalLrlTVebqdww9fFRbMmk3wlVg6IqW59LNaLqd/RZVOJXMU2QquX9v9uMIvre9H3pBrISOolT/d1QAKRaXp1OUumCRHrbEHF3dxfLRuoDklnS1XvFfQr9TaXRv5eMbi/q5p2oVgnLKrR/USxHTNsOPadtiX7P0j6KWr8U17mE+rRJVfz+nyBLsaIYXZygqpW0z5H2nRndT2ZUZpcz7aPoghC1ZNODLtJ9LhWPjlPqpim1hNFFv4yi3wId2+i3wdj3xi04jCmgvgF0skadbKmvAZV1pavKdIChq1P0Gl2pzkg/Ekp7oQCCWhTohJCmQ6lTdIJIaTHS1VJC/UjopJT+pZNACnakq4Pq0LxQSVRKU6ADKl1pptaYzF4Bp+9Cd7b+HEqPoLQn6p9A6TN0UkEnDHQVmjqy0vejII3SF6jjLh1cKR2DDrbUryO9vh3q0HKje5hQ+Wu6siilY1CZaHqNSrtSaWhaTnQApquMlJ9OQSGtGzrZ/h5ofqRAlU4GaRlNnTpVPKcAgh6EghjqCE7lY2m+Kf+drrzScqQyyl+K0qaoczstB7qCS1dSqcQvLUsq/02tdHRSSEEMpfVRUQIpAKdlTcuNxqMyu3RiRPND41EfLULfRbr3EJWAppQlOtGkYID6AH0O9bWhaVAaHbXg0NVl2pYpsMtoH5yMoFQdOrmjFhm68k6tLbT9SkGxdBWefmN0xZ9OTGm7pe9DJ7EUtKXXokloGdDyplbatPqspIfmifYBVByCTswpsKDiGl/aupgVaB4ysr3QeLQt0zqnQIhubkv7NukKflajlE3qK0XbI+1XpH6DdIFAKqdM80xlkqmYAbXe0bzSRQRqXaTh0r1bvhR9LpWup/vUUH8tuhhA2wltu5ndT2ZUZpczLQtKwaX+OvS5VE78c+iYQWWvaVlSEEbFFyiDgfZH9Dui/W1G0b6Bljv9Lhj77r5hxTbGfhhUdrd///6yAgUKyIyNjWUmJiayIkWKyPr16yfKaqZVelkVlV0dNmyYzMXFRWZgYCArWLCgKEmrWHpXKgXau3dvUYqVSu62b99e5ufnl2aZ6EePHokyzzSujY2NbNCgQbLo6OjPfq/05lWirky09F2o3CwtEyota29vL6tataooTyyVaiVXrlwRpWtpHMX5T++zVctEEyp3SvNAy52mRaVvGzduLLt165Z4/fTp07IWLVqIZUuv079UKlW1ROu3JK0fdQ/V0tm0vqgUK5XnNjIyklWsWDHN8quq0luWVMKZStLSOIqlixs2bCi2L9qe8+fPL+vRo4fs5s2bSu998OCBKN9LpYJpvMKFC8vGjRunNM7t27fFtMzNzWWmpqay2rVri3WuSCoTfePGDbXzuGTJElm+fPnE965QoYIod0vlgDNSJlrd91YtfUuofPYvv/wifiP0ven7UqlgGm/btm1inICAANnAgQPFNkbTpfEqV64s27FjhywjqIw5/R7UlYneuXOn0nB134d+x/Xq1RPLkn5PVIaZSr1/6fdO67eb1jyltZ4ysr34+PjItxUar127dvIywer2Warl3KXPpnlOD20TVFadPtvd3V3MD+0rFi1alGpc2g9RiWIan7Yt2jfSvojK31NZZwl9Lq33jKLPa9q0qez48eOyUqVKiWnTNqO6PDO6n0xrPUnzprj8MrqcJSdPnhSvUanxt2/fZui3QqXDaRnRb5KOU7lz5xbfQSofrroc1KGy5PR9V61ale6yZOxb0aH/fe8gizGWMVQIgNK2qHWAO3IyljlUSIBK/VJJa7rS/P+i4hzUEkMl36XKVSxrUVEPKjucXv8w9v1Rihu1NFEFzK9RwIGxzOI+OIwxxjSO6r2EKF2J+qpR2g+lX2UFSqujtCWpzxNj2ojS56jPDqU7c3DDsgvug8MYY0zjUMEFCnKowzP1EdqzZ48ot0sd6LPyJIz6rDCmzajvj2pxEsa+Nw5wGGOMaRzq+E+lng8dOiQqpVEhB2rBoQIVjDHGNBv3wWGMMcYYY4xpDO6DwxhjjDHGGNMYHOAwxhhjjDHGNAYHOIwxxhhjjDGNwQEOY4wxxhhjTGNwFTUm/Hq6Ny+JbGJV3dU44T3ye88G+6RBnplISDrJyyMb0detD92B7t97NtgnSYs9YDCsOi+PbCJ+3iUs8uz/vWeDfTKoFJeS/x64BYcxxhhjjDGmMTjAYYwxxhhjjGkMDnAYY4wxxhhjGoMDHMYYY4wxxpjG4ACHMcYYY4wxpjE4wGGMMcYYY4xpDA5wGGOMMcYYYxqDAxzGGGOMMcaYxuAAhzHGGGOMMaYxOMBhjDHGGGOMZcqFCxfw888/w8XFBTo6Oti3b99n33Pu3DmUK1cORkZGKFCgANatW5dqnMWLF8PV1RXGxsaoXLkyrl+/nrkZ4wCHMcYYY4wxllmRkZEoXbq0CEgy4tWrV2jatClq166Nu3fvYujQofj1119x/Phx+Tjbt2/H8OHDMWHCBNy+fVtMv2HDhvDz88vUvOln+tswxhhjjDHGtFrjxo3FI6OWLVuGfPnyYc6cOeJ50aJFcenSJcybN08EMWTu3Lno06cPevbsKX/P4cOHsWbNGowaNSrDn8UpaowxxhhjjLGvysPDA/Xq1VMaRoENDSdxcXG4deuW0ji6urriuTRORnELDmOMMcYYYwyxsbHioYj6y9Dj//Xhwwc4OTkpDaPnYWFhiI6ORnBwMBITE9WO8+TJk0x9Fgc4jDHGGGOMaQndge5pvjbeoSEmTZqkNIz6w0ycOBE/Eg5wGGOMMcYY0xK6+mn3UBk9erTo5K8oK1pviLOzMz5+/Kg0jJ5bWlrCxMQEenp64qFuHHpvZnAfHMYYY4wxxrSErp5umg8KZijgUHxkVYDj7u6O06dPKw07efKkGE4MDQ1Rvnx5pXGSkpLEc2mcDH/HLJlj9t1QnfD58+fzGmCMMcYYYxlqwUnrkRkRERGi3DM9pDLQ9Le3t7e8Nahbt27y8fv164eXL19ixIgRok/NkiVLsGPHDgwbNkw+DrUerVy5EuvXr8fjx4/Rv39/UY5aqqqWUZyilsVev34tSuDduXMHZcqUkQ/v0aMHQkJCMnQTJG1UO1dtNMzTCFaGVngb8RZbn27Bq7BXaY5fL3c91MpZG7bGtoiIj8Atv5vY/WI3EpISMjRNM30zNHdrgeK2xcU0wuPDcdf/Dva92IfoxGhouwv7X+L0zmcIC4pFzvxWaDuwFFyL2KQ5flREHA6teYx7l30RFR4PG0cTtOlfEsUrpzQphwREY/+qh3h0/SPiYxNh72KOLn+WRZ7CNkhMSMKhtY/x8PpHBH6IhLGpAQqXc0CL3sVgZW/yjb519rZl83msXXMaAQFhKFwkJ8b83Q6lSrmqHXfv3qsYO2aT0jBDQ33cuZdyMeTkibvYsf0SHj70RmhoFHbtGYWiRXMpvSc2Nh6zZu7B0SO3EBefgGrVimLc+A6wt7eEthtQow3+rNcZzpa2uPfuOQbvmIsbbx6pHVdfVw+jG3ZHt8qNkdPaAV4fvTFq/xIcf3RVPo65kSmmNOuLlmVqwNHcFnd8nmLoznm46f1YaVpFnPJiRsuBqFmwrJjuow+v0HblGLwNVk7p0Db9q7XG8Dqd4GxhC0/fFxi6Zx5uqCw7CS23kfW6omvFxshpZY+nfm8x+tBSnHhyTT6Oro4uxjfqhV/KN4CzhR18wwKw4foRTDu5Xj7OuIa90L5sXeS2dkRcYgJu+3hh/OEVuO6tfjvQJp7HfHD7gDeiQuJgn9ccNXoVgnNB9fuNPRNu492jkFTD85a1Q/MxpcXx4eq2l3hzOxChftEwMtVHrpK2qNo5P8xtU1oWgn2jcHnjc7z3ChXvsc9jjiod3ZCrRNrHLm2lm8lAJi03b94U97SRSKlt3bt3FzfwfP/+vTzYIXR+TCWfKaBZsGABcuXKhVWrVslLRJMOHTrA398f48ePF0UJ6Fz62LFjqQoPfA4HOOy7q+hYEe0LdsCmJxvxMuwl6uWuj6FlhmGsx98i8FBVyaky2uRvi7WP1+JF6HM4mTqjV7FekAHY8Wx7hqZpZWQNayNr7Hy+A76RvrAztkOXIl3F8GX3l0Kb3Trng73LH6DD4NLIW9QG5/a8wJLRVzBuTT1Y2KRupk6IT8LikVdgbm2E3uMqwcreGEEfo2FibiAfJyo8DvOGXkDB0g7oP60qzK0M4f8uEiYWhuL1uNhEvH0egkZdCiOnm6UIknYvvY/l469hxJJa0HYUYMyauRcTJnZAyVKu2LjhLH7rsxiHjoyHnZ2F2veYmxuL1yU6OsqvR0fHoWy5/GjYqBwmjN+idhozp+/G+QsPMXd+b1hYmOCfKTswZPAqbN6inJ+tbdqXq4s5rQej/7ZZuPb6IYbW7oBjg+ahyKSO8I8ITjX+1J9/Q+dKjdB3y3Q8+fAGDYtVxp4+M1BtTl/c9XkqxlnZeTRKuLih2/rJ8A0NQJeKDXFy8H8oPuUX+Ib6i3Hc7HPi4vDlWONxEBMPr0JYTCSK58iHmPg4aLN2ZepgdstBGLjzX1x/8wiDa7bH4d/movj0TvCPSH3iPLlJXxG49NsxE15+3mhQuBJ29ZyGGv/1w913z8Q4f9XtjN+qtkSvrf/g0ftXKJ+nCFZ1HCOW+aKLu8Q4z/zfYsieeXgV6AsTAyMMqdkeR/rNRZF/OiIgMvXnaounlz/i4vpnqN23MJwLWOHu4bc48M9ddFlQBaZWyft8RU3+LCkCEklMRDy2/nkDBd0dxfOE2CT4vwxHxbauIliKjUzAhbXPcHimJzrMrCh/36EZ92DlbIpWE8pC31BXfO7BGffQbaE7zNQcu7SZbhYFOLVq1YJMRmdf6lGQo+491AiQnkGDBonH/4NT1L4ARZLVq1eHtbU17Ozs0KxZM7x48UIenZKyZctCR0dHrEiqPEFNbfv37xfD6HHu3Dkx3siRI1GoUCGYmprCzc0N48aNQ3x8vNLnHTx4EBUrVoSxsTHs7e3RqlWrNOeNImGaL9Ucx+ysfp4GuPjuAi6/v4z3ke9FUBKXGIfqLtXVjl/AKj+ehz7H9Y/XEBgTiEdBD3H9wzXks8yX4Wn6Rr7D0vtLcC/gHvyj/fEk+An2vtiL0valxZU7bXZ29wu4N86LKo3yIkdeS3QYUgaGRnrwOP5G7fhXj70RAUzfSZXhVsIOds5mKFjaHrnyW8nHObn9GawdTNHlr3KiJcg+hxmKVnCEg4uZeN3EzACDZlZDuZo54ZTbAvmK2aLdoFJ4+ywEQX5R0Hbr159B23ZV0aq1OwoUyIEJEzvC2NgQe/akfV8A2s84OFjKH6qtLs1bVMKAgY3hXrWw2veHh0dj9x4PjBjZGlWqFEbx4nkwdVoX3L3zEvfupt26qg2G1e2EVVcOYN3Vw3j84TX6bZuFqLhY9HJvpnb8LpUaYfrx9Tj60EOcDC+7uBdHHl7B8LqdxOvGBkZoU6YWRu5djIvP7+KFvw8mHVmN5/4+6P9TK6VA6cijKxi5b7EIjF4GvMPB+5fUBlXaZGitjljtcRDrrx/B44+vMWDnbETFxaBHZfXro3OFhph5aiOOPb4q1sfyK/tw9LEHhtXqKB/H3bUEDj64hKOPPPAm+AP23DuHk17XUTFPUfk4226fxJmnN8U0qCXtz30LYWVijpIu+aHN7h56i+J1XVCstgtsc5uJQIcCjkdnfNWOb2xhIAIQ6fHWMxj6Rroo8CnAMTLTR8vxZVGwqhNscprBuZAVavYuBL+X4Qj3jxHjRIfFIeR9NMq3yiuCIOscpqKFh4KjwLeR3/T7/+h9cDSF5nyTb4hyAakZjprmKJCgmxBR0EEdoa5fvy7GOXXqlGia27NnD/7880+0b98ejRo1EsPoUbVqVTGehYWFiHAfPXokmuso75Du6CqhpjyadpMmTUTES59XqVIltfM1a9YscZfXEydOoG7duvgR6OnoIa9FXjwKSkklkEGGx8GP4Gal/iDxPPSFeI8U0Ngb26OkfUncD/D84mkSU30TxCTEIEmWciVJ21BrzNunISI9TKKrqyOev34UpPY99z0+wLWYLXYsvIcx7Y5iWp/TOL7FC0mJKVd1Hnh8QJ5C1lg9+TpGtzuCmf3O4vKR1+nOS3RkvGh1oOBHm8XFJeDRw7dwd08JRGifU8W9cLqBRlRULOrVGYe6tcdi0MDleP7sfaY+l1LXEuITlT7Xzc0ZOXLY4K4WBzgGevoon7swTj25IR9GVzDpeRW3EmrfY6RvmKqVJTo+FtXzl5anTOnr6SMmIfU41T6NQwFr0xJV8ezjWxwdOA8fZhyGx1+r0KJUDWgzWh/lchXC6ac3ldbHmWc3USVvcbXvMdI3QEyC8n0+YuJjUdWtlPy5x+sHqF2oPAo65BbPS7kUQDW3UiIoSms+fnVvgZDocHj6Poe2SoxPEoFH7lK28mE6ujri+YenYRmaxqPTvihU1QkGxnppjhMblQDoJAc/UpBk7WKKJ+ffIz4mEUmJSXhw0hcmVgZwdFPfyq3N9Az10nxoCk5R+wJt2rRRer5mzRo4ODiIIIX+JdSyo1jSjsrf0Y2TVMvcjR07VqlgAAVD27ZtEx2wyD///IOOHTsq1SQvXTr5gKeIWoI2btyI8+fPo3hx9Tv17MjcwAJ6unoIi1Pe8dFzZ9Mcat9DLTcWBuYYWX6UeK6vq49zPmdx5M2RL56muYE5mrn+jAvvzkObRYbGIilJBksbY6XhlJr28W2E2vcEfIhE0N0oVKibC/3+cYe/bwR2/HcPiYkyNOlaJHmc95G4dPAVarcpgAa/FIK3Vwh2L/aEvr4uKjfIk2qa8XGJOLDqIcrXzqX1AU5ISAQSE5NSpaLZ2Vni1Sv1/S7yuTpiytTOKFQ4JyLCo7F27Wl0/mUO9h/8G87OGctHp74+Bgb6sLQ0Vf5ce0vxmrayN7cWwcjHcOWA3y88CEWc86p9z/HH1zCsbkdceH4HLwLeoW7hCmhdphb0PrUWR8RG4crL+xjbqKdoEfoYFoROFerDPV8J0YpDHC1sYGFshpENumLcwRWiD0+jolWwu8901FkwSExbG9mbWYn1QctfEa2fwo7q18eJJ9cxpFZHXHxxDy8C36FOwfJoWaom9HRTrvnOOr0JlsZmeDBqMxJlSWJdjTuyAltvn1SaVpNiVbG520SYGhjjfVggGi8dhsDIUGir6PB4yJJkqVLR6Hnwu8+3xn94FiZaXOr0T2kpU5UQl4grm16gUDUnGJrqyy8AtBpfBodn3ceybufFc1MrA7T4uwyMFdKlWdamqGVnHOB8gWfPnonOT9euXUNAQIBouSHUkapYsWKZmtb27dvx33//iRQ3qkaRkJAgSvJJqBpFnz590p3GnDlzRKsStShRmtuX3KH2R1LYujCauDbFZq9NeBn6Eo6mjuhYqBOaxTbDodeHMj09Yz1jDC49RPTFOfDqwFeZZ01GBzMLayN0GloWuno6oqUmNCBGFCmQAhy6opqnkA2a907+feQuYI33r8Nw6dCrVAEO5WKvmXIDlNbbfnDqYJ59XpmybuKh+PznZlOwY/tlDB6iPm2HfT1Dd83Dil9G4fH4beK3QEHOOo/D6KmQ0tZt/SSs7vI33k07iATqsP72KbbePCn6fhApdXa/50XMP7tN/H3P5xnc3Urit59aam2A8yWG712AZR1G4MHozcnrI9BXpLf1qNRUqV9Pp3L10XXTJJF+VjpnQcxpORjvwwKw8cYx+Xjnnt9GhX97wt7MGr2r/Iwt3Sej2vy+avv+sM+jNDa7PGZpFiSg48OxuQ9FXkbtPimty7Qez616ChMrQ7SZXA76hnqiJejgDE90mFGB++BoYYCj+d/wK/j5558RFBQk0skoyKEHiYvLXEdPDw8PdO7cWaSfHTp0SKSg/f3330rToZafz/npp5+QmJgoSu19zvTp02FlZaX0oGHfS0R8OBKTEmFpqLwzo+ehceqvgrXI3xIeHzxw0fci3kW+wx3/O9j7Yg8auzaBDnQyNU0jPWNRfCAmMQaL7y9CoiwR2szMykikpIUFJ+c1S8KDY2GZRidNK1tjOOQyF8GNxCmPhajARilvxNLWGM55lFsgaJxgv+jUwc3UG6LfDfXJ0fb0NGJtbQ49PV0EBioX3AgMDMtwNTMDAz0ULZob3t7JndUzgqYdH5+AsDDlq66BARn/XE0UEBEiAhAni5QUHOJoYYsPYYFpvqf1ilEwH1YHruNao+jkjqLV5mXgO/k41J+m9vwBMB9WG3nGtkSV2b1F2hMNl6YRn5iAxx+U0wOffHiNPDaZuwGeJgmIDBXrg5a/Iqf01kdkCNquGQOrkfWRf0pblJj+CyJio/EyKKWPyIyfB2D26c3Ycec0Hrx/ic03j2PB+R0YUber0rSorw8FrNfePETf7TOQkJSInmn0/dEGJhYGIiUtKlT5fIiem1qnLjCgiFLLnl3+iGJ1XNIJbh4gLCAGLcaVlbfeEJ8HwXh9KwCNhhaHSxFrkZZWq09y35/H5zKXnqsNdLkPDlMVGBgILy8vkVpG/VyKFi2K4OCUDp50kyJCAYciGq467MqVK8ibN68IaipUqICCBQvizRvljtylSpX6bMEA6pNz9OhRTJs2Df/++2+641JN8tDQUKUHDfteKKB4E/4GRW1TmqMpSCliUxQvQ5MLN6gy0jVMVbVDsd9MRqdJLTfDyw5HoiwBi+4tVCoxra30DXSRu5A1nt5JORGmlDV6Tv1s1MlX3A4BvhFiPIm/T4QIamh6xK24HT76KKe4+flEwNbJNFVw4/8uQgQ3ZpbpHwy1BZV3LlY8N65e9ZIPo1bja1efonSZlMIa6aEUt2dPfUWxgYyiogL6BnpKn0spce/fB6NMBj9XE1GQceutl0gzk1A6DD2/+vJBuu+NTYgTFdGoz03rsrVxwPNiqnHohJlOzK1NLNCwaGX5OPS5N948RiEn5RbPgo558CboA7QVLZfbPk9Rp1B5pfVRu2B5XH1DV/o/tz4CxPpoVaomDt5PWR+mhsap+mPShbPPFaGh16nPlbbSM9AVwYXP/WClVv6394PhXCj9/c9zDz8kJshQuIZzmsFNyIdotBpXRgRSiqiYgKBSLZK2hXSKfGkt3Sy6D052xilqmWRjYyP616xYsQI5cuQQaWnUsV/i6OgoWl2o0hrV96bKZ9RKQv1rjh8/LoIjej8No4CG3k99bqhKGhUU2Lt3r9LnTZgwQQRS+fPnF31xKIXtyJEjos+NIipaQMMbN24MfX19DB06VO38UzpadktJO+l9Ar2K9cabsNfiPjX18tSDkZ6RqIBG6LWQ2GDsebFHPKfKZ1QlzTvCG68+pai1dGsJz4B7ophARqZJwc2wssNFsLTq4UoY6xuD/iPhceHy6Wij2m3yY9Os2yKlLG9hG5zb+wKxMYmo0jD5xGrDzFuwtjdG897Jfb1++tkVFw+8xO4lnqjZMj/83kXgxNanqNnSTWmac4dcEMUHqFLaG69gXDnyGh2HlpEfvKgAwdvnofhtShVxQAwLSm5FMrUwlAdK2qp79zoYM3ojipfIg5Ilk8tER0fHolWrKuL10SM3wNHJCsOGtxDPlyw+itJlXJEnjwPCw6KxZs0p+PoGoU3b5OImJCQkUgQr/n7JrZqvP/XnodYZCoSoLHSb1u6YNWMPrKzMRNnpaVN3iuAmo4GVppp3eivWdRuHm95PcJ3KRNfpCDMjY6y9mpwiu67bePiG+GPMgeSS85VciyGnlQPu+jwT98GZ0PRX6OroYNbJlHsVNShaWZyMeX18gwIOuTCr1SA8+fgGaz1S0m7/PbUZ23pNwcVnd3H22W00KlYFP5eshtoLBkKbzT+3DWt++Ru33j4RQSCViTYzNMH6a4fF62t/GYt3of4Ye3i5eF4pTzG4WNnjnu9z8e/4hr1E4Y5/z6SUSz/88DJG1e8G75CPokx0mVyFMLRWB6y7dkQeAI2u1w2HHl4WaWuUota/emtxX53d985Cm5VplhunFj+GY34LOBWwFOWaE2ITRVU1cmLhI3H/GqpypujhGV+4VbRPFbzQ8eHonAfwfxWOZqNKiYtpkcHJqfbUv4aCKgqejMwNxOdSOWlKUXt4yhdhftFwLWf3Db/9j0FPXyUS1EAc4GQS7QQpIBk8eDBKlCiBwoULiz40VA5aLFB9ffF88uTJop8OpY9RSWjqR0P/UksN9bU5e/YsmjdvLm52RLW+qV9M06ZNRZloKistoenu3LkTU6ZMwYwZM0T/nBo11FfNodLVFCRRypuenh5+//13/Ahu+N2AuaEFWri1hKWRJd6Gv8X8u/PkRQLsjG2VWmyonw09a+XWEtZGNuK+NhT0UJpaRqdJVdbyf6qoNr3qDKX5GXl5hCg/ra3K18qFiJA4HF7/WKSm0Y0+B0xzlxceCPaLUrqnio2jKQZMr4o9S+9jet8zIvip2coN9TsUko9DgVKfiZVxYPUjHNvkBTtnU7TuXxIV6+aW3wSUqrERqrCmaPC/1cT9c7RZ4yblERQcgUX/HUZAQDiKFM2J5SsGylPF3r8PEmkhEkormzBuixjX0soExYvlEfeuoRLTkrNn7yvdDPTPP9aKf6l09MBByX0RRo5uI6Y7dMgqxMcl3+hz7PgO0HY7bp+Gg4UNJjX7VdwEku6d0njxMPiFJ1+1zmPjpHT131jfCFN+/g1u9i4iFerIQw/R5yY0OqVVk8oLT2veD7msHREUFYY9d8/h7wPLRMqTZN+98+LeO6MadMOCdsPh5fcGbVeNweUXyRUktdXOu2fgYG6NCY1+ld94tdnyP+D3qXx2bpX1YWRgiElN+sDNLnl9UGW0HpunIDQmZX3Q/W0mNe6DhW3+gKO5jbjR58orBzD1RPLvJDEpCYWd8oqbhdqbWyEwMkzclLX2woGiz442o87/0WHxuLb9JSJD4uDgaoHmf5eWp6hFBMSkui9X8LtIvH8SihZjU26QLokMisWrmwHi721/pVQvJK0mlkWu4jYwsTQUn3F160vsm3RHFLmxy2WGpiNLic9nygw1qBx0WnRk6d2hh2mNX0/3/t6zwD5ZVXc1Tngrt9Cx76dBnplISFKunMS+L33d+tAd6M6rIZtIWuwBg2Hq71vGvr34eZewyLM/L/psYlCp7Hfz8CLr2qb52pMeyTey/dFxCw5jjDHGGGNawlChJLqm4gCHMcYYY4wxLWGoUPVUU3GAwxhjjDHGmJYw1II+OBzgMMYYY4wxpiUMuQWHMcYYY4wxpikMuQWHMcYYY4wxpikMuQWHMcYYY4wxpilM9PWg6bgPDmOMMcYYY1rCkFtwGGOMMcYYY5rCkPvgMMYYY4wxxjSFIbfgMMYYY4wxxjSFga7m3+hT8+/0wxhjjDHGGBMMdNN+ZNbixYvh6uoKY2NjVK5cGdevX09z3Fq1akFHRyfVo2nTpvJxevToker1Ro0aZXq+uMgAY4wxxhhjWsIgi1LUtm/fjuHDh2PZsmUiuJk/fz4aNmwILy8vODo6php/z549iIuLkz8PDAxE6dKl0a5dO6XxKKBZu3at/LmRkVGm541bcBhjjDHGGNMSBlnUgjN37lz06dMHPXv2RLFixUSgY2pqijVr1qgd39bWFs7OzvLHyZMnxfiqAQ4FNIrj2djYZPo7coDDGGOMMcaYFvXBMUjjkVHUEnPr1i3Uq1dPPkxXV1c89/DwyNA0Vq9ejY4dO8LMzExp+Llz50QLUOHChdG/f3/R0pNZnKLGGGOMMcaYljBIp3kjNjZWPFRbVFTTxAICApCYmAgnJyel4fT8yZMnn50H6qvz4MEDEeSopqe1bt0a+fLlw4sXLzBmzBg0btxYBE16ehm/QSm34DDGGGOMMaYlDNJpwZk+fTqsrKyUHjQsq1FgU7JkSVSqVElpOLXoNG/eXLzWsmVLHDp0CDdu3BCtOpnBAQ5jjDHGGGNawkAv7cfo0aMRGhqq9KBhquzt7UWLysePH5WG03PqN5OeyMhIbNu2Db179/7svLq5uYnPev78+dcNcOLj45E/f348fvw4s29ljDHGGGOMZdMWHCMjI1haWio91FUxMzQ0RPny5XH69Gn5sKSkJPHc3d093c/fuXOnSIPr0qXLZ+fVx8dH9MHJkSPH1w1wDAwMEBMTk9m3McYYY4wxxjSkitrw4cOxcuVKrF+/XjR8UEEAap2hqmqkW7dualt/KD2N0s/s7OyUhkdEROCvv/7C1atX8fr1axEstWjRAgUKFBDlp796kYGBAwdi5syZWLVqFfT1uU4BY4wxxhhjPwLjLLoPTocOHeDv74/x48fjw4cPKFOmDI4dOyYvPODt7S0qqymie+RcunQJJ06cSDU9Snnz9PQUAVNISAhcXFzQoEEDTJkyJdP3wtGRyWSyzH6hVq1aiajK3NxcdAJSLe9GN/JhjDHGGGOMZS9bvAam+dovhRdDE3xR84u1tTXatGmT9XPDvpszPmN46WcTdXJNg2/kqu89G+wTF7Nfccy2MC+PbKRRkBf0fq/6vWeDfZK48ApaHOjGyyOb2N98A95Hqb/RIvv2cpj2ynaLXV+lVUUTfVGAs3bt2qyfE8YYY4wxxthXpa+b8fvJ/Kj+rw40lHdHuXSE7jbq4OCQVfPFGGOMMcYYy2L6WtCC80XfkCok9OrVS5Rsq1GjhnhQRyCqZx0VFZX1c8kYY4wxxhj7v+nr6KX50OoAh8rCnT9/HgcPHhRVDuixf/9+MeyPP/7I+rlkjDHGGGOMZUkLjn4aD61OUdu9ezd27dqFWrVqyYc1adIEJiYmaN++PZYuXZqV88gYY4wxxhjLAvrcB0c9SkOTalwrcnR05BQ1xhhjjDHGsil9DWqpScsXfUN3d3dMmDABMTEx8mHR0dGYNGmSeI0xxhhjjDGW/ehrQR+cL0pRW7BgARo2bIhcuXKhdOnSYti9e/dgbGyM48ePZ/U8MsYYY4wxxrKAPqeoqVeiRAk8e/YMmzdvxpMnT8SwTp06oXPnzqIfDmOMMcYYYyz70deCFLUvvg+Oqakp+vTpk7VzwxhjjDHGGPtq9LkFJ8WBAwfQuHFjGBgYiL/T07x5c94sGWOMMcYYy2b0dbgFR65ly5b48OGDqJRGf6dFR0cHiYmJ32YNMcYYY4wxxjLMQM9A45dWhlPUkpKS1P7NGGOMMcYY+zHo6XxxD5Ufxhe1Ub19+zbr54QxxhhjjDH21QMcvTQeWh3guLq6ombNmli5ciWCg4Ozfq4YY4wxxhhjWU6PAxz1bt68iUqVKmHy5MnIkSOH6JOza9cuxMbG8mbIGGOMMcZYNqWnq5/mQ6tbcMqWLYvZs2fD29sbR48ehYODA/r27QsnJyf06tUr6+eSMcYYY4wx9n/T4xac9FHFtNq1a4tUtVOnTiFfvnxYv359hhdwrVq1MHToUGRn33IeKfVv/vz53+SzGGOMMcaY9tHTMUjzoSn+r7YoHx8fbNmyRTwePHgAd3d3LF68OMPv37Nnj7ivTnZw7tw5EaxRnyJra+uvOo/r1q0TQVNISIjS8Bs3bsDMzAza6Ny+Fzi54ynCgmKQK78VOvxeBq5FbNMcPyoiDvtXP8TdS76ICo+DraMp2g0shRKVc4jXD61/hMMbHiu9xym3OSaua5hqWjKZDItGX8ajGx/x26QqKFM951f4hj+evdtvY/uGGwgKjET+Qo4YPKIuipZIXr6qjh14gJkTjyoNMzDUw4mrw5WGvXkZiBX/nce922+RmCBDXjc7TJrdAk45LPHBNxSdmq1QO/0JM5ujVv3C0GZ5ev+CfL/3hqGjA8IfPsHjkVMQevt+muM7tWiEgqOHwCRPTkS9fA2vif8i4NQF+esFRg6Cc6umMM7pDFl8PELvPsSzf+Yh9JanfBzT/K4oPGkEbCqXg66hAcIfeuHZtAUIunQN2q7/T63xZ93OcLa0xb13zzFk11zceKO8z1G8qd6oBt3QrVIT5LS2h5efN0bvX4Ljj1OWo7mRKSY37YOWpWvC0dwGd3yeYtju+bjp/Vg+jSnNfkPj4u5ws3NBaEwETnvdxOj9S/E+LADarolrXbQs0AQ2RlZ4HfYWK+5vxLOQl2mO/7NbQzR2rQN7EzuEx4Xjiu8NbHi8E/FJ8eL1Rq51xOuOJg7iuXf4O2x/ug+3/VJ+Hwa6BuhVvBOq56wCA1193PG7j2X31yM0Ngzajo4f29ZfE8ePAnT8GFkPRUu4qB336IH7mDnhSKrjx8lrfyoNe/MyAMsX0PHDW378mPxvK3H8kDy89w6rFl/A4/vvoaunIz579pL2MDLOHuea2YWeBhUTSMsXfcPly5eLoOby5csoUqQIOnfujP379yNv3ryZmo6tbdonsNnFt5xHSvXTRjfPvsXuZZ7oNLQs8hWxxZk9z/DfyEuYuK4BLG2MU42fEJ+E/0ZcgoW1EfpOqAxrexMEfoyCqbnyDiyHqyWGzP5J/lxPT0ft55/Z/Rw66l/SWmeOP8HSuecwbEx9FC2ZA7s238KIgTuxYW9v2NiqD8LNzA2xYU/vlAEqC/Xd22AM7r0FjVuURI9+1WBqZojXLwNhaKQnXndwssDuE/2V3nNwjye2b7iOytXyQZs5t2qMIlNH4+EfExBy6x5c+3VHhV2rcbFSI8QFBKUa37pSWZReOQdPp8yF//GzyNH2Z5TbtBhXardGxONnYpzI56/xeORkRL1+Cz0TY+Tt3wMVdq/BhfL1ER+YXDym/NZliHz5BtdbdEdSTAzy9uuOcluXiXHi/LT3pLp9ubqY02owBmyfjWtvHmJIrQ44OmAeik7pBP+I1IV3KDDpXLEhfts6A08+vkHDopWx+9cZqD7vN9z1eSrGWfnLKBTP4YbuGybDN9QfnSs2wolBC1Din1/gGxoAU0NjlMtdCP8cWysCKhtTC8xrMxT7fpuJyrMVfndaqLpLZfQq/guWeq7D0+AXIniZWOUvDDgzAqFx4anGr5HTHd2KtsPCu6vxJOgZXMydMaRsH8gArHm4RYwTGB2EDY92wDfyI3Sggzq5q2NMpaEYdn4c3oa/E+P0LvELKjiWwaybCxEVH42+JbthdMXBGHVpKrTZmeOPsWTOGQz/u4EIanZtuYm/BuzAxn190j9+7O2jlCGkevz4vddmNGlZCj37V08+frwIkB8/pOBmxKAd+KWnuwio9PR08eKpH3R0+QCvKiv72lDDBnVboXtlli5dGgsXLhT99NO6wN+zZ0+lYUZGRoiJiVG66DxhwgSRHUYNAdWqVcPSpUtRsGDBr98HZ+rUqahcuTJu3bolWm5Gjx6d6eBGNf2L0rOmTZsm+vBYWFggT548WLEi5Wpu1apVMXLkSKX3+/v7i9aVCxeSr0pSkYM///wTOXPmFC0hNI/UMiN58+YNfv75Z9jY2IjXixcvjiNHjuD169ei9YbQa/TD6tGjR6p5JO/fv0fTpk1hYmIiUvIo0FNNLZs7dy5KliwpPiN37twYMGAAIiIixGs0P7RyQ0NDxefQY+LEifJloDgd6uPUokULmJubw9LSEu3bt8fHjx/lr9P7ypQpg40bN4r3WllZoWPHjggPT71Dz85O73qGak1cUbWRqwhKOg0tJ3ZaHsfeqB3/yrHXiAyLQ7/J7shfwh52zmYoVNoBufKntLxJAY2VrbH8YW5llGpab5+H4NTOZ+j6V4Wv9v1+RDs330TTVqVEMOLqZi8OVMbGBji6/0E679KBrb15ysNO+UC2evElVK7mhn5Da6FgESfkzG2DajULyA94dDBSer+9OS6dfYZa9YvAxNQQ2sx1QE+83bAD77bsQaTXCzwcPgGJUTHI2bmN2vHz/tYNAacv4vXC1Yh8+hLPpy1AmOcj5Pm1i3yc97sPIfC8B6Lf+CDiyXM8GTsdBpYWsCie3FJmYGsDswL58Gr+CkQ88kLUyzd4OnkO9M1MYVE0cwcaTTO0dkes8jiAddcO4/GH1+i/fRai4mLR072Z2vG7VGqI6SfW4+gjD7wK9MWyS3tx9NEVDK/TSbxubGCI1qVrYdT+Jbj44i5eBLzD5KOr8dzfB/2qtxbjhMVEouHiodh55wye+nnj2uuHGLxzLirkKYrcNk7QZi3yN8IJ73M4/fYi3kb4ikAnNjEW9fLUVDt+EdsCeBz0DBfeecAvOgB3/R/gwrurKGjtJh/nxse7uOXnifeRH+Eb+QGbnuxCTEIMCtvkF6+b6puI6VNAdD/gMV6EvsZ/d1eiqG0hFPo0jrbauekGmrYujcYtSsE1Px0/Gorjx5F999M9ftjZm8sfqsePVYsuoHL1/Og3tHbK8aNWQaWAadGc02jdsTw696qCfPkdkMfVDrUbFIWhoea3VnyvPjjbt2/H8OHDRUBy+/ZtEeA0bNgQfn5+ab6HzmfpXFp60Lm5olmzZuG///7DsmXLcO3aNXEuTdNUDIK+WoBDJ940A/RFstKcOXNQoUIF3LlzRwQF/fv3h5eXl3iNWom2bdsmIjvFBevi4oKffkq+Sj9o0CB4eHiI8Tw9PdGuXTs0atQIz54lX7EcOHCgCIIoILp//z5mzpwpggcKQnbv3i3Goc+jBb5gwQK189itWzf4+vqKQIXeQ0GY6orU1dUVK+fhw4eiT9KZM2cwYsQIeaBGQYziCqagTBXdTJWCm6CgIJw/fx4nT57Ey5cv0aFDB6XxXrx4gX379uHQoUPiQePOmDEDPwpqjfF+GoIi5Rzlw3R1dcTzl48C1b7H84ov3IrZYtt/dzCizSFM7n0SRzc/QVJiyrZB/N5FYFT7wxjb5SjWTLuOoI9RSq/HxSRgzT/X0XFwGREAsWTx8Yl4+vgDylfOq7ROylXOi4eevmkupujoOHRsshztGy/D38P24tWLlCv8SUkyXL30Arny2uCvATvRqu5i9O+2SQQwafF69AHPvfzQpGVJrV41OgYGsCxdHIHnr6QMlMnEc+uKZdW+x7piGRG8KAo4c0kMT+szcnfvgPjQMIQ/SN7nxgcFI+LpS7h0bAk9UxPo6Okhd48OiPULEOls2spATx/lcxcW6WESOi6d9roBd9cSat9jpG+I2Pg4pWHR8XGo5lZK/K2vqw99PX3ExCtXIo2Oj0W1/MnjqGNlYiaOFSHRP9ZFraykr6OH/FauuOefsk3KIMO9gEcobFNA7XueBD1HfmtXeUDjZOqA8o6lccvvntrxdaGDn1wqw1jPCF5Bz8Uwej+lpSl+7ruI9/CLCkCRND5XW44fXmqOH+Uru+KRZ3LLV1rHjw6Nl6JdoyX4e+huvHrhr3L8eInceej4sR0t6yxE/64bcPFscusnCQ6KFGlpFPAM7L4RreouxJDeW+B5x+crftsfl14WBTh0Qb9Pnz7iwn2xYsVEUGJqaoo1a9ak+R66sO/s7Cx/UIEyxX0pnSOPHTtWnAOXKlUKGzZsEOfddK6bGV8U1kpNh1FRUSLYiYtT3nHTDH2JJk2aiMCGUGvNvHnzcPbsWRQuXFi0XlBLyqVLl+QBDbWedOrUScwPzcfatWvFvxT0EAocjh07JoZT6xC91qZNG9G6Qtzc3FKlojk6Oir1wVH05MkTUUyB+spQIEZWrVqVqtlMscWHWlaoxatfv35YsmQJDA0NRUuLtILTcvr0aRGEvXr1SgRghFYytTrR51esWFEMo4MbNflRqxfp2rWreO8///yDH0FEaKzYeammotHzj2/VH7QD3kfC644/KtXNg4HTq8H/XQS2LbiLxMQkNOtWTIxD/Xe6jagAp1wWCAuKFv1x5gw9j3Gr68HYNDmVbecST7gVt0PpaurzgrVVaEi0CBZtbE2VhtNz79ep06FI7rw2GDGhEfIXdEBERBx2bLiB33tuxtqdvUTqWUhQJKKj4rF17XX0GlAdvw2pgetXXmP8n/swd0VHlCmfvI0rOrL/PvLms0OJ0trdJ8rQzga6+vqI81cO+GP9A2FWKGUfpsjI0T5VClmsX6AYrsihQS2UXjVXBDCxH/xxo3UvEdhIbrTugXIbl6Ce923IkpIQ5x+Em+1+RUKo9vYxsDezFsHIxzDl38LH8CAUdlKfyXDi8TUMrdMRFz61ztQtVAGtSteEnk7yNcaI2ChceXkffzfqiccf3ohpdSpfH+75SohWnLSCpunNB2DbrZMIj1G+eKNNLA0toKerhxCVfi8hsaHIZa6+zyC13FgammN69bHQ+RRgHn19GrueHVQaL69FLsz8aTwMdQ0QnRiD6TcWiBYiYmNkjfjEeEQmRKX6XGsjK2ir0OAocfywVUlFs7Gj44f6i5Z58tpi5IQmcCvkgMjwWGzfeB2DemzC2l294ehkKYKX6Kg4bFl7Db0H/oS+Q2rh+uVXGP/HXsxb0QllKuSBr09yn+Z1yy+h/7DaKFDYCccPPcAfv20Tx6FcebN/l4gfrQ9OXFycyOSiLC7FC/z16tUTjQ1poYwmyvqi89dy5cqJ83M6tyV0zkupbjQNCZ0zU0YWTZOylDLqi74hpYZRChcFD+okJiZ+yWSVAiMpAJBaR6h/SoMGDbB582YR4NBCoC9L/YEIBQP0uYUKFVKaJrXY2NnZib8HDx4sWoVOnDghFh4FO5kJxqh1R19fX6wQSYECBURamyIKgqZPny4CorCwMCQkJIimNQoIKbLNiMePH4vARgpuCEXHFHzRa1KAQwGUFNwQui9Rek2DtDxU71dE+Y8/ElkSYGFjhM7Dy4lOhHkL2SAkIBondzyTBzglKisEj/mt4FrUFn//chS3zvmgWpN8uHfFF153/TBmecqPiH254qVzioekRCkXdG+zBgd33xMBTdKnxrWqtQqgXZfkiwN0AKKc6YO77qYKcGJj4nH66GN06+POq+UromIBV2q2hIGdDXJ3a48ya+bjav128n49xWZNQFxAIK417Yyk6Bjk6tpO9MvxqNsWsR9TrrCy9A3dPR8rOo3Co7FbxRVKCnLWXT2MnlVSUtq6b5yMVb+Mgc8/B5CQmIDbPk+x7dYplMudurgGFRzY3muKOE4O2DGbF38mlbArgrYFf8Zyz/Wiz04OMyf8WqIL2hcKwY6n+5VaZIaeHwszfVNUdamIIWX74u/L0+RBDvtKx4/SOdGtzSpxbOg9sAZknw4g1cTxI/ncp+Cn48cBOn5UyCMf5+c2ZURqnBiniBNuX38jLpb1Haw+XVFb6aXTByet80TVc8WAgABx3q3YAkPoOZ3/qkMNFtS6Q+fe1FXj33//FZlNlPGUK1cuEdxI01CdpvTaV01RoxYKmjHKjaO+KBToUCoWtWQcOHAAX0q1WhntvCnCk1CaGt1QND4+XrTeUEuM1BpDEaGenp6IJu/evSt/UDAgpZv9+uuvIs2LWjkoIKJWGOoMlZWoP0+zZs3EyqMUNpofqbKcaktXVvjcMlNFgRdFw4oPGva9UL8Yar4OC1bOraTnlmmkjVnZGcMxl7kIbiTOeSxFBTZKeVPH1NxQtOb4+0aK59QCFOAbiT+aH8DA+nvEg6yYdBVzh5+HNrOyNhHLNjhI+cokPVfNi06LvoEeChZxFB1DpWnq6evC1S35YoMkTz47fPyQujXg/KmnIshp0Cz5qo42iwsMRlJCAgwdlJedkYMdYj+q7+hPaWSGKq01Ro52YriixKhoRL3yRujNe3gw+G/IEhKQq0tb8ZptjSpwbFgLd38dhpBrt0Ufnkd/TUJidIxIW9NWAZEhIgBxslS+IuxkYZuqVUf+nogQtF45ChZ/1EW+Ca1RbGonRMRG42VgSsrOy4B3qPPfQFj8UQd5x7eC+7+/wkBPT/TZSR3cTEUeW2c0XDREq1tvSFhcOBKTEmFtlFJJi1ArSnBMqNr3/FKkDc75XMFJ7/N4E+6Dqx9uYeOTnWhboJkoKCBJkCXiQ6Sf6F+z8fFOUZ2tmVsD8VpwbAgM9AxE8KP6udSKo62sbEzF8SMoKPlYKwkOzOTxo7AT3r0NkU+Tjh953ZT3aVRFze/T8cPOwfzTMJVx8qWMw1LoJCak+fia54lUbZm6elD/8Zo1a4pKxdSAITVWZKUvCnCoTwnl3VGAQM1R1NTUpUsX0S/na54sUz4etYRQQEUBDgU8ijcfpUiSWi+oVUXxoZgKRi0ilC5GC/WPP/4QVRoIpY59rvWJIk9qjaE+QpLnz5+L0tISCmgowKD+RFWqVBEtSpQ7qIg+63OtXEWLFsXbt2/FQ/Lo0SNRUYJacr4UNSVScKr4UGxe/Nb0DXSRp5C1CDgklLJGz92KKZ/QSSitzP9dpBhP4ucTLgIfmp46MdEJ8PeNkAdNDTsVxt8r62HMirryB2nbvzS6aXnBAQMDPRQq6iyufEloWdPz4qUyls5H6YIvnwfA1t5MPs0ixZzxViXFzcc7CE45Uqdz0BW3qjULwNomYy2emoxKOIfdewi7GgqtWTo6sKvpjpAbKfsiRSE37sKuRhWlYXa1qorh6dHR1YWuUfK+UM/EJHmgwu9Mek7jaav4xATceuuFOoXKK11YqlOoAjxep1eEA4hNiBMV0ShIaV2mFg7cv5hqnKi4GHwIC4S1iQUaFKmMA54XUwU3BRxyo8GiIQiK4hM3CkIoAClln3IxhIKUUvbF4BWc3F9GlZGeIZIoFUCB9Dy9els0XSoNTV6EvEZ8UgJKOaQcj3OaOcPR1B5P0vhcbUD7+sJ0/LimfPy4df01ipXKmYnjhz/sVI8fb5SPH/RcKhHt7GIFewdzvFVJg1MchylISkjzkdHzRHt7e9GwoFj8itDz9LpgqF6kp/N3Opcm0vv+n2lKvugoFRkZKfqqEErPopQ1Qq0pVEXha6FKCi1btsS4ceNEywz1v5FQIEEBD0WGFLxQCtv169dFwHX48GF5y9Px48fFazSf1L+HAglCQRodpKijPn0fqeqZIiqJTaltffv2FdOmQIf+plYsqV8SBVTUwkQtQ9RaRBXOqNOVIkoro+lTXxlq4qPUNVX0ObQ86TvRvNLn0XejiFfq//MlqImRChwoPr53ilrdtgVx6fAreBx/g/dvwrB1/h3ExiTAvWFyPvu6GTewb1XKiUON5m7i3jc7F98T/XTuX32PY1u8ULNFSuUaKjv99J4/Aj9E4sXDQCwf7yFaiirWSU6FoqICOfNZKT2IraMJ7HNo572IFLXrXAGH9nri2MEH4t4186adQEx0PBo1T+5EPW3cYaxcmHJPlfUrruCGxyuRB/308UdMG3sYH9+HiUpskg7dKuLsiSc4tOce3nkHY++227hy4QVatlPu+E6ved5+i6Ytv6wvnyZ6vWQtcnVrL1pOqN9N8TkTRb8ZqqpGSi6ZiULjUu459Gb5BtjX/QmuA3vCrKCbuOeNVZkS8F61SbxO7y04dhisKpSGcS4XUcSgxMJpMMrhhA/7k1OPKRiKDwlDySUzRGU16Z44Jnlzwv9ESnVKbTT/7Db8WrU5ulVqjCJOebGk/V8wMzLGuquHxOvruo7DPz/3k49fKW8x0ecmn50LqucvjSMD5kFXRwezT22Wj0PBDJWPdrXLgXqFK+L04IWipPTaT9Ok4GZn72kon6cIum6YKPrvUKsRPajwgTbb/+IYGuStidq5qyOXuQv6leouCgKcepu8jxpati+6Fm2nVCGtsWtdUTiAApLSDsXRuUgbMTxJFIuGGL+YbWE4mtiLvjj0vIR9EZx/l1zsIyohGqe8z4vy1CXtiopCB4PL9hFlpyntTZtRGtmhvfdw7MB9ce+aedOOi+MHVeUk08YeEvdDk6xfflnh+PEB//x96NPxI6WQVcfulXH2+GMc2nMXPt7B2LPtFq5ceI4W7ZO7DNA5WIfulcTwcyefiHFWL74g+o1SaWmmIjEhzUdGzxPpYn358uXFuayELvDTc2qpyQi62E8ZVdS9glB1YgpkFKdJXT0oYyyj05R80V6RWjKoPwqdqFMlNWpaor/pRF6aya+FTvipGEGNGjVEKWlFVEyAOvRTy8y7d+9EdEmtKJQyJi1IqqRGNyilFUYV1qiQAaHS0pMmTcKoUaNENQgKJqjzvirq6N+7d2/x+bQSKICi3EFj4+SWAVoe1LpFFdoo4qXxaByanoTyDakViSqiBQYGivJ6UqloCf1Y6d5Cv//+u5gGtZTR/GZ1Sl12UKF2blFs4NC6RyI1jW70+fuM6vLWliC/KKWa+HRTT3p951JPTO1zStwHp3brAmjYMSVXPdg/WlRIo3LSlAaXv4QdRiyqLe6dwz6vTsMiorPouqWXk2/0WdgRMxe1lacY+H0IFwGjJCIsBnOmnBDjmlsaiRagRWt/ESWmJT/VKYRhYxpgy9qrWDj7jChMQDf5LFk2V6rWGypMUMHdlVfVJx/2HoWhnS0Kjh4MI0cHhD14LDr7S4UHTHLloCOLfHmFXL+De33/RKExQ1Fo7HBEvnyN210Gyu+BI0tMFIFP2Y6tRBGDuKAQhN65L/raUMloQsUG6DMKjR2KivvXQ9fAABFPnonp0A0/tdmO26dhb26NiU37wNnCFnffPUOTJcPhF57cmk9lmxVbCKgM9OSmfeFm7yJS06hcNN3vJjQ6Qqki2j8/90cuawfRMrPn3jmMPbgcCUnJrf05rR3QvFRygZ07ozYozU+dBQNx/rn61jxtcMn3mig28Evh1uJGn6/CvDHp6mz5DTfpZp5JChVYqZ8N9YXqXLQtbI1tEBYbjhsf72DT413ycawMLTG0XF/YGlkjMiEab8LeYuLV2UpV01Y/2AJZcRlGVvxdtOzc8b+PZZ7roe3qNCyKkOAorF16KflGn4UdMWtxe/nxg9KSFe9NEx4eg38nH/t0/DBG4aJOWLyuiygxrXj8oHLTm9dcxX+zTiN3XltMnt0KpRSOH+06V0RcbCIWzzmD8NAY5C/kgH+XdhAlpZkKaq3JAlQiunv37uLCO937hiqgUSOIdK8bOvel82spu2vy5MnivJwaAygjie6fQ2WiqQsJoXM9aoygc3nq9kIBDzVqUPEwauDIDB2ZYt3lDNq0aZNI1aJCA5SSRSfeVM6YojkKClRLGWsyCpYo7Y0KC9Stm5zm9CM64zPme88C+6ROrmnwjVzFyyObcDH7FcdsU3f0Zt9PoyAv6P1elVdBNpG48ApaHEi5iMe+r/3NN+B9VNpletm3lcO0V/Zb5OE7037NIqW1MyMWLVokv9En9a2h26RQ1TPpXpLUACI1GAwbNkxkWdG4lAFGLUAUzFCamuqNPuk2LBQEVa9eXVQhVi0i9lUCHFWUYkUVE6hFhVpNNBn1P6L0Mkofo3vY0P1tqLXo6dOnqTr8/0g4wMk+OMDJXjjAyX44wMleOMDJXjjAyV6yZYATujXt16xSun/8yLIkcZdKHyuWTtZk1L9mzJgxon8NlWemdDMqXf0jBzeMMcYYY0w7yBLj03wtvUIbGh/gUF8Wam6iTkBUtUy1LDG1cmiqhg0bigdjjDHGGGPa2gdH4wKcIUOGiACnadOmKFGihFIHcMYYY4wxxlg2lcQBjlrbtm3Djh07RDUzxhhjjDHG2A8ikQMctahaGpV4Y4wxxhhjjP1AkjQ/wPmiG33SfWYWLFggSrkxxhhjjDHGfvwbfWpdH5zWrVunKiRw9OhRFC9ePFUFMapxzRhjjDHGGMtmEjUnkPm/AxwrKyul561atfoa88MYY4wxxhj7WhISNX7ZZjjAWbt27dedE8YYY4wxxtjXFZf2fXC0ug8OSUhIwKlTp7B8+XKEh4eLYb6+voiIiMjK+WOMMcYYY4xlZQtOQhoPbb4Pzps3b9CoUSN4e3sjNjYW9evXh4WFBWbOnCmeL1u2LOvnlDHGGGOMMfb/SdCcQCZLW3DoRp8VKlRAcHAwTExMlPrlnD59OivnjzHGGGOMMZZVEhLSfmhzC87Fixdx5coVcT8cRa6urnj37l1WzRtjjDHGGGMsC8kS027B0dHmACcpKQmJahaOj4+PSFVjjDHGGGOMZUMJnKKmVoMGDTB//nz5cx0dHVFcYMKECWjSpMm3W0GMMcYYY4yxjEvgFDW15syZg4YNG6JYsWKIiYnBL7/8gmfPnsHOzg5bt27lTYwxxhhjjLHsKEHzW3C+KEUtV65cuHfvHrZt2wZPT0/RetO7d2907txZqegAY4wxxhhjLBtJ0PwA54uqqAUGBkJfXx9dunTB77//Dnt7e3h5eeHmzZtZP4eMMcYYY4yxrJGg+SlqmQpw7t+/LyqlOTo6okiRIrh79y4qVqyIefPmYcWKFahduzb27dv39eaWMcYYY4wx9uUSNP9GnzoymUyW0ZEbN24sWm5GjRqFjRs34tChQ6IvzsqVK8Xr1Jpz69YtXL169WvOM2OMMcYYY+wLJG7vnuZreh3Wa1+AQ6loZ86cQalSpUS/G0tLS9y4cQPly5cXrz958gRVqlRBSEjI15xn9hUUX9+Ol2s28bD7ThRc0/Z7zwb75FmvXUD8UV4e2YlBY+j0r/K954J9Ilt6lddHNlsf/93r/71ng30yuPTSbLcsEjZ1TfM1/S4bMzWtxYsXY/bs2fjw4QNKly6NhQsXolKlSmrHpQaRDRs24MGDB+I5xQ/Tpk1TGr9Hjx5Yv145yKLGlGPHjn29FLWgoCA4OzuLv83NzWFmZgYbGxv56/R3eHh4pmaAMcYYY4wx9o3EJab9yITt27dj+PDh4jYxt2/fFgEOBSN+fn5qxz937hw6deqEs2fPwsPDA7lz5xa3nnn37p3SeI0aNcL79+/ljy+p0JzpIgN0z5v0njPGGGOMMcayJ1lsYpqPzJg7dy769OmDnj17ilvHLFu2DKamplizZo3a8Tdv3owBAwagTJkyoi//qlWrkJSUhNOnTyuNZ2RkJBpUpIdiY8pXKxNNTUf0wYTugdOvXz/RkkNiY2MzPQOMMcYYY4yxb0MWn3YgQ+fyqufzdN4vnftL4uLiRL/70aNHy4fp6uqiXr16onUmI6KiohAfHw9bW9tULT1U0IwCmzp16mDq1KniXptfrQWne/fu4gOtrKzEg8pEu7i4yJ/Ta926dcvUDDDGGGOMMca+DVl8UpqP6dOny8/rpQcNUxUQEIDExEQ4OTkpDafn1B8nI0aOHCniCAqKFNPTqJ8OterMnDkT58+fF0XO6LO+WgvO2rVrMzVxxhhjjDHGWDYSl5TmS9QiQ/1qFKm23mSFGTNmYNu2baK1xtjYWD68Y8eO8r9LliwpCpvlz59fjFe3bt2vl6LGGGOMMcYY07wUNWM16WhpVVbW09PDx48flYbTc6kgWVr+/fdfEeCcOnVKBDDpcXNzE5/1/PnzTAU4mS4ywBhjjDHGGNO8FLWMMjQ0FGWeFQsESAUD3N3d03zfrFmzMGXKFFH2uUKFCp/9HB8fHwQGBiJHjhzIDG7BYYwxxhhjTEvIMhHIpIdS2ah/PgUqdC+b+fPnIzIyUlRVI9QvP2fOnPI+PNSnZvz48diyZQtcXV3lfXXo1jP0oHtsTpo0CW3atBGtQC9evMCIESNQoEABUX46MzjAYYwxxhhjTFvEZ67Dflo6dOgAf39/EbRQsELln6llRio84O3tLSqrSZYuXSqqr7Vtq3wzc7qPzsSJE0XKm6enp7jRZ0hIiChAQPfJoRafzPYD4gCHMcYYY4wxLSHLohYcMmjQIPFQhwoDKHr9+nW60zIxMcHx48ezZL44wGGMMcYYY0xLJGVhgJNdcYDDGGOMMcaYlkjkAIcxxhhjjDGmKZISsqYPTnbGLTiMMcYYY4xpiSRuwWGMMcYYY4xpiiQOcBhjjDHGGGOaIiGWU9QYY4wxxhhjGiKJW3AYY4wxxhhjmiIpi270mZ1xkQENQTdTql27NoKDg2FtbY0fTafCDdGzRHPYm1jDK+gNpl1fg/sBz9Mcv2vRJuhQuCFymNkjODYMJ99cxbxbWxCXFC9eP9FmMXKaO6Z639YnxzD12mrxt72xNf6o0BVVXUrBVN8Yr8N8scJzD056X4O261y0EX4t0RwOJtZ4EvwGkz1WwzOd9dGjWFN0KtoALrQ+YsJx7PVV/HtrM+ISk9eHro4uBpdtj+b5fxLT9IsKxp5n57D43i75NH4v2x5N81VDDjM7xCcl4EHgS8y7tRX3/J99k++c3W3eehGr156Bf0A4ihR2wbgxbVCqZN7Pvu/wkdsYPmID6tYpgSX//SofHhkViznzDuLUmfsICYlCrpy26Nq5Bjp1qCYfZ/vOKzh0+BYePvZBZGQsblyZBktL06/2HX8kA2q2wV/1u8DZ0hb3fJ7j9+1zcOPNI7Xj6uvqYXSj7uhepQlyWjvA66M3Ru5djOOPrsrHMTcyxZTmfdGqdE04WtjgztunGLJzHm6+eSwfR7Y0ZXxFf+1ZiH9Pboa2+qlAGbEuyucpDBdrB7RcNgL7711I9z01C5bD3LZDUDxHPrwN/oipR9dh/dXDmVrHRvqGmNN2MDqWrw8jfQMcf3wNA7bOhl94ELTd/WM+uHPQG1EhcbDLa44avQrBqYCl2nH3TrwN30chqYbnLWuHZqNLi7+v73iJZ1f8EBEYAz19XTi4WaByRzc4F7RSWwJ555ibCHwTgfazKsLB1eIrfMMfW2IC3weHsa+ukWtVjKjYHZOursB9/+foWqwpltf7G832DUFQTFiq8Zvmq45h5Ttj3OWluOPnBVerHPin2kDIZMCsm+vFOB0OjYaejq78PQVscmN1g/E4/tpDPmzaT4NgaWiGQWdmIjgmDE3dqmNOzeFof3gkngSlf7ddTdYkX1WMqdQd46+sEMFF9+JNsabhWDTYPVjt+vjZrTr+rNAZoy8twW0/L+SzdMGMGgMhgwzTryevj74lW6JTkQYYeWERnoW8RUn7/Jj+00CEx0dhw6MjYpzXob6YfHUV3oZ/hJGeIXoWb4a1Dcei3q7f1X6uNjly9Damz9qHSePbo3SpvFi/8Tx6/7YMxw6OgZ1d2gdvn3eBmDlnPyqUd0v12oxZ+3D12jPMnt4FOXPa4vIVL0yauguOjlaoW7uEGCc6Jg4/VS8qHnPmH/qq3/FH0r58PcxtMwT9ts7EtVcPMbRORxwfPB+FJ3aAf3hwqvGnNu+HLpUbos+m6Xjy8Q0aFquCvb/NQNXZfXHX56kYZ1WXMSjh4oau6ybBNzQAXSo1wqkhC1FsUif4hvqLcZxHNlGabuPi7ljd5W/svnMW2szMyAT33j3DmisHsbffzM+O72qXA4cHzsGyi3vRee0E1C1cAau6jMb70ACceHwtw+t4XruhaFqiKtqtGoPQ6Ags6vAn9vw2A9X/7Qtt9uzKR1za8Ay1+hSGU0Er3Dv8Fgf/uYtf5leBqZVhqvEb/1lS6YQ7Jjwe2/+6gfzuKRcprV1MRZBk6WSChLjE5GlOvYsuC91hYqk8zSubnsPM1kgEOEx7U9RSzgBZhtSqVQu///47hg4dChsbGzg5OWHlypWIjIxEz549YWFhgQIFCuDo0aPy95w/fx6VKlWCkZERcuTIgVGjRiEhIUH+uqurK+bPn6/0OWXKlMHEiRPlz3V0dLBq1Sq0atUKpqamKFiwIA4cOCBee/36tWi9ITRPNG6PHj1+mDXavVgz7Hp2Gvuen8OLUB9M8liBmMQ4tC5QR+34ZRwLi8Dm8KtL8I30xxVfTxx5dRkl7QvIx6FWnYCYEPmjVq7y8A77gBsfU66+lXUojM2Pj4qWIp8IPyz33IPwuEgUt0t9MqhNepX4Gdu9TmH3s7N4HuKD8ZdXIDohFm0LqV8fZR0L45afFw6+vIR3Ef645HsPh15eQimHlPVRzrEwTnvfwDmf22IcauG5/O4eSimsM3r/Fd/7eBvuJz6XgiMLQzMUtvl8K4WmW7vhHNq3dUebVpVRIL8zJo1vB2NjQ+zem3ZrY2JiEv4cuQm/D2iM3LnsUr1+5+4rtGxREZUrFUSunHbo0K6qaBnyvP9GPk6PrrXQ99d6IqhiKYbX7YSVl/djncdhPP7wWpwER8XFoJd7M7WLqWvlRph2bD2OPvTAqwBfLLuwB0ceeuCPer+I140NjNCmbC2M2LsIF5/fxQt/H0w6vArP/X3Qv2Zr+XQ+hgUpPVqUqoGzT2+JaWqzYw89MO7Acuy7dz5D4/f7qTVeBfriz93/4cmH11h8fhd23TmLYXU7ZngdWxqboXfVnzF81wKc9bqF295e6LlhKqrlL4XK+YpDm9099BbF67qgaG0X2OYyE4GOvqEuHp9Vv50amxvAzNpI/vDxDIa+kS4KVEkJcApVd0buUrawcjKBXW5zVO9WEHHRiQhQCWLe3AnEW88gVOuacmxh6gOctB6aggOcL7B+/XrY29vj+vXrItjp378/2rVrh6pVq+L27dto0KABunbtiqioKLx79w5NmjRBxYoVce/ePSxduhSrV6/G1KlTM/25kyZNQvv27eHp6Smm2blzZwQFBSF37tzYvXu3GMfLywvv37/HggUL8CMw0NVHMTs3ePh6yofRlf+rvp4o7VBI7Xvu+nmJ90gBTS5zR/yUsywuvLud5mc0c/sJe56fURp+x99LtB5ZGZpDBzpo7FoVhnoGuPFBfZqJNqBlRQEeBY2K64MCDwoI1aFgs4SdmzxYyW3hiFq5yuH82zvycahlxz1HSbha5hDPi9jmRXmnIrjgcyfN+ehQuD7CYiO1ujWNxMUn4OEjH1StkvJ70NXVFc/v3Et72Sxeehx2tuZo16aK2tfLlsmHM2cf4OPHEMhkMly9/gyvXvujetUiX+V7aAoDPX2RCnXqyQ35MFp+9NzdraTa91AqU0x8nNKw6LhYVC9QWp7Cpq+nr36c/MnjqHK0sEXTktWw+srBLPhW2sXdrYTS+iOULiitv4ys4/J5i8BQ30BpHK+Pb/Am8D3c86nfDrQBtcT4vwxHrpK28mE6ujri+YenGWuJf3TGFwWrOsHAWC/Nz3h4yheGpvqwz2suH07pcGeXP0G9QcVEQMW0O8DhPjhfoHTp0hg7dqz4e/To0ZgxY4YIePr06SOGjR8/XgQyFIgcPHhQBCCLFi0SLStFihSBr68vRo4cKcajE5WMolaZTp06ib+nTZuG//77TwRZjRo1gq1t8s7E0dHxh+qDY21kIQ7ugTGhSsPpeT6rnGrfQy039L6NjaYAOsknw9u8TmDl/b1qx6+Tu6JoCaAWIkV/nJuLOTWH4UqntaLPR0xCHIacmw3v8A/QVjaf1kdAtMr6iA5Bfmv164NaXmyMLbG16RSxjdP62PL4OJZ57pGPs9xzL8wNTXC8zQIkypJE+uDcW1tx4OVFpWnVzl0e82oNhYm+kein0+P4ZATHhkObBQdHitYY1VQ0ev7y1Ue177l5+yV27b2Kfbv+SnO61Idn3MTtqFF3IvT1dcW6mzqxAypWyJ/l30GT2Jtbi2CEWlAUfQwLRhEnV7XvOf74qmgRuPDsLl4E+KBu4YpoXbaWPI02IjYKV154YlyTXqK1gKbdqWIDcSJOrTjqUH+e8JhI7LmjvF9jn+dsaadm/QXBysRctKbZmFp8dh3TNGLj40RqmtI44UHiNW0VExYPWZIMptbKaWP0PNg36rPv//g8DEFvI1Gnf9FUr72+FYDj8x+KFDUza0M0H1tGnp5GAejpJY9Qor4LHPNbIswvOgu/leZJSJBB03GA8wVKlSol/1tPTw92dnYoWTLlig2lrRE/Pz88fvwY7u7u4uRBUq1aNURERMDHxwd58uT5os81MzODpaWl+IzMiI2NFQ9FlDr3I6noVAx9S7XGlGsr4en/HHksnTG6Yk/4l2qDZZ7JLVmK2hSsg0vv7sA/Wjk3/veyHUXg0+v4JITEhqNOnoqiD063o+PxLMT7G36jH1sl5+LoV6oVJnqsEn128lo6Y2zlnhgY1VZeRID69TR3+wnDzy0QfXCK2rri78o94RcVhL3PU9JKrr5/gOb7/oKtsQXaF66HBbWHo+3B0VrfByczIiJjMGL0JkyZ2AG2NilXN1Vt3HwBdz1fY+miX+GSwxY3b73ApH92iz44Vd3Vt9axLzNkxzys7DwaTyZuEydiLwLeYa3HIaWUNup7s6br3/CdcQgJiQm4/dYLW2+cRPk86lvUelVths3XTyA2QbnVh7EfGbXe2OUxU1uQIGdxG3SYXVEEUY9O++L4vAdoO62C6NfjedQH8dGJKNdK/UUGpkyhl4TG4gDnCxgYGCg9F1etFYZJwUxSUsaa+qgVhw56iuLj4zP0uRn9DMn06dNFqpuiCRMmAPnwXVBgkZCUCDtj5Uoo9DwgOnVVFSkwOfDiAnY/S045o2CErvhPdP9N9KOhlCoJVVmrkqOUaJlRlNvCCZ2LNkbz/cPwIiT5CqlX8BuUdyyKTkUaYvLVldBGwZ/Wh72JyvowsYZ/lPr1MbRcR+x/cQE7n54Wz58GJ6+PqdX6Ycm93WJ9jKzYFcvv78PhV5fl4+Q0d8BvpVorBTjU14da0Ohx1/8ZTrZZiHaF6ooWIG1lY2MGPT1dBAYqt2TRc3v71CcBb98G4N27IPQftEo+LCkp+TdRrPRwUZjA0dES8xYcxqIFvVCrZnJ/Aep/8/jJO6xed5YDnHQERISIAMTJMiUFhzhZ2uBDWGCa72m1fKRIVbMzsxJFA2a0HIiXCn1nXga8Q615A2BqaCz6d9C0tvWeKoarotS2Is6u6LAqOZOAZQ4t29Trz1a0xsTExyIgIvGz65j+NTIwFK0+iq04Tha2aW4H2sDY0kCkpFG6mCJ6rtqqoyo+JhHPL39EpQ7q+8FSypq1syngDDgXssKmwR54fMYX5Vu54t2DYHx4Goplvyi3aO4cdROFqjuJtDWWIkHzq0RzH5yvrWjRovDw8FAKYC5fviyKEeTKlUs8d3BwEP1mJGFhYXj16lWmPsfQMHnHkZiY/lZLKXWhoaFKDxr2vVBq2KPAl6iSI6UFjPrDVM5REvf8k6sLqTLWN4JMphzYJX16rthSRloVqI2gmFBc8FHun2Osl9xqpRpY0nSopLG2ovXxMPAl3F2U10dVl5Kiz5I6JvqGSFKzHBXXh7p1RqlquirrSxW9Tv2itJmhgT6KF8sFj2sp5bLpwobHtacoWzr11Uq3fE44uHekSE+THnVqF0flSgXE3845rJGQkIT4hERxIqJIT09HpJewtMUnJuCWt5dIM5PQdk7PPV7eT3fRUWsLBTeUBkpFBfZ7pi5lTB3Z6QTZ2tQCDYtVVjtO76rNRfloz3dpl25nafN4+UBUTlNUv2gl+frLyDq+9eYJ4hLiUbdIyjiFnPIgr10OeLxKfzvQZFIJZ58HKRkTtE+h586F1JeJljy/6ofEBBkK/+Scoc+i4zeVhCY/9SqEDrMrocOsiuLRbHRyxkvDocVRpZN2Fw5KqwUnrYem4Bacr2zAgAGiQhoVIxg0aJAoAkAtJsOHD5f3v6lTpw7WrVuHn3/+WfSfob45lPqWGXnz5hU74EOHDokCBCYmJjA3T52eQulo2S0lbf2jQ5hWfSAeBr4QFc26Fm0qWgD2Pk8ufTqt+iCRyjT/9hbx/Nzbm6Ly2uOgV+LeLHksnPF7mY449/aW/MRaOjGnAGf/i/PiZFrRq9B3eBP2HhPc++LfmxuTU9RyV4S7SykMOD0D2mzNg4OY9dMgPAh4IVIAexRPXh+7nyavj1k1fsfHyEDMuZW8Ps68vYVexZvhUeAreYoateqc8b4pXx9n395E/9Jt4BsRIFLUitnlE+/Z9Sx5mjR9ev2M9w3R94b69HQp2ghOprY4+uoKtF3PbrUw8u8tKFE8N0qVyIP1m84jOjoOrVtWFq9TSpqToxX+GPYzjIwMUKhgcjEHiaWFifhXGk5BU6UK+TF7zgEYGxnAxcUWN24+x74DNzHqrxby9/kHhCEgIAze3gHi+dNn72FmRtUgbWBtZQZtNff0VqzvPg43vR/j+utHGFqnA8yMjLHWI/k+Kuu7j8e7EH+M2b9UPK/kWlzc/4ZKQtO/E5v+Kvb/s05skk+zQdHKYh9OHdULOOTG7NaDREnptVeUy3NbGJuiXbk6+GP3f9/4W2fvMtEFHJIvGJJ8di4onasggiLDxD1uprXoL5Z79/WTxevLLu7BoFptMbPVIFFauk7hCmhfri6aLv4jw+s4LCZSFHiY22YwgiJDxfOF7f8QfamorLQ2K9MsN04vfgxHNws4FrDEvSNvkRCbiKK1XMTrpxY9EmWc3X9R7u9HrTH5KtrD2MIgVcvOzT2vka+CPUxtDEUZ6fvH3iEyKE5eStrC3ljpPVKBAktnE5jbKb/GoFGBTFo4wPnKcubMiSNHjuCvv/4SxQmoGEDv3r3lRQoItaBQi02zZs1gZWWFKVOmZLoFhz6HUs+oBDWVq+7WrZsImn4Ex15fga2xJQaV6SBu9ElVs3479Y+88AClmSm2tCz3TE57Gly2ExxNbcU9bM753MSC21uVpkutEC7mDqmqp5EEWSL6nZqG4eU7Y1GdkeJGn2/DP2DMpcW4+E59ZS9tceRV8voYUq6juCnn46DX6H0iZX3QzTwVW2OW3N0l1s+w8h1FQEL3rKGgZ+6nAIjQjUKHlu+IiVX7wM7YUgQx27xOYtHd5D46FIDmt8qJVnVqis+mVLn7/i/Q6cg4UTJa2zVpXA5BwZH4b9FREXQULZITq5b9Bnv75MID798HQ1elNeZz5v7bHXPnH8KfozYhNDQKLi42GDa4idKNPrdtv4xFS4/Ln3fuvlD8O31qJ3lwpY123DoFB3NrTG7WR3Qov+vzDI0WDpPf4DGPrbNSq6axgSGmNv8NbvYuiIiNxpEHV0SfG8XUJkp1mt6yP3JZOyIoKkzc2+bv/ctEyqiijhXqi0Bo640T3/AbZ28V8hTFueFL5M/p/jSESjz33DAFOazsxTqRvA58L4KZee2GYEjt9vAJ8cOvm6bL74GTkXVMhu2cLy7i7O47XaQfHn90DQO2zYK2owpo0WHxuLbjpUhNs3e1QLMxpeUpauEBMVBtvA/2jcT7J6GicIAqSqoI8Y3CsTn3ER0eLwIgKiTQalI5UTKaZV6CFgQ4OjLVHB2mlYqvb/e9Z4F98rD7ThRc05aXRzbxrNcuID7lvlYsGzBoDJ3+6stfs29PtvQqr49stj7+u9f/e88G+2Rw6eSW3OzklGPahWTq+alPR//RcAsOY4wxxhhjWiJBC1pwtLc3NWOMMcYYY1omIQuLDCxevBiurq4wNjZG5cqVxf0Z07Nz505xT0gan26xQt04FFFiGfVFz5Ejh+hPXq9ePTx7llJkJ6M4wGGMMcYYY0xLZFWAs337dlE0i4pn3b59W/Q1b9iwYZr3aLxy5Yq4YT31Rb9z5w5atmwpHg8ePJCPM2vWLHEj+2XLluHatWvivo80zZiYmEzNGwc4jDHGGGOMadF9cBLSeGTG3Llz0adPH1HcqlixYiIoMTU1xZo1a9SOv2DBAjRq1EgU3qLbqFBRrXLlymHRokXy1huqPEyFuFq0aCFucL9hwwb4+vpi3759mZo3DnAYY4wxxhjTEgnptODExsaK+zEqPmiYqri4ONy6dUukkEmo/D09p/s/qkPDFccn1DojjU8VhD98+KA0DlUXptS3tKaZFi4ywBhjjDHGmJbomJh2pbSJEyeK244oohQ0Gq4oICBA3FzeyclJaTg9f/LkidppU/CibnwaLr0uDUtrnIziAIcxxhhjjDEGujcj9atRlN1uEJ8RHOAwxhhjjDHGQMFMRgIae3t76Onp4ePHj0rD6bmzc8qNdRXR8PTGl/6lYVRFTXGcMmVS3wQ2PdwHhzHGGGOMMZZhhoaGKF++PE6fPi0flpSUJJ67u7urfQ8NVxyfnDx5Uj5+vnz5RJCjOA71AaJqamlNMy3cgsMYY4wxxhjLFEpl6969OypUqIBKlSqJCmiRkZGiqhrp1q0bcubMienTp4vnQ4YMQc2aNTFnzhw0bdoU27Ztw82bN7FixQrxuo6ODoYOHYqpU6eiYMGCIuAZN24cXFxcRDnpzOAAhzHGGGOMMZYpHTp0gL+/v7gxJxUBoDSyY8eOyYsEeHt7i8pqkqpVq2LLli2iDPSYMWNEEEPln0uUKCEfZ8SIESJI6tu3L0JCQlC9enUxTboxaGZwgMMYY4wxxhjLtEGDBomHOufOnUs1rF27duKRFmrFmTx5snj8P7gPDmOMMcYYY0xjcIDDGGOMMcYY0xgc4DDGGGOMMcY0Bgc4jDHGGGOMMY3BAQ5jjDHGGGNMY3CAwxhjjDHGGNMYHOAwxhhjjDHGNAYHOIwxxhhjjDGNwQEOY4wxxhhjTGPoyGQy2feeCcb+X7GxsZg+fTpGjx4NIyMjXqDfGa+P7IfXSfbC6yN74fWRvfD6YP8vDnCYRggLC4OVlRVCQ0NhaWn5vWdH6/H6yH54nWQvvD6yF14f2QuvD/b/4hQ1xhhjjDHGmMbgAIcxxhhjjDGmMTjAYYwxxhhjjGkMDnCYRqDCAhMmTOACA9kEr4/sh9dJ9sLrI3vh9ZG98Ppg/y8uMsAYY4wxxhjTGNyCwxhjjDHGGNMYHOAwxhhjjDHGNAYHOIwxxhjTajo6Oti3b1+Wj8u+LcV18/r1a/H87t27vBq0EAc4TKN07doV06ZNy7LpxcXFwdXVFTdv3syyaWoKf39/9O/fH3ny5BEdQp2dndGwYUNcvnxZPs6VK1fQpEkT2NjYwNjYGCVLlsTcuXORmJj4XeddE9GBPL3HxIkT5Qd86WFnZ4cGDRrgzp0733v2NVqPHj3QsmVL+fMPHz7g999/h5ubm/jt5M6dGz///DNOnz79XeczOy0vaRs1NDREgQIFMHnyZCQkJHy1z3z//j0aN26c5eNq63ozMDBAvnz5MGLECMTExHzvWWNaiAMc9l12fE5OTqhfvz7WrFmDpKQkte+hk2U9PT3cuHEjQ59x7949HDlyBIMHD5YPq1WrltqTPelAuWfPHnGCRyd66q700MH1zz//xMiRI/+v76+J2rRpI06M169fj6dPn+LAgQNieQcGBorX9+7di5o1ayJXrlw4e/Ysnjx5giFDhmDq1Kno2LEjZDLZ9/4KGoVOuqTH/PnzYWlpqTSMtmPJqVOnxLDjx48jIiJCnKyFhIR81/nXFhRkli9fHmfOnMHs2bNx//59HDt2DLVr18bAgQO/9+xlG40aNRLb6LNnz/DHH3+IAJ2Wl7qLUFmBLtBQsJnV42rrenv58iXmzZuH5cuXiwqnjH1zMsa+ge7du8saNWoke//+vczHx0d269Yt2T///CMzNzeXNW7cWBYfH680/ps3b8RrgwcPlvXr1y9Dn9G7d2/Zb7/9pjSsZs2asj59+ojPVXxINmzYIJs0aZJs5cqVdLYtu3PnTqrpBgUFyQwNDWUPHjz44u+vaYKDg8XyOnfunNrXIyIiZHZ2drLWrVuneu3AgQPivdu2bfsGc6qd1q5dK7Oysko1/NWrV6m288uXL4thx44d+8ZzqV37vxYtWoi/aX+XM2dO8RtR97tiystLUr9+fVmVKlXkr02dOlWWI0cOmaurq3jd29tb1q5dO7Hd29jYyJo3by62d0WrV6+WFStWTOzPnZ2dZQMHDpS/Rr+BvXv3ir9jY2PFazSOkZGRLE+ePLJp06apHZd4enrKateuLTM2NpbZ2tqKY054eHiq9T979mwxTRpnwIABsri4OI1fb3QMKFu2rPg7MTFRLEdaZ7SsSpUqJdu5c6fS+HScbdq0qczCwkKcA1SvXl32/Plz8dr169dl9erVE8cWS0tLWY0aNcS5hCLFdaNuf8e0B7fgsG9GSmPKmTMnypUrhzFjxmD//v04evQo1q1bpzTu2rVr0axZM5ECtXXrVkRHR6c7bUp52rVrl0jzUGVqaio+V/GhmNI2fvx41KtXL81pU3pVtWrVsG3bti/63prI3NxcPCjXOTY2NtXrJ06cEC05iq0GElpHhQoVEuuVfX8mJiZZeiWcpS0oKEi01lBLjZmZWarXra2tefGls51K2yil8nl5eeHkyZM4dOgQ4uPjRYu/hYUFLl68KNJkaf9ErQnSe5YuXSqWe9++fUWrGbU4U+qbOv/99594fceOHeJzNm/eLFKV1YmMjBSfTccJyjbYuXOnaCEdNGiQ0njUiv3ixQvxL7V60zFP9binaR48eCDSlCkTgkyfPh0bNmzAsmXL8PDhQwwbNgxdunTB+fPnxevv3r1DjRo1xLkCtXDeunULvXr1kmdchIeHo3v37rh06RKuXr2KggULihRoGs6YKv1UQxj7hurUqYPSpUuLVLFff/1VDKOLMBTgLF68GEWKFBEHIQpeKBhJi6enJ0JDQ1GhQoWvMp+VKlUSB06WTF9fXxyc+/TpIw5WFLBSOhqlnpUqVUqkrJGiRYuqXWS0XqVx2PdDaWlTpkwRJ4O0jbOv6/nz52L/Rts/yxhaXhTQUDol9Vuivn8UHK5atUp+4rxp0yaR6kzDKNWY0DGEAsZz586JNGRKjaVUN0qTlVSsWFHtZ3p7e4uT5+rVq4vp5c2bN83527Jli+hjQifuUtC6aNEicSFn5syZIh2bUABEwyntmtZ/06ZNxfeifagmoYCT9icUlNDFL11dXfG96W/qH0vBn7u7uxiX+qBRsEJpbHT8oGO+lZWVuJhIqeyELoYpni8oWrFihVjHFCDRBVHGFHELDvvuaGdPeekS2gFGRUWJq2KErvCsXr063Wm8efNGHDgcHR1TvbZkyRJ5iwM96CCXWS4uLuIzmHIfHF9fX3Glk66U0okEBTqKVyW5n032VLVqVfFboJMu6ru2fft2+YkY+3r495D5E2UqTkJ9xDp06CD64RAqViIFN4S2YQoeqQVH2s/b2tqKwINaTfz8/MS+qm7duhnuM0r9MQsXLiz6dFKLdFoeP34sLtIptshRiz8FXNT6IylevLg4Rkly5Mgh5kvTUF8yWnbXrl0TrS09e/YUxwpaP3Rcp763isdjCgxpHRF6308//SQPblR9/PhRBIQUfFIgRP0MqQ8hBaSMqeIWHJYtDvrSVTdChQfoYEatBKRTp07466+/xE4wf/78aqdBKWzUrK04HUnnzp3x999//19pIJQeQTtnpoxOPuiARY9x48aJVjjqUEqd3KWDP51Mq6LhxYoV48X5nVBAQ8ufimtwWtS3QydmtI+ighvs8yfKlFZGgQxdYJKOB0Q1vY9OcqlwA6WSqXJwcBCtCJlBF2pevXol0qfpglv79u1FGjNlEnwp1ZN22g7SKrDzI6N1I6X+0bGcgj+6QFmiRAkx7PDhwyJNXZFUsEFKl00LBUyU+rxgwQLRqkbvo9YgTq9l6nALDvvu6GSXyklKOepUfYtaXeiARg/aGVJzN+0s02Jvby8CEHU7OrrSQztc6UHjZhbNFx0oWfropJly0iklhK6gzpkzJ9U41OJDlZEocGXfB5UlposFHNx8W/SboJZpSsWh34kqrmSX+kSZytArBjdpBSS0T6EWfMV9PT1o/08tO9SHJjNluKl1gC60rVy5UlwQ2L17tzgOqKI0XGpBUlyf1AeIgipqAdJmtAyor+3YsWPFsYECEmptUV1HtD8ilN5MqeDUp0odWq7Uokb9bqhFjKYXEBDwjb8V+1FwgMO+K+pISB0+qQmb0BU4KitMBwxqrpYedKJMqU9p3T+lTJky4t9Hjx59tc6SZcuW/SrT/hHRVTTKh6bcd+r/RFc7qXPtrFmz0KJFC3FyQnnVVESCOvXSOJSGSFfyKP2jbdu24qooY9qGghvaj1GfJzppphNzushDHdulvgksc6iVni5c0b6HTpBpf0Qps3Qy7OPjI8ah9DY6jtBypmV++/ZtLFy4UO306F5dVASFWtqoryDt26g4jboLAvTZ1JJNrQt0nKAiAtRXiPqMcton0K5dO5GaR8cDKjpDhQWoyAJlZEjrgJ4TKswQFhYm+nLSvedoPW3cuFGe6kctoPScfi+UAkfL/nOtPkx7cYoa+2aokyHd4I4O7pRLS9WEqKoKdQ7s1q2bGIdOgOnkV2rOltAVntGjR4v3UOdMVdS6QlfxqMOiFOxkBF2RoytKlJ9NpB2parU1OmhSZ2yWjHKnK1euLO5zQAcquuJG64jyo+mKHaH1SAf7f/75R+RVUz48HaAoXXDo0KFq0wkZ03TUsZpO7Oh3Qf0B6Z4htP+iFCtKyWKZR5UyL1y4IO5X1rp1a1FVi1r+qc8NtcQQCkBoH0T7LDrRpoCI9lHqUIsPXayhE2w6OadiBHSPNXWpbvTZVACBihfQePScLthRkMSSC9JQ4ELLkwJP2tbpuE/3yaGAUaqoSihlli56Uko6FR2gZU/Hc+rTJJ0f0AUzeg8db6hogbpKnYwRHaoVzYuCfW101V66SkM7POrcTLm5v/zyizjw0IGDSkJSFbTr16+rrW5DzdJ0pYwqrqlDJwfUYdHDw0M+jG48STtIqU+IKmoVok6QqqgfidShlaZHn01BEF8tYowxxhjL3jjAYRqDCg1QzjPlS2dlqgflYVMwJl1lYowxxhhj2Rf3wWEag1pXqAUnKzsdUtECKklKecOMMcYYYyz74xYcxhhjjDHGmMbgFhzGGGOMMcaYxuAAhzHGGGOMMaYxOMBhjDHGGGOMaQwOcBhjjDHGGGMagwMcxhhjjDHGmMbgAIcxxphG0tHRwb59+773bDDGGPvGOMBhjDH21fTo0UMEGv369Uv12sCBA8VrNE5GnDt3TowfEhKSofHfv3+Pxo0bZ3qeGWOM/dg4wGGMMfZV5c6dG9u2bUN0dLR8WExMDLZs2YI8efJk+efRDXqJs7MzjIyMsnz6jDHGsjcOcBhjjH1V5cqVE0HOnj175MPobwpuypYtKx+WlJSE6dOnI1++fDAxMUHp0qWxa9cu8drr169Ru3Zt8beNjY1Sy0+tWrUwaNAgDB06FPb29mjYsKHaFDUfHx906tQJtra2MDMzQ4UKFXDt2jVe+4wxpmH0v/cMMMYY03y9evXC2rVr0blzZ/F8zZo16Nmzp0g7k1Bws2nTJixbtgwFCxbEhQsX0KVLFzg4OKB69erYvXs32rRpAy8vL1haWoogSLJ+/Xr0798fly9fVvv5ERERqFmzJnLmzIkDBw6I1p3bt2+LoIoxxphm4QCHMcbYV0eByujRo/HmzRvxnAIRSluTApzY2FhMmzYNp06dgru7uxjm5uaGS5cuYfny5SI4oZYX4ujoCGtra6XpU0A0a9asND+f0uH8/f1x48YN+XQKFCjw1b4vY4yx74cDHMYYY18dtcI0bdoU69atg0wmE39TOpnk+fPniIqKQv369VP1p1FMY0tL+fLl03397t27YjpScMMYY0xzcYDDGGPsm6WpUV8Zsnjx4lQpZOTw4cMijUxRRgoFUJ+a9CimszHGGNNsHOAwxhj7Jho1aiRaZKjzv1QIQFKsWDERyHh7e4t0NHUMDQ3Fv4mJiZn+7FKlSmHVqlUICgriVhzGGNNwXEWNMcbYN6Gnp4fHjx/j0aNH4m9FFhYW+PPPPzFs2DBRMODFixeiCMDChQvFc5I3b14RHB06dEj0p5FafTKCqqdRYYGWLVuK/j8vX74URQs8PDyy/Hsyxhj7vjjAYYwx9s1Q9TN6qDNlyhSMGzdOVFMrWrSoaPGhlDUqG00odW3SpEkYNWoUnJyc5OluGUGtPydOnBAFCpo0aYKSJUtixowZqQItxhhjPz4dGfX2ZIwxxhhjjDENwC04jDHGGGOMMY3BAQ5jjDHGGGNMY3CAwxhjjDHGGNMYHOAwxhhjjDHGNAYHOIwxxhhjjDGNwQEOY4wxxhhjTGNwgMMYY4wxxhjTGBzgMMYYY4wxxjQGBziMMcYYY4wxjcEBDmOMMcYYY0xjcIDDGGOMMcYY0xgc4DDGGGOMMcagKf4HeWdplHGi2MwAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAADwCAYAAADM38HGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAhh5JREFUeJztnQV4FEcbx/9xdyVYCO4uAYq7uxeHDytWihWHYi3W4u7u7lYguDsECUmAuHty3/NO2Mvd5RISGiDcvb8+W7K7s7OzM3s7884royOTyWRgGIZhGIZhGIbRAHS/dwEYhmEYhmEYhmGyChZwGIZhGIZhGIbRGFjAYRiGYRiGYRhGY2ABh2EYhmEYhmEYjYEFHIZhGIZhGIZhNAYWcBiGYRiGYRiG0RhYwGEYhmEYhmEYRmNgAYdhGIZhGIZhGI2BBRyGYRiGYRiGYTQGFnAYhvnu9OzZE66urt+7GMx3pFatWmKTePPmDXR0dLB+/fps3S6DBg1C/fr1v3cxmK8IfZuaNWuWZflJ7/Zff/2Fb82UKVPEvbOS+Ph45M6dG0uXLs3SfBnmv8ACDsOo4fXr1xgyZAgKFSoEU1NTsRUrVgyDBw/G/fv3f2hBgjo3S0tLREdHpzr/4sULcf5LO9+oqCjRgZ4/fx7ayI4dO9CtWzcULFhQ1KHigF2V2NhYjBkzBi4uLjAxMUHlypVx6tSpTLWjtBkZGYl3ddKkSYiJicnCJ2I+951YvXo1xo8fzxXFaC0GBgYYOXIk/vjjD/7+MNkGFnAYRoXDhw+jRIkS2LRpE+rVq4cFCxZg0aJFaNy4MY4ePYoyZcrg7du3P2y96evrC0Hk0KFDqc5t2bIFxsbGX5w35Tt16tRMCzirVq3Cs2fP8KOzbNkyHDhwQMxm2tjYfFZImT9/Prp27SreLz09PTRp0gSXLl3K0L1IqKF3lDbKh2aZp0+fjj59+kATyJs3rxDCf/75Z2RXqN3y5cuH2rVrf++iMEyGmDBhgtrJrf9Kr169EBAQgK1bt3JLMNkC/e9dAIbJTnh6eqJTp05icHXmzBnkyJFD6fycOXOEGl5XN/25gcjISJiZmSE7QgPjatWqYdu2bejQoYPSOeqcmjZtij179nyTskj1RDOAmgAJGzlz5hTvBwnJaXH9+nVs374df/75J0aNGiWOde/eXVwzevRoXLlyJUOCKmmLFE2lqlatKtqVBB4nJydkJzL7myDN1H8Rtr82ZJZDEwIDBgz43kVh/iNJSUmIi4vL1u9bVkHfDdqyAplMJjQ2pIG2trZGgwYNhElp7969syR/hvkvsAaHYRSYO3euGIitW7culXBDUMcwdOhQMUOvOBNvbm4uhCOagbewsBCz8gTl9euvv4r0JFgULlxYmH5Rx5ARXwM6TiZfqvbTT58+FcIJmZrZ2dlh2LBhmTIN6NKlC44dO4aQkBD5sRs3bggTNTqnDko7fPhw+bMUKFBACHw0OJCew8HBQfxNWhzJfEoqf3r1pM4Hh/KlGfKSJUuKgQfl3ahRI9y8eVOehky6qlevLjpXypvq93uaC1HdfE74JXbv3i00Nv3795cfo2ck7YuHhwfevXuX6XtTXVNd0Lv16tUrpXPU1j/99JMQMKjeSYh99OhRqjyk94rqmgYtVJ+///67Upo7d+4IbSa9e1TndevWxdWrV5XS0LtM5blw4YIQvBwdHZErVy75+ZUrVyJ//vziHpUqVcK///6bqizqfhfSO+Tj44NWrVqJv6msJCQmJiYqXR8YGCi0P1ROej969OiBe/fupcrzw4cPYvaZykfvNf3uW7ZsKe6fHqRpoxlr0vIqQtpLusfOnTuFyQ7lS21L9fTy5UultPTc7du3R548ecS96f0ZMWJEqhn2jD63om/HkiVL4ObmJsxraeBJ7xS9G6TlozJR3dNzBgUFpXq2jLwvZKpL5aJ70PM5OzuLgS3VuyLSN4uendJTW1hZWYk6J43v5yAzTxL8b926JQR4KjdpzZYvX67W7HPy5Mni2yTVJ00Y0HFFqDxkgkwCavHixUXa48ePf7YsJ0+eFBp8el4yWd67d2+mv5OqSL8FSluxYkXxHc5sPdP3RPq9qbJixQpx7uHDh2n64CQkJIj3QioHfYvpO6pab5Iv0okTJ1ChQgXRFpS/BPmi0e9C3TvFMN8a1uAwjIp5GnVI5A+RGaiDaNiwoRhg0uCCBhU0mGjRogXOnTsnBq7UMVLH8Ntvv4mBCpm+fSk0CKXOZtasWWJw+ffffyM4OBgbN27M0PVt2rQRM8/UQUuzbaS9KVKkCMqVK5cqPQ1EatasKcr9v//9TwzISMswbtw4vH//HgsXLhQDLjLRGjhwIFq3bi3uQZQqVSrdekoLqjMaiNJgum/fvuJaGhDS81LnSgMu6mwp/2nTpomOmQZRly9fRnaHhATymaHBtyI02Cfu3r2rJERnFGlQrmgeR1olGtxTvdNAi9qS2onagMohCZY0kKJBLWnTSPCi4ySMkikjDdQJqnNKQ+WmgSOlpQEODUJpcKX6uyHhht4L8g0iYZ9Ys2aNeIdosEoDQRLG6Hdia2uboWemAT09C92L3qHTp09j3rx5YnBG7x5Bg8nmzZsLTRkdo/eaTAepHlRp27ateK5ffvlFPLOfn58QnL28vNINfEHvPw0Uy5Ytq/b87NmzhbBLQkhoaKiYPCGB/tq1a/I0u3btEu1BZaSJCirvP//8A29vb3Eus88tQQN30kjQM9Fgk+5N34w6deoIAYx8v+i3Qvei8q1duzbT7wvVEbUdCSo06KY6pME6/Uu/UdVBNN2fBBP6Zt2+fVv4LpHgS/f4HPRto0kRyqNz585CeKRnNjQ0lH+/qM3pPaIBNr2/RYsWxYMHD8R39vnz59i/f79SnmfPnhX5kKBjb2//2SAnNPnTsWNH8d2k+qFJMBJOSTCSgkxk5DupCH1zw8PDRVqqL2on+m5SvUpa7YzUMwmgJPTS89D9Vf0CSYhLT6NM39cNGzagXbt2YkKO3lFqpydPnmDfvn1KacmUmNqAytyvXz8xCSJRvnx50e/RM2dlUAaG+SJkDMMIQkNDSa0ia9WqVaoaCQ4Olvn7+8u3qKgo+bkePXqI68aOHat0zf79+8XxGTNmKB1v166dTEdHR/by5Uux//r1a5Fu3bp1qe5LxydPnizfp7/pWIsWLZTSDRo0SBy/d+9euq1JZTUzM5OXo27duuLvxMREmbOzs2zq1Kny8vz555/y66ZPny6ue/78uVJ+9Mx6enoyLy8vsU91o1rmz9WTdC5v3rzy/bNnz4q0Q4cOTZU2KSlJ/LtgwQKRhu6ZHSlevLisZs2aaZ6rU6dOquOPHj0Sz7R8+fIMtaP0PtK79Ndff4n3qkSJEvI6Cg8Pl1lbW8v69eundP2HDx9kVlZWSsdr1Kghs7CwkL19+1YprZQXQb8NQ0NDmaenp/yYr6+vuI6ul6B3mZ6jevXqsoSEBPnxuLg4maOjo6xMmTKy2NhY+fGVK1eK9Ir1pe53Ib1D06ZNUypj2bJlZeXLl5fv79mzR6RbuHCh/Bi941TninnS71r1Xc8o3bp1k9nZ2aU6fu7cOZFn0aJFlZ5x0aJF4viDBw/kxxS/IxKzZs0S7ajYDhl9bqnOHBwcZCEhIfLj48aNE8dLly4ti4+Plx/v3LmzaM+YmJhMvy/qyr5t2zZxn4sXL6b6ZvXu3VspbevWrdXWnyr0TtD18+bNkx+jeqV3iN4leqeITZs2yXR1dWX//vuv0vX0W6LrL1++LD9G+5SWfm8Zgb5NdA29V4r9RY4cOUQbZPY7KbUTPX9QUJA83YEDB8TxQ4cOZbqeqS2pPhR/b+/fvxfPqfjeSO0hcffuXbHft29fpXuMGjVKHKdvsWo9HD9+XG090beAzs+ZMyeNmmSYbwebqDHMJ8LCwsS/NBOmCs1Q00y0tJH5hyqqs6gUkIDMkMikTRGaIaM+lsxAvhSK5qYIzdRK98woZIpGs7lkokOzmfRvWuZpNJtMM/ekGSCzHGkj8xyaWb548WKG76taT+ogHyCamSRzE1WkmWEydSFoZj4t84/sCpkgkcZJFckHICNOwKQRkd5H0jrSTDz5VlF9SHVEs79kMkMzrortRu8laQJIu0j4+/uLNqTZcJp1VkTKi9qZTHTIRIrMZSTIpIveG5o5l35DEjTDS/eSIPNC0pDQLDjNvkuQCQ6ZLWUUVb8XejcVzfJoVp1mwOn+EqRNUf3dkIkNlYN+B6QlyAxkIpReIAmacVd8RiojoVhOur9ie1LbkGaLvg+kLcnsc0uQZkGxPiXNGvlsKfpf0HHS9JDGITPvi2rZyTyW0lWpUkXsk4YmI2WnOlR9Z9RBZSaNgQTVK+3Tu0Sma9I3irQ2pK1TLDtprQjFshOk6SAzs4xC0Q5JMy1BWkzym6N2om/nl3wnSSOk+A597h1Jr54pL6oPxQAvZLpG30Y6lxZSn0FR0FT7KeLIkSNKx0kLR9o9dUjPQmVkmO8Nm6gxzCfI1pyIiIhIVSdkhkOmBB8/flRy7Jb/kPT1lXwMCIq0Rp2ilK8EdcLS+S+FwhArQmYqNID7nN+AIpIfDJkwkEkU2X/TQFldHmSeQSZMko+NKtSxZgR19aQOMo2iuiOzpbSgTpvMXMi8YuzYscLHgcw7yMwiPT8YMtmhQd2XQOVRHLR+KTRoUbVvJyQ/KsVBTVqQMCRFwiOTJjJvoXZQvJbajZAGeapIJnLSgCo9MxYSgsgER9EkRfGdpoEU+XmQOYziYEgR6Z1XfX9JGFEUmtJD8sdSHVgpCih0HxK8VE0g6f1WhIRMMpGiwRwFZaCBI5nW0MCVzIE+h6IvnSqqgqI0+FMsJ5nBkfnewYMHUwlYZNaW2edO696SsKNqAigdl/LI6Psi/Y7I146CZaj+/lXL/rn6UDXVVIW+BaoBKsjEk6DvFbUblZ1MqjL6jVJ9Nz8HvTuqZneKZaD3JbPfyYy8IxmtZ/JPpPak7zl9Cwn6m0yjpXKqg34r9L1U/W3Q89Akkmo/lV69Sb+HrF5nh2G+BBZwGOYT1DnQoEhyxlREmgFNS4CggVJGnMvVkVZnoOo0/SV5pAeVmQQCsr2mAa5iMANVaPBKdubkd6GO9DrQrKonVWggTzOiNDNLs4w0a08dOg3OSNOgqDlQhJ5ZnTNuRqB7pbe2TUah90yaNVeE7PSlAd3noOdTdHCnWVWavaaZbRowE5Jmi/wq1A3YsyqaUlpkRFDLLGm165dCfkDkr0M+GuQjN3HiROF/QFrNtPxrCPKZSU/rk1Y5pUEg/b7pN0UDWPKJobajQTy9F6TRUtVKZua500r7uTJl5n0hfxjytSCfQhpEk+abrqeBtjqN6ufu/V+he1JAEoogqA5V4e5rvJuZ/U5mpE4yWs/0bSXtKvnMUKRPmowjf8SZM2dmaR+SXr1JvwfyaWKY7w0LOAyjADlrklaAnH0lh+8vhUJNkyMwaX4UtTgUqUo6rzhrpxjR7HMaHpopVJxJI4dh6uw+5yirCpkWkYMxCR0UHjstSENEmi3ViFGqZNXMHd2PBps0+EtPi0PlptlK2mhgQ505Rf0iQSStspJjdmbNkSRKly6NrIAGKlRGMs9RnL2WHNDp/JcITRSBi2Z7yfmYZrWpHgly5k6v7STtiTrhXoJmpUkjom69InqnqS0+FyRAeufp/VXUElDIZVo0M6vql+5D9UsaJ0UtjmoUMwmqJ9Li0EZlo/qn92Tz5s1p3oMEEnLmp1n0zJjXSZADPDm/0wQDaYwkMrrY69cgo+8L/X4ojD69a6SBkpA0QFmNr69vqjDjVHeE9M2jslOUPPoWfA0NAr07JHgo5q2uDBn5TmaUzNYzabXpfaJrSJtF5U3PPE36rVDfQXlK1gUECUjUJ0m/2YxAv2FCMR+G+V6wDw7DKEAzbzQgIl8E+sD/l9lGMgGjWdrFixcrHaeoPtRJUnQwgga4NOOlap9Ns3BpoeoDRNGQCCnPjEILFFJ4UCpjeiY5NItI4YtJ6FCFOkGKcEZIg0lVYS2zUGQrqmvq2NNqA3WhSCXBQJ35l2KkHxqAfMn2ucU7MwqZ0dG7QdGQJKjMFJmJtIVfEkFN8sWiNqAIXpJWh94vEvxIiFBndiYJLzVq1BDCLplNqatvmm2mcMPk46OoyaTfCUWDoihbnzM1ouh3dC8K8atoJkjR8v7rO6MIPTc9Ly0gK0GDONXfDQlAquHVaZBKExLpvUOEu7u7qBvJBySzSLP3it8U+ptCo38vMvq+qCs7oRolLKug74tiOGJ6d2if3iX6PUvfKNJ+Kba5BPm0SVH8/ouQpRhRjCYnKGolfXOkb2dGv5MZJbP1TN8omhAiTTZtNEn3OVM86qfU5SlpwmjSL6PQb4H6NvptMMz3hjU4DKMA+QbQYI2cbMnXgMK60qwydTA0O0XnaKY6I34kZPZCAgRpFGhASPmQ6RQNEMksRpotJciPhAal9C8NAknYkWYH1UFloZCoZKZAHSrNNJM2JrMz4PQstLL15yDzCDJ7Iv8EMp+hQQUNGGgWmhxZ6flISCPzBXLcpc6VzDGosyW/jvR8O9RB9UZrmFD4a5pZlMwxKEw0naPQrhQamuqJOmCaZST7dBIKqW1osP09oPJIgioNBqmOZsyYIfZJgKCNICGGHMEpfCyVm+zfaeaV6pHCKH8pZDZFzu1UDzSDSzOpFOKX6pLCf5OWjgaFJMSQWR8FJZAEcKprqjdKR2F2aWBE5aF05KNF0LNIaw9RCGgyWaKBJgkD5AP0OcjXhvIgMzrS4NDsMr3LJNhl1AcnI5CpDg3uSCNDM++kbaH3VxKKpVl4+o3RjD8NTOm9peehQSwJbelpNAmqA6pv0tKm5bOSHlQm+gZQcAgamJNgQcE1vlS7mBVQGTLyvlA6epepzUkQosVt6dsmzeBnNWSySb5S9D7Sd0XyG6QJAimcMpWZwiRTMAPS3lFZaRKBtIt0XFq75Uuh+1Loelqnhvy1aDKA3hN6dzP7ncwoma1nqgsywSV/HbovhRP/HNRnUNhrqksSwij4Alkw0PeIfkf0vc0o9G2geqffBcN8d75hxDaG+WGgsLsDBw6UFShQQGZsbCwzMTGRFSlSRDZgwAARVjOt0MuqUNjVESNGyFxcXGQGBgayggULipC0iqF3pVCgffr0EaFYKeRuhw4dZH5+fmmGiX78+LEI80xpbWxsZEOGDJFFR0d/9rnSK6uEujDR0rNQuFmqEwota29vL6tataoITyyFaiWuXLkiQtdSGsXyp3dv1TDRBIU7pTJQvVNeFPq2cePGslu3bonzZ86ckbVs2VLULZ2nfylUqmqI1m+J1D7qNtXQ2dReFIqVwnMbGRnJKlasmGb4VVXSq0sK4UwhaSmNYujihg0biveL3uf8+fPLevbsKbt586bStQ8fPhTheylUMKUrXLiwbOLEiUppbt++LfIyNzeXmZqaymrXri3aXBEpTPSNGzfUlnHp0qWyfPnyieeuUKGCCHdL4YAzEiZa3XOrhr4lKHx2ly5dxG+Enpuel0IFU7rt27eLNAEBAbLBgweLd4zypXSVK1eW7dy5U5YRKIw5/R7UhYnetWuX0nF1z0O/43r16om6pN8ThWGmUO9f+txp/XbTKlNa7ZSR98Xb21v+rlC69u3by8MEq/tmqYZzl+5NZU4PeicorDrd293dXZSHvhWLFy9OlZa+QxSimNLTu0XfRvoWUfh7CussQfelds8odL+mTZvKTpw4IStVqpTIm94Z1frM6HcyrXaSyqZYfxmtZ4lTp06JcxRq/N27dxn6rVDocKoj+k1SP5U7d27xDFL4cNV6UAeFJafnXb16dbp1yTDfCh363/cWshiGyRgUCIDMtkg7wI6cDJM5KJAAhfqlkNY00/xfoeAcpImhkO9S5Coma6GgHhR2OD3/MOb7QyZupGmiCJhfI4ADw2QW9sFhGIZhNA7VtYTIXIl81cjsh8yvsgIyqyOzJcnniWG0ETKfI58dMndm4YbJLrAPDsMwDKNxUMAFEnLI4Zl8hPbu3SvC7ZIDfVYOwshnhWG0GfL9UQ1OwjDfGxZwGIZhGI2DHP8p1PPhw4dFpDQK5EAaHApQwTAMw2g27IPDMAzDMAzDMIzGwD44DMMwDMMwDMNoDCzgMAzDMAzDMAyjMbCAwzAMwzAMwzCMxsACDsMwDMMwDMMwGgNHUWMEfc/04ZrIJqyuuwYnvcZ872Iwn2iQZw4Skk5xfWQj9HXrQ3ew+/cuBvOJpCUeMBhRnesjmxC/4BIW3x/4vYvBfGJIKQ4l/z1gDQ7DMAzDMAzDMBoDCzgMwzAMwzAMw2gMLOAwDMMwDMMwDKMxsIDDMAzDMAzDMIzGwAIOwzAMwzAMwzAaAws4DMMwDMMwDMNoDCzgMAzDMAzDMAyjMbCAwzAMwzAMwzCMxsACDsMwDMMwDMMwGgMLOAzDMAzDMAzDZIqLFy+iefPmcHFxgY6ODvbv3//Za86fP49y5crByMgIBQoUwPr161OlWbJkCVxdXWFsbIzKlSvj+vXrmSsYCzgMwzAMwzAMw2SWyMhIlC5dWggkGeH169do2rQpateujbt372L48OHo27cvTpw4IU+zY8cOjBw5EpMnT8bt27dF/g0bNoSfn1+myqaf6adhGIZhGIZhGEarady4sdgyyvLly5EvXz7MmzdP7BctWhSXLl3CggULhBBDzJ8/H/369UOvXr3k1xw5cgRr167F2LFjM3wvNlFjGIZhGIZhGOar4uHhgXr16ikdI8GGjhNxcXG4deuWUhpdXV2xL6XJKKzBYRiGYRiGYRgGsbGxYlOE/GVo+698+PABTk5OSsdoPywsDNHR0QgODkZiYqLaNE+fPs3UvVjAYRiGYRiGYRgtQXewe5rnJjk0xNSpU5WOkT/MlClT8CPBAg7DMAzDMAzDaAm6+ml7qIwbN044+SuSFdobwtnZGR8/flQ6RvuWlpYwMTGBnp6e2NSloWszA/vgMAzDMAzDMIyWoKunm+ZGwgwJHIpbVgk47u7uOHPmjNKxU6dOieOEoaEhypcvr5QmKSlJ7EtpMvyMWVJi5rtBccIXLlzILcAwDMMwDMNkSIOT1pYZIiIiRLhn2qQw0PS3l5eXXBvUvXt3efoBAwbg1atXGD16tPCpWbp0KXbu3IkRI0bI05D2aNWqVdiwYQOePHmCgQMHinDUUlS1jMImalnMmzdvRAi8O3fuoEyZMvLjPXv2REhISIYWQdJGaueqjYZ5GsHK0ArvIt5h2/OteB32Os309XLXQ62ctWFrbIuI+Ajc8ruJPZ57kJCUkKE8zfTN0MKtJYrbFhd5hMeH467/Hez33I/oxGhoOxcPvMKZXS8QFhSLnPmt0G5wKbgWsUkzfVREHA6vfYJ7l30RFR4PG0cTtB1YEsUrp6iUQwKicWD1Izy+/hHxsYmwdzFHt1FlkaewDRITknB43RM8uv4RgR8iYWxqgMLlHNCyTzFY2Zt8o6fO3mzdcgHr1p5BQEAYChfJifG/t0epUq5q0+7bdxUTxm9WOmZoqI8791ImQ06dvIudOy7h0SMvhIZGYffesShaNJfSNbGx8Zg7Zy+OHb2FuPgEVKtWFBMndYS9vSW0nUE12mJUva5wtrTFPZ+XGLpzPm68faw2rb6uHsY17IHulRsjp7UDnn30wtgDS3Hi8VV5GnMjU0xv1h+tytSAo7kt7ng/x/BdC3DT64k8zdqfJ6BnlaZKeR9/fBVNlqQMDrSVgdXaYGSdznC2sMV9X08M37sANxTqTrU9xtT7GT9XbIycVvZ47vcO4w4vw8mn1+RpdHV0MalRb3Qp3wDOFnbwDQvAxutHMfPUBnmaiQ17o0PZusht7Yi4xATc9n6GSUdW4rqX+vdAm7h/3Bu3D3ohKiQO9nnNUaN3ITgXVP/d2Dv5Nnweh6Q6nresHVqMLy36h6vbX+Ht7UCE+kXDyFQfuUraomrX/DC3TdEsBPtG4fKml3j/LFRcY5/HHFU6uSFXibT7Lm1FN5OCTFrcvHlTrGkjIZm29ejRQyzg+f79e7mwQ9D4mEI+k0CzaNEi5MqVC6tXr5aHiCY6duwIf39/TJo0SQQloLH08ePHUwUe+Bws4DDfnYqOFdGhYEdsfroJr8JeoV7u+hheZgQmePwuBA9VKjlVRtv87bDuyTp4hr6Ek6kzehfrDRmAnS92ZChPKyNrWBtZY9fLnfCN9IWdsR26FflZHF/+YBm0mVvnvbFvxUN0HFoaeYva4PxeTywddwUT19aDhU1qNXVCfBKWjLkCc2sj9JlYCVb2xgj6GA0TcwN5mqjwOCwYfhEFSztg4MyqMLcyhL9PJEwsDMX5uNhEvHsZgkbdCiOnm6UQkvYse4AVk65h9NJa0HZIwJg7Zx8mT+mIkqVcsWnjOfyv3xIcPjoJdnYWaq8xNzcW5yV0dJTPR0fHoWy5/GjYqBwmT9qqNo85s/bgwsVHmL+wDywsTPDH9J0YNnQ1tmxVts/WNjqUq4t5bYZi4Pa5uPbmEYbX7ojjQxagyNRO8I8ITpV+RvP/oWulRui/dRaefniLhsUqY2+/2ag2rz/uej8XaVZ1HYcSLm7ovmEafEMD0K1iQ5wa+jeKT+8C31B/eV7HHnmg9+YZ8v3Y+HhoO+3L1MGfrYZg8K6/cP3tYwyt2QFH/jcfxWd1hn9E6oHztCb9heAyYOccPPPzQoPClbC710zU+HsA7vq8EGl+q9sV/6vaCr23/YHH71+jfJ4iWN1pPMJiIrH4390izQv/dxi2dwFeB/rCxMAIw2p2wNEB81Hkj04IiEx9X23h+eWP+HfDC9TuXxjOBaxw98g7HPzjLrotqgJTq+RvviJNRpUUAolETEQ8to26gYLujmI/ITYJ/q/CUbGdqxCWYiMTcHHdCxyZcx8d51SUX3d49j1YOZui9eSy0DfUFfc9NPseuv/jDjM1fZc2o5tFAk6tWrUgk9HoSz0k5Ki7hpQA6TFkyBCx/RfYRO0LIEmyevXqsLa2hp2dHZo1awZPT0+5dEqULVsWOjo6oiEp8gSp2g4cOCCO0Xb+/HmRbsyYMShUqBBMTU3h5uaGiRMnIl6lwzp06BAqVqwIY2Nj2Nvbo3Xr1mmWjSRhKpeqjWN2pn6eBvjX5yIuv7+M95HvhVASlxiH6i7V1aYvYJUfL0Nf4vrHawiMCcTjoEe4/uEa8lnmy3CevpE+WPZgKe4F3IN/tD+eBj/FPs99KG1fWszcaTPn9njCvXFeVGmUFznyWqLjsDIwNNKDx4m3atNfPf5WCDD9p1aGWwk72DmboWBpe+TKbyVPc2rHC1g7mKLbb+WEJsg+hxmKVnCEg4uZOG9iZoAhc6qhXM2ccMptgXzFbNF+SCm8exGCIL8oaDsbNpxFu/ZV0bqNOwoUyIHJUzrB2NgQe/emvS4AfWccHCzlm6rWpUXLShg0uDHcqxZWe314eDT27PXA6DFtUKVKYRQvngczZnbD3TuvcO9u2tpVbWBE3c5YfeUg1l89gicf3mDA9rmIiotFb/dmatN3q9QIs05sEMIJDYaX/7sPRx9dwci6ncV5YwMjtC1TC2P2LcG/L+/C098bU4+uwUt/bwz8Sfl7H5sQh49hQfItJDr1JJC2MbxWJ6zxOIQN14/iycc3GLTrT0TFxaBnZfXt0bVCQ8w5vQnHn1wV7bHiyn4ce+KBEbU6ydO4u5bAoYeXcOyxB94Gf8Dee+dx6tl1VMxTVJ5m++1TOPv8psjj8YfXGLX/H1iZmKOkS35oM3cPv0Pxui4oVtsFtrnNhKBDAsfjs75q0xtbGAgBRNre3Q+GvpEuCnwScIzM9NFqUlkUrOoEm5xmcC5khZp9CsHvVTjC/WNEmuiwOIS8j0b51nmFEGSdw1RoeEg4CnwX+U2f/0f3wdEUNOdJviFkC0hqOFLNkSBBixCR0EGOUNevXxdpTp8+LVRze/fuxahRo9ChQwc0atRIHKOtatWqIp2FhYWQcB8/fizUdWR3SCu6SpAqj/Ju0qSJkHjpfpUqVVJbrrlz54pVXk+ePIm6deviR0BPRw95LfLicVCKKYEMMjwJfgw3K/WdxMtQT3GNJNDYG9ujpH1JPAi4/8V5Eqb6JohJiEGSLGUmSdsgbcy75yHCPExCV1dH7L95HKT2mgceH+BazBY7/7mH8e2PYWa/Mzix9RmSElNmdR56fECeQtZYM+06xrU/ijkDzuHy0TfpliU6Ml5oHUj40Wbi4hLw+NE7uLunCCL0zaniXjhdQSMqKhb16kxE3doTMGTwCrx88T5T9yXTtYT4RKX7urk5I0cOG9zVYgHHQE8f5XMXxumnN+THaAaT9qu4lVB7jZG+IWLi45SORcfHonr+0nKTKX09fcQkpE5T7VMaiVoFy+HD7CN4Mmk7lnb6DbZm2m0uSO1RLlchnHl+U6k9zr64iSp5i6u9xkjfADEJyut8xMTHoqpbKfm+x5uHqF2oPAo65Bb7pVwKoJpbKSEUpVWOvu4thcB53/cltJXE+CQheOQuZSs/pqOrI/Y/PA/LUB6Pz/iiUFUnGBjrpZkmNioB0EkWfiQhydrFFE8vvEd8TCKSEpPw8JQvTKwM4OimXsutzegZ6qW5aQpsovYFtG3bVml/7dq1cHBwEEIK/UuQZkcxpB2Fv6OFk1TD3E2YMEEpYAAJQ9u3bxcOWMQff/yBTp06KcUkL11aucOTNEGbNm3ChQsXULy4+o96dsTcwAJ6unoIi1P+8NG+s2kOtdeQ5sbCwBxjyo8V+/q6+jjvfQ5H3x794jzNDczRzLU5LvpcgDYTGRqLpCQZLG2MlY6TadrHdxFqrwn4EImgu1GoUDcXBvzhDn/fCOz8+x4SE2Vo8nOR5DTvI3Hp0GvUblsADboUgtezEOxZch/6+rqo3CBPqjzj4xJxcPUjlK+dS+sFnJCQCCQmJqUyRbOzs8Tr18qhNCXyuTpi+oyuKFQ4JyLCo7Fu3Rl07TIPBw79DmfnjNmjk6+PgYE+LC1Nle9rbynOaSv25tZCGPkYrizw+4UHoYhzXrXXnHhyDSPqdsLFl3fgGeCDuoUroE2ZWtD7pC2OiI3ClVcPMKFRL6ERIs1M5wr14Z6vhNDiyPN5fBX77p7H68D3yG+fE3+0GICjgxag6l/9tHZixt7MSrQH1b8i1D6FHdW3x8mn1zGsVif863kPnoE+qFOwPFqVqgk93ZQ537lnNsPS2AwPx25BoixJtNXEoyux7fYppbyaFKuKLd2nwNTAGO/DAtF42QgERoZCW4kOj4csSZbKFI32g30+r43/8CJMaFzqDEzRlKmSEJeIK5s9UaiaEwxN9eUa69aTyuDI3AdY3v2C2De1MkDL38vAWMFcmslaE7XsDAs4X8CLFy+E89O1a9cQEBAgNDcEOVIVK1YsU3nt2LEDf//9tzBxo2gUCQkJIiSfBEWj6NevX7p5zJs3T2iVSKNEZm5fskLtj0Rh68Jo4toUW55txqvQV3A0dUSnQp3RLLYZDr85nOn8jPWMMbT0MOGLc/D1wa9SZk2GOjMLayN0Hl4Wuno6QlMTGhAjghRIAg7NqOYpZIMWfZJ/H7kLWOP9mzBcOvw6lYBDtthrp98AmfV2GJpamGc+T5mybmJT3G/ebDp27riMocPUm+0wX4/huxdgZZexQutCvwUSctZ7HEEvBZO27humYk233+Ez8xASyGH93XNsu3lK+H5I7Lh1Wv73Q19P3Pd5Cc9pe1CrUDmcfZaiwWDSZ+S+RVjecTQejtuS3B6BvsK8rWelpkp+PZ3L1cfPm6cK87PSOQtiXquheB8WgE03jsvTnX95GxX+6gV7M2v0qdIcW3tMQ7WF/dX6/jCfh8zY7PKYpRmQgPqH4/MfCbuM2v1StMvUjudXP4eJlSHaTisHfUM9oQk6NPs+Os6uwD44WijgaP4TfgWaN2+OoKAgYU5GQg5tRFycsnnB5/Dw8EDXrl2F+dnhw4eFCdrvv/+ulA9pfj7HTz/9hMTERBFq73PMmjULVlZWShsd+15ExIcjMSkRlobKHzPaD41TPwvWMn8reHzwwL++/8In0gd3/O9gn+deNHZtAh3oZCpPIz1jEXwgJjEGSx4sRqIsEdqMmZWRMEkLC062a5YID46FZRpOmla2xnDIZS6EGwmnPBYiAhuZvBGWtsZwzqOsgaA0wX7RqYWbGTeE3w355Gi7eRphbW0OPT1dBAYq+1oEBoZlOJqZgYEeihbNDS+vFGf1z0F5x8cnICxMedY1MCDj99VEAiJChADiZJFigkM4WtjiQ1hgmte0WTkW5iPqwHViGxSd1klobV4F+sjTvArwQe2Fg2A+ojbyTGiFKn/2EWZPdDwtyPfDPzwYBRyUo99pEwGRoaI9qP4VcUqvPSJD0G7teFiNqY/809uhxKwuiIiNxqugFB+R2c0H4c8zW7Dzzhk8fP8KW26ewKILOzG67s9KeZGvDwms194+Qv8ds5GQlIheafj+aAMmFgbCJC0qVHk8RPum1qkDDChCpmUvLn9EsTou6Qg3DxEWEIOWE8vKtTeE98NgvLkVgEbDi8OliLUwS6vVL9n358n5zJnnagO67IPDqBIYGIhnz54J0zLycylatCiCg1Oi5tAiRQQJHIrQcdVjV65cQd68eYVQU6FCBRQsWBBv3yo7cpcqVeqzAQPIJ+fYsWOYOXMm/vrrr3TTUkzy0NBQpY2OfS9IoHgb/hZFbVPU0SSkFLEpilehyYEbVDHSNUwVtUPRPCOjeZLmZmTZkUiUJWDxvX+UQkxrK/oGushdyBrP76QMhMlkjfbJz0Yd+YrbIcA3QqST8PeOEEIN5Ue4FbfDR29lEzc/7wjYOpmmEm78fSKEcGNmmX5nqC1QeOdixXPj6tVn8mOkNb529TlKl0kJrJEeZOL24rmvCDaQUSiogL6BntJ9ySTu/ftglMngfTWR+MQE3Hr3TJiZSZA5DO1fffUw3WspQABFRCOfmzZla+Pg/X9TpaEBMw3MrU0s0LBoZbVpJCjktJ2ZFd6HBkCb2+O293PUKVReqT1qFyyPq29ppv9z7REg2qN1qZo49CClrk0NjVOZ/dHE2eeC0NB58rnSVvQMdIVw4f0gWEnL/+5BMJwLpf/9eenhh8QEGQrXcE5TuAn5EI3WE8sIQUoRCiYgUIkWSe9COkG+tBbdLFoHJzvDJmqZxMbGRvjXrFy5Ejly5BBmaeTYL+Ho6Ci0LhRpjeJ7U+Qz0pKQf82JEyeEcETX0zESaOh68rmhKGkUUGDfvn1K95s8ebIQpPLnzy98cciE7ejRo8LnRhEKWkDHGzduDH19fQwfPlxt+ckcLbuZpJ3yOonexfrgbdgbsU5NvTz1YKRnJCKgEXQuJDYYez33in2KfEZR0rwivPD6k4laK7dWuB9wTwQTyEieJNyMKDtSCEurH62Csb4x6D8iPC5cno82Urttfmyee1uYlOUtbIPz+zwRG5OIKg2TTck2zrkFa3tjtOiT7Ov1U3NX/HvwFfYsvY+arfLDzycCJ7c9R81Wbkp5zh92UQQfoEhpb58F48rRN+g0vIy886IABO9ehuJ/06uIDjEsKFmLZGphKBeUtJUePepg/LhNKF4iD0qWTA4THR0di9atq4jz48ZshKOTFUaMbCn2ly45htJlXJEnjwPCw6Kxdu1p+PoGoW275OAmREhIpBBW/P2StZpvPvnzkHaGBCEKC922jTvmzt4LKyszEXZ65oxdQrjJqGClqSw4sw3ru0/ETa+nuE5hout0gpmRMdZdTTaRXd99EnxD/DH+YHLI+UquxZDTygF3vV8IoWRy077Q1dHB3FMpaxU1KFpZDMaefXwrNDJzWw/B049vsc4jOU8zIxNMbtIHe+6cEwJQfodcmNNqsPDRIR8fbWbh+e1Y2+V33Hr3FDfePhFhos0MTbDh2hFxfl2XCfAJ9ceEIyvEfqU8xeBiZY97vi/Fv5Ma9haBO/46mxIu/cijyxhbvzu8Qj6KMNFlchXC8Fodsf7aUbkANK5edxx+dFmYrZGJ2sDqbcS6OnvunYM2U6ZZbpxe8gSO+S3gVMBShGtOiE0UUdWIk/88FuvXUJQzRR6d9YVbRftUwgv1D8fmPYT/63A0G1tKTKZFBieb2pN/DQlVJDwZmRuI+1I4aTJRe3TaF2F+0XAtZ/cNn/7HQE9fRRLUQFjAyST0ESSBZOjQoShRogQKFy4sfGgoHLSoUH19sT9t2jThp0PmYxQSmvxo6F/S1JCvzblz59CiRQux2BHF+ia/mKZNm4ow0RRWWoLy3bVrF6ZPn47Zs2cL/5waNWqoLRuFriYhiUze9PT08Msvv+BH4IbfDZgbWqClWytYGlniXfg7LLy7QB4kwM7YVkljQ342tNfarRWsjWzEujYk9JCZWkbzpChr+T9FVJtVdbZSecZcHi3CT2sr5WvlQkRIHI5seCJM02ihz0Ez3eWBB4L9opTWVLFxNMWgWVWxd9kDzOp/Vgg/NVu7oX7HQvI0JCj1m1IZB9c8xvHNz2DnbIo2A0uiYt3c8kVAKRobQRHWFBn6VzWxfo4207hJeQQFR2Dx30cQEBCOIkVzYsXKwXJTsffvg4RZiASZlU2euFWktbQyQfFiecTaNRRiWuLcuQdKi4GO+nWd+JdCRw8ekuyLMGZcW5Hv8GGrER+XvNDnhEkdoe3svH0GDhY2mNqsr1gEktZOabxkBPzCk2et89g4Kc3+G+sbYXrz/8HN3kWYQh195CF8bkKjU7SaFF54ZosByGXtiKCoMOy9ex6/H1wuTJ6IxKQkEX6YFgsl7Q5pHk49uYaJh1ciLkG718LZdfcsHMytMblRX/nCq81W/Aq/T2sS5VZpDyMDQ0xt0g9udsntQZHRem6ZjtCYlPag9W2mNu6Hf9r+CkdzG7HQ56orBzHj5Dp5exR2yisWC7U3t0JgZJhYlLX2P4OFz442Q87/0WHxuLbjFSJD4uDgaoEWv5eWm6hFBMSkWpcr2CcS75+GouWElAXSJSKDYvH6ZrKWcvtvKdELidZTyiJXcRuYWBqKe1zd9gr7p94RQW7scpmh6ZhS4v6MMoYaFA46LXRk6a3Qw2gNfc/0+d5FYD6xuu4anPRS1tAx348GeeYgIUk5chLzfdHXrQ/dwe7cDNmEpCUeMBihft0y5tsTv+ASFt8fyFWfTRhSKvstHl5kfbs0zz3tmbyQ7Y8Oa3AYhmEYhmEYRkswVAiJrqmwgMMwDMMwDMMwWoKhQtRTTYUFHIZhGIZhGIbREgy1wAeHBRyGYRiGYRiG0RIMWYPDMAzDMAzDMIymYMgaHIZhGIZhGIZhNAVD1uAwDMMwDMMwDKMpmOjrQdNhHxyGYRiGYRiG0RIMWYPDMAzDMAzDMIymYMg+OAzDMAzDMAzDaAqGrMFhGIZhGIZhGEZTMNDV/IU+NX+lH4ZhGIZhGIZhBAa6aW+ZZcmSJXB1dYWxsTEqV66M69evp5m2Vq1a0NHRSbU1bdpUnqZnz56pzjdq1CjT5eIgAwzDMAzDMAyjJRhkkYnajh07MHLkSCxfvlwINwsXLkTDhg3x7NkzODo6pkq/d+9exMXFyfcDAwNRunRptG/fXikdCTTr1q2T7xsZGWW6bKzBYRiGYRiGYRgtwSCLNDjz589Hv3790KtXLxQrVkwIOqampli7dq3a9La2tnB2dpZvp06dEulVBRwSaBTT2djYZPoZWcBhGIZhGIZhGC3ywTFIY8sopIm5desW6tWrJz+mq6sr9j08PDKUx5o1a9CpUyeYmZkpHT9//rzQABUuXBgDBw4Ump7MwiZqDMMwDMMwDKMlGKSj3oiNjRWbqkZF1UwsICAAiYmJcHJyUjpO+0+fPv1sGchX5+HDh0LIUTVPa9OmDfLlywdPT0+MHz8ejRs3FkKTnl7GFyhlDQ7DMAzDMAzDaAkG6WhwZs2aBSsrK6WNjmU1JNiULFkSlSpVUjpOGp0WLVqIc61atcLhw4dx48YNodXJDCzgMAzDMAzDMIyWYKCX9jZu3DiEhoYqbXRMFXt7e6FR+fjxo9Jx2ie/mfSIjIzE9u3b0adPn8+W1c3NTdzr5cuXX1fAiY+PR/78+fHkyZPMXsowDMMwDMMwTDbV4BgZGcHS0lJpUxfFzNDQEOXLl8eZM2fkx5KSksS+u7t7uvfftWuXMIPr1q3bZ8vq7e0tfHBy5MjxdQUcAwMDxMTEZPYyhmEYhmEYhmE0JIrayJEjsWrVKmzYsEEoPiggAGlnKKoa0b17d7XaHzJPI/MzOzs7peMRERH47bffcPXqVbx580YISy1btkSBAgVE+OmvHmRg8ODBmDNnDlavXg19fY5TwDAMwzAMwzA/AsZZtA5Ox44d4e/vj0mTJuHDhw8oU6YMjh8/Lg884OXlJSKrKUJr5Fy6dAknT55MlR+ZvN2/f18ITCEhIXBxcUGDBg0wffr0TK+FoyOTyWSZfaDWrVsLqcrc3Fw4AamGd6OFfBiGYRiGYRiGyV5sfTY4zXNdCi+BJvBF6hdra2u0bds260vDfDfOeo/n2s8m1Mk1E76Rq793MZhPuJj1xXHbwlwf2YhGQc+gP6za9y4G84mERZfR8mB3ro9swoEWG/E+Sv1Ci8y3J4dp72xX7foqWhVN5IsEnHXr1mV9SRiGYRiGYRiG+aro62Z8PZkflf/kQEN2d2RLR9Bqow4ODllVLoZhGIZhGIZhshh9LdDgfNETUoSE3r17i5BtNWrUEBs5AlE866ioqKwvJcMwDMMwDMMw/xl9Hb00N60WcCgs3IULF3Do0CER5YC2AwcOiGO//vpr1peSYRiGYRiGYZgs0eDop7FptYnanj17sHv3btSqVUt+rEmTJjAxMUGHDh2wbNmyrCwjwzAMwzAMwzBZgD774KiHzNCkGNeKODo6sokawzAMwzAMw2RT9DVIU5MWX/SE7u7umDx5MmJiYuTHoqOjMXXqVHGOYRiGYRiGYZjsh74W+OB8kYnaokWL0LBhQ+TKlQulS5cWx+7duwdjY2OcOHEiq8vIMAzDMAzDMEwWoM8mauopUaIEXrx4gS1btuDp06fiWOfOndG1a1fhh8MwDMMwDMMwTPZDXwtM1L54HRxTU1P069cva0vDMAzDMAzDMMxXQ581OCkcPHgQjRs3hoGBgfg7PVq0aMGvJcMwDMMwDMNkM/R1WIMjp1WrVvjw4YOIlEZ/p4WOjg4SExO/TQsxDMMwDMMwDJNhDPQMNL62MmyilpSUpPZvhmEYhmEYhmF+DPR0vthD5Yfhi3RU7969y/qSMAzDMAzDMAzz1QUcvTQ2rRZwXF1dUbNmTaxatQrBwcFZXyqGYRiGYRiGYbIcPRZw1HPz5k1UqlQJ06ZNQ44cOYRPzu7duxEbG8uvIcMwDMMwDMNkU/R09dPctFqDU7ZsWfz555/w8vLCsWPH4ODggP79+8PJyQm9e/fO+lIyDMMwDMMwDPOf0WMNTvpQxLTatWsLU7XTp08jX7582LBhQ4YruFatWhg+fDiyM9+yjGT6t3Dhwm9yL4ZhGIZhGEb70NMxSHPTFP6TLsrb2xtbt24V28OHD+Hu7o4lS5Zk+Pq9e/eKdXWyA+fPnxfCGvkUWVtbf9Uyrl+/XghNISEhSsdv3LgBMzMzaCPn93vi1M7nCAuKQa78Vuj4Sxm4FrFNM31URBwOrHmEu5d8ERUeB1tHU7QfXAolKucQ5w9veIwjG58oXeOU2xxT1jdMlZdMJsPicZfx+MZH/G9qFZSpnvMrPOGPx74dt7Fj4w0EBUYifyFHDB1dF0VLJNevKscPPsScKceUjhkY6uHk1ZFKx96+CsTKvy/g3u13SEyQIa+bHab+2RJOOSzxwTcUnZutVJv/5DktUKt+YWgzefp0Qb5f+sDQ0QHhj57iyZjpCL39IM30Ti0boeC4YTDJkxNRr97g2ZS/EHD6ovx8gTFD4Ny6KYxzOkMWH4/Qu4/w4o8FCL11X57GNL8rCk8dDZvK5aBraIDwR8/wYuYiBF26Bm1nYPU2+LVOFzhb2uK+z0sM27MAN7yUvzmKi+qNrd8dP1dqjJxW9njm54XxB5fhxNOUetTV0cXkxn3QpUIDOFvYwTcsABuvHcUfJ9fL0yQsuqw2/zEHlmDe2a3QZpq41kWrAk1gY2SFN2HvsPLBJrwIeZVm+uZuDdHYtQ7sTewQHheOK743sPHJLsQnxYvzjVzriPOOJg5i3yvcBzue78dtv5Tfh4GuAXoX74zqOavAQFcfd/weYPmDDQiNDYO2Q/3H9g3XRP9RgPqPMfVQtISL2rTHDj7AnMlHU/Ufp66NUjr29lUAViyi/sNL3n9M+6u16D8kHt3zweolF/HkwXvo6umIe/+5tAOMjLPHWDO7oKdBwQTS4ouecMWKFUKouXz5MooUKYKuXbviwIEDyJs3b6bysbVNewCbXfiWZSRTP23k5rl32LP8PjoPL4t8RWxxdu8L/D3mEqasbwBLG+NU6RPik/D36EuwsDZC/8mVYW1vgsCPUTA1V/6A5XC1xLA/f5Lv6+npqL3/2T0voaP+lNZy9sRTLJt/HiPG10fRkjmwe8stjB68Cxv39YGNrXoh3MzcEBv39kk5oFKpPu+CMbTPVjRuWRI9B1SDqZkh3rwKhKGRnjjv4GSBPScHKl1zaO997Nh4HZWr5YM249y6MYrMGIdHv05GyK17cB3QAxV2r8G/lRohLiAoVXrrSmVRetU8PJ8+H/4nziFHu+Yot3kJrtRug4gnL0SayJdv8GTMNES9eQc9E2PkHdgTFfasxcXy9REfmBw8pvy25Yh89RbXW/ZAUkwM8g7ogXLblos0cX4B0Fbal62Lv1r/gkE7/8T1N48xtFYHHB04H8X+6Az/COWJK2J60/7oUqEhBuyYg6cf36JBkUrY3WcWflr4P9z1SW6P0fW64X/VWqH3lhl49OE1yucugjVdfkdoTAQWX9wt0uSc0Fwp30bFqmBVp3HYe+88tJnqLpXRu3gXLLu/Hs+DPYXwMqXKbxh0djRC48JTpa+R0x3di7bHP3fX4GnQC7iYO2NY2X6QAVj7KFlQDIwOwsbHO+Eb+RE60EGd3NUxvtJwjLgwEe/CfUSaPiW6oIJjGcy9+Q+i4qPRv2R3jKs4FGMvzYA2c/bEEyyddxYjf28ghJrdW2/it0E7sWl/v/T7j339lCyEVPuPX3pvQZNWpdBrYPXk/sMzQN5/SMLN6CE70aWXuxCo9PR04fncDzq63MGrkpW+NqTYILcVWiuzdOnS+Oeff4SffloT/L169VI6ZmRkhJiYGKVJ58mTJwvrMFIEVKtWDcuWLUPBggW/vg/OjBkzULlyZdy6dUtobsaNG5dp4UbV/IvMs2bOnCl8eCwsLJAnTx6sXJkym1u1alWMGTNG6Xp/f3+hXbl4MXlWkoIcjBo1Cjlz5hSaECojaWYk3r59i+bNm8PGxkacL168OI4ePYo3b94I7Q1B5+iH1bNnz1RlJN6/f4+mTZvCxMREmOSRoKdqWjZ//nyULFlS3CN37twYNGgQIiIixDkqDzVuaGiouA9tU6ZMkdeBYj7k49SyZUuYm5vD0tISHTp0wMePH+Xn6boyZcpg06ZN4lorKyt06tQJ4eGpP+jZmTO7X6BaE1dUbeQqhJLOw8uJj5bH8bdq0185/gaRYXEYMM0d+UvYw87ZDIVKOyBX/hTNmyTQWNkayzdzK6NUeb17GYLTu17g598qfLXn+xHZteUmmrYuJYQRVzd70VEZGxvg2IGH6VylA1t785TNTrkjW7PkEipXc8OA4bVQsIgTcua2QbWaBeQdHnVGStfbm+PSuReoVb8ITEwNoc24DuqFdxt3wmfrXkQ+88SjkZORGBWDnF3bqk2f93/dEXDmX7z5Zw0in7/Cy5mLEHb/MfL07SZP837PYQRe8ED0W29EPH2JpxNmwcDSAhbFkzVlBrY2MCuQD68XrkTE42eIevUWz6fNg76ZKSyKZq6j0TRG1OqI1VcOYcO1o3jy8Y0QdKLiYtGrSjO16btWbITZpzbi2GMPvA70xYrL+3HsiQdG1OksT+OerwQOPvwXRx974G3QByG0nHp2HRXzFpOn+RgepLS1KPETzr+8LfLUZlrmb4STXudx5t2/eBfhKwSd2MRY1MtTU236IrYF8CToBS76eMAvOgB3/R/ios9VFLR2k6e58fEubvndx/vIj/CN/IDNT3cjJiEGhW3yi/Om+iYifxKIHgQ8gWfoG/x9dxWK2hZCoU9ptJVdm2+gaZvSaNyyFFzzU//RUPQfR/c/SLf/sLM3l2+q/cfqxRdRuXp+DBheO6X/qFVQSWBaPO8M2nQqj669qyBffgfkcbVD7QZFYWio+dqK7+WDs2PHDowcOVIIJLdv3xYCTsOGDeHn55fmNTSepbG0tNHYXJG5c+fi77//xvLly3Ht2jUxlqY8FYWgrybg0MCbCkAPkpXMmzcPFSpUwJ07d4RQMHDgQDx79kycIy3R9u3bhWSnWLEuLi746afkWfohQ4bAw8NDpLt//z7at2+PRo0a4cWL5BmywYMHCyGIBKIHDx5gzpw5QnggIWTPnj0iDd2PKnzRokVqy9i9e3f4+voKQYWuISFMtSF1dXVF4zx69Ej4JJ09exajR4+WC2okxCg2MAllqtBiqiTcBAUF4cKFCzh16hRevXqFjh07KqXz9PTE/v37cfjwYbFR2tmzZ+NHgbQxXs9DUKSco/yYrq6O2H/1OFDtNfev+MKtmC22/30Ho9sexrQ+p3Bsy1MkJaa8G4SfTwTGdjiCCd2OYe3M6wj6GKV0Pi4mAWv/uI5OQ8sIAYhJJj4+Ec+ffED5ynmV2qRc5bx4dD/tgVR0dBw6NVmBDo2X4/cR+/DaM2WGPylJhquXPJErrw1+G7QLresuwcDum4UAkxbPHn/Ay2d+aNKqpFY3jY6BASxLF0fghSspB2UysW9dsazaa6wrlhHCiyIBZy+J42ndI3ePjogPDUP4w+RvbnxQMCKev4JLp1bQMzWBjp4ecvfsiFi/AGHOpq0Y6OmjXO7COPP8hvwY9Utnnt9EFdcSaq8x0jdATEKc0rHo+FhUy1dKvu/x+iHqFKyAgg65xX4plwKo5lYKxx9fVZuno4UNmhSvirVXD0Ob0dfRQ34rV9zzT3knZZDhXsBjFLYpoPaap0Evkd/aVS7QOJk6oLxjadzyu6c2vS508JNLZRjrGeFZ0EtxjK4nszTF+/pEvIdfVACKpHFfbek/nqnpP8pXdsXj+8mar7T6j46Nl6F9o6X4ffgevPb0V+k/XiF3Huo/dqBVnX8w8OeN+Pfcc3ma4KBIYZZGAs/gHpvQuu4/GNZnK+7f8f6KT/vjopdFAg5N6Pfr109M3BcrVkwIJaampli7dm2a19DEvrOzs3yjAGWK31IaI0+YMEGMgUuVKoWNGzeKcTeNdTPDF4m1kuowKipKCDtxccofbirQl9CkSRMh2BCkrVmwYAHOnTuHwoULC+0FaVIuXbokF2hIe9K5c2dRHirHunXrxL8k9BAkOBw/flwcJ+0QnWvbtq3QrhBubm6pTNEcHR2VfHAUefr0qQimQL4yJIgRq1evTqU2U9T4kGaFNF4DBgzA0qVLYWhoKDQtUgOnxZkzZ4QQ9vr1ayGAEdTIpHWi+1esWFEuCJHKj7RexM8//yyu/eOPP/AjEBEaKz5eqqZotP/xnXpNVMD7SDy7449KdfNg8Kxq8PeJwPZFd5GYmIRm3ZNnO8l/p/voCnDKZYGwoGjhjzNv+AVMXFMPxqbJpmy7lt6HW3E7lK6m3i5YWwkNiRbCoo2tqdJx2vd6k9ocisid1wajJzdC/oIOiIiIw86NN/BLry1Yt6u3MD0LCYpEdFQ8tq27jt6DquN/w2rg+pU3mDRqP+av7IQy5ZPfcUWOHniAvPnsUKK0dvtEGdrZQFdfH3H+ygJ/rH8gzAqlfMMUMXK0T2VCFusXKI4r4tCgFkqvni8EmNgP/rjRprcQbCRutOmJcpuWop7XbciSkhDnH4Sb7fsiIVR7fQzszayhr6cPv3Dl3wLtF3HMo/aak0+vYXitTvjX8y48A3xQt1AFtC5VE3q6KXOMc05vgqWxKR6N34pEWRL0dHQx8chKbLt1Um2e3Ss2RnhMFPbduwBtxtLQAnq6eghR8XsJiQ1FLnP1PoOkubE0NMes6hNAoxl9XX0ce3MGu18cUkqX1yIX5vw0CYa6BohOjMGsG4uEhoiwMbJGfGI8IhOiUt3X2sgK2kpocJToP2xVTNFs7Kj/UD9pmSevLcZMbgK3Qg6IDI/Fjk3XMaTnZqzb3QeOTpZCeImOisPWddfQZ/BP6D+sFq5ffo1Jv+7DgpWdUaZCHvh6J5uGrl9xCQNH1EaBwk44cfghfv3fdtEP5cqb/V0ifjQfnLi4OGHJRVZcihP89erVE8qGtCCLJrL6ovFruXLlxPicxrYEjXnJ1I3ykKAxM1lkUZ5kpZRRvugJyTSMTLhIeFBHYmLil2SrJBhJAoCkHSH/lAYNGmDLli1CwKFKoIclfyCChAG6b6FChZTyJI2NnZ2d+Hvo0KFCK3Ty5ElReSTsZEYYI+2Ovr6+aBCJAgUKCLM2RUgImjVrlhCIwsLCkJCQIFRrJBCSZJsRnjx5IgQbSbghSDom4YvOSQIOCVCScEPQukTpqQapPlTXKyL7xx8JWRJgYWOEriPLCSfCvIVsEBIQjVM7X8gFnBKVFYTH/FZwLWqL37scw63z3qjWJB/uXfHFs7t+GL8i5UfEfDnFS+cUm0SJUi7o0XYtDu25JwSapE/Ktaq1CqB9t+TJAeqAyGb60O67qQSc2Jh4nDn2BN37uXOzfEUoWMCVmq1gYGeD3N07oMzahbhav73cr6fY3MmICwjEtaZdkRQdg1w/txd+OR512yH2Y8oMK5M+I/YswopOY4TwQjOUngG+WH/tCHpVTjFpa1+mDjqXb4BuG6fg8YfXKJ2zIOa3GQbf0ABsuqEcwIPoWaUZtt46iVgVzRDzeUrYFUG7gs2x4v4G4bOTw8wJfUt0Q4dCIdj5/ICSRmb4hQkw0zdFVZeKGFa2P36/PFMu5DBfqf8onRPd264WfUOfwTUg+9SBVBP9R/LYp+Cn/uMg9R8V8sjTNG9bRpjGiTRFnHD7+lsxWdZ/qHpzRW1FLx0fnLTGiapjxYCAADHuVtTAELRP4191kMKCtDs09iZXjb/++ktYNpHFU65cuYRwI+Whmqd07quaqJGGggpGtnHki0KCDplikSbj4MGD+FJUo5WRkEMSngSZqdGCovHx8UJ7Q5oYSRtDEqGenp6QJu/evSvfSBiQzM369u0rzLxIy0ECEWlhyBkqKyF/nmbNmonGIxM2Ko8UWU5V05UVfK7OVCHBi6RhxY2OfS/IL4bU12HByraVtG+ZhtmYlZ0xHHOZC+FGwjmPpYjARiZv6jA1NxTaHH/fSLFPGqAA30j82uIgBtffKzZi5dSrmD9Su2dEraxNRN0GBynPTNK+ql10Wugb6KFgEUfhGCrlqaevC1e35MkGiTz57PDxQ2ptwIXTz4WQ06BZ8qyONhMXGIykhAQYOijXnZGDHWI/qnf0JzMyQxVtjZGjnTiuSGJUNKJeeyH05j08HPo7ZAkJyNWtnThnW6MKHBvWwt2+IxBy7bbw4Xn821QkRscIszVtJSAyBAmJCXC0UJ4Rpv0PKlodxWvarhkHy9/qwW1qWxSf2RmRsdF4peA7M6flYMw9vRk775zBw/evsOXmCSw6vwNj6v+cKr/qbqVRxCkv1nooaxy0kbC4cCQmJcLaKCWSFkFalOCYULXXdCnSFue9r+CU1wW8DffG1Q+3sOnpLrQr0EwEFJBIkCXiQ6Sf8K/Z9GSXiM7WzK2BOBccGwIDPQMh/Kjel7Q42oqVjanoP4KCkvtaieDATPYfhZ3g8y5Enif1H3ndlL9pFEXN71P/Yedg/umYSpp8KWmYFHQSE9LcvuY4kaItk6sH+Y/XrFlTRComBYakrMhKvkjAIZ8SsrsjAYHUUaRq6tatm/DL+ZqDZbLHI00ICVQk4JDAo7j4KEmSpL0grYripmgKRhoRMhejSv31119FlAaCTMc+p30iyZO0MeQjJPHy5UsRWlqCBBoSMMifqEqVKkKjRLaDitC9PqflKlq0KN69eyc2icePH4uIEqTJ+VJIlUjCqeKmqF781ugb6CJPIWshcEiQyRrtuxVTHtBJkFmZv0+kSCfh5x0uBB/KTx0x0Qnw942QC00NOxfG76vqYfzKuvKNaDewNLprecABAwM9FCrqLGa+JKiuab94qYyZ85G54KuXAbC1N5PnWaSYM96pmLh5ewXBKUdqcw6acataswCsbTKm8dRkKIRz2L1HsKuhoM3S0YFdTXeE3Ej5FikScuMu7GpUUTpmV6uqOJ4eOrq60DVK/hbqmZgkH1T4nUn7lE5biU9MwO13z1CnUAWliaU6hcrj6pv0gnBAaFtII0Nho1uXroVDD/+VnzM1NEYSqacVSExKgq6aEI8UzOCm11Pc9032B9FmSAghAaSUfcpkCAkppeyL4Vmw+vox0jNMVdfSfnrxtihfCg1NeIa8QXxSAko5pPTHOc2c4Whqj6dp3FcboG99Yeo/rin3H7euv0GxUjkz0X/4w061/3ir3H/QvhQi2tnFCvYO5ninYganmIZRICkhzS2j40R7e3uhWFAMfkXQfnouGKqT9DR+p7E0IV33X/KU+KJeKjIyUviqEGSeRSZrBGlTKIrC14IiKbRq1QoTJ04Umhnyv5EgQYIEHpIMSXghE7br168LgevIkSNyzdOJEyfEOSon+feQIEGQkEadFDnq0/NIUc8UoZDYZNrWv39/kTcJOvQ3abEkvyQSqEjDRJoh0hZRhDNyulKEzMoof/KVIRUfma6pQveh+qRnorLS/ejZSOKV/H++BFIxUoADxe17m6jVbVcQl468hseJt3j/NgzbFt5BbEwC3BsmOymun30D+1enDBxqtHATa9/sWnJP+Ok8uPoex7c+Q82WKZFrKOz083v+CPwQCc9HgVgxyUNoiirWSTaFoqACOfNZKW2EraMJ7HNo51pEirTvWgGH993H8UMPxdo1C2aeREx0PBq1SHainjnxCFb9k7KmyoaVV3DD47Wwg37+5CNmTjiCj+/DRCQ2iY7dK+Lcyac4vPcefLyCsW/7bVy56IlW7ZUd3+nc/dvv0LTVl/nyaSJvlq5Dru4dhOaE/G6Kz5si/GYoqhpRcukcFJqYsubQ2xUbYV/3J7gO7gWzgm5izRurMiXgtXqzOE/XFpwwAlYVSsM4l4sIYlDin5kwyuGEDweSTY9JGIoPCUPJpbNFZDVpTRyTvDnhf1K7wxIvOL8Dfd2b4+eKjYUmZUn7UTAzNBZmZ8S6rhPwR7MB8vSV8hZDq1I1kc/ORWhfKKQ0CS5/ntkiT3P44WWMa9ADTYq5I6+tM1qWqoHhtTviwP2U3xlhYWSKdmVqY+1V1t5IHPA8jgZ5a6J27urIZe6CAaV6iIAAp98l193wsv3xc9H2ShHSGrvWFYEDSCAp7VAcXYu0FceTRLBoiPTFbAvD0cRe+OLQfgn7IrjgkxzsIyohGqe9Lojw1CXtiopAB0PL9hNhp8nsTZshM7LD++7h+MEHYu2aBTNPiP6DonISMyccFuuhSWxYcVmh//iAP34//Kn/SAlk1alHZZw78QSH996Ft1cw9m6/hSsXX6Jlh2SXARqDdexRSRw/f+qpSLNmyUXhN0qhpRkVEhPS3DI6TqTJ+vLly4uxrARN8NM+aWoyAk32k0UVuVcQFJ2YBBnFPMnVgyzGMprnf/LBIU0G+aPQQJ0iqZFqif6mgbxUyK8FDfgpGEGNGjVEKGlFKJgAOfSTZsbHx0dIl6RFIZMxqSIpkhotUEoNRhHWKJABQaGlp06dirFjx4poECRMkPO+KuTo36dPH3F/agQSoMh20Ng4WTNA9UHaLYrQRhIvpaM0lJ8E2RuSFokiogUGBorwelKoaAn6sdLaQr/88ovIgzRlVN6sNqnLDlSonVsEGzi8/rEwTaOFPn+ZXV2ubQnyi1KKiU+LetL5XcvuY0a/02IdnNptCqBhp5SFIIP9o0WENAonTWZw+UvYYfTi2mLtHObz1GlYRDiLrl92OXmhz8KOmLO4ndzEwO9DuBAYJSLCYjBv+kmR1tzSSGiAFq/rIkJMS/xUpxBGjG+Areuu4p8/z4rABLTIZ8myuVJpbygwQQV3V26qT3zYdwyGdrYoOG4ojBwdEPbwiXD2lwIPmOTKQT2LvL5Crt/Bvf6jUGj8cBSaMBKRr97gdrfB8jVwZImJQvAp26m1CGIQFxSC0DsPhK8NhYwmKNgA3aPQhOGoeGADdA0MEPH0hciHFvzUZnbdOQMHc2tMadJXLPR5z/sFmi7/FX7hydr8PDZOSFKI+Gmsb4hpTfvBzc4FEbHRIlx0j03TERqdMpFGC4VObdIP/7QfBUdzG7HQ56rLBzD9xDqle3csV098D7ffOvUNnzh7c8n3mgg20KVwG7HQ5+swL0y9+qd8wU1azFOxPcjPhnyhuhZtB1tjG4TFhuPGxzvY/CR5vSHCytASw8v1h62RNSITovE27B2mXP1TKWramodbISsuw5iKvwjNzh3/B1h+fwO0nToNiyIkOArrll1KXuizsCPmLukg7z/ILFlxbZrw8Bj8Ne34p/7DGIWLOmHJ+m4ixLRi/0HhpresvYq/555B7ry2mPZna5RS6D/ad62IuNhELJl3FuGhMchfyAF/LesoQkozKpC2JgugENE9evQQE++09g1FQCMliLTWDY19aXwtWXdNmzZNjMtJGUAWSbR+DoWJJhcSgr5tpIygsTy5vZDAQ0oNCh5GCo7MoCNTjLucQTZv3ixMtSjQAJlk0cCbwhmTNEdCgWooY02GhCUye6PAAnXrJps5/Yic9R7/vYvAfKJOrpnwjVzN9ZFNcDHri+O2KcIz8/1pFPQM+sOqfe9iMJ9IWHQZLQ+mTOIx35cDLTbifVTaYXqZb0sO097Zr8rDd6V9ziJF25kRFi9eLF/ok3xraJkUinomrSVJChBJYTBixAhhZUVpyQKMNEAkzJCZmupCn7QMCwlB1atXF1GIVYOIfRUBRxUysaKICaRRIa2JJkP+R2ReRuZjtIYNrW9D2qLnz5+ncvj/kWABJ/vAAk72ggWc7AcLONkLFnCyFyzgZC+ypYATui3tc1Yp7h8/MlmyvCuFPlYMnazJkH/N+PHjhX8NhWcmczMKXf0jCzcMwzAMwzCMdiBLjE/zXHqBNjRewCFfFlI3kRMQRS1TDUtMWg5NpWHDhmJjGIZhGIZhGG31wdE4AWfYsGFCwGnatClKlCih5ADOMAzDMAzDMEw2JYkFHLVs374dO3fuFNHMGIZhGIZhGIb5QUhkAUctFC2NQrwxDMMwDMMwDPMDkaT5As4XLfRJ68wsWrRIhHJjGIZhGIZhGObHX+hT63xw2rRpkyqQwLFjx1C8ePFUEcQoxjXDMAzDMAzDMNmMRM0RZP6zgGNlZaW037p1669RHoZhGIZhGIZhvhYJiRpftxkWcNatW/d1S8IwDMMwDMMwzNclLu11cLTaB4dISEjA6dOnsWLFCoSHh4tjvr6+iIiIyMryMQzDMAzDMAyTlRqchDQ2bV4H5+3bt2jUqBG8vLwQGxuL+vXrw8LCAnPmzBH7y5cvz/qSMgzDMAzDMAzz30jQHEEmSzU4tNBnhQoVEBwcDBMTEyW/nDNnzmRl+RiGYRiGYRiGySoSEtLetFmD8++//+LKlStiPRxFXF1d4ePjk1VlYxiGYRiGYRgmC5Elpq3B0dFmAScpKQmJairH29tbmKoxDMMwDMMwDJMNSWATNbU0aNAACxculO/r6OiI4AKTJ09GkyZNvl0DMQzDMAzDMAyTcRLYRE0t8+bNQ8OGDVGsWDHExMSgS5cuePHiBezs7LBt2zZ+xRiGYRiGYRgmO5Kg+RqcLzJRy5UrF+7du4ft27fj/v37QnvTp08fdO3aVSnoAMMwDMMwDMMw2YgEzRdwviiKWmBgIPT19dGtWzf88ssvsLe3x7Nnz3Dz5s2sLyHDMAzDMAzDMFlDguabqGVKwHnw4IGIlObo6IgiRYrg7t27qFixIhYsWICVK1eidu3a2L9//9crLcMwDMMwDMMwX06C5i/0qSOTyWQZTdy4cWOhuRk7diw2bdqEw4cPC1+cVatWifOkzbl16xauXr36NcvMMAzDMAzDMMwXkLijR5rn9Dpu0D4Bh0zRzp49i1KlSgm/G0tLS9y4cQPly5cX558+fYoqVaogJCTka5aZ+QoMOd+P6zWbsLjWKhRc2+57F4P5xIveu4H4Y1wf2QmDxtAd7P69S8F8ImmJB3QGVuH6yCbIll3F3/cGfu9iMJ8YWnpZtquLhM0/p3lOv9umTOW1ZMkS/Pnnn/jw4QNKly6Nf/75B5UqVVKblhQiGzduxMOHD8U+yQ8zZ85USt+zZ09s2KAsZJEy5fjx41/PRC0oKAjOzs7ib3Nzc5iZmcHGxkZ+nv4ODw/PVAEYhmEYhmEYhvlGxCWmvWWCHTt2YOTIkWKZmNu3bwsBh4QRPz8/tenPnz+Pzp0749y5c/Dw8EDu3LnF0jM+Pj5K6Ro1aoT379/Lty+J0JzpIAO05k16+wzDMAzDMAzDZE9ksYlpbplh/vz56NevH3r16iWWjlm+fDlMTU2xdu1atem3bNmCQYMGoUyZMsKXf/Xq1UhKSsKZM2eU0hkZGQmFirQpKlO+WphoUh3RjQlaA2fAgAFCk0PExsZmugAMwzAMwzAMw3wbZPFpCzI0llcdz9O4Xxr7S8TFxQm/+3HjxsmP6erqol69ekI7kxGioqIQHx8PW1vbVJoeCmhGgk2dOnUwY8YMsdbmV9Pg9OjRQ9zQyspKbBQm2sXFRb5P57p3756pAjAMwzAMwzAM822QxSeluc2aNUs+rpc2OqZKQEAAEhMT4eTkpHSc9skfJyOMGTNGyBEkFCmap5GfDml15syZgwsXLoggZ3Svr6bBWbduXaYyZxiGYRiGYRgmGxGXlOYp0siQX40iqtqbrGD27NnYvn270NYYGxvLj3fq1En+d8mSJUVgs/z584t0devW/XomagzDMAzDMAzDaJ6JmrEac7S0Iivr6enh48ePSsdpXwpIlhZ//fWXEHBOnz4tBJj0cHNzE/d6+fJlpgScTAcZYBiGYRiGYRhG80zUMoqhoaEI86wYIEAKGODunnYY/7lz52L69Oki7HOFChU+ex9vb28EBgYiR44cyAyswWEYhmEYhmEYLUGWCUEmPciUjfzzSVChtWwWLlyIyMhIEVWNIL/8nDlzyn14yKdm0qRJ2Lp1K1xdXeW+OrT0DG20xubUqVPRtm1boQXy9PTE6NGjUaBAARF+OjOwgMMwDMMwDMMw2kJ85hz206Jjx47w9/cXQgsJKxT+mTQzUuABLy8vEVlNYtmyZSL6Wrt2youZ0zo6U6ZMESZv9+/fFwt9hoSEiAAEtE4OaXwy6wfEAg7DMAzDMAzDaAmyLNLgEEOGDBGbOigwgCJv3rxJNy8TExOcOHEiS8rFAg7DMAzDMAzDaAlJWSjgZFdYwGEYhmEYhmEYLSGRBRyGYRiGYRiGYTSFpISs8cHJzrAGh2EYhmEYhmG0hCTW4DAMwzAMwzAMoykksYDDMAzDMAzDMIymkBDLJmoMwzAMwzAMw2gISazBYRiGYRiGYRhGU0jKooU+szMcZEBDoMWUateujeDgYFhbW+NH4905X7w59Q5xoXEwz2WOIp3ywyqfpdq0N+fdQ/Dz0FTH7UvYouwvJcTfsWFxeLH3NQIfByMhKgE2Ba1QuFMBmDmZKIVJfL7LEx9v+iMpIQl2xWxRpEsBGFkaQtvpWrQR+pZoAQcTazwNfotpHmtwP+Blmul7FmuKzkUbwMXMHsEx4Tj+5ir+urUFcYnx4ryuji6Glu2AFvl/Enn6RQVj74vzWHJvtzyPX8p2QNN81ZDDzA7xSQl4GPgKC25twz3/F9/kmbM7W7b9izXrzsI/IBxFCrtg4vi2KFUy72evO3L0NkaO3oi6dUpg6d995ccjo2Ixb8EhnD77ACEhUciV0xY/d62Bzh2rydPs2HUFh4/cwqMn3oiMjMWNKzNhaWn61Z7xR2JQjbYYVa8rnC1tcc/nJYbunI8bbx+rTauvq4dxDXuge+XGyGntgGcfvTD2wFKceHxVnsbcyBTTm/VHqzI14GhuizvezzF81wLc9HqiNs9lnUbjfz+1xojdC7Ho3A5oMz8VKIPf6ndD+TyF4WLtgFbLR+PAvYvpXlOzYDnMbzcMxXPkw7vgj5hxbD02XD2ilGZQzbYiX9HG3i/xy455Sm1spG+Iee2GolP5+jDSN8CJJ9cwaNuf8AsPgrbz4Lg37hzyQlRIHOzymqNG70JwKqC+T9835TZ8H4ekOp63rB2ajSst/r6+8xVeXPFDRGAM9PR14eBmgcqd3OBc0CrVddS37xp/E4FvI9BhbkU4uFp8hSf8sUlM0Px1cHS/dwEY5sMNPzzb7Qm3pnlR+fdysMhlhtt/P0RcWJzayik9oBhqzK0i39wnl4eOLuBU3l6cl8lkuLf0EaL9o1FmUHFUmVAOxnZGuL3wPhIV7E6f7/REwP0glOpfFBV+LY3YkFjcW65+gKJNNMlXFeMr9cDiu7vQ6uBoPAl6g7UNJ8DWWH3n1NytOkZV6IrFd3ah0d7hGH9pGZq4VcWv5bvI0/Qv2QqdizQQghKl+fPmZvQt1RLdizWRp3kT6otpV1ej2f6R6HRkAnzC/bAunftqE0eP3casufsxeGAj7Ns1CkUK50Sf/y1HYGB4utd5+wRizrwDqFDeLdW52XP3499LT/HnrG44enAsevxcE9Nn7sGZcw/laaJj4vBT9aIY0K/+V3muH5UO5epiXpuhmHZ0DcrP7on73i9wfMgCOJjbqE0/o/n/0L96KwzdNR/Fp3fBikv7sLffbJTJVUieZlXXcahXtCK6b5iGUjO74dSTazg19G+4WDmkyq9V6ZqonK84fEL8v+pz/iiYGZngns8LDN7+V4bSu9rlwJHB83Du+S2UmdkdC8/uwOpu49CgaGV5mg7l62F+22GYemQ1ys3sgXveL3Bi6EI4WKS08YL2w9G8ZHW0Xz0eNRcMhIuVPfb+bza0nRdXPuLSxheo2M4VHeZUhH1ecxz64y6iQtX36Y1HlUTPldXkW6d5laCjq4P87o7yNNYupkJI6vRXZbSeVg4WDsY4NOMuotWME65sfgkzW6Ov+oyaYKKWlMamKbCAk0lq1aqFX375BcOHD4eNjQ2cnJywatUqREZGolevXrCwsECBAgVw7Ngx+TUXLlxApUqVYGRkhBw5cmDs2LFISEiQn3d1dcXChQuV7lOmTBlMmTJFvq+jo4PVq1ejdevWMDU1RcGCBXHw4EFx7s2bN0J7Q1CZKG3Pnj3xo/D2tA9yVc+BnNWcYe5ihqJdC0LPUBc+Vz6oTW9gZgAjK0P5RloaXUM9OJVPHghE+UUj9HW4yMfK1QJmzqYo2qWgmNV5f8NPpImPToDP5Q8o1N4NtkVsYJnXAsV7FkaoZxhCXoVBm+ldojl2PDuNPS/O4WWINyZdXonohFi0K1RHbfqyjoVxy+8ZDr26BJ8If1zyvYfDry6hlEMBeZpyjoVxxusGznvfFmlIw3PZ5x5K2aekoeuv+D7Au3A/cd9Z1zfAwtAMhW0+r6XQdNZtPI8O7dzRtnVlFMjvjKmT2sPY2BB79l1L85rExCSMGrMZvwxqjNy57FKdv3P3NVq1rIjKlQoiV047dGxfVWiG7j94K0/T8+da6N+3HkqX4jZQZETdzlh95SDWXz2CJx/eYMD2uYiKi0Vv92Zq26JbpUaYdWIDjj3ywOtAXyz/dx+OPrqCkXU7i/PGBkZoW6YWxuxbgn9f3oWnvzemHl2Dl/7eGPhTa6W8SOD5u/1IdFs/BfGJKf2INnP8kQcmHlyB/fcuZCj9gJ/aiHYYtedvPP3wBksu7MbuO+cwom4neRpqm1WXD2C9x6c23jYHUXEx8ja2NDZDn6rNMXL3Ipx7dgu3vZ6h18YZqJa/lBA+tZm7h9+heF0XFK3tAttcZqjVrzD0DXXx5Jyv2vTG5gYwszaSb973g6FvpIsCVVIEnELVnZG7lC2snExgl9sc1bsXRFx0IgLeRijl9fZOIN7dD0K1n1P6FiY1LOAwatmwYQPs7e1x/fp1IewMHDgQ7du3R9WqVXH79m00aNAAP//8M6KiouDj44MmTZqgYsWKuHfvHpYtW4Y1a9ZgxowZma7dqVOnokOHDrh//77Is2vXrggKCkLu3LmxZ88ekebZs2d4//49Fi1a9EO0HpmGhXuFw7ZoilkdzdzYFrFG6Kv0Z6clfC9/gHMFB+gZ6X3KUyb+1TXQVcpTV18HIS+ThZfwt+GQJcpgWzRlNo4EIWNbI4RqsYBjoKuP4nZuuOJ7X35MBpkQPMo6FFZ7zR2/Zyhh5yYXVnJbOKJWrnK48O6OPM1tv2dwz1ESrpY5xH4R27wo71QEF73vpFmOjoXrIyw2Ek+D3kCbiYtPwKPH3qhaJWW2X1dXV+zfuZd23SxZdgJ2tuZo37aK2vNly+TD2XMP8fFjiNB6Xr3+Aq/f+KN61SJf5Tk0BQM9fZTPXRinn96QH6P6o/0qbskmsqqQKVNMvPJMc3R8LKrnLy03YdPX00dMQuo01T6lIWjyamOPSfjr9BY8fv86i59Me3B3K6HUfgSZC7q7lUxp4zzq21hKUz5vERjqGyilefbxLd4Gvod7vuQ02mr65P8qHLlK2ir1v7T/4XnG+tbHZ31RsKoTDIz10rzHo9O+MDTVF9ohCTKHO7fiKeoNKSYEKka7BRz2wfkCSpcujQkTJoi/x40bh9mzZwuBp1+/fuLYpEmThCBDgsihQ4eEALJ48WLRORUpUgS+vr4YM2aMSEcDlYxCWpnOnZNn/GbOnIm///5bCFmNGjWCrW3yx8TR0fGH8sGJi4iHLAkwtFD2ezG0NETkh9R+NqqEvg5DhG8UinVPGfyZOZsIQeXlvtfJ2iAjPaElig2OEz4+RGxYPHT0dWBgqp/qvlIabcTGyEIMtgKiles+MDoE+a1zqr2GNC82xpbY1nS6eMdJONn65ASW398rT7Pi/j6YG5rgRNtFSJQlQU9HF/NvbcPBV/8q5VU7d3ksqDUcJvpGwk+n54lpCI7NmKCrqQQHRwptjJ2dsh057b96/VHtNTdvv8LufVexf/dvaeZLPjwTp+xAjbpToK+vK9puxpSOqFghf5Y/gyZhb24thJGPKn4W5HdRxFm9pot8M0g7cPHlHXgG+KBu4QpoU6aW+B0QEbFRuPLqASY06iW0BR/DgtC5Qn245yshtDgSY+r/jISkRPx9fudXfkrNxtnSTtSxIrRvZWIutGk2phbJbZwqTTCKOLnK84iNj0NotLIGgd4LOqetxIRRny6DqbVyn077wb5Rn73+48swBL2LRJ2BRVOde3MrACcWPkJCXCLMrA3RYkIZmHzymSUB9MzSxyhR3wWO+S0R5hedhU+leSR8mgjWZFjA+QJKlSol/1tPTw92dnYoWTJlxobM1gg/Pz88efIE7u7uYvAgUa1aNURERMDb2xt58uT5ovuamZnB0tJS3CMzxMbGik0RMp37USEzM/OcZkoBCXT1dIWfzqONz3F+pIfwzyEzNLsSNqSOYLKYSs7FMaBUa0zxWC0CAuS1dMaEyr0wOKqdPIgA+fW0cPsJI88vwouQdyhq64rfK/eCX1QQ9r1MMSu5+v4hWuz/DbbGFuhQuB4W1R6JdofGIShGe7VqmSUiMgajx23G9CkdYWuTMrupyqYtF3H3/hssW9wXLjlscfOWJ6b+sQeOjlao6q5eW8d8GcN3L8DKLmPxZNJ2MRAjIYdMn3opmLR13zAVa7r9Dp+Zh5CQmIDb755j281TKJ8nWaNWLndhDK3dQfj8MIymQtobuzxmagMS5Cxug45/VhRC1OMzvjix4CHazawAUytD3D/mjfjoRJRrnSyAMumj4CWhsbCA8wUYGBgo7YtZa4VjkjCTlJQxVR9pcajTUyQ+Pj5D983oPSRmzZolTN0UmTx5MlAL3wVDcwMhgMSFK2tNKMAA+dekBwUM+HjDH/lbpP6gkU+N+8TywtdGlpAkNETXZt2B5Sd1tpGlAWQJMsRHJShpcei+hp+5ryZD2hKaIbY3UY5MY2diDf+o1FFuiOHlOuGA50Xsen5G7D8P9hIamBnVBmDpvT3CxG1MxZ+x4sF+HHl9WZ4mp7kD/leqjZKAQ74+XuEfxHbX/wVOtf0H7QvVFRogbcXGxgx6erqpAgrQvr196kHAu3cB8PEJwsAhq+XHkpKSvy/FSo/E8UPj4ehoiQWLjmDxot6oVTPZX4D8b5489cGa9edYwEmHgIgQIYA4WaSY4BCOFrb4EBaY5jVtVo4Vpmp2ZlbwDfXH7JaD8CrQR57mVYAPai8cBFNDY+HfQXlt6z1dHJcihTma2+Dt9JTfAmkZ/mrzC4bV7gi3SW3Sf5EYOVS3TpbK7Uf7pI2JiY9FQERichunSmMjb2P618jAUGh9FLU4Tum8B9qAsSX16TrCXEwR2lfV6qgSH5OIl5c/olLH1EFRCDJZs3Y2BZwB50JW2DzUA0/O+qJ8a1f4PAzGh+ehWN7lvNI1u8beRKHqTsJsjUkhQfOjRLOA87UpWrSo8I8hAUYSfC5fviyCEeTKlUvsOzg4CL8ZibCwMLx+nTn7akPD5A9HYmL6by2Z1I0cOTKVBudXjyH4Hujq68IijwWCnoTAscynKGhJMgQ9DUHu2i7pXvvxVnJ4Z+fKKY6IqhiYJAsvkR+jEfY2HAVaJpuQWOS1gI6eDoKeBsOpXHJwgsgPUYgJioWVm/ZG7aLwzI8CX8HdpSROeyXblutAB1VdSmLTk5TAGYqY6BsiSUVATyK7w09COL37xvpGkH06JkGmaroKmk110HlDPWXBXtswNNBH8WK54HHtBerVTdbi0sSGx7Xn6Nb5p1Tp3fI54dC+MUrHFv5zRIR5/n1sGzjnsEZcbALiExLFQEQRPT0d8ftj0oYc+2+9eybMzA7cvyh/z2mfnNXTIzYhTgg3ZAbapmxt7LqdPCmgCDmy02ZtYoGGRStjzP4l4vim68dS+Y0cH7IQm68fwzoP5fDGTPp4vHqIJiXclY7VL1oJHq8epLSxF7VxRXm46eQ2rojF53eJ/VtvnyIuIR51i1TE3jvnxLFCTnmQ1y4HPF4n56ONSCGcvR8Gw61Sct9K3xTaL9lIvZmzxMurfkhMkKHwT84Zuhf1LRQ8iPipdyERNloiMjgWh/64h4bDi8OpoPb26WnBGhzmPzNo0CARIY2CEQwZMkQEASCNCQkZkv9NnTp1sH79ejRv3lz4z5BvDpm+ZYa8efOKD/Dhw4dFAAITExOYm6c2TyFhJruZpOWtlxOP1j+Dpas5LF0t4XXGG4lxSXCpmvyRe7juKYysjVCwdb5U5mkOZeyFFkid8GNgbiB8cSJ8IvFsp6cQoGitG0nwoahtz3e9ElHZ9I318HS7pxBurLVYwCHWPjyEuT8NwcMAT9z3f4mexZsKjcye58md+Nwav+BjZCDm3doq9s++u4XexZvhceBruYkaaXXOet2UCzrn3t3EwNJt4RsRIEzUitnlE9fsfpGcJ+VP58963RC+N+TT061oIziZ2uLY6yvQdnp1r4Uxv29FieK5UapEHmzYfAHR0XFo0yo5rC2ZpDk5WuHXEc1hZGSAQgWTgzlIWFokr/8kHSehqVKF/Phz3kEYGxnAxcUWN26+xP6DNzH2t5by6/wDwhAQEAYvrwCx//zFe5iZUTRIG1hbmUFbWXBmG9Z3n4ibXk9x/c0jDK/TCWZGxlh39bA4v777JPiG+GP8wWViv5JrMeS0csBd7xdiHZzJTfsK4X3uqc3yPClEMX3DyVG9gEMuzG09BE8/vsU6j+Q8gyLDxKYIDcQ/hAXhuZ8XtD1MNNWZRD47F5TOVVDUF61xM7PlQFHvPTZME+eX/7sXQ2q1w5zWQ7D2yiHUKVxBhP5uuuRXeR7zz2zDhh7Uxk9w/c1jDK/TMbmNPwmTYTGRWHPlEOa3HYqgyFCx/0+HX3HF8z6uvX4EbaZMs9w4s+QJHN0s4FjAEveOvkNCbCKK1kqetDy9+LEI4+zeRdnfj7Qx+Sraw9jCIJVm5+beN8hXwR6mNoaICY/Hg+M+iAyKk4eStrA3VrpGClBg6WwCczvlcwxYwGH+Ozlz5sTRo0fx22+/ieAEFAygT58+8iAFklaFNDbNmjWDlZUVpk+fnmkNDt2HTM8oBDWFq+7evbsQmn4EnCs6imADngffigU6LXKZo9zQEvIFN0mrApWZftK2UES0csPUR6uJDY3Ds12eiAuLF6ZuOao4wa2psr9ToQ75ST0h1r4hTZB9MRsU6VIQ2s7R11fE2jPDynUSi3LSOjh9Tv6BwJjkwAO0mKeiNmbp3d1iJm1E+U5CICF/GRJ65n8SgAha/2Z4+U6YUrUf7IwthRCz/dkpLL67W67NyW+VE63r1BT3JlO5B/6e6Hx0oggZre00aVwOQcGR+HvxMSF0FC2SE6uX/w/29smBB96/D4auijbmc8z/qwfmLzyMUWM3IzQ0Ci4uNhgxtInSQp/bd1zG4mUn5Ptde/wj/p01o7NcuNJGdt4+I9ZDmdqsL5wt7HDX5wUaLxkBv/BgcT6PjZNcuCdIgzm9+f/gZu+CiNhoHH3kIXxuFE2byNRpZosByGXtiKCoMOy9ex6/H1wuTEaZ9KmQpyjOj1yqtD4NIfycNk5HDit75LFN0Qq8CXwvhJkF7YdhWO0O8A7xQ9/Ns3DySUrY9Z23TsPB3BrTmvUTQQNIOG30D7VxSuCBEbsWinbe03+WMD888fgaBm2fq/XNRRHQosPicW3nK2GaZu9qgWbjS8tN1MIDYlS7dAT7RuL901AROEAVMmMP8Y3C8XkPEB0eLwQgCiTQemo5ETKayTwJWuCDoyNTdf5gtJIh55MjwDHfn8W1VqHg2nbfuxjMJ1703g3EqzfPY74TBo2hO1jZxIj5fiQt8YDOQPXhyJlvj2zZVfx9byBXfTZhaOlkTW524rRj2oFk6vk9gybAQQYYhmEYhmEYRktI0AINDq+ExDAMwzAMwzBaJOAkpLFlliVLlsDV1RXGxsaoXLmyWJ8xPXbt2iXWhKT0tMQKuXEoQoZl5IueI0cO4U9er149vHjxItPlYgGHYRiGYRiGYbSErBJwduzYIYJmUfCs27dvC1/zhg0bprlG45UrV8SC9eSLfufOHbRq1UpsDx8+lKeZO3euWMh++fLluHbtmlj3kfKMiYnJVNlYwGEYhmEYhmEYLVoHJyGNLTPMnz8f/fr1E8GtihUrJoQSU1NTrF27Vm36RYsWoVGjRiLwFi2jQkG1ypUrh8WLF8u1NxR5mAJxtWzZUixwv3HjRvj6+mL//v2ZKhsLOAzDMAzDMAyjJSSko8GJjY0V6zEqbnRMlbi4ONy6dUuYkEnQ8ie07+Hhofa+dFwxPUHaGSk9RRD+8OGDUhqKLkymb2nlmRYcZIBhGIZhGIZhtIROiWlHSpsyZYpYdkQRMkGj44oEBASIxeWdnJyUjtP+06dP1eZNwou69HRcOi8dSytNRmEBh2EYhmEYhmEY0NqM5FejSHZbID4jsIDDMAzDMAzDMAxImMmIQGNvbw89PT18/PhR6TjtOzunLKyrCB1PL730Lx2jKGqKacqUSb0IbHqwDw7DMAzDMAzDMBnG0NAQ5cuXx5kzZ+THkpKSxL67u/qFmOm4Ynri1KlT8vT58uUTQo5iGvIBomhqaeWZFqzBYRiGYRiGYRgmU5ApW48ePVChQgVUqlRJRECLjIwUUdWI7t27I2fOnJg1a5bYHzZsGGrWrIl58+ahadOm2L59O27evImVK1eK8zo6Ohg+fDhmzJiBggULCoFn4sSJcHFxEeGkMwMLOAzDMAzDMAzDZIqOHTvC399fLMxJQQDIjOz48ePyIAFeXl4isppE1apVsXXrVhEGevz48UKIofDPJUqUkKcZPXq0EJL69++PkJAQVK9eXeRJC4NmBhZwGIZhGIZhGIbJNEOGDBGbOs6fP5/qWPv27cWWFqTFmTZtmtj+C+yDwzAMwzAMwzCMxsACDsMwDMMwDMMwGgMLOAzDMAzDMAzDaAws4DAMwzAMwzAMozGwgMMwDMMwDMMwjMbAAg7DMAzDMAzDMBoDCzgMwzAMwzAMw2gMLOAwDMMwDMMwDKMxsIDDMAzDMAzDMIzGoCOTyWTfuxAM81+JjY3FrFmzMG7cOBgZGXGFfme4PbIf3CbZC26P7AW3R/aC24P5r7CAw2gEYWFhsLKyQmhoKCwtLb93cbQebo/sB7dJ9oLbI3vB7ZG94PZg/itsosYwDMMwDMMwjMbAAg7DMAzDMAzDMBoDCzgMwzAMwzAMw2gMLOAwGgEFFpg8eTIHGMgmcHtkP7hNshfcHtkLbo/sBbcH81/hIAMMwzAMwzAMw2gMrMFhGIZhGIZhGEZjYAGHYRiGYRiGYRiNgQUchmEYhmG0Gh0dHezfvz/L0zLfFsW2efPmjdi/e/cuN4MWwgIOo1H8/PPPmDlzZpblFxcXB1dXV9y8eTPL8tQU/P39MXDgQOTJk0c4hDo7O6Nhw4a4fPmyPM2VK1fQpEkT2NjYwNjYGCVLlsT8+fORmJj4XcuuiVBHnt42ZcoUeYcvbXZ2dmjQoAHu3LnzvYuv0fTs2ROtWrWS73/48AG//PIL3NzcxG8nd+7caN68Oc6cOfNdy5md6kt6Rw0NDVGgQAFMmzYNCQkJX+2e79+/R+PGjbM8rba2m4GBAfLly4fRo0cjJibmexeN0UJYwGG+y4fPyckJ9evXx9q1a5GUlKT2Ghos6+np4caNGxm6x71793D06FEMHTpUfqxWrVpqB3tSR7l3714xwKOBnrqZHupcR40ahTFjxvyn59dE2rZtKwbGGzZswPPnz3Hw4EFR34GBgeL8vn37ULNmTeTKlQvnzp3D06dPMWzYMMyYMQOdOnWCTCb73o+gUdCgS9oWLlwIS0tLpWP0HkucPn1aHDtx4gQiIiLEYC0kJOS7ll9bICGzfPnyOHv2LP788088ePAAx48fR+3atTF48ODvXbxsQ6NGjcQ7+uLFC/z6669CQKf6UjcJlRXQBA0Jm1mdVlvb7dWrV1iwYAFWrFghIpwyzDdHxjDfgB49esgaNWoke//+vczb21t269Yt2R9//CEzNzeXNW7cWBYfH6+U/u3bt+Lc0KFDZQMGDMjQPfr06SP73//+p3SsZs2asn79+on7Km4SGzdulE2dOlW2atUqGm3L7ty5kyrfoKAgmaGhoezhw4df/PyaRnBwsKiv8+fPqz0fEREhs7Ozk7Vp0ybVuYMHD4prt2/f/g1Kqp2sW7dOZmVller469evU73nly9fFseOHz/+jUupXd+/li1bir/pe5czZ07xG1H3u2KU60uifv36sipVqsjPzZgxQ5YjRw6Zq6urOO/l5SVr3769eO9tbGxkLVq0EO+7ImvWrJEVK1ZMfM+dnZ1lgwcPlp+j38C+ffvE37GxseIcpTEyMpLlyZNHNnPmTLVpifv378tq164tMzY2ltna2oo+Jzw8PFX7//nnnyJPSjNo0CBZXFycxrcb9QFly5YVfycmJop6pDajuipVqpRs165dSumpn23atKnMwsJCjAGqV68ue/nypTh3/fp1Wb169UTfYmlpKatRo4YYSyii2DbqvneM9sAaHOabIZkx5cyZE+XKlcP48eNx4MABHDt2DOvXr1dKu27dOjRr1kyYQG3btg3R0dHp5k0mT7t37xZmHqqYmpqK+ypuiiZtkyZNQr169dLMm8yrqlWrhu3bt3/Rc2si5ubmYiNb59jY2FTnT548KTQ5iloDCWqjQoUKiXZlvj8mJiZZOhPOpE1QUJDQ1pCmxszMLNV5a2trrr503lPpHSVTvmfPnuHUqVM4fPgw4uPjhcbfwsIC//77rzCTpe8TaROka5YtWybqvX///kJrRhpnMn1Tx99//y3O79y5U9xny5YtwlRZHZGRkeLe1E+QtcGuXbuEhnTIkCFK6UiL7enpKf4lrTf1ear9nqbx8OFDYaZMlhDErFmzsHHjRixfvhyPHj3CiBEj0K1bN1y4cEGc9/HxQY0aNcRYgTSct27dQu/eveUWF+Hh4ejRowcuXbqEq1evomDBgsIEmo4zjCr6qY4wzDekTp06KF26tDAV69u3rzhGkzAk4CxZsgRFihQRnRAJLySMpMX9+/cRGhqKChUqfJVyVqpUSXScTDL6+vqic+7Xr5/orEhgJXM0Mj0rVaqUMFkjihYtqrbKqF2lNMz3g8zSpk+fLgaD9I4zX5eXL1+K7xu9/0zGoPoigYbMKclviXz/SDhcvXq1fOC8efNmYepMx8jUmKA+hATG8+fPCzNkMo0lUzcyk5WoWLGi2nt6eXmJwXP16tVFfnnz5k2zfFu3bhU+JjRwl4TWxYsXi4mcOXPmCHNsggQgOk5m19T+TZs2Fc9F31BNggRO+p6QUEKTX7q6uuK56W/yjyXhz93dXaQlHzQSVsiMjfoP6vOtrKzEZCKZshM0GaY4XlBk5cqVoo1JQKIJUYZRhDU4zHeHPvZkly5BH8CoqCgxK0bQDM+aNWvSzePt27ei43B0dEx1bunSpXKNA23UyWUWFxcXcQ9G2QfH19dXzHTSTCkNJEjQUZyVZD+b7EnVqlXFb4EGXeS7tmPHDvlAjPl68O8h8wNlCk5CPmIdO3YUfjgEBSuRhBuC3mESHkmDI33nbW1theBBWhM/Pz/xrapbt26GfUbJH7Nw4cLCp5M00mnx5MkTMUmnqJEjjT8JXKT9kShevLjooyRy5MghyqVpkC8Z1d21a9eEtqVXr16ir6D2oX6dfG8V+2MSDKmNCLrup59+kgs3qnz8+FEIhCR8kiBEfobkQ0gCKcOowhocJlt0+tKsG0GBB6gzIy0B0blzZ/z222/iI5g/f361eZAJG6m1FfOR6Nq1K37//ff/ZAZC5hH0cWaUocEHdVi0TZw4UWjhyKGUnNylzp8G06rQ8WLFinF1fidIoKH6p+AabBb17aCBGX2jKOAG8/mBMpmVkSBDE0xSf0ComvfRIJcCN5ApmSoODg5Ci5AZaKLm9evXwnyaJtw6dOggzJjJkuBLUR2003uQVoCdHxlqG8n0j/pyEv5ogrJEiRLi2JEjR4SZuiJSwAbJXDYtSGAi0+dFixYJrRpdR9ogNq9l1MEaHOa7Q4NdCicp2ahT9C3SulCHRht9DEndTR/LtLC3txcCiLoPHc300AdX2ihtZqFyUUfJpA8NmskmnUxCaAZ13rx5qdKQxociI5HgynwfKCwxTRawcPNtod8EaabJFId+J6pwJLvUA2UKQ68o3KQlkNA3hTT4it962uj7T5od8qHJTBhu0g7QRNuqVavEhMCePXtEP6AKmeGSBkmxPckHiIQq0gBpM1QH5Gs7YcIE0TeQQELaFtU2ou8RQebNZApOPlXqoHoljRr53ZBGjPILCAj4xk/F/CiwgMN8V8iRkBw+SYVN0AwchRWmDoPU1dJGA2UyfUpr/ZQyZcqIfx8/fvzVnCXLli37VfL+EaFZNLKHJtt38n+i2U5yrp07dy5atmwpBidkV01BJMipl9KQGSLN5JH5R7t27cSsKMNoGyTc0HeMfJ5o0EwDc5rkIcd2yTeByRykpaeJK/r20ACZvkdkMkuDYW9vb5GGzNuoH6F6pjq/ffs2/vnnH7X50VpdFASFNG3kK0jfNgpOo25CgO5NmmzSLlA/QUEEyFeIfEbZ7BNo3769MM2j/oCCzlBgAQqyQBYZUhvQPkGBGcLCwoQvJ609R+20adMmuakfaUBpn34vZAJHdf85rQ+jvbCJGvPNICdDWuCOOneypaVoQhRVhZwDu3fvLtLQAJgGv5I6W4JmeMaNGyeuIedMVUi7QrN45LAoCTsZgWbkaEaJ7LMJ6UOqGm2NOk1yxmaSIdvpypUri3UOqKOiGTdqI7KPphk7gtqROvs//vhD2FWTPTx1UGQuOHz4cLXmhAyj6ZBjNQ3s6HdB/oC0Zgh9v8jEikyymMxDkTIvXrwo1itr06aNiKpFmn/yuSFNDEECCH2D6JtFA20SiOgbpQ7S+NBkDQ2waXBOwQhojTV1pm50bwqAQMELKB3t04QdCUlMckAaElyoPknwpHed+n1aJ4cERimiKkEmszTpSSbpFHSA6p76c/JpksYHNGFG11B/Q0EL1EXqZBhCh2JFc1UwXxuatZdmaeiDR87NZJvbpUsX0fFQx0EhISkK2vXr19VGtyG1NM2UUcQ1ddDggBwWPTw85Mdo4Un6QEo+IaqQVoicIFUhPxLJoZXyo3uTEMSzRQzDMAzDMNkbFnAYjYECDZDNM9lLZ6WpB9lhkzAmzTIxDMMwDMMw2Rf2wWE0BtKukAYnK50OKWgBhSQlu2GGYRiGYRgm+8MaHIZhGIZhGIZhNAbW4DAMwzAMwzAMozGwgMMwDMMwDMMwjMbAAg7DMAzDMAzDMBoDCzgMwzAMwzAMw2gMLOAwDMMwDMMwDKMxsIDDMAzDaCQ6OjrYv3//9y4GwzAM841hAYdhGIb5avTs2VMIGgMGDEh1bvDgweIcpckI58+fF+lDQkIylP79+/do3LhxpsvMMAzD/NiwgMMwDMN8VXLnzo3t27cjOjpafiwmJgZbt25Fnjx5svx+tEAv4ezsDCMjoyzPn2EYhsnesIDDMAzDfFXKlSsnhJy9e/fKj9HfJNyULVtWfiwpKQmzZs1Cvnz5YGJigtKlS2P37t3i3Js3b1C7dm3xt42NjZLmp1atWhgyZAiGDx8Oe3t7NGzYUK2Jmre3Nzp37gxbW1uYmZmhQoUKuHbtGrc+wzCMhqH/vQvAMAzDaD69e/fGunXr0LVrV7G/du1a9OrVS5idSZBws3nzZixfvhwFCxbExYsX0a1bNzg4OKB69erYs2cP2rZti2fPnsHS0lIIQRIbNmzAwIEDcfnyZbX3j4iIQM2aNZEzZ04cPHhQaHdu374thCqGYRhGs2ABh2EYhvnqkKAybtw4vH37VuyTIEJma5KAExsbi5kzZ+L06dNwd3cXx9zc3HDp0iWsWLFCCCekeSEcHR1hbW2tlD8JRHPnzk3z/mQO5+/vjxs3bsjzKVCgwFd7XoZhGOb7wQIOwzAM89UhLUzTpk2xfv16yGQy8TeZk0m8fPkSUVFRqF+/fip/GkUztrQoX758uufv3r0r8pGEG4ZhGEZzYQGHYRiG+WZmauQrQyxZsiSVCRlx5MgRYUamSEYCBZBPTXoomrMxDMMwmg0LOAzDMMw3oVGjRkIjQ87/UiAAiWLFiglBxsvLS5ijqcPQ0FD8m5iYmOl7lypVCqtXr0ZQUBBrcRiGYTQcjqLGMAzDfBP09PTw5MkTPH78WPytiIWFBUaNGoURI0aIgAGenp4iCMA///wj9om8efMK4ejw4cPCn0bS+mQEip5GgQVatWol/H9evXolghZ4eHhk+XMyDMMw3xcWcBiGYZhvBkU/o00d06dPx8SJE0U0taJFiwqND5msUdhogkzXpk6dirFjx8LJyUlu7pYRSPtz8uRJEaCgSZMmKFmyJGbPnp1K0GIYhmF+fHRk5O3JMAzDMAzDMAyjAbAGh2EYhmEYhmEYjYEFHIZhGIZhGIZhNAYWcBiGYRiGYRiG0RhYwGEYhmEYhmEYRmNgAYdhGIZhGIZhGI2BBRyGYRiGYRiGYTQGFnAYhmEYhmEYhtEYWMBhGIZhGIZhGEZjYAGHYRiGYRiGYRiNgQUchmEYhmEYhmE0BhZwGIZhGIZhGIbRGFjAYRiGYRiGYRgGmsL/Ad5e5Q+Sx/0eAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -248,7 +248,7 @@ }, { "cell_type": "markdown", - "id": "d73a293f", + "id": "2a726be7", "metadata": {}, "source": [ "### 1.4 Metric Explanations\n", @@ -260,14 +260,14 @@ "| **TP** | Temporal Precision | Are bout boundaries (start/end frames) accurate? |\n", "| **IC** | Intra-bout Continuity | Is each detected bout clean, without label fragmentation? |\n", "\n", - "**Absent-behavior rule (v0.2.0):**\n", + "**Absent-behavior rule (since v0.2.0; per-file scoring, the default):**\n", "- GT has no bouts **and** machine predicts nothing → all metrics = **1.0** (correct absence) \n", "- GT has no bouts **but** machine predicts bouts → all metrics = **0.0** (false detection)" ] }, { "cell_type": "markdown", - "id": "723dc4da", + "id": "ccdbc5e0", "metadata": {}, "source": [ "---\n", @@ -276,7 +276,7 @@ }, { "cell_type": "markdown", - "id": "129853f9", + "id": "5a0759b7", "metadata": {}, "source": [ "### 2a. Frame-based F1: our implementation\n", @@ -291,7 +291,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "cc4eee5e", + "id": "01bf9d01", "metadata": {}, "outputs": [ { @@ -334,7 +334,7 @@ }, { "cell_type": "markdown", - "id": "6b969b09", + "id": "ecbcba3b", "metadata": {}, "source": [ "### 2b. BANOS metrics: same recordings, same behaviors" @@ -342,18 +342,18 @@ }, { "cell_type": "code", - "execution_count": 9, - "id": "f1ba3a31", + "execution_count": 8, + "id": "85f77926", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "BANOS DA (F1): 0.774\n", + "BANOS DA (F1): 0.749\n", "BANOS SO: 0.698\n", "BANOS TP: 0.338\n", - "BANOS IC: 0.995\n" + "BANOS IC: 0.992\n" ] } ], @@ -373,7 +373,7 @@ }, { "cell_type": "markdown", - "id": "88c1634e", + "id": "c07f61bc", "metadata": {}, "source": [ "### 2c. Side-by-side comparison" @@ -381,8 +381,8 @@ }, { "cell_type": "code", - "execution_count": 10, - "id": "3415f1b7", + "execution_count": 9, + "id": "363063a5", "metadata": {}, "outputs": [ { @@ -392,10 +392,10 @@ "Metric Value What it captures\n", "----------------------------------------------------------------------\n", "Frame-based F1 0.815 Frame overlap; correct absence = 1\n", - "BANOS DA (F1) 0.774 Bout-level detection (did system find right bouts?)\n", + "BANOS DA (F1) 0.749 Bout-level detection (did system find right bouts?)\n", "BANOS SO 0.698 Temporal quality of overlapping bouts\n", "BANOS TP 0.338 Boundary precision\n", - "BANOS IC 0.995 Label stability within detected bouts\n" + "BANOS IC 0.992 Label stability within detected bouts\n" ] } ], @@ -417,7 +417,7 @@ }, { "cell_type": "markdown", - "id": "b52dbd23", + "id": "e501f21e", "metadata": {}, "source": [ "### 2d. Why our approach differs from CalMS21 — and why intentionally\n", @@ -433,7 +433,9 @@ "\n", "2. **Explicit absent-behavior scoring**: we explicitly reward correct absence (score 1) and penalize false detection (score 0). CalMS21’s pooled approach implicitly handles this by having behaviors present in at least some videos.\n", "\n", - "We do **not** align with CalMS21’s pooled approach because BANOS’s purpose is to quantify annotation quality **per recording** also, not globally only. Collapsing all recordings into one global pool hides per-recording variability that is central to BANOS’s value.\n", + "By default, we do **not** align with CalMS21’s pooled approach because BANOS’s purpose is to quantify annotation quality **per recording** also, not globally only. Collapsing all recordings into one global pool hides per-recording variability that is central to BANOS’s value.\n", + "\n", + "To compare or rank models on a whole dataset, `banos.score(data, aggregation=\"pooled\")` scores each behavior once on the bout counts and terms of all recordings, without the absent-behavior rule. The per-file mean, by contrast, mixes detection quality with the share of recordings without the behavior (see the README).\n", "\n", "> For direct comparison with CalMS21 leaderboard numbers, use the CalMS21 dataset with their exact metric. \n", "> For publications, state both the frame F1 methodology and the BANOS metrics explicitly." @@ -441,7 +443,7 @@ }, { "cell_type": "markdown", - "id": "4a840d3e", + "id": "199e3622", "metadata": {}, "source": [ "### 2e. Thought experiment — the shifted annotator\n", @@ -451,8 +453,8 @@ }, { "cell_type": "code", - "execution_count": 11, - "id": "534a6e39", + "execution_count": 10, + "id": "e4a47340", "metadata": {}, "outputs": [ { @@ -460,7 +462,7 @@ "output_type": "stream", "text": [ "Pred A (identical): Frame F1=1.00 BANOS DA=1.000 SO=1.000 TP=1.000\n", - "Pred B (shift 15f): Frame F1=0.00 BANOS DA=NaN SO=NaN TP=NaN\n", + "Pred B (shift 15f): Frame F1=0.00 BANOS DA=0.000 SO=NaN TP=NaN\n", "Pred C (shift 2f): Frame F1=0.80 BANOS DA=1.000 SO=0.667 TP=0.200\n" ] } @@ -515,23 +517,23 @@ }, { "cell_type": "markdown", - "id": "6f8ac5d5", + "id": "3bd70b20", "metadata": {}, "source": [ "**Interpretation:**\n", - "- **Frame F1** tells you how many frames overlap — a 15-frame shift collapses the score to 0.\n", - "- **BANOS DA** tells you whether the system found the right *bouts* (count-level matching). Both shifted predictions still detect 2 bouts matching 2 GT bouts, so DA=1.0.\n", - "- **BANOS TP** quantifies the temporal shift magnitude — a small shift gives high TP (~1.0), a large shift gives low TP.\n", - "- **BANOS SO** measures fractional temporal IoU of matched bouts.\n", + "- **Frame F1** tells you how many frames overlap — a 15-frame shift (B) collapses the score to 0, a 2-frame shift (C) gives 0.80.\n", + "- **BANOS DA** tells you whether the system found the right *bouts*: a predicted bout detects a GT bout when they share at least one frame. C detects both GT bouts, so DA = 1.0. B shares no frame with them: both GT bouts are missed and both predicted bouts are false, so DA = 0, and SO and TP are undefined (NaN), having no overlapping pair (with `missed_bouts=\"zero\"`, they would be 0).\n", + "- **BANOS TP** quantifies the boundary error, 1 / (1 + |Δstart| + |Δend|): C, 2 frames late at both ends, gets 1 / 5 = 0.2. It falls quickly with the shift.\n", + "- **BANOS SO** measures the temporal IoU of overlapping bouts: for C, 8 shared frames out of 12, 0.667.\n", "\n", - "Frame F1 says “shifted by 15 frames is equally wrong as completely missing”. \n", - "BANOS tells you *how* the system is wrong — and by how much." + "Frame F1 gives C a single 0.80. \n", + "BANOS tells you *how* the system is wrong — every bout found (DA), but late (SO, TP)." ] } ], "metadata": { "kernelspec": { - "display_name": "banos", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -545,7 +547,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.9" + "version": "3.12.12" } }, "nbformat": 4, diff --git a/scripts/create_tutorial_notebook.py b/scripts/create_tutorial_notebook.py index f59b498..cce8050 100644 --- a/scripts/create_tutorial_notebook.py +++ b/scripts/create_tutorial_notebook.py @@ -29,11 +29,11 @@ nbformat.v4.new_markdown_cell("---\n## Section 1 — Classic BANOS Usage"), nbformat.v4.new_markdown_cell("### 1.1 Setup"), nbformat.v4.new_code_cell( - "from pathlib import Path\n" - "import math\n\n" + "import math\n" + "from pathlib import Path\n\n" + "import matplotlib.pyplot as plt\n" "import numpy as np\n" "import pandas as pd\n" - "import matplotlib.pyplot as plt\n" "import seaborn as sns\n\n" "import banos\n\n" 'print(f"banos version: {banos.__version__}")\n\n' @@ -43,7 +43,7 @@ nbformat.v4.new_markdown_cell( "### 1.2 Single Recording\n\n" "Recording_1, behaviors `attack`/`investigation`/`mount` (dropping `other`).\n\n" - "> **Note (v0.2.0 absent-behavior rule):** \n" + "> **Note (absent-behavior rule, since v0.2.0; per-file scoring, the default):** \n" "> - GT has no bouts AND machine predicts nothing \u2192 all metrics = **1.0** (correct absence) \n" "> - GT has no bouts BUT machine predicts bouts \u2192 all metrics = **0.0** (false detection)" ), @@ -117,15 +117,12 @@ "| **SO** | Segment Overlap | Temporal IoU \u2014 how well do detected bouts overlap GT? |\n" "| **TP** | Temporal Precision | Are bout boundaries (start/end frames) accurate? |\n" "| **IC** | Intra-bout Continuity | Is each detected bout clean, without label fragmentation? |\n\n" - "**Absent-behavior rule (v0.2.0):**\n" + "**Absent-behavior rule (since v0.2.0; per-file scoring, the default):**\n" "- GT has no bouts **and** machine predicts nothing \u2192 all metrics = **1.0** (correct absence) \n" "- GT has no bouts **but** machine predicts bouts \u2192 all metrics = **0.0** (false detection)" ), # ── Section 2 ────────────────────────────────────────────────────────── - nbformat.v4.new_markdown_cell( - "---\n" - "## Section 2 \u2014 Frame-based F1 vs BANOS: our approach and why it differs from CalMS21" - ), + nbformat.v4.new_markdown_cell("---\n## Section 2 \u2014 Frame-based F1 vs BANOS"), nbformat.v4.new_markdown_cell( "### 2a. Frame-based F1: our implementation\n\n" "We compute per-recording frame F1 for each behavior, macro-average across behaviors, " @@ -157,7 +154,7 @@ " f1s = [frame_f1(p[b].values, g[b].values) for b in BEHAVIORS]\n" " per_rec_f1.append(np.nanmean(f1s))\n\n" "frame_f1_score = np.nanmean(per_rec_f1)\n" - 'print(f"Frame-based F1 (our approach): {frame_f1_score:.3f}") # Expected: ~0.791' + 'print(f"Frame-based F1: {frame_f1_score:.3f}")' ), nbformat.v4.new_markdown_cell("### 2b. BANOS metrics: same recordings, same behaviors"), nbformat.v4.new_code_cell( @@ -194,12 +191,15 @@ "- Macro-average across 3 behaviors (attack/investigation/mount)\n\n" "**Our approach differs in two ways:**\n\n" "1. **Per-recording then averaged**: we treat each recording equally; CalMS21 weights by recording length. " - "Per-recording aggregation preserves recording-level variability \u2014 which is scientifically meaningful for ethological studies.\n\n" + "Per-recording aggregation preserves recording-level variability.\n\n" "2. **Explicit absent-behavior scoring**: we explicitly reward correct absence (score 1) and penalize false detection (score 0). " "CalMS21\u2019s pooled approach implicitly handles this by having behaviors present in at least some videos.\n\n" - "We do **not** align with CalMS21\u2019s pooled approach because BANOS\u2019s purpose is to quantify annotation quality " - "**per recording**, not globally. Collapsing all recordings into one global pool hides per-recording variability " + "By default, we do **not** align with CalMS21\u2019s pooled approach because BANOS\u2019s purpose is to quantify annotation quality " + "**per recording** also, not globally only. Collapsing all recordings into one global pool hides per-recording variability " "that is central to BANOS\u2019s value.\n\n" + 'To compare or rank models on a whole dataset, `banos.score(data, aggregation="pooled")` scores each behavior once ' + "on the bout counts and terms of all recordings, without the absent-behavior rule. The per-file mean, by contrast, " + "mixes detection quality with the share of recordings without the behavior (see the README).\n\n" "> For direct comparison with CalMS21 leaderboard numbers, use the CalMS21 dataset with their exact metric. \n" "> For publications, state both the frame F1 methodology and the BANOS metrics explicitly." ), @@ -246,14 +246,17 @@ ), nbformat.v4.new_markdown_cell( "**Interpretation:**\n" - "- **Frame F1** tells you how many frames overlap \u2014 a 15-frame shift collapses the score to 0.\n" - "- **BANOS DA** tells you whether the system found the right *bouts* (count-level matching). " - "Both shifted predictions still detect 2 bouts matching 2 GT bouts, so DA=1.0.\n" - "- **BANOS TP** quantifies the temporal shift magnitude \u2014 a small shift gives high TP (~1.0), " - "a large shift gives low TP.\n" - "- **BANOS SO** measures fractional temporal IoU of matched bouts.\n\n" - "Frame F1 says \u201cshifted by 15 frames is equally wrong as completely missing\u201d. \n" - "BANOS tells you *how* the system is wrong \u2014 and by how much." + "- **Frame F1** tells you how many frames overlap \u2014 a 15-frame shift (B) collapses the score to 0, " + "a 2-frame shift (C) gives 0.80.\n" + "- **BANOS DA** tells you whether the system found the right *bouts*: a predicted bout detects a GT bout " + "when they share at least one frame. C detects both GT bouts, so DA = 1.0. B shares no frame with them: " + "both GT bouts are missed and both predicted bouts are false, so DA = 0, and SO and TP are undefined (NaN), " + 'having no overlapping pair (with `missed_bouts="zero"`, they would be 0).\n' + "- **BANOS TP** quantifies the boundary error, 1 / (1 + |\u0394start| + |\u0394end|): C, 2 frames late at both ends, " + "gets 1 / 5 = 0.2. It falls quickly with the shift.\n" + "- **BANOS SO** measures the temporal IoU of overlapping bouts: for C, 8 shared frames out of 12, 0.667.\n\n" + "Frame F1 gives C a single 0.80. \n" + "BANOS tells you *how* the system is wrong \u2014 every bout found (DA), but late (SO, TP)." ), ] From 52df89e7ac4cefb386cca19f22491fa50068db97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 00:29:01 +0100 Subject: [PATCH 20/25] chore: set the version to 1.1.0 pyproject.toml, banos.__version__, the banos entry of uv.lock and the version line of the MATLAB Contents.m. Its date, required by the format of that line, is the date of this change; it is set again, with the changelog entry, on release day. The tutorial notebook prints the new version. --- demo/tutorial_python.ipynb | 2 +- matlab/BANOS/Contents.m | 2 +- pyproject.toml | 2 +- src/banos/__init__.py | 2 +- uv.lock | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/demo/tutorial_python.ipynb b/demo/tutorial_python.ipynb index f08d7f9..8445d01 100644 --- a/demo/tutorial_python.ipynb +++ b/demo/tutorial_python.ipynb @@ -43,7 +43,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "banos version: 1.0.0\n" + "banos version: 1.1.0\n" ] } ], diff --git a/matlab/BANOS/Contents.m b/matlab/BANOS/Contents.m index 761285e..7cf7830 100644 --- a/matlab/BANOS/Contents.m +++ b/matlab/BANOS/Contents.m @@ -1,5 +1,5 @@ % BANOS Behavior ANnOtation Score toolbox. -% Version 0.2.0 23-Jan-2025 +% Version 1.1.0 30-Oct-2026 % % High-level entry point (mirrors Python banos.score): % BANOS_score - Score one or multiple annotation pairs. diff --git a/pyproject.toml b/pyproject.toml index 317677b..479746e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "banos" -version = "1.0.0" +version = "1.1.0" description = "Behavior ANnOtation Score: ethological metrics for evaluating behavior annotation quality" readme = "README.md" license = "MIT" diff --git a/src/banos/__init__.py b/src/banos/__init__.py index 7f35875..da5ad98 100644 --- a/src/banos/__init__.py +++ b/src/banos/__init__.py @@ -14,7 +14,7 @@ from __future__ import annotations -__version__ = "1.0.0" +__version__ = "1.1.0" import pandas as pd diff --git a/uv.lock b/uv.lock index 412dab0..47b13aa 100644 --- a/uv.lock +++ b/uv.lock @@ -221,7 +221,7 @@ wheels = [ [[package]] name = "banos" -version = "1.0.0" +version = "1.1.0" source = { editable = "." } dependencies = [ { name = "numpy" }, From 0e898e8d5b6c59c21636cb50438879827598746b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:08:12 +0100 Subject: [PATCH 21/25] fix: give the matlab errors ids and drop the toolbox dependency BANOS_score raised three errors without an identifier: optimal matching requested, a single prediction without its ground truth, and a first argument that is neither a cell array nor a struct. They are BANOS:optimalNotImplemented, BANOS:missingGroundTruth and BANOS:invalidData, in the scheme of the other errors and warnings; its help lists them. BANOSaggregateMetrics and BANOSpoolComponents averaged with nanmean, a function of the Statistics and Machine Learning Toolbox, so scoring several recordings needed that toolbox. They use mean(..., 'omitnan'), which is base MATLAB and gives the same values: the MATLAB golden fixture regenerates byte for byte. The BANOS toolbox now needs base MATLAB only; the README and the changelog say so, and the help texts no longer name nanmean. --- CHANGELOG.md | 7 ++++++- README.md | 8 +++++--- matlab/BANOS/BANOS_score.m | 18 +++++++++++++----- matlab/BANOS/BANOSaggregateMetrics.m | 4 ++-- matlab/BANOS/BANOScalculateIC.m | 4 ++-- matlab/BANOS/BANOScalculatePrecisionRecallF1.m | 2 +- matlab/BANOS/BANOScalculateSO.m | 2 +- matlab/BANOS/BANOSpoolComponents.m | 5 +++-- matlab/BANOS/Contents.m | 2 +- 9 files changed, 34 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ab64d6..509aa49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,7 +85,9 @@ options and valid input, precision, recall, SO and TP are unchanged. `BANOS:lengthMismatch`, `BANOS:noFrame`, `BANOS:nanReplaced`, `BANOS:columnDropped`). Options are stricter: an unknown option name, which was ignored, and an unknown `'matching'` value, which was treated as greedy, raise an error (`BANOS:unknownOption`, - `BANOS:missingOptionValue`, `BANOS:invalidOption`). + `BANOS:missingOptionValue`, `BANOS:invalidOption`). The other errors of `BANOS_score` + get identifiers too: `BANOS:optimalNotImplemented`, `BANOS:missingGroundTruth` and + `BANOS:invalidData`. - **Imports.** `calculate_so`, `calculate_tp_metric` and `calculate_ic` are no longer importable from `banos.pipeline`, which only used them; import them from `banos.core`. - **Packaging.** matplotlib and seaborn move from the runtime dependencies to the @@ -122,6 +124,9 @@ options and valid input, precision, recall, SO and TP are unchanged. compares `banos.__version__` with the version in `pyproject.toml`. - Type errors reported by mypy: `is_logical_column` returns a Python `bool`, and `drop_non_logical_columns` is annotated with the column labels it returns. +- The MATLAB code averaged with `nanmean`, a function of the Statistics and Machine + Learning Toolbox, so scoring several recordings needed that toolbox. It now uses + `mean(..., 'omitnan')` of base MATLAB, which gives the same values. - `SECURITY.md` said that BANOS reads CSV files (it reads no file: it takes DataFrames or MATLAB cell arrays) and that `uv.lock` pins the dependencies users get (it pins the development and CI environment). `SETUP.md` said to allow all actions (the repository diff --git a/README.md b/README.md index cd6dd52..9c36a88 100644 --- a/README.md +++ b/README.md @@ -290,7 +290,8 @@ Import the metric functions from `banos.core`. ## MATLAB -A MATLAB implementation is provided in `matlab/BANOS/`. Usage mirrors the Python API. +A MATLAB implementation is provided in `matlab/BANOS/`. It needs base MATLAB only, no +toolbox. Usage mirrors the Python API. ```matlab % Add the toolbox to the path for this session @@ -314,8 +315,9 @@ tests compare the Python results with golden values written by the MATLAB code, 1e-6. MATLAB uses greedy matching only (`'matching', 'optimal'` raises an error). A single recording is named `single_recording` in messages, and errors and warnings carry identifiers: `BANOS:lengthMismatch`, `BANOS:noFrame`, `BANOS:nanReplaced`, -`BANOS:columnDropped`, `BANOS:unknownOption`, `BANOS:missingOptionValue` and -`BANOS:invalidOption`. `BANOScalculateBANOSForEachFile` returns the components of each +`BANOS:columnDropped`, `BANOS:unknownOption`, `BANOS:missingOptionValue`, +`BANOS:invalidOption`, `BANOS:optimalNotImplemented`, `BANOS:missingGroundTruth` and +`BANOS:invalidData`. `BANOScalculateBANOSForEachFile` returns the components of each recording as a second output, and `BANOSpoolComponents` pools them. See [`demo/tutorial_matlab.m`](https://github.com/BelloneLab/BANOS/blob/main/demo/tutorial_matlab.m) diff --git a/matlab/BANOS/BANOS_score.m b/matlab/BANOS/BANOS_score.m index 1f7fb78..3aaff3a 100644 --- a/matlab/BANOS/BANOS_score.m +++ b/matlab/BANOS/BANOS_score.m @@ -35,12 +35,20 @@ % or a name without a value, raises an error (BANOS:unknownOption, % BANOS:invalidOption, BANOS:missingOptionValue). % +% ERRORS: +% BANOS:optimalNotImplemented - 'matching', 'optimal' (Python only). +% BANOS:missingGroundTruth - a single prediction without its ground truth. +% BANOS:invalidData - a first argument that is neither a cell +% array nor a struct. +% BANOSpreprocessData raises and warns about the data itself. +% % NaN VALUES: % With 'per_file', a behavior with no ground-truth bouts in a recording % scores 1.0 on all metrics if nothing is predicted there and 0.0 -% otherwise; groupMetrics and overallMetrics average with nanmean. +% otherwise; groupMetrics and overallMetrics average with +% mean(..., 'omitnan'), leaving NaN out. % With 'pooled', this 1.0/0.0 rule does not apply and overallMetrics -% averages the pooled behaviors with nanmean. +% averages the pooled behaviors, leaving NaN out. % A metric is NaN when it is undefined: precision when nothing is % predicted, recall when there is no GT bout (pooled), F1 when there is no % bout at all (pooled), SO, TP and IC when they have no term (no overlap, @@ -76,7 +84,7 @@ pooled = strcmp(options.aggregation, 'pooled'); if strcmp(options.matching, 'optimal') - error('BANOS_score: optimal matching is not yet implemented in MATLAB. Use the Python package (banos.score(..., matching=''optimal'')).'); + error('BANOS:optimalNotImplemented', 'BANOS_score: optimal matching is not yet implemented in MATLAB. Use the Python package (banos.score(..., matching=''optimal'')).'); end % --- Detect single vs multi-recording mode ---------------------------- @@ -97,7 +105,7 @@ % Single-recording: data = pred cell array, gt = first cell array in % varargin (taken out above, before the options were parsed) if isempty(gt) - error('BANOS_score: single-recording mode requires two cell array arguments: BANOS_score(pred, gt).'); + error('BANOS:missingGroundTruth', 'BANOS_score: single-recording mode requires two cell array arguments: BANOS_score(pred, gt).'); end tmpData.single_recording = {data, gt}; @@ -111,6 +119,6 @@ end else - error('BANOS_score: first argument must be a cell array (single recording) or a struct (multiple recordings).'); + error('BANOS:invalidData', 'BANOS_score: first argument must be a cell array (single recording) or a struct (multiple recordings).'); end end diff --git a/matlab/BANOS/BANOSaggregateMetrics.m b/matlab/BANOS/BANOSaggregateMetrics.m index 9956d8c..e3c079f 100644 --- a/matlab/BANOS/BANOSaggregateMetrics.m +++ b/matlab/BANOS/BANOSaggregateMetrics.m @@ -37,8 +37,8 @@ for j = 1:length(metricFields) field = metricFields{j}; for behavior = fieldnames(groupMetrics)' - groupMetrics.(behavior{1}).(field) = nanmean(groupMetrics.(behavior{1}).(field)); + groupMetrics.(behavior{1}).(field) = mean(groupMetrics.(behavior{1}).(field), 'omitnan'); end - overallMetrics.(field) = nanmean(overallMetrics.(field)); + overallMetrics.(field) = mean(overallMetrics.(field), 'omitnan'); end end \ No newline at end of file diff --git a/matlab/BANOS/BANOScalculateIC.m b/matlab/BANOS/BANOScalculateIC.m index 55016e1..b300ca2 100644 --- a/matlab/BANOS/BANOScalculateIC.m +++ b/matlab/BANOS/BANOScalculateIC.m @@ -5,8 +5,8 @@ % boutLength = end - start is the number of frame pairs of the bout. % A GT bout is found when at least one of its frames is predicted (value 1). % One-frame bouts (boutLength = 0) are always excluded from the average. - % Returns NaN if there is no term (matches Python behaviour — NaN - % propagates through nanmean). + % Returns NaN if there is no term (matches Python behaviour — the means + % over recordings and behaviors leave NaN out). % % Arguments: % predSequence - The sequence of predictions (1-based index vector). diff --git a/matlab/BANOS/BANOScalculatePrecisionRecallF1.m b/matlab/BANOS/BANOScalculatePrecisionRecallF1.m index 2e2e1a9..b87280e 100644 --- a/matlab/BANOS/BANOScalculatePrecisionRecallF1.m +++ b/matlab/BANOS/BANOScalculatePrecisionRecallF1.m @@ -5,7 +5,7 @@ % matching Python behaviour. F1 is 0 when no bout is matched but some % bouts were predicted or missed (TP = 0 and FP + FN > 0), and NaN only % when all three counts are zero. - % NaN values propagate correctly through BANOSaggregateMetrics (nanmean). + % BANOSaggregateMetrics leaves NaN values out of its means. % % Arguments: % tp - Count of True Positives (matched bout pairs). diff --git a/matlab/BANOS/BANOScalculateSO.m b/matlab/BANOS/BANOScalculateSO.m index 4a93174..fdb620c 100644 --- a/matlab/BANOS/BANOScalculateSO.m +++ b/matlab/BANOS/BANOScalculateSO.m @@ -30,7 +30,7 @@ end end % Return NaN if there is no term (matches Python behaviour). - % NaN propagates correctly through BANOSaggregateMetrics (nanmean). + % BANOSaggregateMetrics leaves NaN out of its means. if isempty(soTerms) so = NaN; else diff --git a/matlab/BANOS/BANOSpoolComponents.m b/matlab/BANOS/BANOSpoolComponents.m index 9f888b6..25fef46 100644 --- a/matlab/BANOS/BANOSpoolComponents.m +++ b/matlab/BANOS/BANOSpoolComponents.m @@ -16,7 +16,8 @@ % % Returns: % groupMetrics - Pooled metrics for each behavior. - % overallMetrics - Mean of the pooled metrics over the behaviors (nanmean). + % overallMetrics - Mean of the pooled metrics over the behaviors, NaN left + % out (mean with 'omitnan'). pooled = struct(); fileNames = fieldnames(fileComponents); @@ -64,6 +65,6 @@ end for k = 1:length(metricFields) - overallMetrics.(metricFields{k}) = nanmean(overallMetrics.(metricFields{k})); + overallMetrics.(metricFields{k}) = mean(overallMetrics.(metricFields{k}), 'omitnan'); end end diff --git a/matlab/BANOS/Contents.m b/matlab/BANOS/Contents.m index 7cf7830..cb7b6b2 100644 --- a/matlab/BANOS/Contents.m +++ b/matlab/BANOS/Contents.m @@ -7,7 +7,7 @@ % Pipeline functions: % BANOSpreprocessData - Fill NaN, drop non-binary columns, align headers. % BANOScalculateBANOSForEachFile - Compute metrics for each recording. -% BANOSaggregateMetrics - Average metrics across recordings (nanmean). +% BANOSaggregateMetrics - Average metrics across recordings (NaN left out). % BANOSpoolComponents - Pool bout counts and terms across recordings. % BANOScomputeBehaviorMetrics - Per-behavior metric computation for one recording. % From 29e9505216cba284602f59feb50276bdad19e553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:17:43 +0100 Subject: [PATCH 22/25] docs: re-run the calms21 tutorial with 1.1.0 The CalMS21 tutorial is executed again with this code, TensorFlow 2.21.0 and the other versions of uv.lock (Python 3.12), so its stored outputs are those of 1.1.0: detection F1 of mount 0.7006 -> 0.6638 and overall 0.6398 -> 0.6286; IC attack 0.7287 -> 0.7228, investigation 0.9734 -> 0.9652, mount 0.9374 -> 0.9308, overall 0.8798 -> 0.8719. The predictions are those of the earlier run: the 1.0.0 code gives the stored values on them. Precision, recall, SO and TP are unchanged. The note that the outputs came from 0.2.0 is removed, and the comparison table prints the corrected SO wording. The setup cell printed the absolute paths of the model and of the data cache on the machine that ran it; it prints them relative to the repository. data.caltech.edu answers 403 to the default user agent of urllib, so the automatic download failed for every user; the requests now send a User-Agent that names the tutorial and the repository. The stored output is that of a first run, which downloads the data, with its stream outputs merged as Jupyter displays them (one final line for the progress bar). --- CHANGELOG.md | 4 +++ demo/tutorial_calms21_task1.ipynb | 50 ++++++++++++++----------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 509aa49..f7488cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -117,6 +117,10 @@ options and valid input, precision, recall, SO and TP are unchanged. - Tutorials: the text said that the prediction shifted by 15 frames still detects both bouts (DA 1.0); it shares no frame with them, so DA is 0 (it printed NaN before this release). +- The CalMS21 tutorial printed the absolute local paths of its model and data cache; it + prints them relative to the repository. Its download failed: data.caltech.edu refuses + the default user agent of urllib (HTTP 403), so the requests now send a User-Agent + that names the tutorial and the repository. Its stored outputs are those of 1.1.0. - `compare_fixtures.py` swapped the `VALUE->NULL` and `NULL->VALUE` tags and misdescribed the v0.1.5 to v0.2.0 changes in its legend; it no longer fails on a Windows console. - `test_preprocess_data_does_not_mutate_caller_dict` compared a dict's id with itself and diff --git a/demo/tutorial_calms21_task1.ipynb b/demo/tutorial_calms21_task1.ipynb index 11c7443..414524b 100644 --- a/demo/tutorial_calms21_task1.ipynb +++ b/demo/tutorial_calms21_task1.ipynb @@ -41,18 +41,6 @@ "the repo at `demo/models/task1_seed_42_model.h5`." ] }, - { - "cell_type": "markdown", - "id": "a1000000-note", - "metadata": {}, - "source": [ - "> **Note on the stored outputs.** The outputs stored in this notebook were produced\n", - "> with an earlier version of BANOS (0.2.0, as the setup cell prints). With BANOS 1.1.0,\n", - "> the detection F1 (DA) and IC values differ (see `CHANGELOG.md`, 1.1.0, Numerical\n", - "> changes); precision, recall, SO and TP do not. Run the notebook to get the values of\n", - "> 1.1.0." - ] - }, { "cell_type": "markdown", "id": "a1000001", @@ -73,9 +61,9 @@ "output_type": "stream", "text": [ "TensorFlow 2.21.0\n", - "BANOS 0.2.0\n", - "Model: C:\\Users\\GA401\\Documents\\Pro\\Code\\GithubPerso\\Tl_BANOS_dev\\demo\\models\\task1_seed_42_model.h5 (959 KB)\n", - "Cache: C:\\Users\\GA401\\Documents\\Pro\\Code\\GithubPerso\\Tl_BANOS_dev\\.cache\\calms21\n" + "BANOS 1.1.0\n", + "Model: demo/models/task1_seed_42_model.h5 (959 KB)\n", + "Cache: .cache/calms21\n" ] } ], @@ -119,6 +107,7 @@ "\n", "\n", "DEMO_DIR = _find_demo_dir()\n", + "REPO_DIR = DEMO_DIR.parent\n", "\n", "MODEL_PATH = DEMO_DIR / \"models\" / \"task1_seed_42_model.h5\"\n", "\n", @@ -136,8 +125,8 @@ " f\"Model not found at {MODEL_PATH}\\n\"\n", " f\"Make sure you cloned the full repo (demo/models/task1_seed_42_model.h5 must exist).\"\n", ")\n", - "print(f\"Model: {MODEL_PATH} ({MODEL_PATH.stat().st_size / 1e3:.0f} KB)\")\n", - "print(f\"Cache: {CACHE_DIR}\")" + "print(f\"Model: {MODEL_PATH.relative_to(REPO_DIR).as_posix()} ({MODEL_PATH.stat().st_size / 1e3:.0f} KB)\")\n", + "print(f\"Cache: {CACHE_DIR.relative_to(REPO_DIR).as_posix()}\")" ] }, { @@ -172,7 +161,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "CalMS21 task1 zip: 457MB [00:41, 11.1MB/s] \n" + "CalMS21 task1 zip: 457MB [00:13, 34.5MB/s] \n" ] }, { @@ -186,6 +175,13 @@ } ], "source": [ + "# data.caltech.edu refuses the default user agent of urllib (HTTP 403):\n", + "# the requests name this tutorial instead.\n", + "_opener = urllib.request.build_opener()\n", + "_opener.addheaders = [(\"User-Agent\", \"BANOS tutorial (https://github.com/BelloneLab/BANOS)\")]\n", + "urllib.request.install_opener(_opener)\n", + "\n", + "\n", "def download_with_progress(url, dest_path, label):\n", " \"\"\"Download a file once; skip if already present.\"\"\"\n", " dest_path = Path(dest_path)\n", @@ -592,17 +588,17 @@ "text": [ "BANOS per-behavior metrics (nanmean across 19 recordings):\n", " DA (F1) SO TP IC Precision Recall\n", - "attack 0.5745 0.5272 0.4800 0.7287 0.5548 0.6076\n", - "investigation 0.6476 0.4556 0.0651 0.9734 0.6166 0.7061\n", - "mount 0.7006 0.6357 0.2586 0.9374 0.6801 0.7264\n", + "attack 0.5745 0.5272 0.4800 0.7228 0.5548 0.6076\n", + "investigation 0.6476 0.4556 0.0651 0.9652 0.6166 0.7061\n", + "mount 0.6638 0.6357 0.2586 0.9308 0.6801 0.7264\n", "\n", "BANOS overall metrics (nanmean across behaviors and recordings):\n", " precision 0.6172\n", " recall 0.6801\n", - " f1_score 0.6398\n", + " f1_score 0.6286\n", " so 0.5378\n", " tp 0.2681\n", - " ic 0.8798\n" + " ic 0.8719\n" ] } ], @@ -658,10 +654,10 @@ "---------------------------------------------------------------------------\n", "CalMS21 F1 (pooled) 0.7792 Global frame overlap; matches leaderboard method\n", "Our Frame F1 0.7267 Per-recording avg; explicit absent-behavior scoring\n", - "BANOS DA (F1) 0.6398 Bout-level detection accuracy\n", - "BANOS SO 0.5378 Temporal overlap quality of matched bouts\n", + "BANOS DA (F1) 0.6286 Bout-level detection accuracy\n", + "BANOS SO 0.5378 Temporal quality of overlapping bouts\n", "BANOS TP 0.2681 Boundary precision (start/end frame accuracy)\n", - "BANOS IC 0.8798 Label stability within detected bouts\n" + "BANOS IC 0.8719 Label stability within detected bouts\n" ] } ], @@ -714,7 +710,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.9" + "version": "3.12.12" } }, "nbformat": 4, From e27a12a23a12f84e4b9b966534191a400521a888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:17:57 +0100 Subject: [PATCH 23/25] build: extend the package keywords Eight keywords follow the existing six: computational ethology, animal behavior, behavioral neuroscience, action segmentation, bout detection, evaluation metrics, inter-annotator agreement and rodent. They are written in the Keywords field of the package metadata, and CITATION.cff lists the same fourteen. --- CITATION.cff | 8 ++++++++ pyproject.toml | 17 ++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 45b0802..d3b6143 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -16,6 +16,14 @@ keywords: - video - scoring - metrics + - computational ethology + - animal behavior + - behavioral neuroscience + - action segmentation + - bout detection + - evaluation metrics + - inter-annotator agreement + - rodent preferred-citation: type: article title: "From eye to AI: Studying rodent social behavior in the era of machine learning" diff --git a/pyproject.toml b/pyproject.toml index 479746e..de37896 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,22 @@ license-files = ["LICENSE"] requires-python = ">=3.11" authors = [{ name = "Benoît J. T. Girard" }] maintainers = [{ name = "Benoît J. T. Girard" }] -keywords = ["behavior", "annotation", "ethology", "video", "scoring", "metrics"] +keywords = [ + "behavior", + "annotation", + "ethology", + "video", + "scoring", + "metrics", + "computational ethology", + "animal behavior", + "behavioral neuroscience", + "action segmentation", + "bout detection", + "evaluation metrics", + "inter-annotator agreement", + "rodent", +] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", From 8bd327ebdb29ead0f9dd35f995de7dd7d0f5d150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:18:02 +0100 Subject: [PATCH 24/25] chore: set the release date of 1.1.0 The changelog entry and CITATION.cff (date-released) are dated 2026-10-30, the date already in the version line of Contents.m. RELEASE.md adds two steps: the release pull request is merged as a single squashed commit, subject "chore: release vX.Y.Z (#N)" and an empty body, and once the package is on PyPI the GitHub release of the tag is created with the changelog entry as notes. --- CHANGELOG.md | 2 +- CITATION.cff | 1 + RELEASE.md | 8 +++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7488cc..a2f8567 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. -## [1.1.0] — Unreleased +## [1.1.0] — 2026-10-30 Two metrics are corrected (detection F1 and IC), two options are added (`missed_bouts` and `aggregation`), and invalid input now raises an error or a warning. With the default diff --git a/CITATION.cff b/CITATION.cff index d3b6143..dff310b 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -4,6 +4,7 @@ title: BANOS abstract: "Behavior ANnOtation Score: ethological metrics for evaluating behavior annotation quality." type: software version: 1.1.0 +date-released: 2026-10-30 license: MIT repository-code: "https://github.com/BelloneLab/BANOS" authors: diff --git a/RELEASE.md b/RELEASE.md index d6c95be..f46e116 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -31,7 +31,9 @@ uv lock # 2. Date the CHANGELOG.md entry -# 3. Commit, open a pull request, and merge it once CI is green +# 3. Commit, open a pull request, and merge it once CI is green, as a single +# squashed commit whose subject is "chore: release vX.Y.Z (#N)", N being +# the pull request number, and whose body is empty git commit -am "chore: release vX.Y.Z" # 4. Tag the merged commit on main @@ -40,6 +42,10 @@ git tag -a vX.Y.Z origin/main -m "BANOS X.Y.Z" # 5. Push the tag — this triggers the publish workflow git push origin vX.Y.Z + +# 6. Once the package is on PyPI, create the GitHub release for the tag, +# with the CHANGELOG.md entry of the version as notes (notes.md) +gh release create vX.Y.Z --verify-tag --title "vX.Y.Z" --notes-file notes.md ``` The `publish.yml` workflow: From 155984b345699455e37097aa041e68fb19d128c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20J=2E=20T=2E=20GIRARD?= <53222414+BenoitJT-GIRARD@users.noreply.github.com> Date: Fri, 30 Oct 2026 22:30:13 +0100 Subject: [PATCH 25/25] build: upgrade locked packages with security advisories Upgrade pillow, virtualenv, filelock, tornado, nbconvert, certifi and setuptools in uv.lock to releases without the published advisories; the new virtualenv also moves distlib and packaging. None of them is a runtime dependency of banos: they come from the notebook and calms21 extras and from the development tools. numpy, pandas and scipy are unchanged. --- uv.lock | 268 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 153 insertions(+), 115 deletions(-) diff --git a/uv.lock b/uv.lock index 47b13aa..4bbae78 100644 --- a/uv.lock +++ b/uv.lock @@ -313,11 +313,11 @@ css = [ [[package]] name = "certifi" -version = "2026.2.25" +version = "2026.10.23" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/af/2d/7bf41579a8986e348fa033a31cdd0e4121114f6bce2457e8876010b092dd/certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7", size = 155029, upload-time = "2026-02-25T02:54:17.342Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3c/c17fb3ca2d9c3acff52e30b309f538586f9f5b9c9cf454f3845fc9af4881/certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa", size = 153684, upload-time = "2026-02-25T02:54:15.766Z" }, + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, ] [[package]] @@ -746,11 +746,11 @@ wheels = [ [[package]] name = "distlib" -version = "0.4.0" +version = "0.4.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/d36966de7304047a1001a7559fbf266c464bb5d8cc85464536ef8c9bcc32/distlib-0.4.4.tar.gz", hash = "sha256:ca4175b4ac7f71223cc30ed7248c45d2b38c2439d8ee6e498ab67ed63cf3508d", size = 621304, upload-time = "2026-10-16T13:56:34.203Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, + { url = "https://files.pythonhosted.org/packages/c1/2b/238f02a4927fa2e27aa995371d240a468b52e42eae44a678434aff736b38/distlib-0.4.4-py2.py3-none-any.whl", hash = "sha256:cf09d55b1e85a89576b036aeb71550bfd68320f3f21761ca329eadadad918638", size = 472099, upload-time = "2026-10-16T13:56:32.053Z" }, ] [[package]] @@ -773,11 +773,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.25.2" +version = "4.2.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/b8/00651a0f559862f3bb7d6f7477b192afe3f583cc5e26403b44e59a55ab34/filelock-3.25.2.tar.gz", hash = "sha256:b64ece2b38f4ca29dd3e810287aa8c48182bbecd1ae6e9ae126c9b35f1382694", size = 40480, upload-time = "2026-03-11T20:45:38.487Z" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/4b/2320d9fa03b65b4beb43b521fe1b12b7f87ab91ee42de3a286075016d205/filelock-4.2.5.tar.gz", hash = "sha256:37d8fa4f8b5e9233612e68ab23bf64cae41771322c8797361088ae2895b12894", size = 659943, upload-time = "2026-10-30T19:27:22.237Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/a5/842ae8f0c08b61d6484b52f99a03510a3a72d23141942d216ebe81fefbce/filelock-3.25.2-py3-none-any.whl", hash = "sha256:ca8afb0da15f229774c9ad1b455ed96e85a81373065fb10446672f64444ddf70", size = 26759, upload-time = "2026-03-11T20:45:37.437Z" }, + { url = "https://files.pythonhosted.org/packages/14/2d/2cf36eca330b7a97c0e9076e8fc4272127c29398fa2bb7b24d4321b8b476/filelock-4.2.5-py3-none-any.whl", hash = "sha256:46ce42d27e5cdde09ee7197349663e97cb106679bd9e75db3a09679fbc7aa1a8", size = 163877, upload-time = "2026-10-30T19:27:20.409Z" }, ] [[package]] @@ -2024,7 +2024,7 @@ wheels = [ [[package]] name = "nbconvert" -version = "7.17.1" +version = "7.17.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "beautifulsoup4" }, @@ -2042,9 +2042,9 @@ dependencies = [ { name = "pygments" }, { name = "traitlets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/01/b1/708e53fe2e429c103c6e6e159106bcf0357ac41aa4c28772bd8402339051/nbconvert-7.17.1.tar.gz", hash = "sha256:34d0d0a7e73ce3cbab6c5aae8f4f468797280b01fd8bd2ca746da8569eddd7d2", size = 865311, upload-time = "2026-04-08T00:44:14.914Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/6a/252c7620305a9661205aed7a2a4b3592f5207bfca56ad11b7dea2c5ec2f6/nbconvert-7.17.2.tar.gz", hash = "sha256:344a218e8595117ce37396ed025d994f146d076226ebac59255a2fa085ab24d8", size = 865989, upload-time = "2026-10-12T09:11:19.583Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/67/f8/bb0a9d5f46819c821dc1f004aa2cc29b1d91453297dbf5ff20470f00f193/nbconvert-7.17.1-py3-none-any.whl", hash = "sha256:aa85c087b435e7bf1ffd03319f658e285f2b89eccab33bc1ba7025495ab3e7c8", size = 261927, upload-time = "2026-04-08T00:44:12.845Z" }, + { url = "https://files.pythonhosted.org/packages/81/fd/a8d457af68848a8e5c12dd1d84ff35c415db471af78280e0703db4efbb12/nbconvert-7.17.2-py3-none-any.whl", hash = "sha256:6a143c11c00bf831604da4b1be4347d36ea52367057eb920135d69bc2862fed2", size = 262094, upload-time = "2026-10-12T09:11:17.215Z" }, ] [[package]] @@ -2326,11 +2326,11 @@ wheels = [ [[package]] name = "packaging" -version = "26.0" +version = "26.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] [[package]] @@ -2446,87 +2446,92 @@ wheels = [ [[package]] name = "pillow" -version = "12.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/c8/0a78b0e02d7ac54bc03e5321c9220da52f0c2ea83b21f7c40e7f3169c502/pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756", size = 5392415, upload-time = "2026-07-01T11:53:47.162Z" }, - { url = "https://files.pythonhosted.org/packages/b2/5b/a02d30018abd97ced9f5a6c63d28597694a00d066516b9c1c6de45859fc9/pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6", size = 4785266, upload-time = "2026-07-01T11:53:49.079Z" }, - { url = "https://files.pythonhosted.org/packages/c8/98/766667a4be768150a202836acd9fad19c06824ca86c4286d3cf6b274964e/pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd", size = 6263814, upload-time = "2026-07-01T11:53:51.32Z" }, - { url = "https://files.pythonhosted.org/packages/3b/2d/ede717bc1144f63886c21fd349bb95860b0d1a21149ff16f2bb362b612b6/pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd", size = 6934408, upload-time = "2026-07-01T11:53:53.487Z" }, - { url = "https://files.pythonhosted.org/packages/a3/48/9c58b685e69d49c31af6c8eb9012055fab7e665785165c84796e2c73ce72/pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c", size = 6337160, upload-time = "2026-07-01T11:53:55.457Z" }, - { url = "https://files.pythonhosted.org/packages/ff/fa/dc2a5c0ba6df93f67c31d34b808b7ce440b40cdbf96f0b81cde1d1e6fa93/pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5", size = 7045172, upload-time = "2026-07-01T11:53:57.736Z" }, - { url = "https://files.pythonhosted.org/packages/86/a5/444817a4d4c4c2417df00513086ca196f388d8f9ef40c2e4ccd1ad1af54b/pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b", size = 6472232, upload-time = "2026-07-01T11:53:59.767Z" }, - { url = "https://files.pythonhosted.org/packages/63/c6/4bad1b18d132a50b27e1365e1ab163616f7a5bb56d330f66f9d1d9d4f9d4/pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a", size = 7233653, upload-time = "2026-07-01T11:54:02.066Z" }, - { url = "https://files.pythonhosted.org/packages/fd/16/00f91ab7760dc842f5aad55217e80fc4a7067a0604535249bc8a2d6d9870/pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26", size = 2568195, upload-time = "2026-07-01T11:54:04.622Z" }, - { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, - { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, - { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, - { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, - { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, - { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, - { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, - { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, - { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, - { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, - { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, - { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" }, - { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" }, - { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, - { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, - { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, - { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, - { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" }, - { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" }, - { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, - { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, - { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, - { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" }, - { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" }, - { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, - { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, - { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, - { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, - { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" }, - { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" }, - { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" }, - { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" }, - { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, - { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, - { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, - { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, - { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, - { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, - { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, - { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, - { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, - { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, - { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, - { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, - { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, - { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, - { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, - { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, - { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, - { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, - { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, - { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, - { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, - { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, - { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, - { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, - { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, - { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, - { url = "https://files.pythonhosted.org/packages/75/18/2e8b40223153ccbc60df07f9e8928dc0c76202aa4e55ae9f53962b6510d6/pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468", size = 5302510, upload-time = "2026-07-01T11:56:25.736Z" }, - { url = "https://files.pythonhosted.org/packages/46/3e/51fabf59d5ab801ceab709453d3ab6b180083496579549de4c45ced6528a/pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94", size = 4736058, upload-time = "2026-07-01T11:56:28.041Z" }, - { url = "https://files.pythonhosted.org/packages/bf/20/22fe9384b7949e25fb1293bcfc84fb82590ff4ea6b37c95b24d26d793d86/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e", size = 5237776, upload-time = "2026-07-01T11:56:30.263Z" }, - { url = "https://files.pythonhosted.org/packages/08/14/f6ba68107680ffa74b39985f3f30884e41318fbc4250caa423c79b4788bb/pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3", size = 5860358, upload-time = "2026-07-01T11:56:32.68Z" }, - { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/d7/af6af1b8cc55e07cb0489d5b0950bf5a9bb1461eeba825700b6d8d0e26ea/pillow-13.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb", size = 5592324, upload-time = "2026-10-16T22:15:24.672Z" }, + { url = "https://files.pythonhosted.org/packages/0b/d2/2c12cb4fbb5d95f38d1001ff3d95fafcb830077018cba40e56a8f932dfbc/pillow-13.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061", size = 4856298, upload-time = "2026-10-16T22:15:27.669Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ee/229b27472483a4801ea94990815e710391836afdf987b4b722b62bd75db6/pillow-13.0.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0", size = 6517304, upload-time = "2026-10-16T22:15:29.43Z" }, + { url = "https://files.pythonhosted.org/packages/36/46/f509b74787f40ce0627501089d630b2c741b63b244c1f40b640b3b01e181/pillow-13.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c", size = 7140667, upload-time = "2026-10-16T22:15:31.368Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/5a398b8ec2c37145397d6461c0b551c23ec03072aa64f357aed25e582b00/pillow-13.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f", size = 6610583, upload-time = "2026-10-16T22:15:34.025Z" }, + { url = "https://files.pythonhosted.org/packages/4f/7b/f2f1109b230fd4e2d57ab2305de31adc37ea9b4ff8719436c623cb484415/pillow-13.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806", size = 7218241, upload-time = "2026-10-16T22:15:36.214Z" }, + { url = "https://files.pythonhosted.org/packages/ab/d3/069edb9083b267746baf59756282cacef904d3475c5288c7216fb5195a98/pillow-13.0.0-cp311-cp311-win32.whl", hash = "sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5", size = 6515278, upload-time = "2026-10-16T22:15:38.129Z" }, + { url = "https://files.pythonhosted.org/packages/7d/f6/6e2d96413e4c9dbec12ea584d9e646d602877d4cce3ea669099f959deac6/pillow-13.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3", size = 7278471, upload-time = "2026-10-16T22:15:40.286Z" }, + { url = "https://files.pythonhosted.org/packages/70/e7/baff21f8cb60496c8c250609501b88919eeedb4088d3da106f6c62f3f3c0/pillow-13.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13", size = 2698981, upload-time = "2026-10-16T22:15:42.637Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/9132d38df320e8099e2642d9fa538f18a7dfebf68a875a31725dc2a115aa/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00", size = 4190620, upload-time = "2026-10-16T22:16:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/cb/be/436dac0996876f65ef1916ffb711c5ef4af2ebc57414b6f16642e07f6b47/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033", size = 4290286, upload-time = "2026-10-16T22:16:04.861Z" }, + { url = "https://files.pythonhosted.org/packages/60/16/7c1007162e4169ea0d5bec47e2e0b2174f56c50cf5b8c02a1dc23e41037e/pillow-13.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599", size = 3723169, upload-time = "2026-10-16T22:16:06.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/bc/578ab2eaa04705a8978bd9be0fe15c3d43a8fc95097c4f7e39540e79e4fa/pillow-13.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe", size = 5568469, upload-time = "2026-10-16T22:16:08.664Z" }, + { url = "https://files.pythonhosted.org/packages/98/c8/ccede3dcc4f274b80fb49199eed1ef34ba05566c8b71976b9d1f111da2e3/pillow-13.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313", size = 4852187, upload-time = "2026-10-16T22:16:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3c/cd0770e96d20bfd990db024bcbd43897ff3e88da1f4782cf90ea231b1d4f/pillow-13.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369", size = 6515908, upload-time = "2026-10-16T22:16:12.896Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/f5cdaa2d1c4086e757430f368c271a58ee29cfe7378e7ae7b9d964015c25/pillow-13.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582", size = 7142544, upload-time = "2026-10-16T22:16:14.844Z" }, + { url = "https://files.pythonhosted.org/packages/fb/b5/a54f6d44adbcd23737c8c12953f42be107e4cca64e247c0ca3a5b563061c/pillow-13.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81", size = 6613097, upload-time = "2026-10-16T22:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/9e6c1207c21b2e91db0d15c16d30f3701ce5add1bd4da0e787761e66cb6e/pillow-13.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502", size = 7224846, upload-time = "2026-10-16T22:16:19.195Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/d3c3cb0518f3dd1eee0ef501535f53d15597a53c5c4118d178086143397a/pillow-13.0.0-cp313-cp313-win32.whl", hash = "sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8", size = 6516013, upload-time = "2026-10-16T22:16:21.389Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/1b16fb66a37e1b9e41370b93b2c614c7da761a5b2a41dc333f7af4639962/pillow-13.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42", size = 7284777, upload-time = "2026-10-16T22:16:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/7d/8d02eba9f7ecc87f8e90f731e032cc4773802f2a843763afd0c66a535ff4/pillow-13.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f", size = 2698826, upload-time = "2026-10-16T22:16:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, + { url = "https://files.pythonhosted.org/packages/6e/eb/e86a41962a3a31ad495a3d6a912d9002b01bba4a6fa71c28703abdca7f69/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521", size = 4191213, upload-time = "2026-10-16T22:17:09.449Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/7e7445e706afec84ef06dc25b2c77f9cecff242c23730c62928ad1089d4a/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc", size = 4290650, upload-time = "2026-10-16T22:17:11.196Z" }, + { url = "https://files.pythonhosted.org/packages/46/85/d534089068ac40c0f61f650389a296c812eab745a8a733f81feec39ae955/pillow-13.0.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac", size = 3723078, upload-time = "2026-10-16T22:17:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2b/00b5c3b401df538b5dbe311d3899730fae5ba1e21cb8b80a128887ac6747/pillow-13.0.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d", size = 5572292, upload-time = "2026-10-16T22:17:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/72/f5/a6b0610f946e9b54ae888ba7ad29d6e2f8d9b11f69e212d70416b7186c09/pillow-13.0.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782", size = 4852222, upload-time = "2026-10-16T22:17:16.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d0/c0ff25bb0427ec70f32aac995453321fc3c4494a46c7292ff02d9d260b04/pillow-13.0.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071", size = 6527349, upload-time = "2026-10-16T22:17:18.534Z" }, + { url = "https://files.pythonhosted.org/packages/85/11/e46fa0bcc0fc117f1eacc206b3d51f04d583fab328b4b740672d0438d83f/pillow-13.0.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf", size = 7152873, upload-time = "2026-10-16T22:17:20.415Z" }, + { url = "https://files.pythonhosted.org/packages/7c/53/2bdeca705f3a58679fa9201233193c86a30238340ca29b3a152c3fcc719b/pillow-13.0.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3", size = 6626871, upload-time = "2026-10-16T22:17:22.361Z" }, + { url = "https://files.pythonhosted.org/packages/14/de/3829030f209722ade3753bb456b805a0e7f885cf885b483407365c82e9ca/pillow-13.0.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932", size = 7236439, upload-time = "2026-10-16T22:17:24.352Z" }, + { url = "https://files.pythonhosted.org/packages/44/b1/67fc37388382bc036a74f04da5bb6fcbf90ec3ac1f45893e06176c373fd3/pillow-13.0.0-cp315-cp315-win32.whl", hash = "sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81", size = 6508688, upload-time = "2026-10-16T22:17:26.314Z" }, + { url = "https://files.pythonhosted.org/packages/44/05/fdb6a79aee10861165526b41b7b440e36ee1ff58613b2fbefe3acbf06d4f/pillow-13.0.0-cp315-cp315-win_amd64.whl", hash = "sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6", size = 7281107, upload-time = "2026-10-16T22:17:28.198Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9c/6c6f7dc2a566467baf121394e83de25ac287cffe34cc030726beb4be0224/pillow-13.0.0-cp315-cp315-win_arm64.whl", hash = "sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0", size = 2695513, upload-time = "2026-10-16T22:17:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/23/16/e9120defdb8bff693b582d035312acc6f530cdc2bc5238a87f410efccd82/pillow-13.0.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f", size = 5578129, upload-time = "2026-10-16T22:17:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/22/76faf2bb2448f48a84cf894c1e2bb1090d7d2b1aa521ece131cb2f05d631/pillow-13.0.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88", size = 4857428, upload-time = "2026-10-16T22:17:33.43Z" }, + { url = "https://files.pythonhosted.org/packages/b3/64/0861ea4e03c50b14af9ed9dd8b9f388ab42f445c623a9fcbab19aef7aa55/pillow-13.0.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb", size = 6612582, upload-time = "2026-10-16T22:17:35.415Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/cd1e9b3f60a9f3d5ebe46da36955c26004092a501f7f2c42f9fcc26611bc/pillow-13.0.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe", size = 7230176, upload-time = "2026-10-16T22:17:37.362Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ef/ffb0a2fc85690714fb7ac662956949645d57f15bf4426f520da361e7b6e3/pillow-13.0.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22", size = 6703640, upload-time = "2026-10-16T22:17:39.313Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c2/6402ee3ccb3363aeed8fdb3f9fe56703ac5377a539f0b81b53999d7e121f/pillow-13.0.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630", size = 7309231, upload-time = "2026-10-16T22:17:41.021Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/d6b82e8b8c06ad143b66b12e0b31051e4bf489b4fabca1f390a14fe1ebde/pillow-13.0.0-cp315-cp315t-win32.whl", hash = "sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2", size = 6517200, upload-time = "2026-10-16T22:17:42.886Z" }, + { url = "https://files.pythonhosted.org/packages/60/d4/0062e6005e9b270b165015aafd343364ac378ec5a685431ea42baad33357/pillow-13.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43", size = 7287488, upload-time = "2026-10-16T22:17:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/e4/81/b8eb349746be7bce47dd95337c6634e84910af91b59aaa9159c034c614fa/pillow-13.0.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8", size = 2698073, upload-time = "2026-10-16T22:17:47.556Z" }, + { url = "https://files.pythonhosted.org/packages/68/c0/fb7b1caf25ed5ac5d46116f09b8b0706b9559833c65c667cecb1a53153f2/pillow-13.0.0-pp311-pypy311_pp80-macosx_10_15_x86_64.whl", hash = "sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca", size = 5521630, upload-time = "2026-10-16T22:17:50.008Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e6/4d11d5b1137f4fc080605cce410cfaae4f303ad400bdec75d27d3083fa29/pillow-13.0.0-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039", size = 4806023, upload-time = "2026-10-16T22:17:51.932Z" }, + { url = "https://files.pythonhosted.org/packages/f1/45/0923202a658e8ff6aafd72c1a4395ebadcdebaa182c437d8595d3dd3db32/pillow-13.0.0-pp311-pypy311_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad", size = 5464831, upload-time = "2026-10-16T22:17:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/07/52/d3e38f0cad1dadea2a658b9a7d6ba8a04048f5ae14f011b806e5c3bd6bcb/pillow-13.0.0-pp311-pypy311_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2", size = 6043133, upload-time = "2026-10-16T22:17:55.625Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f9/2ed7685980c1d64f8131a7d87a19c369424747577152622b73ddeff163b7/pillow-13.0.0-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c", size = 7365208, upload-time = "2026-10-16T22:17:57.509Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, + { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, ] [[package]] @@ -3234,11 +3239,11 @@ wheels = [ [[package]] name = "setuptools" -version = "84.0.0" +version = "84.2.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } +sdist = { url = "https://files.pythonhosted.org/packages/70/b3/c19426f783614d625699a4b247faf628ea9ab1db228674491767b417f9e1/setuptools-84.2.1.tar.gz", hash = "sha256:3ff8b3b5492a0aa66960f4419365a86b84e013070d4313e4d4a6894f5d64d6bc", size = 1230289, upload-time = "2026-10-20T19:01:37.742Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, + { url = "https://files.pythonhosted.org/packages/ef/55/6dc6d6bb953bdfc52519a0059592c52b6e3b8ad51cf8256dffceb55b0610/setuptools-84.2.1-py3-none-any.whl", hash = "sha256:ca140b45d5325a9478f880e351484f62fef15f336462e449ca0c03c873d589c2", size = 876263, upload-time = "2026-10-20T19:01:35.678Z" }, ] [[package]] @@ -3414,19 +3419,52 @@ wheels = [ [[package]] name = "tornado" -version = "6.5.10" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/61/53d562a57b28c08eda40b258c0f975e360541943ad7c7bef897a40caafda/tornado-6.5.10.tar.gz", hash = "sha256:a6b1ccd08c04b4a06fb5aeb381be99de5ad1e5375c1785e31d78c880feb57687", size = 537910, upload-time = "2026-09-15T13:47:48.73Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/5b/ff5fc58fa2427c30dea74c90053f4fc5eda1e7f3833ed3ecc7147fe2b311/tornado-6.5.10-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9261783640e23258694a9ff0795df430a5a7b0a651d3dd53dd0969ad6be16da7", size = 465883, upload-time = "2026-09-15T13:47:35.463Z" }, - { url = "https://files.pythonhosted.org/packages/ad/f5/cd7be26c34a3315532f3aef5f092465da8f59c334dd439d3c14aaef16461/tornado-6.5.10-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:83e6cf438b106c6b3852d70960967bb1b70c87438050dca0981e4b9aa751a4c1", size = 464046, upload-time = "2026-09-15T13:47:37.178Z" }, - { url = "https://files.pythonhosted.org/packages/60/33/df6d7d04854a58619f8349a51e3edb138324130a7562b0bb21f115bb940f/tornado-6.5.10-cp39-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bdf942448169e5336451d0494d7e3d81cfa726d5aa312affdc4682dd62a62f6d", size = 467096, upload-time = "2026-09-15T13:47:38.559Z" }, - { url = "https://files.pythonhosted.org/packages/29/17/cc35dff68272d685cffd8600ffafbd8067e7d05e7348d9f80caddffbbd5f/tornado-6.5.10-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69acca6501eed74582b76dbbceee2a91613f54728e3e418346000d7103101676", size = 468067, upload-time = "2026-09-15T13:47:40.085Z" }, - { url = "https://files.pythonhosted.org/packages/c3/01/6e5349b4e1a53a4b4972a6716785e1fe7407f312063c3972690af8ff301b/tornado-6.5.10-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:66aaa3f57d30c6e6becee83ff28055d5930ac724214bde99393eefda83d5e015", size = 467901, upload-time = "2026-09-15T13:47:41.576Z" }, - { url = "https://files.pythonhosted.org/packages/28/5e/b4facf94370dba006819c8d304376f8b9fbec6b935b5e51bf45823a9790b/tornado-6.5.10-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4bd192b959f9128fb99b8898148070ba4574c9589b78bce42d1851131fe85828", size = 467308, upload-time = "2026-09-15T13:47:43.145Z" }, - { url = "https://files.pythonhosted.org/packages/56/ae/047938e828cafc8eca4c908fafb6588fee944e3af39a0af9d7b602499ae5/tornado-6.5.10-cp39-abi3-win32.whl", hash = "sha256:302eb1e0e3e159314eb591920529fdea80acca92df5510a2cec5bbd4f099ec72", size = 468387, upload-time = "2026-09-15T13:47:44.556Z" }, - { url = "https://files.pythonhosted.org/packages/d8/d4/5901517f05affd752490f6a654ba31b7474664e8dd80bd045a00c220bd88/tornado-6.5.10-cp39-abi3-win_amd64.whl", hash = "sha256:37ae8f150cecfdbf747fc4e12f5e9a97ecd8cf1d4cdb3f119e2de84b11196918", size = 468828, upload-time = "2026-09-15T13:47:45.961Z" }, - { url = "https://files.pythonhosted.org/packages/f3/1a/fd497f3a7f7b74bb04f4b94536b5c9f80742b5d50501fd27977652ddec16/tornado-6.5.10-cp39-abi3-win_arm64.whl", hash = "sha256:ce045d3c298fddd30e89a2777f97039d1b641eb9518ac7b26a4721903539c694", size = 467847, upload-time = "2026-09-15T13:47:47.283Z" }, +version = "6.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/02/5f549b1ad5da6267ed1e1fac2eef69257b1979cc812497002499b7895ff9/tornado-6.6.tar.gz", hash = "sha256:34c4aa916d7f2b2d5cadee2c6850f5a4a7f3d688cbf58399ea9bd3057af4abd5", size = 617998, upload-time = "2026-10-21T15:31:04.733Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/f5/d3e6f76b945f9812842db742d0708a3b6287e850e81be3ca10e290c103c6/tornado-6.6-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:01abf15429d64f04ccfafc238b191cc2c177108b7a331f4461707ad77198c8c2", size = 528326, upload-time = "2026-10-21T15:30:05.528Z" }, + { url = "https://files.pythonhosted.org/packages/30/69/b20cf3a8bf641ec792a5a6050aa153e8f120604f9c23471142de535113e5/tornado-6.6-cp311-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:11bb97c00981eb8254e55bd6a5ad960c983ae343fef8ae3ae5584cd1fb70236a", size = 529544, upload-time = "2026-10-21T15:30:07.126Z" }, + { url = "https://files.pythonhosted.org/packages/0f/f4/d9d8d3ac345aa9df3a132ad03bd443160c2ab24ad4003b4e8d6d5af06371/tornado-6.6-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e152a3b2d3eae113b836d979823f7bb2320f7e334a84449729964921f374a4c", size = 530523, upload-time = "2026-10-21T15:30:08.478Z" }, + { url = "https://files.pythonhosted.org/packages/ad/82/1296ade1c14dee5a4906e5ca092997622ece7c4dc768c68ed9caaf895d1b/tornado-6.6-cp311-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e912d4c4babd04db6f34b25b748174a4a176debe86c8610beedf1815f9af7fdd", size = 530739, upload-time = "2026-10-21T15:30:09.98Z" }, + { url = "https://files.pythonhosted.org/packages/66/30/6ac1eab66c81af46dafd93c9b24bd311f82e199d9d7b8ce2a64a91b6a200/tornado-6.6-cp311-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33aabe8a03d426cef4e0fe5669f0643cadd983b54957f1a7ac2789ec326e9644", size = 529738, upload-time = "2026-10-21T15:30:11.498Z" }, + { url = "https://files.pythonhosted.org/packages/24/b4/6c1f6aa1fd148e4d0f500a10cd0c9949e7bf385e161ff497c81de5b3ae4f/tornado-6.6-cp311-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9aaf0e1c2fb9f046f30e2036b8d2060666399e8464d0a72769e43c754089567b", size = 527908, upload-time = "2026-10-21T15:30:13.298Z" }, + { url = "https://files.pythonhosted.org/packages/54/5b/67d3d452b1179bc9e265cebd381a56ab4cf2d7b2a54607aedd8d6e1ebc2e/tornado-6.6-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:61fc5513cea97720ab9bf54b04827ac0f4500c63f0b072b963d6145c798b4661", size = 530367, upload-time = "2026-10-21T15:30:15.183Z" }, + { url = "https://files.pythonhosted.org/packages/74/14/4e8e9a14f77bcafb2408634b2d4c8c676ad7a43bf8dac4ee2d48370698f9/tornado-6.6-cp311-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:31e1a0d5cd704d2fe8a008c1cc27a0e2b67269ca263630a27985dddc160b5537", size = 530456, upload-time = "2026-10-21T15:30:16.741Z" }, + { url = "https://files.pythonhosted.org/packages/56/12/e3ddef99a2089ccfc86f32bd3f39caaad87bee4868be1bacbf213a0b07a2/tornado-6.6-cp311-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:74416e41e9bdf70fbc2fb242f6fbbc7a58ec11e59d39f15fc5a421ecb6845118", size = 529610, upload-time = "2026-10-21T15:30:18.031Z" }, + { url = "https://files.pythonhosted.org/packages/35/3d/62a692343d46b37e852e5078e0f6513534d925f70a04580072e3c127b129/tornado-6.6-cp311-abi3-musllinux_1_2_s390x.whl", hash = "sha256:8cdd69104ae9d705b3f04d2563aa96e357043e267a455802daae9a1e633a37c8", size = 529803, upload-time = "2026-10-21T15:30:19.326Z" }, + { url = "https://files.pythonhosted.org/packages/49/04/6c59a9413adfe4810f648a4dd9fdf476f13b0495f9cc25ef856b71f9ce1e/tornado-6.6-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:fc7c8a3263501fbe97518cd699101bc3c5f499a7dd4f75fc6cbb3bd9ee1db1f1", size = 529751, upload-time = "2026-10-21T15:30:20.826Z" }, + { url = "https://files.pythonhosted.org/packages/31/36/a1b3b84cd347ae43f982d3bf95628de6c6365668dba2543af47c6a1c888e/tornado-6.6-cp311-abi3-win32.whl", hash = "sha256:b54ca69ff93d0374efb083a12a1e9ec2dd2036e8496d197e0e8ef77b8d9135c3", size = 530643, upload-time = "2026-10-21T15:30:22.428Z" }, + { url = "https://files.pythonhosted.org/packages/b7/50/dc813fe35a672e14ee8bed8b5c634eeb343a05c98eb455318a169de24567/tornado-6.6-cp311-abi3-win_amd64.whl", hash = "sha256:48f26ed9d7beb64e5f06adc864524e581ab6d62ba8ff46eaaa1ac2d0e6d753fd", size = 530886, upload-time = "2026-10-21T15:30:23.975Z" }, + { url = "https://files.pythonhosted.org/packages/0d/71/62a9b9e980026d38874740d0a7c162f53eb0563f6a441e3aff84c48db7e8/tornado-6.6-cp311-abi3-win_arm64.whl", hash = "sha256:29114ffd4e4de869d390393df9e9a0f191065aae1a96f38a58d35e464c04a867", size = 530899, upload-time = "2026-10-21T15:30:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4c/633e3568eb574df555d6ddd2dac44b7793a5c26cd63b444e653fcfb1d6ec/tornado-6.6-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:63fc8d243b52fd8dd9ed4ad02d0df08f2024f9aea632a433a8a6c91075076d44", size = 528432, upload-time = "2026-10-21T15:30:26.578Z" }, + { url = "https://files.pythonhosted.org/packages/37/eb/b525d1870ad28a56f47b664dd8fd3fab4f7ddd237c87c14e97248bf4762f/tornado-6.6-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8d4b85cac795bfc6a7f20ba81871abb8c2ae8fba019061b8a9ae1c2f97e29637", size = 532586, upload-time = "2026-10-21T15:30:28.027Z" }, + { url = "https://files.pythonhosted.org/packages/b7/35/909a33c253352346ca081a27ca66b45c1398468adf784b767597ca773c28/tornado-6.6-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e035c065edadfa696529d05eebcbeb25154c129b5aee6dbfb697a6abf5d4f697", size = 533641, upload-time = "2026-10-21T15:30:29.318Z" }, + { url = "https://files.pythonhosted.org/packages/6f/64/100202666d6e3d96288ab184827762415456c139798a15c962547f9fabf9/tornado-6.6-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5b94e589ab855090c448b5c0937b3a75f1c1a604b7b36207e8236d209f720b9c", size = 533869, upload-time = "2026-10-21T15:30:30.882Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e0/af3ed62bc15830d60328d5b4a1a9076b35ea0ba142cb17b1c1772cb15c59/tornado-6.6-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a394c8c9db9137f2a2b42869d9f7b91398fcf19da120d0dde4c4546172710f7", size = 532767, upload-time = "2026-10-21T15:30:32.273Z" }, + { url = "https://files.pythonhosted.org/packages/24/d7/1e2e1a6932eb62d63324b64dba2f62e6eaf79ad883dd209352c297b7848c/tornado-6.6-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0215d1df6e4b5e736ddad5adf0af4138cf7437fdd39c77c5e69768e37850a632", size = 530956, upload-time = "2026-10-21T15:30:33.746Z" }, + { url = "https://files.pythonhosted.org/packages/70/65/3b9a0e53d7a4c39f2ebe95486540fa8eeb67bfb6e8c437d603c00d35b128/tornado-6.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4d711790e0a9576ad924d942c4607e4e401d697588ea6f30f6662824008a7add", size = 533372, upload-time = "2026-10-21T15:30:35.085Z" }, + { url = "https://files.pythonhosted.org/packages/44/48/90979cbfaa3ff941324a2605eb794af0f087c670cd99f18a69ab40498073/tornado-6.6-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:034a66bae0f55421d007803e2385d2a59707d8829b2f026373d42f08256e9922", size = 533425, upload-time = "2026-10-21T15:30:36.425Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c4/ce743a86d91f85c255b77a4282eb5b2716b61e74430eb9d1df8578a539d2/tornado-6.6-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5cb38483b077129056a94dd415a71d29ab4e42b46da894fee8b4657003f10b21", size = 532545, upload-time = "2026-10-21T15:30:37.789Z" }, + { url = "https://files.pythonhosted.org/packages/02/5e/a82131e35778540fc19139229d02aa8cf2453a8022e202be50c88f0f8ac3/tornado-6.6-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:46bbbd107efa37777ac4b83601866d19b4dbd5b770a7a9123582633d95959ad3", size = 532700, upload-time = "2026-10-21T15:30:39.336Z" }, + { url = "https://files.pythonhosted.org/packages/3a/09/6dac8c2c677c65d60b3d7dae94176dbfe392f314e7c8e2d709623edd6a26/tornado-6.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b85e9d0263cdbc027b95947c8b873a7bc7fd6a08d16bfb4f5047350b0808e464", size = 532712, upload-time = "2026-10-21T15:30:40.698Z" }, + { url = "https://files.pythonhosted.org/packages/b2/ac/71e5776cbcdc3264e81aa6db28c69717ba50607a9eb8d5b766f11a4b020e/tornado-6.6-cp314-cp314t-win32.whl", hash = "sha256:4276eef0880e5fe4488e6799c2842f59af3b06637e9313ab4fb506f9c9c2d6cb", size = 530491, upload-time = "2026-10-21T15:30:42.115Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0f/0b6a6908a5b5aa8320009dfdb84434b1cf160b577a00456e59ab9e6ea598/tornado-6.6-cp314-cp314t-win_amd64.whl", hash = "sha256:10d4af98c6df09f8429b1340ccaea3a72887b86f695dfb8056bda517c8115a00", size = 530780, upload-time = "2026-10-21T15:30:43.414Z" }, + { url = "https://files.pythonhosted.org/packages/3c/39/c152e1cc5371f75f5f9f0a4f44bf2dc9396be9407d384de7428446fb4651/tornado-6.6-cp314-cp314t-win_arm64.whl", hash = "sha256:b1f0ef20a5278854f5fdac31637bafadb055405df28b7fc945e54add12b172e3", size = 530781, upload-time = "2026-10-21T15:30:44.741Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c5/f96c7d7a8895745bb37dfb6e6107e1ee40ad02a4b162d7d6c143644455ab/tornado-6.6-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:8f8e382da30fe042f27f27a871e21aefc2613870e5a41648de18446c72d6ea59", size = 528446, upload-time = "2026-10-21T15:30:46.112Z" }, + { url = "https://files.pythonhosted.org/packages/c0/dd/ab8f6500d274b9c55b025798eb54e92591bcff27515799bec443bd54e35a/tornado-6.6-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8c7a8a328396c2195454a2ecd9499512441bbce99505544a717e6a87d16ff6f9", size = 532774, upload-time = "2026-10-21T15:30:47.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/23/b78e7d1bf329160b8157284005395421ea941864c16c7f04c3e48c98bcf9/tornado-6.6-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ca025bb0da81c294aef26f27427316df1525220820315f4bf32dc1e84a4d2232", size = 533872, upload-time = "2026-10-21T15:30:48.884Z" }, + { url = "https://files.pythonhosted.org/packages/e0/4f/ab66b614ad0bc9a219484be6bc2c002c8de9916a7b6c7b27fe6273838e5c/tornado-6.6-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:59c22ebc905ac2a267f8d9d8a878f86afbcbbde329d75c485f1422fc45a81b83", size = 534027, upload-time = "2026-10-21T15:30:50.157Z" }, + { url = "https://files.pythonhosted.org/packages/da/63/501fa15a36bace2f4b9158228cf922d606403261bf71c960c433ec3ca0a4/tornado-6.6-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f89802bc0bcd15f988d9325b4e6d2171bb04ea68d188732556335e10d3fd00bd", size = 532936, upload-time = "2026-10-21T15:30:51.45Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b7/82d643f635f0300bda98613ca2660b4f3134b6fb594c6fddfa4de706806c/tornado-6.6-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:70043256d5958f9c655066153d23d60ea3e29bf9854f7fb75850e223c477758a", size = 531254, upload-time = "2026-10-21T15:30:52.865Z" }, + { url = "https://files.pythonhosted.org/packages/d4/53/92c40891a94750bc09257310af964bf25c59e3531d6199736c318baa864f/tornado-6.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:82b49495e3849d644b841ef33b35491160055997deaa93720ff647ef019421db", size = 533566, upload-time = "2026-10-21T15:30:54.15Z" }, + { url = "https://files.pythonhosted.org/packages/7e/88/b82f5c5efde731b88d839ad99cb6ace2a521f03bb6d7346d2de089c76da8/tornado-6.6-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:07b2bcaa4bf447bdbfb664e4c28344226540db3f0ebccd74886528338a240765", size = 533567, upload-time = "2026-10-21T15:30:55.451Z" }, + { url = "https://files.pythonhosted.org/packages/02/2c/e9a7f5d0fca0bdbcd48b2d9c7884dd0b32669c9945d98bc6d303a80ecfea/tornado-6.6-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:c36b921374c302e54909bae17d9f354061d80e7ae7d8c94f73abbcc217866460", size = 532849, upload-time = "2026-10-21T15:30:56.729Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f5/4ade737340ca6ce9c677ea0c2843591c8ad065d07f0913c28d922c7e078e/tornado-6.6-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:7412d9e37aa9f7048e4b591f520a95f7a097278ffab15f1ddf3f5c2320cf6648", size = 532873, upload-time = "2026-10-21T15:30:58.067Z" }, + { url = "https://files.pythonhosted.org/packages/8e/04/6acdc937d0bb6189ddb34ceded517d7bd1c7b3bded5d4e1b3442160e6b6f/tornado-6.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:15bc95112b2d24b0bfaee9a004e3f974da929e33b4244a7f695a22f45b08a5e5", size = 532883, upload-time = "2026-10-21T15:30:59.395Z" }, + { url = "https://files.pythonhosted.org/packages/ab/27/0a766fac6b1cc629d96346329a0485287fd2857c51a887a3a154f2a69938/tornado-6.6-cp315-cp315t-win32.whl", hash = "sha256:f661c21117b875d612371f3eb243d0c69f56d9bdd6aee38b7c24453419dfe554", size = 530488, upload-time = "2026-10-21T15:31:00.77Z" }, + { url = "https://files.pythonhosted.org/packages/5c/80/3e4142570883da27b3f60ae82c638ac96e60997eac684740613557519c96/tornado-6.6-cp315-cp315t-win_amd64.whl", hash = "sha256:ee8853ab35da510dc952879ebd9995c82bc10ffb7925fd87930dc0d6916f4b39", size = 530777, upload-time = "2026-10-21T15:31:02.087Z" }, + { url = "https://files.pythonhosted.org/packages/17/57/4a85c617b6349962320b4afbe1386d5b05fd57218ee8ad6e82e0110ec9ee/tornado-6.6-cp315-cp315t-win_arm64.whl", hash = "sha256:7de62e2dfdf559fd7154d2b9aa58ff3d9cff8304ae0519341feb082618bc2a33", size = 530780, upload-time = "2026-10-21T15:31:03.425Z" }, ] [[package]] @@ -3488,7 +3526,7 @@ wheels = [ [[package]] name = "virtualenv" -version = "21.14.5" +version = "21.15.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib" }, @@ -3497,9 +3535,9 @@ dependencies = [ { name = "platformdirs" }, { name = "python-discovery" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c4/f9/f323b3b6058cff3853b31cf6a49c0425ed797bf9611572ec10d065e08bac/virtualenv-21.14.5.tar.gz", hash = "sha256:c4cb6c13e46b57225a999c7e22a09b163393878facc7ac4c059a57f46faa1647", size = 5467127, upload-time = "2026-10-02T17:42:25.613Z" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9e/40c7098240cedd3a2f32e42a74b35c4563c617fdd7e41ae6268d4fdd8915/virtualenv-21.15.2.tar.gz", hash = "sha256:dbd886e6e5689eb4dde6a9bbc478ad14b4fc8ed0852690d8273ccfb8c467f635", size = 5520597, upload-time = "2026-10-30T19:18:04.759Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/df/93/084f10ebbecdedf7902ed68fbecc903f0e9c5742155bfffb05eccbc071da/virtualenv-21.14.5-py3-none-any.whl", hash = "sha256:b0651e0174982bba17cc6f0aaa2ef496730cb85a3f35eb5d1119d1c904f7d1da", size = 5487482, upload-time = "2026-10-02T17:42:23.38Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d8/09c9d6df253b2016312dd5ccbb962dfa86a5202f2452604b52e0a9844661/virtualenv-21.15.2-py3-none-any.whl", hash = "sha256:fb8c62c7c328e0238061c721068f06b7bc6bd10f7a0f5987195cef1008837ee6", size = 5559530, upload-time = "2026-10-30T19:18:01.134Z" }, ] [[package]]