diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8c14665..e09302f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -5,6 +5,7 @@ on:
branches: [main]
pull_request:
branches: [main]
+ workflow_call: # Run by publish.yml before it publishes
permissions:
contents: read
@@ -15,6 +16,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
@@ -25,10 +28,29 @@ jobs:
run: uv sync --locked --dev
- name: Run ruff check
- run: uv run ruff check src/ tests/
+ run: uv run ruff check src/ tests/ demo/quickstart.py
- name: Run ruff format check
- run: uv run ruff format --check src/ tests/
+ run: uv run ruff format --check src/ tests/ demo/quickstart.py
+
+ typecheck:
+ name: Type check (mypy)
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+ with:
+ python-version: "3.11"
+
+ - name: Install dependencies
+ run: uv sync --locked --dev
+
+ - name: Run mypy
+ run: uv run mypy src/
test:
name: Test (Python ${{ matrix.python-version }})
@@ -40,6 +62,8 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
@@ -50,4 +74,4 @@ jobs:
run: uv sync --locked --dev
- name: Run tests
- run: uv run pytest tests/ --ignore=tests/test_parity.py -v
+ run: uv run pytest tests/ -v
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 3d9b3f5..a3d6a2d 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -5,9 +5,37 @@ on:
tags:
- 'v*'
+permissions:
+ contents: read
+
jobs:
+ check-version:
+ name: Check tag against package version
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Compare the tag with the version in pyproject.toml
+ env:
+ TAG: ${{ github.ref_name }}
+ run: |
+ version=$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')
+ if [ "$TAG" != "v$version" ]; then
+ echo "::error::Tag $TAG does not match the package version $version (expected v$version)"
+ exit 1
+ fi
+ echo "Tag $TAG matches the package version $version"
+
+ ci:
+ name: CI
+ needs: check-version
+ uses: ./.github/workflows/ci.yml # Every job of ci.yml, test matrix included
+
build-and-publish:
name: Build and publish to PyPI
+ needs: [check-version, ci]
runs-on: ubuntu-latest
environment: pypi
permissions:
@@ -16,18 +44,14 @@ jobs:
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
with:
python-version: "3.11"
- - name: Install dependencies
- run: uv sync --locked --dev
-
- - name: Run tests before publishing
- run: uv run pytest tests/ --ignore=tests/test_parity.py --cov=banos --cov-fail-under=80
-
- name: Build package
run: uv build
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 2bca525..53dc5f4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,10 +1,11 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.4.4
+ rev: v0.15.7 # Same ruff version as uv.lock (and CI)
hooks:
- - id: ruff
+ - id: ruff-check
args: [--fix]
- id: ruff-format
+ types_or: [python, pyi] # Notebooks are not reformatted
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.6.0
hooks:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e5b9cd7..a2f8567 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,140 @@
All notable changes to this project will be documented in this file.
+## [1.1.0] — 2026-10-30
+
+Two metrics are corrected (detection F1 and IC), two options are added (`missed_bouts` and
+`aggregation`), and invalid input now raises an error or a warning. With the default
+options and valid input, precision, recall, SO and TP are unchanged.
+
+### Numerical changes
+- **Detection F1 is 0, not NaN, when no real bout is matched.** F1 is now 0 when TP = 0
+ and FP + FN > 0 (a recording with real bouts and nothing predicted, or only false
+ bouts), the value of 2 TP / (2 TP + FP + FN). It stays NaN only when TP = FP = FN = 0;
+ every other value is unchanged. Such a recording now stays in the means instead of
+ leaving them. Precision and recall are unchanged (NaN on a zero denominator). This
+ brings back the 0 of v0.1.5, which v0.2.0 had turned into NaN.
+- **IC averages over the found real bouts only.** A real bout with no predicted frame
+ inside it holds no switch and scored IC 1.0: missing a bout raised IC, and a model that
+ predicted nothing scored IC 1.0. IC is now the mean of `1 - switches / (bout length - 1)`
+ over the real bouts with at least one predicted frame, one-frame bouts still left out,
+ and NaN when no such bout is found.
+- **IC reads the prediction by position.** IC read the prediction by index label, so
+ DataFrames indexed other than 0..n-1 (frames numbered from 1, reversed, float or string
+ labels, concatenated recordings) got a wrong IC or an exception. Input with the default
+ index is unchanged.
+- **Golden values.** The fixtures move to `golden_python_v110.json` and
+ `golden_matlab_v110.json`; the v0.2.0 files are kept as archives. 41 of the 300
+ per-recording values change: 2 F1 values (Recording_8 mount F1 from NaN to 0,
+ Recording_8 overall F1 from 0.6062 to 0.4546) and 39 IC values, in every recording
+ (Recording_8 mount IC from 1.0 to NaN: both its real bouts are missed). The other 259
+ are unchanged. `python tests/fixtures/compare_fixtures.py --mode 3` shows them.
+- **Bundled data**, the 10 recordings scored together with the default options
+ (`banos.score(data)`, four behaviors), 1.0.0 → 1.1.0: F1 mount 0.8780 → 0.7902, overall
+ 0.7746 → 0.7553; IC attack 0.9956 → 0.9949, investigation 0.9918 → 0.9875, mount
+ 0.9969 → 0.9942, other 0.9945 → 0.9927, overall 0.9947 → 0.9923. Precision, recall, SO
+ and TP are unchanged.
+- **A model that predicts nothing**, on the same data, per file: F1 attack 1.0 → 0.5,
+ mount 1.0 → 0.4, investigation and other NaN → 0, overall 1.0 → 0.225; IC investigation
+ and other 1.0 → NaN. The 0.5 and 0.4 that remain are the share of recordings without
+ the behavior (absent-behavior rule, unchanged). Pooled, its F1 is 0 for every behavior.
+- The MATLAB code gives the same numbers (parity tests within 1e-6).
+
+### Added
+- `missed_bouts` option of `banos.score()`: `"exclude"` (default) leaves a missed real
+ bout out of SO, TP and IC; `"zero"` adds a term equal to 0 to SO, to TP and, for a bout
+ longer than one frame, to IC. Precision, recall and F1 do not depend on it.
+- `aggregation` option of `banos.score()`: `"per_file"` (default) scores each recording,
+ then averages; `"pooled"` sums the TP, FP and FN counts of all recordings and averages
+ all their SO, TP and IC terms, for each behavior, without the absent-behavior rule, and
+ `overall_metrics` is the mean over the behaviors. The result has the same form in both.
+- Functions `calculate_so_terms`, `calculate_tp_terms` and `calculate_ic_terms`
+ (`banos.core`), `compute_behavior_components`, `calculate_components_for_each_file` and
+ `pool_components` (`banos`). `calculate_so`, `calculate_tp_metric`, `calculate_ic`,
+ `compute_behavior_metrics` and `calculate_banos_for_each_file` take the keyword
+ `missed_bouts`.
+- MATLAB: the `'missed_bouts'` and `'aggregation'` options of `BANOS_score`;
+ `BANOSparseOptions` and `BANOSpoolComponents`; the terms as second output of
+ `BANOScalculateSO`, `BANOScalculateTP` and `BANOScalculateIC`, and the components as
+ second output of `BANOScomputeBehaviorMetrics` and `BANOScalculateBANOSForEachFile`.
+- Golden values of the 10 recordings scored together, for each combination of the two
+ options (`all_recordings` in the v110 fixtures), checked by the regression and parity
+ tests.
+- `py.typed`: type checkers read the annotations of the package.
+- `CITATION.cff`, with the article to cite.
+- Mode 3 of `tests/fixtures/compare_fixtures.py` (v0.2.0 against v1.1.0), and a test that
+ runs the script and compares its output with `reports/fixture_comparison.html`.
+
+### Changed
+- **Input errors.** `ValueError` when the prediction and the ground truth of a recording
+ have different numbers of frames (a shorter prediction raised `KeyError` when a real
+ bout longer than one frame reached past its last frame, and was scored otherwise; a
+ longer one was scored) or when a recording has no frame (it scored 1.0 on all six
+ metrics).
+- **Warnings.** A `UserWarning` names the recording and the column for each column
+ dropped (values other than 0 and 1, or missing or invalid on the other side) and for
+ each column whose NaN values are replaced by 0; both were silent. The warnings point at
+ the line that called `banos.score()` or `preprocess_data()`. Valid input gives no
+ warning.
+- **Options checked on entry.** `banos.score()` and the pipeline functions raise
+ `ValueError` for an unknown `matching`, `missed_bouts` or `aggregation` value even when
+ nothing would be scored; an unknown `matching` used to pass unnoticed when no behavior
+ had real bouts.
+- **MATLAB.** `BANOSpreprocessData` raises and warns in the same way (identifiers
+ `BANOS:lengthMismatch`, `BANOS:noFrame`, `BANOS:nanReplaced`, `BANOS:columnDropped`).
+ Options are stricter: an unknown option name, which was ignored, and an unknown
+ `'matching'` value, which was treated as greedy, raise an error (`BANOS:unknownOption`,
+ `BANOS:missingOptionValue`, `BANOS:invalidOption`). The other errors of `BANOS_score`
+ get identifiers too: `BANOS:optimalNotImplemented`, `BANOS:missingGroundTruth` and
+ `BANOS:invalidData`.
+- **Imports.** `calculate_so`, `calculate_tp_metric` and `calculate_ic` are no longer
+ importable from `banos.pipeline`, which only used them; import them from `banos.core`.
+- **Packaging.** matplotlib and seaborn move from the runtime dependencies to the
+ `notebook` extra: `pip install banos` installs numpy, pandas and scipy only. The `dev`
+ extra is removed; the `dev` dependency group, which `uv sync` installs, stays. Authors
+ and maintainers are set, the license is an SPDX expression (`License-Expression: MIT`,
+ with `License-File`) and the license classifier is removed. The build backend is
+ `hatchling>=1.27,<2`. The sdist holds the package, its tests with the data they read,
+ `demo/quickstart.py`, README, LICENSE and CHANGELOG (about 70 kB instead of 1.2 MB).
+- **CI.** The test matrix runs the parity tests; a `Type check (mypy)` job runs mypy on
+ `src/`; checkouts do not keep credentials. The publish workflow checks that the tag is
+ `v` followed by the version in `pyproject.toml` and runs the whole CI before building
+ and publishing.
+- **Development tools.** Pre-commit runs ruff 0.15.7, the version of the lock and CI. CI
+ also lints `demo/quickstart.py`. mypy, pandas-stubs and scipy-stubs join the dev group;
+ pytest-html leaves it.
+- **Tests.** `tests/test_comparison_report.py`, which was always skipped, is removed:
+ `test_parity_group_metrics` checks the same values, and now also fails when Python and
+ MATLAB report different behaviors.
+
+### Fixed
+- The `score()` docstring and the `BANOS_score` help said that a metric is NaN when the
+ ground truth has no bout; per file, such a behavior scores 1.0 or 0.0.
+- README: the IC formula (the denominator is the bout length minus 1), SO and TP (averaged
+ over every overlapping pair, not over matched pairs), the return value for several
+ recordings, the dependencies, and links that did not work on the PyPI page.
+- Tutorials: the text said that the prediction shifted by 15 frames still detects both
+ bouts (DA 1.0); it shares no frame with them, so DA is 0 (it printed NaN before this
+ release).
+- The CalMS21 tutorial printed the absolute local paths of its model and data cache; it
+ prints them relative to the repository. Its download failed: data.caltech.edu refuses
+ the default user agent of urllib (HTTP 403), so the requests now send a User-Agent
+ that names the tutorial and the repository. Its stored outputs are those of 1.1.0.
+- `compare_fixtures.py` swapped the `VALUE->NULL` and `NULL->VALUE` tags and misdescribed
+ the v0.1.5 to v0.2.0 changes in its legend; it no longer fails on a Windows console.
+- `test_preprocess_data_does_not_mutate_caller_dict` compared a dict's id with itself and
+ could not fail; it now compares the caller's data before and after. `test_score_version`
+ compares `banos.__version__` with the version in `pyproject.toml`.
+- Type errors reported by mypy: `is_logical_column` returns a Python `bool`, and
+ `drop_non_logical_columns` is annotated with the column labels it returns.
+- The MATLAB code averaged with `nanmean`, a function of the Statistics and Machine
+ Learning Toolbox, so scoring several recordings needed that toolbox. It now uses
+ `mean(..., 'omitnan')` of base MATLAB, which gives the same values.
+- `SECURITY.md` said that BANOS reads CSV files (it reads no file: it takes DataFrames or
+ MATLAB cell arrays) and that `uv.lock` pins the dependencies users get (it pins the
+ development and CI environment). `SETUP.md` said to allow all actions (the repository
+ allows selected actions, pinned by SHA).
+
## [1.0.0] — 2025-02-27
First stable release. Metric computation and the golden fixtures are unchanged from 0.3.0.
@@ -36,10 +170,17 @@ No change to metric computation: `src/banos/` and the golden fixtures are unchan
- **Absent-behavior scoring**: when GT has no bouts, metrics are no longer `NaN`/`0`:
- Machine also has no bouts → all metrics = **1.0** (correct absence, rewarded)
- Machine predicts bouts that don't exist in GT → all metrics = **0.0** (false detection, penalized)
+- **Undefined metrics are NaN instead of 0** (missing from this entry until 1.1.0): for a
+ behavior with real bouts in a recording and no predicted bout matching any of them (so
+ none overlapping them), F1, SO and TP became NaN instead of 0, and so did precision when
+ nothing was predicted. The averages leave NaN out, so such a recording no longer
+ counted in these means. 1.1.0 scores this F1 as 0 again.
- Python minimum version raised to 3.9
### Bug Fixes
- Fixed IC boundary: last transition within GT bout was not counted (`range(gt_end-1)` → `range(gt_end)`)
+- **One-frame real bouts are left out of IC** (missing from this entry until 1.1.0): such a
+ bout has no frame pair to compare; v0.1.5 gave it an IC of 0, whatever was predicted.
- Fixed variable shadowing of `tp` count and `tp` metric in `compute_behavior_metrics`
- Fixed Matlab `BANOSmatchHeaders`: now matches by column name, not column index
- Fixed `aggregate_metrics`: NaN values now excluded from averages (nanmean behavior)
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 0000000..dff310b
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,56 @@
+cff-version: 1.2.0
+message: "If you use BANOS, please cite the article given as preferred citation."
+title: BANOS
+abstract: "Behavior ANnOtation Score: ethological metrics for evaluating behavior annotation quality."
+type: software
+version: 1.1.0
+date-released: 2026-10-30
+license: MIT
+repository-code: "https://github.com/BelloneLab/BANOS"
+authors:
+ - given-names: "Benoît J. T."
+ family-names: Girard
+keywords:
+ - behavior
+ - annotation
+ - ethology
+ - video
+ - scoring
+ - metrics
+ - computational ethology
+ - animal behavior
+ - behavioral neuroscience
+ - action segmentation
+ - bout detection
+ - evaluation metrics
+ - inter-annotator agreement
+ - rodent
+preferred-citation:
+ type: article
+ title: "From eye to AI: Studying rodent social behavior in the era of machine learning"
+ authors:
+ - given-names: Giuseppe
+ family-names: Chindemi
+ - given-names: Camilla
+ family-names: Bellone
+ - given-names: Benoit
+ family-names: Girard
+ journal: "Neuroscience & Biobehavioral Reviews"
+ volume: 188
+ start: 106766
+ year: 2026
+ doi: 10.1016/j.neubiorev.2026.106766
+references:
+ - type: article
+ title: "From eye to AI: studying rodent social behavior in the era of machine Learning"
+ authors:
+ - given-names: Giuseppe
+ family-names: Chindemi
+ - given-names: Camilla
+ family-names: Bellone
+ - given-names: Benoit
+ family-names: Girard
+ journal: "arXiv preprint arXiv:2508.04255"
+ year: 2025
+ doi: 10.48550/arXiv.2508.04255
+ url: "https://arxiv.org/abs/2508.04255"
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9846aca..76ad4dd 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -25,10 +25,9 @@ Contact: open a [GitHub Issue](https://github.com/BelloneLab/BANOS/issues).
## How to Report a Bug
-Open a [GitHub Issue](https://github.com/BelloneLab/BANOS/issues) using the bug report
-template. Please include:
+Open a [GitHub Issue](https://github.com/BelloneLab/BANOS/issues). Please include:
-- BANOS version (`pip show banos` or check `pyproject.toml`)
+- BANOS version (`pip show banos`, or `banos.__version__`)
- Python version (or MATLAB version, if applicable)
- A minimal reproducible example (small CSV + code that demonstrates the wrong output)
@@ -41,8 +40,19 @@ template. Please include:
Code style:
- Python: follow existing style enforced by `ruff` (lint + format)
-- Use type hints consistent with the existing codebase
+- Use type hints consistent with the existing codebase; CI runs `mypy` on `src/`
- Tests are required for bug fixes (see `tests/`)
+- A fix that changes a metric changes both implementations, Python (`src/banos/`) and
+ MATLAB (`matlab/BANOS/`), and regenerates the golden fixtures (see
+ [`RELEASE.md`](RELEASE.md)); the parity tests compare the two
+
+Before opening the PR, run the checks of CI (see [`SETUP.md`](SETUP.md)):
+```bash
+uv run ruff check src/ tests/ demo/quickstart.py
+uv run ruff format --check src/ tests/ demo/quickstart.py
+uv run mypy src/
+uv run pytest
+```
## What We Accept
diff --git a/README.md b/README.md
index 8b02c77..9c36a88 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
[](https://pypi.org/project/banos/)
[](https://pepy.tech/project/banos)
-[](LICENSE)
+[](https://github.com/BelloneLab/BANOS/blob/main/LICENSE)
[](https://www.python.org/downloads/)
[](https://github.com/BelloneLab/BANOS/actions/workflows/ci.yml)
[](https://www.mathworks.com/matlabcentral/fileexchange/157916-banos)
@@ -26,7 +26,7 @@ temporal qualities: does the predicted segment overlap well with the ground-trut
bout? Does the prediction start and end at the right time? Is the prediction
internally consistent, or does it flicker on and off inside the bout?
-BANOS addresses these questions with four complementary metrics.
+BANOS addresses these questions with four complementary metrics, computed for each behavior.
## The 4 BANOS Metrics
@@ -34,17 +34,65 @@ BANOS addresses these questions with four complementary metrics.
-1. **Detection Accuracy (DA)** — Precision, Recall, and F1 score over matched
- behavioral segments (TP/FP/FN at the bout level).
-
-2. **Segment Overlap (SO)** — Temporal Intersection over Union (tIoU) between each
- matched predicted segment and its ground-truth counterpart.
-
-3. **Temporal Precision (TP)** — Penalizes start/end time errors:
- `1 / (1 + |start deviation| + |end deviation|)`.
-
-4. **Intra-bout Continuity (IC)** — Measures label stability inside each
- ground-truth bout: `1 - (label switches / bout length)`.
+Annotations are binary: one column per behavior, one row per frame, 1 when the behavior is
+present. A **bout** is a run of consecutive frames with value 1. Real bouts come from the
+ground truth, predicted bouts from the annotation being evaluated. A predicted bout
+**overlaps** a real bout when they share at least one frame.
+
+1. **Detection Accuracy (DA)** — precision, recall and F1 over bouts. Predicted bouts are
+ taken in time order, and each is matched to the first real bout it overlaps that is not
+ matched yet (greedy matching). TP is the number of matched pairs, FP the number of
+ predicted bouts left unmatched, FN the number of real bouts left unmatched (TP here is
+ a count, not the Temporal Precision metric).
+ Precision = TP / (TP + FP), recall = TP / (TP + FN),
+ F1 = 2 · precision · recall / (precision + recall). F1 is 0 when no bout is matched
+ (TP = 0) but some bouts were predicted or missed (FP + FN > 0).
+
+2. **Segment Overlap (SO)** — the temporal intersection over union (shared frames / frames
+ in the union) of every overlapping (predicted, real) bout pair, averaged over all such
+ pairs.
+
+3. **Temporal Precision (TP)** — `1 / (1 + |start difference| + |end difference|)`, in
+ frames, for every overlapping (predicted, real) bout pair, averaged over all such pairs.
+ It falls quickly: a bout predicted 2 frames late at both ends scores 1 / 5 = 0.2.
+
+4. **Intra-bout Continuity (IC)** — label stability inside the real bouts that are found,
+ that is, with at least one predicted frame inside. For each found real bout longer than
+ one frame: `1 - switches / (bout length - 1)`, where the bout length is in frames and
+ the switches are the changes of the predicted value between consecutive frames of the
+ bout (a bout has bout length - 1 such frame pairs). IC is the mean over these bouts.
+ One-frame real bouts have no frame pair and are left out.
+
+SO and TP average over pairs, not over real bouts: a real bout overlapped by two predicted
+bouts gives two terms, and a predicted bout that overlaps two real bouts gives one term with
+each. A real bout with no predicted frame inside it (a missed bout) gives no SO, TP or IC
+term by default (see the `missed_bouts` option below).
+
+**Undefined values.** For a behavior with real bouts in a recording (or, with
+`aggregation="pooled"`, in at least one recording), recall and F1 are always defined, and
+a metric is NaN when it is undefined:
+
+- precision, when nothing is predicted (TP + FP = 0);
+- SO and TP, when no predicted bout overlaps a real bout;
+- IC, when no real bout longer than one frame is found.
+
+With `missed_bouts="zero"`, each missed bout adds a term equal to 0: SO and TP are then
+always defined, and IC is NaN only when no real bout is longer than one frame. Per
+recording, a behavior without real bouts follows the absent-behavior rule (see
+`aggregation` below). Pooled, a behavior with no real bout in any recording has an
+undefined recall, SO, TP and IC, and an undefined precision and F1 too when nothing is
+predicted either (TP = FP = FN = 0).
+
+NaN values are left out of every mean, over recordings or behaviors: a NaN never counts as
+0 or 1, and a mean with no defined value is NaN.
+
+**Matching.** `matching="optimal"` replaces greedy matching with the Hungarian algorithm
+(`scipy.optimize.linear_sum_assignment`), which finds the largest possible number of
+matched pairs. On bouts read from 0/1 columns, as in `banos.score()` and the pipeline
+functions, greedy matching already finds that number, so both options give the same counts
+and the same metrics. They can differ only when `banos.core.calculate_tp_fp_fn` is given
+bouts directly and the bouts of one side overlap one another or are not in time order;
+`banos.score()` always passes disjoint bouts in time order, so it is unaffected.
## Installation
@@ -52,8 +100,17 @@ BANOS addresses these questions with four complementary metrics.
pip install banos
```
-Requires Python 3.11+. Core dependency: `pandas`. Optional: `scipy` (for
-`matching='optimal'`).
+Requires Python 3.11+. Dependencies: `numpy` (>= 1.24), `pandas` (>= 2.0) and
+`scipy` (>= 1.10).
+
+Two optional extras install what the tutorial notebooks need (the notebooks are in the
+[GitHub repository](https://github.com/BelloneLab/BANOS/tree/main/demo), not in the
+package):
+
+- `pip install "banos[notebook]"`: jupyter, nbformat, matplotlib and seaborn, for
+ `demo/tutorial_python.ipynb`;
+- `pip install "banos[calms21]"`: tensorflow, scikit-learn and tqdm, for
+ `demo/tutorial_calms21_task1.ipynb` (together with the `notebook` extra).
## Quick Start
@@ -61,76 +118,183 @@ Requires Python 3.11+. Core dependency: `pandas`. Optional: `scipy` (for
import banos
import pandas as pd
-# Load annotations as DataFrames (rows=frames, columns=behaviors, values=0/1)
+# One recording: one row per frame, one 0/1 column per behavior
pred = pd.read_csv("machine_annotations.csv")
gt = pd.read_csv("human_annotations.csv")
-# Score a single recording
metrics = banos.score(pred, gt)
-# {'behavior1': {'precision': 0.9, 'recall': 0.85, 'f1_score': 0.87,
-# 'so': 0.76, 'tp': 0.82, 'ic': 0.91}, ...}
+# {'attack': {'precision': ..., 'recall': ..., 'f1_score': ..., 'so': ..., 'tp': ..., 'ic': ...}, ...}
-# Score multiple recordings
+# Several recordings: a dict of (prediction, ground truth) pairs
group_metrics, overall_metrics = banos.score({
- 'recording_1': (pred_1, gt_1),
- 'recording_2': (pred_2, gt_2),
+ "recording_1": (pred_1, gt_1),
+ "recording_2": (pred_2, gt_2),
})
```
-`banos.score()` returns:
+**Input.** For each recording, the prediction and the ground truth are DataFrames with the
+same number of rows (frames). Rows are read by position: the index is ignored. Columns are
+matched by name and hold 0 and 1 (integers, floats or booleans). NaN values are replaced by
+0, and columns that cannot be scored are dropped, each with a warning (see Errors and
+warnings below).
-- **Single recording**: `dict[behavior → dict[metric → float]]`
-- **Multiple recordings**: `(per-recording metrics, overall averaged metrics)`
+**Returns.** `banos.score()` returns, with either aggregation:
-Absent-behavior scoring (v0.2.0): when a behavior has no ground-truth bouts,
-BANOS checks whether the machine also predicted nothing — if so, all metrics = **1.0**
-(correct absence); if the machine predicted bouts that don't exist in GT, all metrics = **0.0**
-(false detection). This is excluded from NaN-averaging correctly.
+- for one recording (a DataFrame pair): `{behavior: {metric: value}}`, with the metrics
+ `precision`, `recall`, `f1_score` (DA), `so`, `tp` and `ic`;
+- for several recordings (a dict): a tuple `(group_metrics, overall_metrics)`, where
+ `group_metrics` is `{behavior: {metric: value}}` over all recordings and
+ `overall_metrics` is `{metric: value}`.
-## Examples & Tutorials
+## Options
-| Resource | Description |
-|----------|-------------|
-| [`demo/quickstart.py`](demo/quickstart.py) | Minimal Python example — score annotations in two lines |
-| [`demo/quickstart.m`](demo/quickstart.m) | Minimal MATLAB example |
-| [`demo/tutorial_python.ipynb`](demo/tutorial_python.ipynb) | Full Python tutorial: classic BANOS usage + frame F1 vs BANOS comparison |
-| [`demo/tutorial_matlab.m`](demo/tutorial_matlab.m) | Full MATLAB tutorial mirroring the Python notebook |
-| [`demo/tutorial_calms21_task1.ipynb`](demo/tutorial_calms21_task1.ipynb) | Human vs machine: CalMS21 Task 1 baseline, F1 + BANOS (Python only) |
+`banos.score(data, gt=None, *, matching="greedy", missed_bouts="exclude", aggregation="per_file")`
+
+### Combining recordings: `aggregation`
+
+**`aggregation="per_file"` (default).** Each recording is scored on its own, then the
+values are averaged: `group_metrics` is the mean of each behavior over the recordings,
+`overall_metrics` the mean of every (recording, behavior) value, NaN left out. Each
+recording weighs the same, whatever its number of bouts.
-## Advanced Usage
+*Absent-behavior rule:* when the ground truth of a recording has no bout of a behavior,
+that behavior scores 1.0 on all six metrics if nothing is predicted (correct absence), and
+0.0 on all six if at least one bout is predicted (false detection).
-### Optimal Matching
+The per-file mean therefore mixes detection quality with the share of recordings without
+the behavior. A model that predicts nothing scores 1.0 on every recording without the
+behavior and F1 = 0 on the others: on the bundled data, its per-file F1 is 0.5 for attack
+(absent from 5 of the 10 recordings) and 0.4 for mount (absent from 4), while it finds no
+bout at all.
-By default, predicted segments are matched to ground-truth bouts greedily (by
-overlap). For a globally optimal assignment, use the Hungarian algorithm:
+Use it to know how good the annotation of each recording is: look at the value of each
+recording (`calculate_banos_for_each_file`, see Lower-level API) and at their mean, with
+the share of recordings without each behavior in mind.
+
+**`aggregation="pooled"`.** Each behavior is scored once over all recordings: TP, FP and
+FN are summed over the recordings and give precision, recall and F1, and the SO, TP and IC
+terms of all recordings are averaged together. There is no absent-behavior rule: a
+recording without the behavior only adds its predicted bouts to FP. A behavior with no
+real and no predicted bout in any recording is NaN on all six metrics. `overall_metrics` is
+the mean over the behaviors, NaN left out. Every bout weighs the same, and predicting
+nothing gives F1 = 0: this is the right choice to compare or rank models and methods on
+the same recordings.
```python
-metrics = banos.score(pred, gt, matching='optimal')
+import pandas as pd
+import banos
+
+gt = pd.DataFrame({"attack": [0, 1, 1, 1, 0, 0, 1, 1, 1, 0]}) # two real bouts
+silent = pd.DataFrame({"attack": [0] * 10}) # a model that predicts nothing
+
+data = {
+ "video_1": (silent, gt), # both real bouts missed
+ "video_2": (silent, silent), # no attack in this video: correct absence
+}
+
+group, overall = banos.score(data) # per_file
+# group["attack"]: precision 1.0, recall 0.5, f1_score 0.5, so 1.0, tp 1.0, ic 1.0
+
+group, overall = banos.score(data, aggregation="pooled")
+# group["attack"]: precision nan, recall 0.0, f1_score 0.0, so nan, tp nan, ic nan
```
-Requires `scipy` (`pip install scipy`).
+Per file, `video_2` scores 1.0 everywhere, while `video_1` has recall and F1 0 and its
+precision, SO, TP and IC undefined: every mean but recall and F1 is 1.0. Pooled, the model
+has found 0 of the 2 bouts.
+
+### Missed bouts: `missed_bouts`
-### Lower-level API
+**`missed_bouts="exclude"` (default).** A real bout with no predicted frame inside it gives
+no SO, TP or IC term. SO, TP and IC then describe the quality of the bouts that were found,
+and are read together with recall, the share of real bouts found.
+
+**`missed_bouts="zero"`.** Each missed real bout adds a term equal to 0 to SO, to TP and,
+if it is longer than one frame, to IC, so these metrics also drop when bouts are missed.
+
+Precision, recall, F1 and the absent-behavior rule do not depend on this option.
```python
-from banos import preprocess_data, calculate_banos_for_each_file, aggregate_metrics
+gt = pd.DataFrame({"attack": [0, 1, 1, 1, 0, 0, 1, 1, 1, 0]}) # two real bouts
+pred = pd.DataFrame({"attack": [0, 1, 1, 1, 0, 0, 0, 0, 0, 0]}) # finds the first only
+
+banos.score(pred, gt)["attack"]
+# precision 1.0, recall 0.5, f1_score 0.667, so 1.0, tp 1.0, ic 1.0
+
+banos.score(pred, gt, missed_bouts="zero")["attack"]
+# precision 1.0, recall 0.5, f1_score 0.667, so 0.5, tp 0.5, ic 0.5
+```
+
+The two options combine: `banos.score(data, missed_bouts="zero", aggregation="pooled")`.
-data_dict = {
- 'file1': (pred_df_1, gt_df_1),
- 'file2': (pred_df_2, gt_df_2),
+### Errors and warnings
+
+`banos.score()` raises `ValueError`:
+
+- for an unknown `matching`, `missed_bouts` or `aggregation` value, before reading the data;
+- when `data` is a DataFrame and `gt` is missing;
+- when the prediction and the ground truth of a recording have different numbers of
+ frames, or a recording has no frame.
+
+It emits a `UserWarning` naming the recording and the column:
+
+- for each column whose NaN values are replaced by 0;
+- for each column dropped because it holds values other than 0 and 1 (probabilities, for
+ example);
+- for each column dropped because it is missing, or was dropped, on the other side.
+
+A single DataFrame pair is named `'single'` in these messages. Valid input gives no
+warning. `preprocess_data()` raises and warns in the same way.
+
+## Lower-level API
+
+`banos.score()` chains three functions; each step can be called on its own, for example
+to keep the results of each recording.
+
+```python
+from banos import (
+ preprocess_data,
+ calculate_banos_for_each_file, aggregate_metrics, # per_file
+ calculate_components_for_each_file, pool_components, # pooled
+)
+
+data = {
+ "recording_1": (pred_1, gt_1),
+ "recording_2": (pred_2, gt_2),
}
+clean, dropped = preprocess_data(data) # dropped: the columns removed from each recording
+
+# Per file: the metrics of each recording, then their means
+file_metrics = calculate_banos_for_each_file(clean) # {recording: {behavior: {metric: value}}}
+group_metrics, overall_metrics = aggregate_metrics(file_metrics)
-preprocessed, dropped = preprocess_data(data_dict)
-per_file_metrics = calculate_banos_for_each_file(preprocessed)
-group_metrics, overall = aggregate_metrics(per_file_metrics)
+# Pooled: the bout counts and terms of each recording, then one score per behavior
+file_components = calculate_components_for_each_file(clean) # {recording: {behavior: components}}
+group_metrics, overall_metrics = pool_components(file_components)
```
+`calculate_banos_for_each_file` and `calculate_components_for_each_file` take the
+`matching` and `missed_bouts` options of `banos.score()`. The components of a behavior in a
+recording are what its metrics are computed from:
+`{"n_tp": ..., "n_fp": ..., "n_fn": ..., "so_terms": [...], "tp_terms": [...], "ic_terms": [...]}`,
+the bout counts and the lists of terms that SO, TP and IC average. For a single recording
+that has already been preprocessed, `compute_behavior_metrics(pred, gt)` and
+`compute_behavior_components(pred, gt)` return its per-behavior metrics and components.
+
+`banos.core` holds the building blocks: `identify_bouts`, `calculate_tp_fp_fn`,
+`calculate_precision_recall_f1`, `calculate_tiou`, `calculate_so` and
+`calculate_so_terms`, `calculate_tp_metric` and `calculate_tp_terms`, `calculate_ic` and
+`calculate_ic_terms`, `count_switches_within_bout`, and the preprocessing functions
+`is_logical_column`, `drop_non_logical_columns`, `match_headers` and `preprocess_data`.
+Import the metric functions from `banos.core`.
+
## MATLAB
-A MATLAB implementation is provided in `matlab/BANOS/`. Usage mirrors the Python API.
+A MATLAB implementation is provided in `matlab/BANOS/`. It needs base MATLAB only, no
+toolbox. Usage mirrors the Python API.
```matlab
-% Add toolbox to path
+% Add the toolbox to the path for this session
run('matlab/BANOS/setup.m');
% Score a single recording (pred/gt = cell arrays: row 1 = headers, rows 2+ = binary data)
@@ -140,33 +304,80 @@ metrics = BANOS_score(pred, gt);
dataStruct.Recording_1 = {pred1, gt1};
dataStruct.Recording_2 = {pred2, gt2};
[groupMetrics, overallMetrics] = BANOS_score(dataStruct);
+
+% Options as name-value pairs (names and values are case-insensitive)
+metrics = BANOS_score(pred, gt, 'missed_bouts', 'zero');
+[groupMetrics, overallMetrics] = BANOS_score(dataStruct, 'aggregation', 'pooled');
```
-See [`demo/tutorial_matlab.m`](demo/tutorial_matlab.m) for a full walkthrough.
+The MATLAB code computes the same metrics with the same options and rules; the parity
+tests compare the Python results with golden values written by the MATLAB code, within
+1e-6. MATLAB uses greedy matching only (`'matching', 'optimal'` raises an error). A
+single recording is named `single_recording` in messages, and errors and warnings carry
+identifiers: `BANOS:lengthMismatch`, `BANOS:noFrame`, `BANOS:nanReplaced`,
+`BANOS:columnDropped`, `BANOS:unknownOption`, `BANOS:missingOptionValue`,
+`BANOS:invalidOption`, `BANOS:optimalNotImplemented`, `BANOS:missingGroundTruth` and
+`BANOS:invalidData`. `BANOScalculateBANOSForEachFile` returns the components of each
+recording as a second output, and `BANOSpoolComponents` pools them.
+
+See [`demo/tutorial_matlab.m`](https://github.com/BelloneLab/BANOS/blob/main/demo/tutorial_matlab.m)
+for a full walkthrough.
Also available on
[MATLAB File Exchange](https://www.mathworks.com/matlabcentral/fileexchange/157916-banos).
+## Examples & Tutorials
+
+| Resource | Description |
+|----------|-------------|
+| [`demo/quickstart.py`](https://github.com/BelloneLab/BANOS/blob/main/demo/quickstart.py) | Minimal Python example — score annotations in two lines |
+| [`demo/quickstart.m`](https://github.com/BelloneLab/BANOS/blob/main/demo/quickstart.m) | Minimal MATLAB example |
+| [`demo/tutorial_python.ipynb`](https://github.com/BelloneLab/BANOS/blob/main/demo/tutorial_python.ipynb) | Full Python tutorial: classic BANOS usage + frame F1 vs BANOS comparison (`notebook` extra) |
+| [`demo/tutorial_matlab.m`](https://github.com/BelloneLab/BANOS/blob/main/demo/tutorial_matlab.m) | Full MATLAB tutorial mirroring the Python notebook |
+| [`demo/tutorial_calms21_task1.ipynb`](https://github.com/BelloneLab/BANOS/blob/main/demo/tutorial_calms21_task1.ipynb) | Human vs machine: CalMS21 Task 1 baseline, F1 + BANOS (Python only; `calms21` and `notebook` extras) |
+
+The demos run from a clone of the repository. The quickstarts and the two full tutorials
+score the example recordings in `data/dataset_human_vs_human/`; the CalMS21 notebook
+downloads its test data on first run.
+
## Contributing
BANOS is functionally complete by design. We welcome **bug reports** and
-**documentation improvements**. See [CONTRIBUTING.md](CONTRIBUTING.md) for details on
-how to report issues, submit fixes, and the project's maintenance lifecycle.
+**documentation improvements**. See
+[CONTRIBUTING.md](https://github.com/BelloneLab/BANOS/blob/main/CONTRIBUTING.md) for
+details on how to report issues, submit fixes, and the project's maintenance lifecycle.
-To report a security vulnerability, see [SECURITY.md](SECURITY.md) — please do not use
-public GitHub Issues for security reports.
+To report a security vulnerability, see
+[SECURITY.md](https://github.com/BelloneLab/BANOS/blob/main/SECURITY.md) — please do not
+use public GitHub Issues for security reports.
## Citation
-If you use BANOS, please cite:
+If you use BANOS, please cite the article:
-> Chindemi G., Bellone C., Girard B. *From eye to AI: studying rodent social behavior
-> in the era of machine learning.* arXiv:2508.04255 (2025).
+> Chindemi G, Bellone C, Girard B. *From eye to AI: Studying rodent social behavior in the
+> era of machine learning.* Neuroscience & Biobehavioral Reviews 188 (2026) 106766.
+> https://doi.org/10.1016/j.neubiorev.2026.106766
+
+The preprint of this article is:
+
+> Chindemi G, Bellone C, Girard B. *From eye to AI: studying rodent social behavior in the
+> era of machine Learning.* arXiv:2508.04255 (2025).
> https://doi.org/10.48550/arXiv.2508.04255
```bibtex
+@article{chindemi2026eye,
+ title = {From eye to {AI}: Studying rodent social behavior in the era of machine learning},
+ author = {Chindemi, Giuseppe and Bellone, Camilla and Girard, Benoit},
+ journal = {Neuroscience \& Biobehavioral Reviews},
+ volume = {188},
+ pages = {106766},
+ year = {2026},
+ doi = {10.1016/j.neubiorev.2026.106766}
+}
+
@article{chindemi2025eye,
- title = {From eye to {AI}: studying rodent social behavior in the era of machine learning},
+ title = {From eye to {AI}: studying rodent social behavior in the era of machine {L}earning},
author = {Chindemi, Giuseppe and Bellone, Camilla and Girard, Benoit},
journal = {arXiv preprint arXiv:2508.04255},
year = {2025},
@@ -174,10 +385,15 @@ If you use BANOS, please cite:
}
```
+The same references are in
+[CITATION.cff](https://github.com/BelloneLab/BANOS/blob/main/CITATION.cff), which GitHub
+offers as "Cite this repository".
+
## Changelog
-See [CHANGELOG.md](CHANGELOG.md) for release history.
+See [CHANGELOG.md](https://github.com/BelloneLab/BANOS/blob/main/CHANGELOG.md) for release
+history.
## License
-MIT — see [LICENSE](LICENSE).
+MIT — see [LICENSE](https://github.com/BelloneLab/BANOS/blob/main/LICENSE).
diff --git a/RELEASE.md b/RELEASE.md
index 94ef1c6..f46e116 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -4,46 +4,65 @@
Before tagging a release, verify:
-- [ ] All CI tests pass on `main` (`pytest` green, coverage >= 80%)
-- [ ] `uv run pytest tests/test_regression.py` passes (verifies golden fixtures)
-- [ ] Version bumped consistently in `pyproject.toml` and `src/banos/__init__.py`
-- [ ] `CHANGELOG.md` has an entry for this version with a summary of changes
+- [ ] CI is green on `main`: `Lint (ruff)`, `Type check (mypy)` and `Test (Python 3.11)`
+ to `Test (Python 3.14)`, which run the whole suite, regression and parity tests
+ included, with coverage >= 80%
+- [ ] The version is the same in `pyproject.toml`, `src/banos/__init__.py`, `uv.lock`
+ (run `uv lock`), `CITATION.cff` and the `Version` line of `matlab/BANOS/Contents.m`
+ (`test_score_version` compares the first two)
+- [ ] `CHANGELOG.md` has an entry for this version, dated on release day; the same date
+ is set in `CITATION.cff` (`date-released`) and in `matlab/BANOS/Contents.m`
- [ ] README examples are up to date
---
## Python Package — PyPI
-Publishing is fully automated via GitHub Actions on version tag push.
+Publishing is automated by GitHub Actions when a version tag is pushed. `main` only
+accepts changes through a pull request, so the version is bumped on a branch.
```bash
-# 1. Bump version in two places
-# pyproject.toml: version = "X.Y.Z"
-# src/banos/__init__.py: __version__ = "X.Y.Z"
+# 1. On a branch, bump the version
+# pyproject.toml: version = "X.Y.Z"
+# src/banos/__init__.py: __version__ = "X.Y.Z"
+# CITATION.cff: version: X.Y.Z and date-released: YYYY-MM-DD
+# matlab/BANOS/Contents.m: % Version X.Y.Z DD-Mon-YYYY
+uv lock
-# 2. Update CHANGELOG.md
+# 2. Date the CHANGELOG.md entry
-# 3. Commit
-git commit -am "chore: bump version to vX.Y.Z"
+# 3. Commit, open a pull request, and merge it once CI is green, as a single
+# squashed commit whose subject is "chore: release vX.Y.Z (#N)", N being
+# the pull request number, and whose body is empty
+git commit -am "chore: release vX.Y.Z"
-# 4. Tag
-git tag vX.Y.Z
+# 4. Tag the merged commit on main
+git fetch origin
+git tag -a vX.Y.Z origin/main -m "BANOS X.Y.Z"
-# 5. Push tag — this triggers the publish workflow automatically
+# 5. Push the tag — this triggers the publish workflow
git push origin vX.Y.Z
+
+# 6. Once the package is on PyPI, create the GitHub release for the tag,
+# with the CHANGELOG.md entry of the version as notes (notes.md)
+gh release create vX.Y.Z --verify-tag --title "vX.Y.Z" --notes-file notes.md
```
The `publish.yml` workflow:
-1. Runs the full test suite (excluding parity tests)
-2. Builds the package with `uv build`
-3. Publishes to PyPI using OIDC trusted publishing (no API token required)
+1. Checks that the tag is `v` followed by the version in `pyproject.toml`, and stops
+ otherwise
+2. Runs the whole CI workflow (`ci.yml`): lint, type check, and the test matrix on
+ Python 3.11 to 3.14, parity tests included
+3. Builds the package with `uv build` and publishes it to PyPI using OIDC trusted
+ publishing (no API token required). This job runs in the `pypi` environment, which
+ waits for the maintainer's approval
The `README.md` is automatically bundled because `pyproject.toml` specifies
`readme = "README.md"` — no manual copying needed.
**Note:** OIDC trusted publishing requires a one-time setup in your PyPI account:
-go to the project page → Publishing → Add a new pending publisher, and configure
-it for this GitHub repository and the `publish.yml` workflow.
+go to the project page → Publishing → Add a new publisher, and configure it for this
+GitHub repository, the `publish.yml` workflow and the `pypi` environment.
---
@@ -52,8 +71,10 @@ it for this GitHub repository and the `publish.yml` workflow.
MATLAB File Exchange does not have a public API for automated uploads.
Updates are done manually when a new version is released.
-1. Download the release ZIP from the GitHub releases page:
- `https://github.com/BelloneLab/BANOS/releases`
+1. Download the source ZIP of the tag:
+ `https://github.com/BelloneLab/BANOS/archive/refs/tags/vX.Y.Z.zip`
+ (also linked from the tag on the GitHub releases page,
+ `https://github.com/BelloneLab/BANOS/releases`)
2. Log in to MATLAB File Exchange:
`https://www.mathworks.com/matlabcentral/fileexchange/`
@@ -72,12 +93,28 @@ Updates are done manually when a new version is released.
| File | Represents |
|------|-----------|
-| `tests/fixtures/golden_python_v020.json` | **Current Python (v0.2.0)** — score 1.0 for correct absence, 0.0 for false detection, corrected IC counting. Canonical reference. |
-| `tests/fixtures/golden_matlab_v020.json` | **Current Matlab (v0.2.0)** — same semantics as Python. Used for parity tests. |
-| `tests/fixtures/golden_python_v015.json` | **Original Python (v0.1.5)** — returns 0 for absent behaviors, IC misses last frame pair. Kept for historical comparison only. |
-
-To regenerate fixtures after a deliberate algorithm change:
+| `tests/fixtures/golden_python_v110.json` | **Current Python (v1.1.0)** — detection F1 0 when no real bout is matched, IC averaged over the found real bouts. Each `Recording_N` entry scores one recording alone with the default options; `all_recordings` scores the 10 recordings together for each combination of `missed_bouts` and `aggregation`. Canonical reference, used by the regression tests. |
+| `tests/fixtures/golden_matlab_v110.json` | **Current MATLAB (v1.1.0)** — the same entries, computed by the MATLAB code. Used by the parity tests. |
+| `tests/fixtures/golden_python_v020.json` | **Python v0.2.0** (archive) — 1.0 for correct absence, 0.0 for false detection, NaN for undefined metrics (F1 included), IC counting the last frame pair. |
+| `tests/fixtures/golden_matlab_v020.json` | **MATLAB v0.2.0** (archive) — same semantics as Python v0.2.0. |
+| `tests/fixtures/golden_python_v015.json` | **Original Python (v0.1.5)** (archive) — 0 for absent behaviors and undefined metrics, IC misses the last frame pair. Kept for historical comparison only. |
+
+To regenerate the current fixtures after a deliberate algorithm change, from the project
+root:
```bash
-python tests/fixtures/python/generate_golden_fixtures.py
+uv run python tests/fixtures/python/generate_golden_fixtures.py
matlab -batch "run('tests/fixtures/matlab/generate_matlab_fixtures.m')"
```
+
+Then regenerate the comparison report (`reports/fixture_comparison.html`), which
+`tests/test_compare_fixtures.py` checks against the fixtures:
+```bash
+uv run python tests/fixtures/compare_fixtures.py --html
+```
+
+The archives are not regenerated. The v0.1.5 file can be reproduced with the BANOS 0.1.5
+package from PyPI and Python 3.12 or later (with Python 3.11, about a hundred values
+differ in their last digit):
+```bash
+uv run --isolated --no-project --python 3.12 --with "BANOS==0.1.5" python tests/fixtures/python/generate_archive_fixtures.py
+```
diff --git a/SECURITY.md b/SECURITY.md
index a988ae6..e8ee213 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -32,22 +32,26 @@ BANOS is a pure-Python / MATLAB computation library. It:
- Has **no network access**
- Has **no authentication or authorization layer**
- Has **no persistent data storage**
-- Reads only local CSV files passed explicitly by the user
+- Reads **no file**: it takes pandas DataFrames (Python) or cell arrays (MATLAB) that the
+ caller has already loaded. Only the demos and tutorials read files, the CSV files of
+ the repository and, for the CalMS21 tutorial, a dataset it downloads
The attack surface is minimal. Security reports are most relevant to dependency
vulnerabilities (e.g., malicious `pandas` or `scipy` releases) or unsafe handling of
-user-supplied CSV input.
+user-supplied data.
## Dependency Policy
BANOS accompanies a peer-reviewed publication and is maintained in security-only mode.
Dependency handling follows two rules.
-**Numerical results must not change.** Runtime dependencies are pinned in `uv.lock`.
-Every update is validated by the golden regression tests
+**Numerical results must not change.** `uv.lock` pins the development and CI
+environment. Every update of the lock is validated by the golden regression tests
(`tests/test_regression.py`, tolerance `1e-10`) across the whole supported Python
matrix before it is merged. An update that moves a metric is rejected — the golden
-fixtures are never regenerated to accommodate a dependency bump.
+fixtures are never regenerated to accommodate a dependency bump. Users who install
+`banos` are not bound by the lock: they get any version of its dependencies within the
+bounds declared in `pyproject.toml`.
**Updates are deliberate, never automatic.** Dependabot alerts are enabled for
monitoring, but automatic security updates and version updates are disabled. Each
@@ -61,5 +65,6 @@ publishing, and unlike Python packages they cannot affect a computed result.
One caveat when reading alerts: Dependabot reports every entry of `uv.lock` as
`runtime` scope. In practice the BANOS runtime closure is limited to `numpy`,
-`pandas`, `scipy`, `matplotlib` and `seaborn`. The `notebook` and `calms21` extras
-are optional tutorial environments and are not needed to import or use BANOS.
+`pandas` and `scipy`, and the packages they depend on. The `notebook` and `calms21`
+extras are optional tutorial environments, and the `dev` dependency group is the
+development environment; neither is needed to import or use BANOS.
diff --git a/SETUP.md b/SETUP.md
index ef68420..572dd42 100644
--- a/SETUP.md
+++ b/SETUP.md
@@ -2,8 +2,18 @@
## GitHub Repository Settings
-- **Branch protection on `main`**: require PR review + CI passing before merge
-- **Actions permissions**: allow all actions (needed for CI and PyPI publish)
+- **Branch protection on `main`**: require PR review + CI passing before merge. The
+ required checks are the CI job names `Lint (ruff)` and `Test (Python 3.11)` to
+ `Test (Python 3.14)`, so these jobs must keep their names. `Type check (mypy)` runs
+ too, and blocks a merge only once it is added to the required checks
+- **Tags `v*`**: only the maintainer may create them, since a pushed tag starts the
+ publish workflow
+- **Actions permissions**: allow only selected actions — actions created by GitHub, plus
+ `actions/checkout`, `astral-sh/setup-uv` and `pypa/gh-action-pypi-publish` — and
+ require actions to be pinned to a full commit SHA. The workflows pin each action by
+ SHA, with its version in a comment; Dependabot updates them
+- **Environment `pypi`**: requires the maintainer's approval before the publishing job
+ runs
- **Secrets**: none required (OIDC replaces API tokens for PyPI)
---
@@ -14,23 +24,54 @@
# Install uv (package manager)
pip install uv
-# Install the package in editable mode with dev dependencies
-uv sync --extra dev
+# Install the package in editable mode with the dev dependency group
+uv sync
-# Verify everything works
+# Verify everything works (the parity tests need no MATLAB)
uv run pytest -q
```
+The checks run by CI:
+```bash
+uv run ruff check src/ tests/ demo/quickstart.py
+uv run ruff format --check src/ tests/ demo/quickstart.py
+uv run mypy src/
+uv run pytest tests/ -v
+```
+
+`uv build` builds the sdist and the wheel into `dist/`.
+
### Optional: pre-commit hooks
```bash
uv run pre-commit install
```
+### Python tutorial
+
+`demo/tutorial_python.ipynb` needs the `notebook` extra:
+```bash
+uv run --extra notebook jupyter notebook demo/tutorial_python.ipynb
+```
+
+The notebook is generated by `scripts/create_tutorial_notebook.py`. To regenerate it and
+store its outputs, from the project root:
+```bash
+uv run python scripts/create_tutorial_notebook.py
+uv run --extra notebook jupyter nbconvert --to notebook --execute --inplace \
+ --ExecutePreprocessor.record_timing=False demo/tutorial_python.ipynb
+```
+
---
## Human-vs-Machine Tutorial — CalMS21 Task 1
-The `demo/tutorial_calms21_task1.ipynb` notebook requires:
+The `demo/tutorial_calms21_task1.ipynb` notebook runs with the `calms21` and `notebook`
+extras:
+```bash
+uv run --extra calms21 --extra notebook jupyter notebook demo/tutorial_calms21_task1.ipynb
+```
+
+It requires:
### 1. Model weights (already in repo)
@@ -74,13 +115,14 @@ cp results/task1_baseline/task1_seed_42_model.h5 ../../demo/models/
### 2. CalMS21 test data (downloaded at runtime)
-The test JSON (~600 MB) is **not** included in the repo. The notebook downloads
-it automatically on first run from CaltechDATA to `.cache/calms21/` (gitignored).
+The test JSON (~600 MB) is **not** included in the repo. On first run, the notebook
+downloads the CalMS21 Task 1 zip (~457 MB) from CaltechDATA to `.cache/calms21/`
+(gitignored) and extracts the test JSON from it.
-If the automatic download fails, download manually from:
-https://data.caltech.edu/records/s0vdx-0k302
+If the automatic download fails, download `task1_classic_classification.zip` manually
+from: https://data.caltech.edu/records/s0vdx-0k302
-Place the file at: `.cache/calms21/calms21_task1_test.json`
+Place the file at: `.cache/calms21/task1_classic_classification.zip`
### 3. Optional — publish model to GitHub Releases
@@ -93,4 +135,6 @@ gh release create calms21-assets-v1 demo/models/task1_seed_42_model.h5 \
--notes "Pre-trained Conv-1D baseline model (seed 42) for tutorial_calms21_task1.ipynb"
```
-Then update `MODEL_URL` in the notebook's setup cell to the release asset URL.
+The notebook loads the model from `demo/models/` (`MODEL_PATH` in its setup cell); it
+has no download step for the model, so a copy downloaded from the release must be placed
+there, or `MODEL_PATH` changed.
diff --git a/demo/quickstart.py b/demo/quickstart.py
index a59f29f..5912de3 100644
--- a/demo/quickstart.py
+++ b/demo/quickstart.py
@@ -7,9 +7,10 @@
from pathlib import Path
-import banos
import pandas as pd
+import banos
+
DATASET = Path(__file__).parent.parent / "data" / "dataset_human_vs_human"
diff --git a/demo/tutorial_calms21_task1.ipynb b/demo/tutorial_calms21_task1.ipynb
index baa7e92..414524b 100644
--- a/demo/tutorial_calms21_task1.ipynb
+++ b/demo/tutorial_calms21_task1.ipynb
@@ -61,18 +61,18 @@
"output_type": "stream",
"text": [
"TensorFlow 2.21.0\n",
- "BANOS 0.2.0\n",
- "Model: C:\\Users\\GA401\\Documents\\Pro\\Code\\GithubPerso\\Tl_BANOS_dev\\demo\\models\\task1_seed_42_model.h5 (959 KB)\n",
- "Cache: C:\\Users\\GA401\\Documents\\Pro\\Code\\GithubPerso\\Tl_BANOS_dev\\.cache\\calms21\n"
+ "BANOS 1.1.0\n",
+ "Model: demo/models/task1_seed_42_model.h5 (959 KB)\n",
+ "Cache: .cache/calms21\n"
]
}
],
"source": [
+ "import math\n",
"import sys\n",
+ "import urllib.request\n",
"import zipfile\n",
"from pathlib import Path\n",
- "import math\n",
- "import urllib.request\n",
"\n",
"import numpy as np\n",
"import pandas as pd\n",
@@ -84,9 +84,10 @@
" print(\"TensorFlow not found. Install with: pip install banos[calms21]\")\n",
" sys.exit(1)\n",
"\n",
- "import banos\n",
"from sklearn.metrics import f1_score\n",
"\n",
+ "import banos\n",
+ "\n",
"print(f\"BANOS {banos.__version__}\")\n",
"\n",
"\n",
@@ -106,6 +107,7 @@
"\n",
"\n",
"DEMO_DIR = _find_demo_dir()\n",
+ "REPO_DIR = DEMO_DIR.parent\n",
"\n",
"MODEL_PATH = DEMO_DIR / \"models\" / \"task1_seed_42_model.h5\"\n",
"\n",
@@ -123,8 +125,8 @@
" f\"Model not found at {MODEL_PATH}\\n\"\n",
" f\"Make sure you cloned the full repo (demo/models/task1_seed_42_model.h5 must exist).\"\n",
")\n",
- "print(f\"Model: {MODEL_PATH} ({MODEL_PATH.stat().st_size / 1e3:.0f} KB)\")\n",
- "print(f\"Cache: {CACHE_DIR}\")"
+ "print(f\"Model: {MODEL_PATH.relative_to(REPO_DIR).as_posix()} ({MODEL_PATH.stat().st_size / 1e3:.0f} KB)\")\n",
+ "print(f\"Cache: {CACHE_DIR.relative_to(REPO_DIR).as_posix()}\")"
]
},
{
@@ -159,7 +161,7 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "CalMS21 task1 zip: 457MB [00:41, 11.1MB/s] \n"
+ "CalMS21 task1 zip: 457MB [00:13, 34.5MB/s] \n"
]
},
{
@@ -173,6 +175,13 @@
}
],
"source": [
+ "# data.caltech.edu refuses the default user agent of urllib (HTTP 403):\n",
+ "# the requests name this tutorial instead.\n",
+ "_opener = urllib.request.build_opener()\n",
+ "_opener.addheaders = [(\"User-Agent\", \"BANOS tutorial (https://github.com/BelloneLab/BANOS)\")]\n",
+ "urllib.request.install_opener(_opener)\n",
+ "\n",
+ "\n",
"def download_with_progress(url, dest_path, label):\n",
" \"\"\"Download a file once; skip if already present.\"\"\"\n",
" dest_path = Path(dest_path)\n",
@@ -288,7 +297,7 @@
"for rec in recordings[:3]:\n",
" n = dataset[rec][\"gt\"].shape[0]\n",
" labels, counts = np.unique(dataset[rec][\"gt\"], return_counts=True)\n",
- " beh_str = \" \".join(f\"{list(VOCAB)[l]}={int(c)}\" for l, c in zip(labels, counts))\n",
+ " beh_str = \" \".join(f\"{list(VOCAB)[label]}={int(c)}\" for label, c in zip(labels, counts))\n",
" print(f\" {rec.split('/')[-1]}: {n} frames | {beh_str}\")\n",
"print(\" ...\")"
]
@@ -579,17 +588,17 @@
"text": [
"BANOS per-behavior metrics (nanmean across 19 recordings):\n",
" DA (F1) SO TP IC Precision Recall\n",
- "attack 0.5745 0.5272 0.4800 0.7287 0.5548 0.6076\n",
- "investigation 0.6476 0.4556 0.0651 0.9734 0.6166 0.7061\n",
- "mount 0.7006 0.6357 0.2586 0.9374 0.6801 0.7264\n",
+ "attack 0.5745 0.5272 0.4800 0.7228 0.5548 0.6076\n",
+ "investigation 0.6476 0.4556 0.0651 0.9652 0.6166 0.7061\n",
+ "mount 0.6638 0.6357 0.2586 0.9308 0.6801 0.7264\n",
"\n",
"BANOS overall metrics (nanmean across behaviors and recordings):\n",
" precision 0.6172\n",
" recall 0.6801\n",
- " f1_score 0.6398\n",
+ " f1_score 0.6286\n",
" so 0.5378\n",
" tp 0.2681\n",
- " ic 0.8798\n"
+ " ic 0.8719\n"
]
}
],
@@ -645,10 +654,10 @@
"---------------------------------------------------------------------------\n",
"CalMS21 F1 (pooled) 0.7792 Global frame overlap; matches leaderboard method\n",
"Our Frame F1 0.7267 Per-recording avg; explicit absent-behavior scoring\n",
- "BANOS DA (F1) 0.6398 Bout-level detection accuracy\n",
- "BANOS SO 0.5378 Temporal overlap quality of matched bouts\n",
+ "BANOS DA (F1) 0.6286 Bout-level detection accuracy\n",
+ "BANOS SO 0.5378 Temporal quality of overlapping bouts\n",
"BANOS TP 0.2681 Boundary precision (start/end frame accuracy)\n",
- "BANOS IC 0.8798 Label stability within detected bouts\n"
+ "BANOS IC 0.8719 Label stability within detected bouts\n"
]
}
],
@@ -671,7 +680,7 @@
" (\"BANOS DA (F1)\", banos_da,\n",
" \"Bout-level detection accuracy\"),\n",
" (\"BANOS SO\", banos_so,\n",
- " \"Temporal overlap quality of matched bouts\"),\n",
+ " \"Temporal quality of overlapping bouts\"),\n",
" (\"BANOS TP\", banos_tp,\n",
" \"Boundary precision (start/end frame accuracy)\"),\n",
" (\"BANOS IC\", banos_ic,\n",
@@ -701,7 +710,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.12.9"
+ "version": "3.12.12"
}
},
"nbformat": 4,
diff --git a/demo/tutorial_matlab.m b/demo/tutorial_matlab.m
index 59a9b10..e115b66 100644
--- a/demo/tutorial_matlab.m
+++ b/demo/tutorial_matlab.m
@@ -14,7 +14,7 @@
clear; clc;
-% Detect project root (works whether run from root or example/)
+% Detect project root from the location of this file (demo/)
thisDir = fileparts(mfilename('fullpath'));
if exist(fullfile(thisDir, 'matlab'), 'dir')
projectRoot = thisDir;
@@ -40,9 +40,9 @@
%% 1.1 Setup — already done above (addpath, BEHAVIORS, datasetDir)
%% 1.2 Single Recording -------------------------------------------------------
-% Recording_1, behaviors event1/event2/event3 (dropping 'other')
+% Recording_1, behaviors attack/investigation/mount (dropping 'other')
%
-% v0.2.0 absent-behavior rule:
+% Absent-behavior rule (since v0.2.0; per-file scoring, the default):
% - GT has no bouts AND machine predicts nothing -> all metrics = 1.0 (correct absence)
% - GT has no bouts BUT machine predicts bouts -> all metrics = 0.0 (false detection)
@@ -50,7 +50,7 @@
predTable = readtable(fullfile(recDir, 'humanAnnotation_2.csv'));
gtTable = readtable(fullfile(recDir, 'humanAnnotation_1.csv'));
-% Keep only event1/event2/event3
+% Keep only attack/investigation/mount
predTable = predTable(:, BEHAVIORS);
gtTable = gtTable(:, BEHAVIORS);
@@ -125,7 +125,7 @@
fprintf(' SO : Temporal IoU -- how well do detected bouts overlap GT?\n');
fprintf(' TP : Boundary precision -- are bout start/end frames accurate?\n');
fprintf(' IC : Intra-bout continuity -- is each detected bout clean?\n\n');
-fprintf('Absent-behavior rule (v0.2.0):\n');
+fprintf('Absent-behavior rule (since v0.2.0; per-file scoring, the default):\n');
fprintf(' GT has no bouts AND machine predicts nothing -> all metrics = 1.0 (correct absence)\n');
fprintf(' GT has no bouts BUT machine predicts bouts -> all metrics = 0.0 (false detection)\n\n');
@@ -171,7 +171,7 @@
end
framef1_overall = mean(perRecF1, 'omitnan');
-fprintf('Frame-based F1 (our approach): %.3f\n', framef1_overall); % Expected: ~0.791
+fprintf('Frame-based F1 (our approach): %.3f\n', framef1_overall); % Expected: 0.815
%% 2b. BANOS metrics: same recordings, same behaviors ------------------------
@@ -217,8 +217,13 @@
fprintf(' 2. Explicit absent-behavior scoring: we reward correct absence (score 1)\n');
fprintf(' and penalize false detection (score 0). CalMS21 implicitly handles\n');
fprintf(' this via pooling.\n\n');
-fprintf('We do NOT align with CalMS21 pooled approach because BANOS quantifies\n');
-fprintf('annotation quality per recording, not globally only.\n\n');
+fprintf('By default, we do NOT align with CalMS21 pooled approach because BANOS\n');
+fprintf('quantifies annotation quality per recording, not globally only.\n');
+fprintf('To compare or rank models on a whole dataset,\n');
+fprintf('BANOS_score(data, ''aggregation'', ''pooled'') scores each behavior once on\n');
+fprintf('the bout counts and terms of all recordings, without the absent-behavior\n');
+fprintf('rule. The per-file mean, by contrast, mixes detection quality with the\n');
+fprintf('share of recordings without the behavior (see the README).\n\n');
fprintf('For publications: state both the frame F1 methodology and BANOS metrics.\n\n');
%% 2e. Thought experiment -- the shifted annotator ---------------------------
@@ -255,7 +260,7 @@
fprintf('%s\n', repmat('-', 1, 52));
for pi = 1:3
p = preds{pi};
- ff1 = local_frame_f1(p, gt_s);
+ ff1 = local_frame_f1_raw(p, gt_s);
pred_cell = [{'beh'}; num2cell(p)];
b_m = BANOS_score(pred_cell, gt_cell);
b = b_m.beh;
@@ -264,18 +269,25 @@
end
fprintf('\nInterpretation:\n');
-fprintf(' Frame F1 collapses to 0 for a 15-frame shift -- same as completely missing.\n');
-fprintf(' BANOS DA = 1.0 for all predictions (both bouts are still detected).\n');
-fprintf(' BANOS TP quantifies the temporal shift: large shift -> low TP.\n');
-fprintf(' BANOS SO measures fractional temporal IoU of matched bouts.\n\n');
-fprintf('Frame F1 says "shifted 15f is equally wrong as missing entirely".\n');
-fprintf('BANOS tells you HOW the system is wrong -- and by how much.\n\n');
+fprintf(' Frame F1 collapses to 0 for a 15-frame shift (B); a 2-frame shift (C)\n');
+fprintf(' gives 0.80.\n');
+fprintf(' BANOS DA counts bouts: a predicted bout detects a GT bout when they share\n');
+fprintf(' at least one frame. C detects both GT bouts: DA = 1.0. B shares no frame\n');
+fprintf(' with them: both GT bouts are missed and both predicted bouts are false,\n');
+fprintf(' so DA = 0, and SO and TP are undefined (NaN), having no overlapping pair\n');
+fprintf(' (with ''missed_bouts'', ''zero'' they would be 0).\n');
+fprintf(' BANOS TP quantifies the boundary error, 1 / (1 + |start diff| + |end diff|):\n');
+fprintf(' C, 2 frames late at both ends, gets 1 / 5 = 0.2.\n');
+fprintf(' BANOS SO measures the temporal IoU of overlapping bouts: for C, 8 shared\n');
+fprintf(' frames out of 12, 0.667.\n\n');
+fprintf('Frame F1 gives C a single 0.80.\n');
+fprintf('BANOS tells you HOW the system is wrong -- every bout found (DA), but late (SO, TP).\n\n');
fprintf('Tutorial complete!\n');
%% =========================================================================
-%% Local helper: frame-level F1 with explicit absent-behavior scoring
+%% Local helpers: frame-level F1, with and without absent-behavior scoring
%% =========================================================================
function f1 = local_frame_f1(pred_col, gt_col)
@@ -310,3 +322,24 @@
f1 = NaN;
end
end
+
+function f1 = local_frame_f1_raw(pred_col, gt_col)
+ % Frame-level F1 without absent-behavior scoring, as frame_f1_raw in the
+ % Python tutorial: precision, recall and F1 are 0 when undefined.
+ tp_f = sum(pred_col == 1 & gt_col == 1);
+ fp_f = sum(pred_col == 1 & gt_col == 0);
+ fn_f = sum(pred_col == 0 & gt_col == 1);
+ prec = 0;
+ rec = 0;
+ if (tp_f + fp_f) > 0
+ prec = tp_f / (tp_f + fp_f);
+ end
+ if (tp_f + fn_f) > 0
+ rec = tp_f / (tp_f + fn_f);
+ end
+ if (prec + rec) > 0
+ f1 = 2 * prec * rec / (prec + rec);
+ else
+ f1 = 0;
+ end
+end
diff --git a/demo/tutorial_python.ipynb b/demo/tutorial_python.ipynb
index 627be15..8445d01 100644
--- a/demo/tutorial_python.ipynb
+++ b/demo/tutorial_python.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "markdown",
- "id": "7766bc43",
+ "id": "bc5c61b1",
"metadata": {},
"source": [
"# BANOS Tutorial — Python\n",
@@ -18,7 +18,7 @@
},
{
"cell_type": "markdown",
- "id": "85d2fa80",
+ "id": "2b5469e0",
"metadata": {},
"source": [
"---\n",
@@ -27,7 +27,7 @@
},
{
"cell_type": "markdown",
- "id": "9ff378d4",
+ "id": "087abc03",
"metadata": {},
"source": [
"### 1.1 Setup"
@@ -36,24 +36,24 @@
{
"cell_type": "code",
"execution_count": 1,
- "id": "c2ce5a40",
+ "id": "e11e85f3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "banos version: 0.2.0\n"
+ "banos version: 1.1.0\n"
]
}
],
"source": [
- "from pathlib import Path\n",
"import math\n",
+ "from pathlib import Path\n",
"\n",
+ "import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import pandas as pd\n",
- "import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"\n",
"import banos\n",
@@ -66,14 +66,14 @@
},
{
"cell_type": "markdown",
- "id": "67656986",
+ "id": "65aef272",
"metadata": {},
"source": [
"### 1.2 Single Recording\n",
"\n",
"Recording_1, behaviors `attack`/`investigation`/`mount` (dropping `other`).\n",
"\n",
- "> **Note (v0.2.0 absent-behavior rule):** \n",
+ "> **Note (absent-behavior rule, since v0.2.0; per-file scoring, the default):** \n",
"> - GT has no bouts AND machine predicts nothing → all metrics = **1.0** (correct absence) \n",
"> - GT has no bouts BUT machine predicts bouts → all metrics = **0.0** (false detection)"
]
@@ -81,7 +81,7 @@
{
"cell_type": "code",
"execution_count": 2,
- "id": "eb84e32f",
+ "id": "d671d64c",
"metadata": {},
"outputs": [
{
@@ -110,7 +110,7 @@
{
"cell_type": "code",
"execution_count": 3,
- "id": "f59929aa",
+ "id": "f68e54f7",
"metadata": {},
"outputs": [
{
@@ -118,17 +118,17 @@
"output_type": "stream",
"text": [
"Per-behavior metrics — Recording 1\n",
- " precision recall da (F1) so tp ic\n",
+ " precision recall DA (F1) so tp ic\n",
"attack 1.0 1.0000 1.0000 1.0000 1.0000 1.0000\n",
- "investigation 0.8 0.7347 0.7660 0.5444 0.0846 0.9829\n",
- "mount 1.0 0.5333 0.6957 0.7333 0.0785 0.9909\n"
+ "investigation 0.8 0.7347 0.7660 0.5444 0.0846 0.9806\n",
+ "mount 1.0 0.5333 0.6957 0.7333 0.0785 0.9829\n"
]
}
],
"source": [
"metrics = banos.score(pred, gt)\n",
"\n",
- "metrics_df = pd.DataFrame(metrics).T.rename(columns={\"f1_score\": \"da (F1)\"})\n",
+ "metrics_df = pd.DataFrame(metrics).T.rename(columns={\"f1_score\": \"DA (F1)\"})\n",
"print(\"Per-behavior metrics — Recording 1\")\n",
"print(metrics_df.round(4).to_string())"
]
@@ -136,12 +136,12 @@
{
"cell_type": "code",
"execution_count": 4,
- "id": "f202751f",
+ "id": "8d710a0b",
"metadata": {},
"outputs": [
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAADwCAYAAADM38HGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbe5JREFUeJzt3Qd0FGUXBuA3vfeQ0HvvvYkUAakqiDRBiorSBUQFf+lNVLoIAtKRXkSQLiAl0qtAqAFCT0JII33/c7+4m91kAwkGWHbf55xRdnZ2djKzyc6de787VhqNRgMiIiIiIiIzYP2yN4CIiIiIiCi7MMAhIiIiIiKzwQCHiIiIiIjMBgMcIiIiIiIyGwxwiIiIiIjIbDDAISIiIiIis8EAh4iIiIiIzAYDHCIiIiIiMhsMcIiIiIiIyGwwwCEiegZWVlbo27fvC993CxcuVO8dFBQESzVy5Ei1D8j0yHGR46PFzysRvQwMcIjoudOe5GgnR0dHFC9eXAUI9+7deyEnw9rJ2toauXLlQsuWLfH3338/1/d+1Wn3neyzmzdvpns+IiICTk5O/ynYGz9+PDZs2ABLVLBgQYPPpouLC6pXr47Fixe/7E0zC3fu3MGQIUPQoEEDuLm5qX28Z8+el71ZRPQCMMAhohdm9OjRWLJkCX788UfUrl0bs2bNQq1atRATE/Pc31veS95bgi05GT979izq1q2LkydP4lXywQcf4PHjxyhQoMALe08HBwcsX7483fx169b953U/S4DzzTffqH1gDipWrKg+lzJJQPno0SN07doVc+fOhTl4GZ9XrcDAQEycOBG3bt1CuXLlXvj7E9HLY/sS35uILEyzZs1QtWpV9e+PP/4YPj4+mDx5Mn777Td07NjxP61bgiRnZ+cMn3/vvffg6+ure9yqVSuULVsWq1evVieZrwobGxs1ZZen7TfRvHlzFeB8+eWXBvN//fVXtGjRAmvXrsWLEB0drbIctra2ajIHefLkQefOnXWPu3XrhsKFC2PKlCno0aMHTI32GLysz2tWVKlSBaGhofD29saaNWvQtm3bl7IdRPTiMYNDRC/NG2+8of5/7do13bylS5eqExMpfZITkw4dOqQrj6pfv74KTo4dO6ayMHKC/vXXX2fpvXPmzKn+n/ZEOS4uDiNGjEDRokVV5iJfvnzqxF7mGyPZB9kWWbZMmTLYunWrwfPXr19H7969UaJECfUzSVAnJ1r6Y2iOHj2qymcWLVqUbv3btm1Tz23atOmJYxp++ukn9f6yHblz50afPn0QHh6eLfvt/fffV5muCxcu6ObdvXsXf/75p3rOmMzsR/k55IRZfm5tmZac4OuXx507d069h5eXF+rUqWPwXFry2ZESL/m5ZHn5Gbdv326wn5s0aaICXTkWhQoVwocffghTkiNHDpQsWRJXrlwxmJ+cnIypU6eqYywlnv7+/vj000/x8OHDdOvYsmUL6tWrp8qy3N3dUa1aNRWM6pPAXvt7JvtDgizJdOiTY+Hq6qq2RYJcWV+nTp3Uc3IcBw4cqLZX5r/99tsIDg5Oty3GPq9Smiclovv371fHS34eCeqMleadPn1a/SyynXnz5sXYsWOxYMGCTI1Dk+2SvyFEZHnM4xIYEb2StCdxctIvxo0bh2HDhqFdu3Yqw/PgwQPMmDFDnaieOHECnp6eutfKlVnJCEkAJCdncsL3JGFhYboTRTmRGzNmjDqxkvfSkufkRE1OvD755BOUKlUKZ86cUVfTL168mK6USpaTMi0JYORkavr06WjTpg1u3Lih+5mOHDmCgwcPqu2UEzQ5KZNyOQk25ORdTsYlqyUneKtWrVLlSfpWrlypTtblxDwjcsI/atQoNGrUCL169VKlOfIe8t4HDhyAnZ3dM+83Iftftl1OkqXMULtdcvIrGZy0MrsfpSxLjrOc5MpyokiRIgbrkmCwWLFiqpRNo9FkuI3y88t+kNJH2UZ7e3scOnRIBWFvvvkm7t+/r/4vJ+QyLkM+S3IssqPMLjslJiaqQEGOuT4JZiRY6N69O/r3768uCkipp/xe6B9jWUaCNgmEhg4dqn5OWUYCb20wql2PBD4TJkxQ4+CmTZum1pP290y2Rz57Elz+8MMPumyfHDcJKGWdss9lPxv7LGTk8uXLKqv60Ucfqc/8/PnzVUAlQZdsu5DfUxk/I8GM/CySOZo3b54KmImInkhDRPScLViwQM5MNTt37tQ8ePBAc/PmTc2KFSs0Pj4+GicnJ01wcLAmKChIY2Njoxk3bpzBa8+cOaOxtbU1mF+vXj21vtmzZz/1vUeMGKGWTTt5enpqtm7darDskiVLNNbW1pp9+/YZzJf3kdccOHBAN08e29vbay5fvqybd+rUKTV/xowZunkxMTHptikgIEAtt3jxYt28oUOHauzs7DRhYWG6eXFxcWo7P/zww3T78tq1a+rx/fv31Xa8+eabmqSkJN1yP/74o1pu/vz5z7Tf9PedHLPBgwdrihYtqnuuWrVqmu7du+v2RZ8+fZ5pP7q4uGi6du2a4Xt37Ngxw+e0Ll26pN6vdevWBvtAJCcnq/+vX79evebIkSMaU1GgQAF13GT/yiSf9Q8++CDd/pT9KPOWLVtm8Hr5/OrPDw8P17i5uWlq1Kihefz4sdH9EB8fr/Hz89OULVvWYJlNmzapdQ0fPlw3T46LzBsyZIjBuk6ePKnm9+7d22D++++/r+bL8cno86r9uWXeX3/9pZsnn2MHBwfN559/rpvXr18/jZWVlebEiRO6eaGhoRpvb+9063ya1atXq9fs3r07068holcXS9SI6IWRDINcQZdyJckgSAZg/fr1ahyCXEmXK/+SUQkJCdFNUkomV/B3795tsC65iitXoTNLxons2LFDlSxJiYt0cZNsi2RX9Mt2JNsgJUL626AtpUu7DfLz6Gccypcvr0qCrl69qpsnpTVaCQkJKoMiZVtylfz48eO659q3b6+e188oyLZKmZk8l5GdO3ciPj4eAwYMUN3OtGT8hmzL5s2b/9N+05Ir9XLVXbJC2v9nVJ6W1f34JD179nzqMpIRks/O8OHDDfaB0JayabMSUuon+9lUyDGW3wmZZCC8ZLXk+Hz//fcG+9PDwwONGzc22J+S7ZDfIe3+lM93ZGSkylBJdtLYfpAyPclmSdZRfxnJvsjxSvt5EZIV1PfHH3+o/0smSZ98BjOrdOnSeP3113WP5eeXMk793x3JOkkTEv0xclJypi2TIyLKCEvUiOiFmTlzpgosZNyLlEbJCY32hPTSpUuqBEmCGWP0y6yEBEVShqQl3af0O2vJc/r191Jmpd9kQMpj5L369eunxqRot+H8+fPqZMsYOTHUlz9//nTLSGmR/rgI2SYpA5KgSkpu9MusZJu1KlSooE4wpfRLynaE/Fu2WRsYGCNjfITsS33y80vZm/b5jPZbZlWqVEltn5SpSbAggWdG25XV/fgkMk4mM6WO8jmSk+aMyDgOCWillE1K5aREUBpNSJD2pJKntJ+rrJCgRD/ANaZGjRpqXElSUpLq7Cf/ls+P/jGS/Snb4efn98T9qS35lHFWWf28CDm+UlaoT35XpTwx7Tpkf6ctJzS2zoxk5ndH3kcCnLTkAgER0ZMwwCGiF0bGWmi7qKUlV+DlKrMMkDbWdUmuVOtLe+L42WefGQzSlxPaJ93zQtYnJ5fSwU3bGUq2Qa6iS2c3YyTzpC+j7lD6QYwEUBLcyNVtOVmTk175OSWDJe+nTzI1Mg5Jrs7LmJ6NGzeq7nLZ2THsaSfcTyLBgIztkW2TbU2bLdHK6n58XturT/a5dNKSex/9/vvvqnmDjFWZNGmSmpf285XR5yor5LhrmyZkRAJYyQQKGesiQYYMwJcxMYMGDdLtTwluli1bZnQdGQWS2UGCv4yO83+Rmd8dIqJnxQCHiEyCXA2Wkxu5Yi9ZnqySDl367XbTDtI2RgZQi6ioKBXgyDacOnUKDRs2NNql61nISbUMopYTaa3Y2Nh0Hc6EBA2SYZByOslwyY00JRB6Eu39RaSxgGRstKRsTQaia0+es4MEOFIGJjdQlFKqjGRlP2bHfpb3kyBAmjY8reV3zZo11SSBpGSjpNxpxYoVatB8Zj5XWaEdLJ8VUiomwbk0VZDGAtrPpZQivvbaa08M+LQZFckEZZTl0P+8pM3AybzM3K9GlpH9LRkj/ayNvD47yftIOWRaxuYREenjGBwiMgnvvvuuuqorJ/hpr+LKYxm78iRSniQn89pJxic8rauajL+RUitt6Y+M/5EyMmM3WZQyJcn0ZJX8TGl/HukMJyVJacm4Fcl8SGmaTLly5VKldU8iP6uUM0kHN/33+eWXX1RZU1Y6Wz2NnEBLq2IpuZNsXEaysh/lBN5YsJcVUmomWQbpnpY2K6bdJ1L6lPY4aIOhjFqAG/tcZWWS4/csvvrqK/V51+4/2Z/yeZHOf8aCdO3+ky5xkl2T4yNBtLH9IBlU+bzPnj3b4OeWzKmUFWbm8yJd+IR85vTJZyM7SUYrICDA4Ga88nubUSaLiEiLGRwiMgly8izjD6QdrLTvlZNWOVmTLIQ0IpA2woMHD/5PmRQpQ5ITvdu3b6sAQE565URPm0WQu65Lq2YZ2C4Dt+WKuZxYyv1fZL6UNWVUYpcRKTeSbIeUpsnJspywydV4bRtpY1kcyZLIAHAZi/O08iApT5J9JoFh06ZNVXtmuZIu98WRNsDPmn3IiJRsPU1W9qMEorI/pJxN7t8jGTwpHcwKyVb873//UwGADFyXYFlKq6QRgqxTTvilzEz2SevWrdVnTQbjSwAhjRjkHi+mRAIIGUcj+0TuZyQZHcnmyM8hJ/sSyMiYNBmbIw0IpJxNxpTJzyLjiyQbJcdee/8gyabJDV1lH8jrJk6cqBoZyHqlBFLbJlruTyP3tnkaCQzldbI/JYiWNtG7du3K9syKZM+kFbU0V5BST22baBm/I4FOZrJ/8jdF/PPPP+r/8ruoHWf0zTffZOv2EpEJedlt3IjI/GlbxWamRe/atWs1derUUe2DZSpZsqRqmRsYGGjQ7rhMmTKZem9jbaJlvbVq1dKsWrUq3fLSRnfixIlq/dK21svLS1OlShXNqFGjNI8ePdItl7aVr34LXP22xw8fPlTtlH19fTWurq6aJk2aaC5cuJBuOf2Wx9rt3L9/f7rnjbXd1baFln0lrab9/f01vXr1Uu+tLyv7TX/fSQvjJzG2LzK7H2Vf1K1bV7ULl/Vo98mT3jttm2gtaYldqVIl3fvJz7tjxw713PHjx1XL6fz586vnpVVyy5YtNUePHtW8LPIZaNGihdHnFi5cqH5GOd5ac+bMUftQ9pW0gy5Xrpzmyy+/1Ny+fdvgtRs3btTUrl1bLefu7q6pXr26Zvny5QbLrFy5UrevpO1yp06dVLt2fXIs5HfFGGkx3b9/f9XqXZZ56623VPv3zLaJNvZzy/GSSZ+0iH799dfVdubNm1czYcIEzfTp09U67969q3kaYy3itRMRmS8r+c/LDrKIiIiIMkMadvz8889q7FxGzQqIyLJxDA4RERGZpLQtumVskpSZ1alTh8ENEWWIY3CIiIjIJElrdblnkTTgkLFCMnZOugsOGzbsZW8aEZkwBjhERERkkqQBhDQImTNnjmoqULlyZRXkPK27IBFZNo7BISIiIiIis8ExOEREREREZDYY4BARERERkdlggENERERERGaDAQ4REREREZkNdlEjxapXTe4JE6GZ9TePhwnh8TA9PCamhcfDtPB4mN7xoBePGRwiIiIiIjIbDHCIiIiIiMhsMMAhIiIiIiKzwQCHiIiIiIjMBgMcIiIiIiIyGwxwiIiIiIjIbDDAISIiIiIis8EAh4iIiIiIzAYDHCIiIiIiMhsMcIiIiIiIKEv++usvvPXWW8idOzesrKywYcOGp75mz549qFy5MhwcHFC0aFEsXLgw3TIzZ85EwYIF4ejoiBo1auDw4cNZ2zAGOERERERElFXR0dGoUKGCCkgy49q1a2jRogUaNGiAkydPYsCAAfj444+xbds23TIrV67EoEGDMGLECBw/flytv0mTJrh//36Wts02yz8NERERERFZtGbNmqkps2bPno1ChQph0qRJ6nGpUqWwf/9+TJkyRQUxYvLkyejRowe6d++ue83mzZsxf/58DBkyJNPvxRI1IiIiIiJ6rgICAtCoUSODeRLYyHwRHx+PY8eOGSxjbW2tHmuXySxmcIiIiIiICHFxcWrSJ+NlZPqv7t69C39/f4N58jgiIgKPHz/Gw4cPkZSUZHSZCxcuZOm9GOAQEREREVkI6z61MnxueI4mGDVqlME8GQ8zcuRIvEoY4BARERERWQhr24xHqAwdOlQN8teXHdkbkTNnTty7d89gnjx2d3eHk5MTbGxs1GRsGXltVnAMDhERERGRhbC2sc5wkmBGAg79KbsCnFq1amHXrl0G83bs2KHmC3t7e1SpUsVgmeTkZPVYu0ymf8Zs2WJ6aaRP+NSpU3kEiIiIiChTGZyMpqyIiopS7Z5l0raBln/fuHFDlw3q0qWLbvmePXvi6tWr+PLLL9WYmp9++gmrVq3CwIEDdctI9mju3LlYtGgRzp8/j169eql21NquapnFACebBQUFqZsdaQ+2Vrdu3dCqVavsfjuz8HrRitjY6wfcmvA7NLP+xjsV6j71NfWKVcaxoYsQO/0vXBq1Gl1rtki3TO96bXBt7Ho8nr4Xf3/5C6oVKG3wvIOtPX7sMBgh329D5JQ/seaTCfBz84al4/EwLTwepoXHw/TwmJgWHg/LCXCOHj2KSpUqqUkbnMi/hw8frh7fuXNHF+wIaREtLZ8layP3t5F20fPmzdO1iBbt27fHDz/8oNZRsWJFdT69devWdI0HnvozZmlpoufAxcEJp25dQp8VP2Rq+YI+ubC5zyTsvngMFcd3wdQ/V2Je56F4s1QN3TLtqjTC5DafYdTmeag8vitOBV/Ctv5TkcPNS7fMlLYD8Fa5Omg772vUm9ILuT18se7Tby3+GPN4mBYeD9PC42F6eExMC4+H5QQ49evXh0ajSTctXLhQPS//37NnT7rXnDhxQnVqu3LlikoApNW3b19cv35dLXPo0CHUqFEj6z9jll9BKpKsU6cOPD094ePjg5YtW6qDpI1OhUSwksmRAymdJyTV9ttvv6l5MmkP+FdffYXixYvD2dkZhQsXxrBhw5CQkGCwl3///XdUq1YNjo6O8PX1RevWrTM8ChIJy3alrXE0ZVv/CcCwjT9jw6m9mVq+5+vv4lrobQxeOx0X7gZh5t41WHNiNwY27KBbZlDDjph74DcsDNiM83eD0HP5RMTEx+LDWi3V8+6OLvio9lsYtGYadgcew/Ebgei+eCxeK1IeNQqVgSXj8TAtPB6mhcfD9PCYmBYej1d7DI65MJ+f5AWSWkBJw0lqTgIJuQmRBB0yEOrw4cNqmZ07d6rU3Lp16zB48GC0a9cOTZs2VfNkql27tlrOzc1NRbjnzp3DtGnTVN2h3NFVS1J5su7mzZuriFfer3r16ka367vvvlN3ed2+fTsaNmwIc1WrcFnsvHDEYN62c3+jVuFy6t92Nraokr+EwTJyRUEea5epUqAk7G3tDJYJvHcd10PvoFahlGWIx+NVxN8P08LjYXp4TEwLj8eLZ2Nvk+FkLtgm+hm0adPG4PH8+fORI0cOFaTI/4VkdvRb2kn7O0m1pW1z98033xg0DJBgaMWKFWoAlhg3bhw6dOhg0JNc6hbTkkzQkiVLsHfvXpQpY94ZiJzuPrgXEWYwTx57OLnC0c4BXs5usLWxNbLMQ5T0L6hbR1xCPB49jjJcJjJMPUc8Hq8q/n6YFh4P08NjYlp4PF486yyWor2KGOA8g0uXLqnBT1IXGBISojI3QgZSlS5tOJD9aVauXInp06erEjfpRpGYmKha8mnJ4KoePXo8cR0ySEuySpJRkjK3Z7lDLRERERGZP2sLCHDM/yd8Dt566y2EhYWpcjIJcmQS8fHxWVpPQEAAOnXqpMrPNm3apErQ/ve//xmsRzI/T/P6668jKSlJtdp7mgkTJsDDw8NgknmvkrsRofB3N+x2Jo8lGxObEIeQqHAkJiUaWcZLvVa7Dgc7e5X1MVjGzVu3DPF4vIr4+2FaeDxMD4+JaeHxePGsOQaH0goNDUVgYKAqLZNxLqVKlcLDhw91z8tNioQEHPpkftp5Bw8eRIECBVRQU7VqVRQrVkx1jdBXvnz5pzYMkDE5W7Zswfjx41VrvSeRnuSPHj0ymGTeqyTg6lk0LFHVYF7jUtURcPWM+ndCUiKO3QhEwxLVdM9LYwd5rF3m2PULiE9MQMOSqcsU98+PAj65EHAtZRni8XgV8ffDtPB4mB4eE9PC4/HqdlEzZSxRyyIvLy81vmbOnDnIlSuXKkuTgf1afn5+Kusindby5s2rOp9JlkTG12zbtk0FR/J6mScBjbxextxIlzRpKLB+/XqD9xsxYoQKpIoUKaLG4kgJ2x9//KHG3OiTpgUyv1mzZrC1tcWAAQOMbr+Uo5laSZq0lCyaI6/ucSGf3KiQtxjCoiNw8+E9jH+nF/J45kDXRaPV87P3rUPf+u9hYuu+mH/wd7xRoiraVW6IFjM/161j8q7lWNR1GI7eOI/DQecw4I32cHFwxIKAzer5iNho/HLwd0xu0x9h0Y/U4xntPsfBK6dx6No/sGQ8HqaFx8O08HiYHh4T08LjYfpsbK1g7hjgZJF0TJOApH///ihbtixKlCihxtBIO2i1Q21t1ePRo0ercTpSPiYtoWUcjfxfMjUy1mb37t14++231d1bpd+3jItp0aKFahMtbaW1ZL2rV6/GmDFj8O2336rxOXXrGr8RprSuliBJSt5sbGzQr18/vAqq5i+FPYN+Mrg/jZAWz90Xj0EuD1/k905tzhAUekcFM1PafobPGrRDcPh9fLx0ArafTykVFKuO7UQOV0+MbtlDDWA8GXwJTWcMxP3I1MYDA1dPRbImGWs/maBu+rnt3CH0XvEdLB2Ph2nh8TAtPB6mh8fEtPB4mD57M2oHnRErjfTPJYtn1aumxe8DU6GZ9TePhwnh8TA9PCamhcfDtPB4mN7xMDUlF76X4XMXuq2BOWAGh4iIiIjIQthbm38GhwEOEREREZGFsLfhGBwiIiIiIjIT9hYwBocZHCIiIiIiC2HPDA4REREREZkLe2ZwiIiIiIjIXNgzg0NERERERObCydYG5o5jcIiIiIiILIQ9MzhERERERGQu7DkGh4iIiIiIzIU9MzhERERERGQu7KzN/0af5n+nHyIiIiIiUuysM56yaubMmShYsCAcHR1Ro0YNHD58OMNl69evDysrq3RTixYtdMt069Yt3fNNmzbN8naxyQARERERkYWwy6YStZUrV2LQoEGYPXu2Cm6mTp2KJk2aIDAwEH5+fumWX7duHeLj43WPQ0NDUaFCBbRt29ZgOQloFixYoHvs4OCQ5W1jBoeIiIiIyELYZVMGZ/LkyejRowe6d++O0qVLq0DH2dkZ8+fPN7q8t7c3cubMqZt27Nihlk8b4EhAo7+cl5dXln9GBjhERERERBY0BscugymzJBNz7NgxNGrUSDfP2tpaPQ4ICMjUOn755Rd06NABLi4uBvP37NmjMkAlSpRAr169VKYnq1iiRkRERERkIeyekN6Ii4tTU9qMStoysZCQECQlJcHf399gvjy+cOHCU7dBxuqcPXtWBTlpy9PeffddFCpUCFeuXMHXX3+NZs2aqaDJxibzNyhlBoeIiIiIyELYPSGDM2HCBHh4eBhMMi+7SWBTrlw5VK9e3WC+ZHTefvtt9VyrVq2wadMmHDlyRGV1soIBDhERERGRhbCzyXgaOnQoHj16ZDDJvLR8fX1VRuXevXsG8+WxjJt5kujoaKxYsQIfffTRU7e1cOHC6r0uX778fAOchIQEFClSBOfPn8/qS4mIiIiIyEQzOA4ODnB3dzeYjHUxs7e3R5UqVbBr1y7dvOTkZPW4Vq1aT3z/1atXqzK4zp07P3Vbg4OD1RicXLlyPd8Ax87ODrGxsVl9GRERERERmUkXtUGDBmHu3LlYtGiRSnxIQwDJzkhXNdGlSxej2R8pT5PyMx8fH4P5UVFR+OKLL/D3338jKChIBUvvvPMOihYtqtpPP/cmA3369MHEiRMxb9482NqyTwERERER0avAMZvug9O+fXs8ePAAw4cPx927d1GxYkVs3bpV13jgxo0bqrOaPrlHzv79+7F9+/Z065OSt9OnT6uAKTw8HLlz58abb76JMWPGZPleOFYajUaT1R+odevWKqpydXVVg4DStneTG/kQEREREZFp+TWwT4bPvV9iJszBM6VfPD090aZNm+zfGnpp/hfwCfe+iRhXaw4exq182ZtB//JyaI8TpUpyf5iQSucvwG5gnZe9GfSvhCn78emfH3N/mIif35iHKSd7vuzNoH8NrDjb5PaFbZqsijl6pgBnwYIF2b8lRERERET0XNlaZ/5+Mq+q/zSARurupJZOyN1Gc+TIkV3bRURERERE2czWAjI4z/QTSoeEDz/8ULVsq1u3rppkIJD0s46Jicn+rSQiIiIiov/M1somw8miAxxpC7d37178/vvvqsuBTL/99pua9/nnn2f/VhIRERERUbZkcGwzmCy6RG3t2rVYs2YN6tevr5vXvHlzODk5oV27dpg1a1Z2biMREREREWUDW47BMU7K0LQ9rvX5+fmxRI2IiIiIyETZmlGmJiPP9BPWqlULI0aMQGxsrG7e48ePMWrUKPUcERERERGZHlsLGIPzTCVq06ZNQ5MmTZA3b15UqFBBzTt16hQcHR2xbdu27N5GIiIiIiLKBrYsUTOubNmyuHTpEpYtW4YLFy6oeR07dkSnTp3UOBwiIiIiIjI9thZQovbM98FxdnZGjx49sndriIiIiIjoubFlBifVxo0b0axZM9jZ2al/P8nbb7/NjyURERERkYmxtWIGR6dVq1a4e/eu6pQm/86IlZUVkpKSXswRIiIiIiKiTLOzsTP7vZXpErXk5GSj/yYiIiIioleDjdUzj1B5ZTxTjurmzZvZvyVERERERPTcAxybDCaLDnAKFiyIevXqYe7cuXj48GH2bxUREREREWU7GwY4xh09ehTVq1fH6NGjkStXLjUmZ82aNYiLi+PHkIiIiIjIRNlY22Y4WXQGp1KlSvj+++9x48YNbNmyBTly5MAnn3wCf39/fPjhh9m/lURERERE9J/ZMIPzZNIxrUGDBqpUbefOnShUqBAWLVqU6R1cv359DBgwAKbsRW6jlP5NnTr1hbwXEREREVkeGyu7DCdz8Z9yUcHBwfj111/VdPbsWdSqVQszZ87M9OvXrVun7qtjCvbs2aOCNRlT5Onp+Vy3ceHChSpoCg8PN5h/5MgRuLi4wBJd3nkLF7fcROyjeHjkd0WlzkXhXdjd6LJ7JpxESOCjdPNzlvdGnUHldI8jbkfjzKpreBAYDk2SBu55XFCrb2k4+zjqlgm9/Ahn1wYh7EoErKyt4JnfFa8PLgcbextYujUrDmHpwgMIC4lC0eL++HxoC5Qpl9fospt+O4Gxw9YbzLO3t8VfR4cbXX7imI1Yv/ooBnzRFB0+qJ3u+fj4RHzUaQ4uBd7F4lW9ULxkLlg63/ffh9+HH8HO1xePL1xA8LixiDlzJsPlPZs0Qa7+n8E+Tx7EXb+O25N+QMRff+met3Z2Ru5Bn8OjYUPYenoiLjgYD5YuQejKlbplii5aDLfq1Q3WG7JiBW6OGglL1+u1dzHojY7I6eaN07evYMC6KThy43yGN9X7qtEH+KBaM+Tx8MXF+zcxdNMsbL9wSLeMtZU1hjf9EO9XeRM53XxwOyIEiw//gfE7Ui8aDmvyIdpVaoh8nn6IT0rE8eBADN88B4dvnIOlq5+nARrnbwIPew8ER93EiovLERR5LcPlG+ZthLp56sPb0RtRCVE4fv8Y1l9di8TkxEyv09baFm2LtkNV/+qwtbLFubB/8GvgMkQmRMDSnd0WjJO/38Tj8Hj4FHDBa92Lw7+o8e/030adwJ1zhudDIn8lbzQfUkH9+8jqa7hy8D6iQmNhbWuNHIXcUL1DIfgX89Atv7RvAKIexBqso0bHwqjUqkC2/3yvOhszaiaQkWf6CX/++WcV1Bw4cAAlS5ZEp06d8Ntvv6FAgax9iLy9vWHqXuQ2SqmfJbp56D5Or7iCyl2Lw7uwGy5tv4V9P5xBk2+rwdHdPt3ytfuVQXKiRvc4LjoBO4cdRd5qqfsv6v5j7Bl3EgXr5kTp1gVg52SLiFvRsLazNghu9k06g5It8qNi56KwtrZC+M0oSU3C0u3YegbTvt+Kr4a9pYKaFUsDMKDnYqzc2B/ePq5GX+Pi6oBVG/unzshgP+7ZdQ5nTwcjh59bhu//4+Tt8M3hpgIcAjybNUOer4bg5siRiDl9Cjm6dEWRufNwvnkzJIaFpT8WFSuh4A+TcHvKZDzaswfeLVui0IwfEfheG8ReuqSWkfW51aiB619+ifhbt+D22mvIN3w4Eu7fR8Tu3bp1haxahTszpuseJz9+bPGHpG3FN/B9q77os/oHHL5+Dv3rtcPmTyejzISOeBCV/kRtdPNPVODSc9VEBN6/gTdLVMea7uNRd3pPnLyVcjy+aNgJn9ZuhQ+Xj8O5O9dQJX9JzOvwNSJio/HjvjVqmUsPbuKzdVNwLfQ2nOwc8Fm9dvij52SUHNcBIdHp39dSVPWrhveKtcOvgUtx7dFVNMzXCP0rDsCIv79BZEJkuuWr+VdH6yJtsOjCAlx9dAV+zv7oVkrK6zVYfXlVptfZrmgHlPMthzlnZ+Nx4mN0LP4+epbrje+PfwtLdvngPRxcfBl1Py4Bv2LuOPPHTWwefwodp9SAk0f67/Qmn5dFcmLq7UdiIxOx+ssjKFzTTzfPM5cz6nQvBnd/JyTGJ+P05pvYPO4UOk6vCSe984Rq7QqhVMPUC2J2juZ/Iv8sbLJxrI0kNmTYitwrs0KFCpgxY4Yap5/RBf7u3bsbzHNwcEBsbGpgqtFoMGLECFUdJomA1157DbNmzUKxYsWe/xicsWPHokaNGjh27JjK3AwdOjTLwU3a8i8pzxo/frwaw+Pm5ob8+fNjzpw5umVr166Nr776yuD1Dx48UNmVv/69KilNDgYPHow8efKoTIhso2RmtK5fv4633noLXl5e6vkyZcrgjz/+QFBQkMreCHlOSu+6deuWbhvFnTt30KJFCzg5OamSPAn00paWTZ48GeXKlVPvkS9fPvTu3RtRUVHqOdkeObiPHj1S7yPTyJEpV0PTrkfGOL3zzjtwdXWFu7s72rVrh3v37umel9dVrFgRS5YsUa/18PBAhw4dEBmZ/g+6Kbu4LRiF6uVCwddzqixL5a7FYGNvjaC/jJ/c2rvawdHTXjfdP/tQZVzyVk8NcM6uuaYyOuXbF4FXATe4+jkhdyVfg4Dp1K9XULRRHpRsmR8eeVzglssZ+ar7wUYvCLJUyxcfxDttqqBlq8ooVMRPBTqOTnbYtOF4hq+Rz7KPr1vqZCQQun8vApMm/IFRE96Dja3xLNnBfRdxKOAy+n/eJFt/pleZX9duCF29GmHr1yH2yhXcHDkCybGx8Hm3jdHlc3T5ABH79+P+/PmIu3oVd6ZPx+Pz55Dj/U66ZVwqVUTobxsQdeQw4m/fQujqVXgcGAiX8uUN1pUc+xiJISG6KTk6GpZuQP0O+CXgdyw6/AfO3wtC79XfIyY+Ft1qtDS6fKeqTTBx5xJsPf+3Ck5+PrgBW84HYGD9DrplahUsi9/P7seWcwG4/vAu1p3agx2Bh1EtfyndMiuO78CfF4+qdZy7ew2DN8yAh5MryuUuAkvWKF9j7L+9DwfvHMCdmDtYFrgU8cnxqJ27jtHli3gUxZVHl3Hk3mGExobifNg59e+C7oUyvU5HGye8lrsOVl9ahcCHF3Aj8joWnl+Aop5FUci9MCyZBB+lGuZGyQa54J3XRQU6tvbWuLD7jtHlHV3t4OzpoJuCT4fB1sEaRfQCnGJ1/JG3vLcKcLzzuaB2l6KIf5yE0Osp51Zado42BuuSx/T8xuCsXLkSgwYNUgHJ8ePHVYDTpEkT3L9/P8PXyPmsnEtrJzk31/fdd99h+vTpmD17Ng4dOqTOpWWd+kFQZjzTmZyceMsGyA+SnSZNmoSqVavixIkTKijo1asXAgMD1XOSJVqxYoWK7PR3bO7cufH666+rx3379kVAQIBa7vTp02jbti2aNm2KS/9esezTp48KgiQgOnPmDCZOnKiCBwlC1q5dq5aR95MdPm3aNKPb2KVLF9y+fVsFKvIaCcLSHkhra2t1cP755x81JunPP//El19+qQvUJIjRP8ASlKUlN1OV4CYsLAx79+7Fjh07cPXqVbRv395guStXrmDDhg3YtGmTmmTZb799da4eyVWb8KBI+JX20s2TUjH/Ml4IvZK5NP+1fXeRr4YfbB1S/pBpkjW4ezoMrjmdse+H0/i930HsGn0ct46F6F4TGxGPsKuRKuD5c+wJ/N7/YErp28X0pW+WJiEhEYHn76BazSIGn+lqNYrgzKngDF/3OCYerZpMwtuNf8AX/X/F1cv3032mR329Fp27vYbCRVO/uPSFhkZhwqiNGDm+DRwcTaN89WWzsrODc5kyiAw4mDpTo0FkQACcK1Y0+hqXChUNl5eSzf0H4KK3fPSJk/Bo8Abs/FKOhWv1GnAoWBARBw4YvM6r5VsodzAAJTduRK6Bg2DlmFriaYnsbGxROW9x7Lp4VDdPvpf+vHQUNQuUMfoaB1s7xCYadhmNTYhD7cKpwWRA0Fk0KF4FxXLkU4/L5y6K1wqXV0FRRtvxca13EP44EqdvX4alsrGyQX63AipI0dJAgwth51E4g0BDght5TUG3lIDG19EXZX3K4WzomUyvs4B7AVWidv5h6jL3Yu6qgKmwh+UGnEmJyXhwNQp5yxl+p+ct5417lzL3nS6BUNHafhkGJ/Ie53bdhr2zLXwKGF5IO/HbDSz4aB9Wf3UEJzfeQHISb0z/PAMcuaDfo0cPdeG+dOnSKihxdnbG/Pnzn3gxNGfOnLpJGpTp/y2Vc+RvvvlGnQOXL18eixcvVufdcq6bFc+Uo5KNEzExMSrYiY+PN3heNuhZNG/eXAU2QrI1U6ZMwe7du1GiRAmVvZBMyv79+3UBjWRPOnbsqLZHtmPBggXq/xL0CAkctm7dquZLdkiea9OmjcquiMKFC6crRfPz8zMYg6PvwoULqpmCjJWRQEzMmzcvXdpMP+MjmRXJePXs2RM//fQT7O3tVaZFe4AzsmvXLhWEXbt2TQVgQg6yZJ3k/atVq6Y7aZSUn2S9xAcffKBeO27cOLwK4iIToEkGHD0MT2Yd3O0QcSfmqa8PuxqBiOBoVP2weOo6IxKQGJuEwM03UKZNIZRrWxh3z4Qh4Md/UO+rCshR0hPR91OuBJzbEITyHYrAI78Lrh+4h7++O4XGY6vCLaczLFX4wxgkJSXD28dwPJiXjwuCrj0w+poCBX3wv1Gt1FidqKhYLFt4AD26zMXydX3hlzOlRnrJ/P2wsbVGu041ja5D/rCN+WYdWrerilJl8uD2Ld5jS9h4esHK1hYJoaEG+ysxNASOhVKvOOuz9fVFYkj65WW+VvDYMcg3egzK7v0LmoQEtf9vDh+G6KOpJ+4PN21C/O3bqmzNqURx5P58MBwLFcS1/nqliBbG18UDtja2uB9pWBp4LzIMJfyMVzJsv3AYn9XvgH1XTuFK6C28UawKWpWvBxvr1GuM3+1aCndHF5wdsgxJmmTYWFlj2B9zsPz4DoN1NS9dG8u6jISznSPuRISi2ayBCI223AszrnausLG2QWS84clzRHwEcjob/46VbI2rnRu+qPIVrP4t19kbvAdbrv+R6XW627sjITlBlaalXcbD3vhYE0sQGyHf6Zp0pWhOHnYIv/307O+9yxEIuxmNej1Lpnvu+rEQ7Jh2DonxSXD2tEfL/1UwKE8r1zQPfAu5qYzQ3YuPcGj5VcSEx6F2l6yVNlkCm2wYgyPn/lLJJVVc+hdDGzVqpJINGZGKJqn6kvPXypUrq/NzObcVcs4rpW6yDi05Z5aKLFmnVCll1jP9hFIaJiVcEjwYk5SU9CyrNQiMtAGANjsi41PefPNNLFu2TAU4shPkh5XxQEKCAXnf4sVTT3SFZGx8fHzUv/v376+yQtu3b1c7T4KdrARjkt2xtbVVB0SraNGiqqxNnwRBEyZMUAFRREQEEhMTVWpNAkKJbDPj/PnzKrDRBjdComMJvuQ5bYAjAZQ2uBFyX6InpQZlf6S9X5HUP76qrv11Fx55XQwaEmizfLkr+6J4k5RB8Z4FXBF6OQJXd99WAY52mUINUkrjhJSyPTgXjqB9d1VQRJlXrkJ+NWmVr5AfHVrNwPo1R/Fp34a4cO42Vi77G4tW9tRdIElr1a+HEBMTj64f1eWufwFydP4ALhUq4EqvXqpEzbVqNeQdljIGR7JDQsrWtGIvXUTCgwcotnAR7PPlQ/zNmzxOmTRo/TTMbv8lzg5dpv72XAm9rcrbulVvYTCup2Plxvhg6ShVflYhTzFMatUfdyJCsORI6nftnsvHUfWH7vB18cRHNd/Cr11H47Wpnxgd+0PGFfcsgWYFmquGANcirsLPyQ/ti3dA8/iW+CNoE3fbS3Thzzvwzu9itCFB7jJeaPtdVRVEnf/zDnZM/QfvjquiC6YqtEz9DpLMjlxQ+2tuIGp0LMLS8yyMwcnoPDHtuWJISIg679bPwAh5LOe/xkjCQrI7cu4tQzV++OEHVdkkFU958+ZVwY12HWnXqX3uuZaoSYZCNkxq42QsigQ6UoolmYyNGzfiWaXtViYnQhLhaUmZmtxQNCEhQWVvJBOjzcZIRGhjY6OiyZMnT+omCQa05WYff/yxKvOSLIcERJKFkcFQ2UnG87Rs2VIdPClhk+3RdpZLm+nKDk/bZ2lJ4CXRsP4k814WBzc7WFkDsY8SDOZLFsbRyGBEfYlxSapBgTQSSLdOGyu45zYMJt1yOyMmNOWX1skzZd3uuV0yXMZSeXo5w8bGGmGhhlfbHoZGq7E1mWFrZ6M6nwXfSLnKffJYEB6GRaNVk8l4rdJINd29HY7pk7ahVdPJapljh6/i7KmbqFt1tHq+bcuU39vuHX/G6P+tg6VKCn8ITWIi7P69UKNl6+OLhJDUskt9MlbG1jf98jJfWDk4INeAAbg18VtE7NmN2IsXEfLrMjzc8gf8umd8L7OY06fV/x3yW25XopDoR0hMSoSfm2EDGn83b9yNCM3gNeF4b/7X8PiqMYqMeQ9lJ7yPqLjHuBp2W7fMt2/1xve7lmHViV04e+cqlh3dhml7V+HLhh8YrEvG+lwJuYVD1//BJyu/RWJyErpnMPbHEkgHtKTkJLilyZpIhuVRvPHM1tuF38GhuwE4cGcfbkffwsmQE9hwZT2aFWgGK1hlap2SqbGztoOTrZORZSy3i5qju3ynW+HxI8PzncePEtSYmCdJiE3ClYP31NgdY6RkzSOnM/yLe6B+z5Lqe14CnYz4FXVHcpIGkWk6qxFglZSY4fQ8zxOl27IM9ZDx4/Xq1VOdiiWBoU1WZKdnCnBkTInU3UmAIOkoSTV17txZjct5nifLUo8nmRAJqCTAkYBH/+ajEklK9kKyKvqTfimYZESkXEx26ueff666NAgpHXta9kkiT8nGyBghrcuXL6vW0loS0EiAIeOJatasqTJKUjuoT97raVmuUqVK4ebNm2rSOnfunOooIZmcZyWpRAlO9Sf99OKLJu0ePQu64f651H0o6W157FPkyWn+4MMPkJyQjPy1/dOt06uQGyLTlLhF3Y2Bs2/K+AH5vzQoSL/MYzj7vroZrexgZ2eLEqVy4cihq7p58pmWx+UqGG8TnZaUuF25dA8+OVLqo5u9VRFL1/RWLZ+1k3RR69TtNUyb1UUtM2hIcyxZnbrM5Jmd1fwx37VFz34NYamkfCzmn3/gVrNW6kwrK7jVrImYkyeNvib61EnD5SV4r10b0f8uLyVv1vI3L+3FEKlX1yubSsupZErZSMKDjLPE5i5BtWe+iDeKVzG8J1yxKvj7+j9PfG1cYjxuPwpRbaNbl6+H38/s0z3nbO+IZKnX1SMn2dI++knkeQfbJ18MMmdJmiQ1wL+UV2ozBglSSnqVxNWI1L9h+uytHdSYGn3JSM7SOq9HXFctpUvqLePv7A8fRx/Vmc1SSdYkR2FX3Dpj+J1+6+xD+Bd78nf6lb/vIylRg+L/VlU8lUajxuNkJCQoUjXzdHLneM50pB16BlNmzxN9fX1VYkG/+ZWQx08agpH2Ir2cv8u5tNC+7r+s8z+VqEVHR6uxKkLKs6RkTU7kJZsiXRSeF+mk0KpVKwwbNkxlZmT8jZa8vwQ8EhlKcCE7TLZLxqNINkU6n0nmqVmzZmpZCUpkfI8EEkKCNPmSkoH6MhZIMlPSgECftMSW0rZPPvlEtayTAyNBkiyrLbuRgEoyTJIZko5t0kpbBl3pk7IyyTjJtkmjBilbS1u6Ju8j+1N+JhlwJYGVjE+SiFc7/udZGEszvmxSRnZk7gUVlGjbRCfGJetKxw7PuQAnL/t0ZWPX9t1RZWgOrun/eJVolg9//3QOviU84VfKU43BuXMyFPWGpAyyluMly/yzIUjd+0amoP131bifmn2fPYA0Fx271MaYb9ajVOncKF0uL1YuDUDs43i0aJVSninNAnL4u6P3Z43V419m70bZ8vmQN783IiNSxuDcvROOd95NOQn08HRWkz7poiad1goUShkXkjOX4dg3J+eUk7a8+bx143gs1f1FC1FgwreIOXsW0WdOw69LV1g7OSF0fUpmq8C33yL+3n3cmZKSDXuweAmKLV4Mv27d8WjvHng1b6EaFdwckXJfIumEFnn4MHJ/8QWSY+NSStSqVYf3O++orI6QMjSvli0RsfcvJIWHw7FEceQdMhSRR46ojI8lm7pnBea//z8cu3kBR66fV22iXeydsOjQZvX8gve/wa1HD/DN5pSrktXzl0ZuD1+cun1Z/X94kw/VxcEf/vxVt87N/xzAkMZdcCP8nmoTXTFvcQyo3x4LD/2hC4CGNuqCTf8cUGVrUqLWq8676r46a0+ltvW2RDtv7lBtnoMiryMo4ppq6Wxv44CDt1MaZshz4XHh2HA15ffldOgp1SXtRuQNXIu4pkrU3i7UCqdDTusCn6etMzbpMQ7c3o+2xdojOiEasUmx6FC8o2pgIGVvlqx8i3zY/dMF5CjiBr8i7jj9RzAS4pJQon5KZubPH8/BxdsBNd4vkq65QMGqvnB0s0uX2Tm+PggFq/jC2csBsZEJ6j470WHxuk5rMubm/qUIVcZm72SjHkur6mKv5zR6jmDxklLv95SWg2PmzhPlYn2VKlXUuaycm2svhspjafqVGXKxXyqq5LxbSHdiCWRkHZLlETLUQyrGZIjJcw9wJJMh41HkRF1O0CW1JP+WE3kZA/I8yQm/7Ii6deuqVtL6pJmADOiXoOPWrVsqupQsipSMaXekdFKTG5RKFzPpsCaNDIS0lh41ahSGDBmiukFIoCSD99OSgf4fffSRen85CJKxktpBx387C8n+kOyWdGiTiFeWk2VkfVpSbyhZJOmIFhoaqtrraVtFa8kJuNxbqF+/fmod8mUo25vdJXWmQDqgSbOBc+uDdDf6rPN5OV2JWkxobLpbqkjmJfRihLoppzF5qviqdtOBm2/i5LLLcMvphFp9y8C3eOqJcrEmeZGUkIxTy68gPipBvW/dL8qrltKWrnHTcqrZwNyf/kRoSBSKlciJKbM+0LV+vnv3kSpD0JKgZsKo39Sybu5OKFk6F+Ys7qFaTNN/F75lC2y9vJGrfz/Y+ubA4/PnceWTHkj8t/GAXa7c6iqpVvTJEwj6YjByfTYAuQYORNz1IFzr11d3DxwR9Pkg5B44CAW+/x62Hh6qmcDtqVPVjTy1mSO3WrV1wVT83TsI37Edd2fNsvhDuvrkn8jh6okRTT9GTndvnLp1GS1//hz3o1KuWufz8jfIxjjY2WNU8x4o7JNblaZJZ7Ruy8bgUWxqi1u5v82oZj0wo83n8HP1Ujf6nHtwI8ZuX6CeT0pORgn/Aupmob6uHgiNjsDRG+fRYEYfNWbHkh29f0Q1BpDSMykRC468iemnpupuuOnt6GOQsVHjbDQavFO4NTwdPBGVEInTIaew4er6TK9TrLq8Ahokq3vfSEe1c6H/4NeLS2Hpitb2V+Nkjqy6hpjwePgWdEWLoeVVYwARKWXget8fIvx2DO5eeIQW/0vfnVeSmOG3YrBt71kV3EgAJIHTOyMrqZbR2szR5YP3cXRNkPped/dzRPnm+VChZeo4ZtKjd0Pb/0JaRHft2lVdeJd738gFeUmCaO91I+e+cn6tre4aPXq0Oi+XZIBUJMn9c6RNtAwh0Z77SjJCzuVl2IsEPJLUkOZh2iAqs6w0+n2XM2np0qUqoyCNBqQkS068pZ2xRHMSFKRtZWzOJFiSsjdpLNCw4atbRvO/gE9e9ibQv8bVmoOHcal3k6eXy8uhPU6USt/Rh16eSucvwG6g8Xuc0IuXMGU/Pv0z5QSFXr6f35iHKSd7vuzNoH8NrGhYxWMSIldn/Jxb2yyt6scff9Td6FOyLnKbFOl6pr2XpCRAtAmDgQMHqiEisqxUgEkGSIIZqbpKe6NPuQ2LBEF16tRRXYjTNhF7LgFOWtIdTDomSEZFsibmTMYfSXmZlI/JPWzk/jaSLbp48WK6Af+vEgY4poMBjmlhgGN6GOCYFgY4poUBjmkxyQDn0fKMn/NIHf7xKvvvjbClNtjZ2aB1sjmT8TVff/216sYm7Zml3ExaV7/KwQ0RERERWQZNkmHnWn3Gb+RgIQGOjGWRdJMMApKuZWnbEkuWw1w1adJETUREREREljoGx+wCnM8++0wFONKZrGzZshneuI+IiIiIiExIMgMco1asWIFVq1bp2roREREREdGr3SbaojM40i1NWrwREREREdErJNn8A5wn3yI5A3KfmWnTpqlWbkRERERE9AplcJIymCwtg/Puu++maySwZcsWlClTJl0HMelxTUREREREJibJfAKZ/xzgeHik3gFetG7d+nlsDxERERERPS+JSWa/bzMd4CxYsOD5bgkRERERET1f8RnfB8eix+CIxMRE7Ny5Ez///DMiIyPVvNu3byMqKio7t4+IiIiIiLIzg5OYwWTJXdSuX7+Opk2b4saNG4iLi0Pjxo3h5uaGiRMnqsezZ8/O/i0lIiIiIqL/JtF8AplszeDIjT6rVq2Khw8fwsnJyWBczq5du7Jz+4iIiIiIKLskJmY8WXIGZ9++fTh48KC6H46+ggUL4tatW9m1bURERERElI00SRlncKwsOcBJTk5GkpGdExwcrErViIiIiIjIBCWyRM2oN998E1OnTtU9trKyUs0FRowYgebNm7+4A0RERERERJmXyBI1oyZNmoQmTZqgdOnSiI2Nxfvvv49Lly7Bx8cHy5cv50eMiIiIiMgUJZp/BueZStTy5s2LU6dOYcWKFTh9+rTK3nz00Ufo1KmTQdMBIiIiIiIyIYnmH+A8Uxe10NBQ2NraonPnzujXrx98fX0RGBiIo0ePZv8WEhERERFR9kg0/xK1LAU4Z86cUZ3S/Pz8ULJkSZw8eRLVqlXDlClTMGfOHDRo0AAbNmx4fltLRERERETPLtH8b/RppdFoNJlduFmzZipzM2TIECxZsgSbNm1SY3Hmzp2rnpdszrFjx/D3338/z20mIiIiIqJnkLSya4bP2bRfZHkBjpSi/fnnnyhfvrwad+Pu7o4jR46gSpUq6vkLFy6gZs2aCA8Pf57bTM/Biot9uV9NRIfiP2L6qV4vezPoX/0rzMKxEiW5P0xIlcALsOlX+2VvBv0racZBWPWqyf1hIjSz/kZkwvqXvRn0Lze71ia3LxKXfpDhc7adl2RpXTNnzsT333+Pu3fvokKFCpgxYwaqV69udFlJiCxevBhnz55VjyV+GD9+vMHy3bp1w6JFhkGWJFO2bt36/ErUwsLCkDNnTvVvV1dXuLi4wMvLS/e8/DsyMjJLG0BERERERC9IfFLGUxasXLkSgwYNUreJOX78uApwJBi5f/++0eX37NmDjh07Yvfu3QgICEC+fPnUrWdu3bplsFzTpk1x584d3fQsHZqz3GRA7nnzpMdERERERGSaNHFJGU5ZMXnyZPTo0QPdu3dXt46ZPXs2nJ2dMX/+fKPLL1u2DL1790bFihXVWP558+YhOTkZu3btMljOwcFBJVS0k34y5bm1iZbUkbyxkHvg9OzZU2VyRFxcXJY3gIiIiIiIXgxNQsaBjJzLpz2fl/N+7bm/Vnx8vBp3P3ToUN08a2trNGrUSGVnMiMmJgYJCQnw9vZOl+mRhmYS2LzxxhsYO3asutfmc8vgdO3aVb2hh4eHmqRNdO7cuXWP5bkuXbpkaQOIiIiIiOjF0CQkZzhNmDBBd16vnWReWiEhIUhKSoK/v7/BfHks43Ey46uvvlJxhARF+uVpMk5HsjoTJ07E3r17VZMzea/nlsFZsGBBllZOREREREQmJD45w6ckIyPjavSlzd5kh2+//RYrVqxQ2RpHR0fd/A4dOuj+Xa5cOdXYrEiRImq5hg0bPr8SNSIiIiIiMr8SNUcj5WgZdVa2sbHBvXv3DObLY21Dsoz88MMPKsDZuXOnCmCepHDhwuq9Ll++nKUAJ8tNBoiIiIiIyPxK1DLL3t5etXnWbxCgbRhQq1atDF/33XffYcyYMartc9WqVZ/6PsHBwQgNDUWuXLmQFczgEBERERFZCE0WApknkVI2GZ8vgYrcy2bq1KmIjo5WXdWEjMvPkyePbgyPjKkZPnw4fv31VxQsWFA3VkduPSOT3GNz1KhRaNOmjcoCXblyBV9++SWKFi2q2k9nBQMcIiIiIiJLkZC1AfsZad++PR48eKCCFglWpP2zZGa0jQdu3LihOqtpzZo1S3Vfe++99wzWI/fRGTlypCp5O336tLrRZ3h4uGpAIPfJkYxPVscBMcAhIiIiIrIQmmzK4Ii+ffuqyRhpDKAvKCjoietycnLCtm3bsmW7GOAQEREREVmI5GwMcEwVAxwiIiIiIguRxACHiIiIiIjMRXJi9ozBMWXM4BARERERWYhkZnCIiIiIiMhcJDPAISIiIiIic5EYxxI1IiIiIiIyE8nM4BARERERkblIzqYbfZoyNhkwE3IzpQYNGuDhw4fw9PTEq+bQ5ps4uC4IUQ/j4V/IFc0/LYm8xT0yXP5xVAJ2LbmM8wH38TgyAZ5+TmjaoziKV82hno+LScSfy66o56MfxSNXYTc061ECedKs88HNKOxYeAlBZ8ORnJSMHPlc0X5oebU+S3dmazBO/H4DMeHx8CngirofFod/UXejy64feRy3z4Wnm1+gkg9aDq2g/n141VVcOngfUaGxsLG1Ro7CbqjRoTByFks9JpsnnkZIUCQeRyTAwcUWect5oXanonDxztodjM1Rjvffh/9HH8Euhy8eX7iAG2PGIubMmQyX92zaBHk++wz2efIgLug6gn/4ARF//aV7vkrgBaOvC/7uO9z7Zb76t0PBgsj75RdwrVwZVnZ2eBwYiFvTpiPq0CFYul6vv4vBDTshp7s3Tt26jM/WTMaR6+eNLmtrbYMhb3ZBl+rNkcfTF4H3b2Dobz9h2/nU/ejq4IzRLXqgVYV68HP1wongixi4diqO3khdZ+sK9fDpa61ROX8J+Lh4oPK3XXHq1iVYuteLVsQXjTujSv4SyO2ZA61mf4nfTqV+1o2pV6wyJr/3GcrkKoSbD+9h7JaFWPT3ZoNletdro9arjnHwZfRbOQlHrp/TPe9ga49J7/VHhyqN4WBrp45n7+Xf435kGCzdquUBWLJgL0JDolCsRC588fXbKFsun9Flf99wFKO+WWMwz97eFgePj9U9/nnmDmzfehr37obDzs4GpUrnRe/+b6Js+fy6ZQb2XYSLF27jYVg03NydUL1mUfQf1Aw5/Ix/b1mypETeB4fouTu77y62zQvEW31KqQDk7403sGT4cfSb/RpcPe3TLZ+YkIzFw47DxdMe7YdUgJuPAx7dfwxHVzvdMr/NOIf716Pw7qCycPN2wOk9d7Bo2HH0/akW3H0c1TJhd2Lwy1dHUblxbjR4vwgcnG1x/0YUbO1tLP6oXzp4D/sXX0L9HiXgX8wDpzbfxO/jTuL9qTXh7JH+mDQbXM7gD2ZsZAJWfnEERWr56eZ55nZWQZK7vxMS45NS1jn2JDrPqAUn95R15injiSqtC8DZyx7RYfE4uOQStk4+gzZjq1r0MfFq1gx5hw7BjREjEX3qFPy6dkWxX+bhn6bNkBiW/mTKpVIlFJ40CbcmT8aj3Xvg/VZLFJn5I86/2waxl1JOiE+9VsfgNR5166LAuLF4uG27bl7R2bMRdz0IF7t2RXJsHPy6dkHR2bNwtvGbSAwJgaVqV7khJrXuj94rv8eh6//gs/rtsaX3FJQa0xEPoh6mW35My0/RqVoTfLr8W1y4dx1NStXA2o+/RZ0pn+Jk8EW1zNz3h6BMrsLoung0bj96gE7VmmJ732koO+593H6Usq9d7J2w/+oprD6xC3PeH/rCf25T5eLgpAK9+Qd/x/qeE5+6fEGfXNjcZxJm71uPTgtGoGGJqpjXeSjuPArB9n+DznZVGmFym8/Qc/lEHLr2Dwa80QHb+k9FiZHt8SAy5RhPaTsALcrWRtt5X+PR4yj82H4w1n36Ler88Aks2fYtpzDlu00YOrw1ypbPh+VLDqDfp79g7e+D4e3javQ1Lq4OWLtpsO6xVZrnCxTMgS+/fht58nojLi4Rvy7ehz6f/IINf3wBL++UdVatXhgf9mgA3xxuuH8vAtN+2IyvBi7F/GW9n+vP+ypKtoASNeuXvQGvmvr166Nfv34YMGAAvLy84O/vj7lz5yI6Ohrdu3eHm5sbihYtii1btuhes3fvXlSvXh0ODg7IlSsXhgwZgsTERN3zBQsWxNSpUw3ep2LFihg5cqTusZWVFebNm4fWrVvD2dkZxYoVw8aNG9VzQUFBKnsjZJtk2W7duuFVcXDDdVRpkheVGuWBX35XtOxdCnYONjix45bR5U/svKUyOB3/VwH5S3vCy98JBct5I2chN/V8QlwSzh+8jze7F0PBsl7wye2sAhjvXE448kewbj2SASpWxRdvdi+OXEXc4Z3LGSVr+BkNqizNyU03UaZhbpRqkBveeV1UoGNrb43zu28bXV6CSxdPB90UfPohbB2sUbRmaoBTvE5O5CvvDQ9/J/jkc0WdLsUQ/zgJIdejdMtUbJkfOYt7wD2HE3KV8EDlVgVw91KERVxtehL/7t0Qsmo1QtetQ+yVK7gxYgSSY2Ph06aN0eX9unyAR/v2q0xM7NWruD1tOmLOnYNf5066ZSRA0Z88G76ByEOHEB+c8jti4+UJx0IFcXfOXDwOvIi469dxa9Jk2Dg7w6lYMViyAQ06YF7ARiw8tBnn7wah18rvEBMfh+61WhpdvnP1JpiwfRG2nAvAtdDbmL1/PbacO4hBb3RUzzva2ePdCvUx5LefsO/KSVwJuYXRW37B5QfB6FnnXd16lh7ZirFbF2Bn4JEX9rO+Crb+E4BhG3/GhlN7M7V8z9ffVcdh8NrpuHA3CDP3rsGaE7sxsGEH3TKDGnbE3AO/YWFAyjGWQCcmPhYf/nuM3R1d8FHttzBozTTsDjyG4zcC0X3xWLxWpDxqFCoDS7Zs8X60eq863m5dFYWL+GPo8FZwdLTHxvVHM3yNnLf4+rrpJh/flO9zraYtKqJGrWLIm88HRYr6Y+CXLREdFYdLF+/qlunU5XWUq5AfuXJ7oUKlAuj6cX2cOX0TiRZQjvUsAU5yBpO5YIDzDBYtWgRfX18cPnxYBTu9evVC27ZtUbt2bRw/fhxvvvkmPvjgA8TExODWrVto3rw5qlWrhlOnTmHWrFn45ZdfMHZsauo1s0aNGoV27drh9OnTap2dOnVCWFgY8uXLh7Vr16plAgMDcefOHUybNg2vAsnG3LkcicIVvHXzrK2tULiiN24GPjL6msBDD5CvpAc2z76A7z7Yi5l9DuKvVdeQnKRRz8v/k5M16oRcn529DW78W0Ylz188GgKfPM5YPPw4vuu8B3M+P6RK2iydBBMPrkYib7nUY2JlbaUe370Ykal1nPvzNorV9oedo02G7/HPztuwd7aFbwHjV/RioxJwcd895CruoUraLJWUhjmXKYOIgwdTZ2o0iDwYANdKFY2+xrViRUQG6C0PIGL/AbhUNL68rY8PPOrVQ8ialL8jIulhuAqOvFu9A2snJ8DGBjnat0dCSAhi/vkHlsrOxhZV8pXArsDUkzWNRoNdgUdQq2BZo6+RUqa4hHiDeY8T4vFa4fLq37bWtrC1sUVsQlyaZeLUCTNlr1qFy2LnBcMgcdu5v1GrcLnUY5y/hMEycozlsXaZKgVKwt7WzmCZwHvXcT30DmoVSlnGEiUkJOLCuVuoUbOobp61tbUqFzt96nqGr3scE4+Wjb9Fi4YTMKjfIly5fO+J77F+9WG4ujmieIlcRpd59CgGWzedRPmK+WFrx6oMSwxwOAbnGVSoUAHffPON+vfQoUPx7bffqoCnR48eat7w4cNVICOByO+//64CkB9//FFdoShZsiRu376Nr776Si0nv/iZJVmZjh1TrviNHz8e06dPV0FW06ZN4e2dcjLq5+f3So3BiYmIV8GGq5dh1kSyKCHB0UZf8/DuY1w7/RDl6udE5xGVVKnZplkXkJSUjAYdU0rNJADau+IafPO6wNXTAWf+uoubgeEqSyNkXI5kD/avuYY3OhdF427FcPlYCFZOOIVu46qojJClio1IgCZZA+c0mSx5/PB2zFNff+9yBMJuRuONXqXSPRd0LATbpv6jStSkxPDtbyrqytO0Di69jDPbgpEYlwz/Yu5oOSRlDI+lspWsrK0tEkNDDeYnhIbAsXAh46/x9UVCiOHyiaEhsPP1Nbq8T+tWSIqORvj21PI0cbFbdxT5aSYqHj8mVwWQEBaGSx/3QFJE5gJdc+Tr4qmCkXsRhqWB9yLDUMK/gNHXSNmTlDj99W92pmHxqmo8jY1Vyt//qLgYHLx6Bv9r2h3n715X6+pYpTFqFSqrsjiUvXK6+6Q/fhFh8HByhaOdA7yc3Ywf44iHKOlfULcOCVqlNM1gmcgw9ZylCn8Yo76L05aiyeOgaw+MvkbKz4aNbqPG6kRFxmLpwr/wYeefsGrDIPjnTB2juW/PeXz9xXLExiaoMrSZcz6Cp5eLwbqmT96CVcsPIvZxgsrmTJnZ9Tn9pK+2xMSUC8LmzHIvi/4H5cunXlGzsbGBj48PypVLvWIjZWvi/v37OH/+PGrVqqWCG63XXnsNUVFRCP63FORZ3tfFxQXu7u7qPbIiLi4OERERBpPMe5VoNICLhz3e7lMauYu6o+zrOVG3XSEc3ZK6P2XsjVxxm9RtH8a8uwuHfr+BcnVz6o6DnMALKUmr3aqAakLwettCKF7NF0e28oTiv5DsjU9+F6MNCfKU8UL776uhzZgqyF/RB9umnEXMI8Mr25Xezo/2E6ur4EeyeTt/PKeOJT0/vm3aIOz3TdDEGx6L/COGq8AqsFMnnG/bDuE7d6oxOLY5Upp5UOYMWDtVBSrnvlmO2Cl7Mb3tICz8ezOS9T7XXZeMhhWsEDxuIx5P2YO+9dtixbGdBssQmaPyFQug5TtVUKJkblSpVhjfT/0AXl6uWLfasJlJ1epF8Ova/pi/tBdqvVYcQwf/irBQwwCzS/e6WLa6P36c85H6/hgxdBW/P4yQURIZTeaCGZxnYGeXOphdyEmz/jztSXRycuZSfZLFSXsCl5CQkKn3zex7aE2YMEGVuukbMWIESr6Pl8LZ3V79EZLuafqiwuPh6mW8c5Zke6RkydomNWjMkddFrUNK3mztrFWm5sNvqyE+Nkl1VJNGA6smnoZXTqfU97WxQo78hld/fPO54sa59IOELYmju50qSZPuafrkcdqsTloJsUm4fOAeqrcvbPR5KVnzzOkM5IQaa7O0fwDO/3kbVVqnXBUVktGRSZoSeOVxxqJeB3HvUoRa3hIlPnwITWKiKiPTZ+cjWRrjA/1lTI2dr+Hythks71qlChwLF8bVAQMN5rvVrAmP+vVxslp1JEenZFNvjhoN99q14dOqFe7NnQtLFBIdjsSkRPi7G2Z5/d28013x170mKhzvzh2iStV8XNxV04AJb/fG1dDUcYZXQ27hjel94GzvqMZ33I0IxfLuo9VYEcpesm/THT93b5WNkTLBkKgk48fY3Uu9VrsOBzt7lfXRz+LI50C7jCXy9HKGjY11usBDHvv4Gi9HTktKykqUyo2bNwz3o5OzPfLl90W+/FDZmdbNv8dv646ge48Geu/voibJChUq7IcWjSbgzKkbKoiiVIkWMCyJGZznrFSpUggICDAIYA4cOKCaEeTNm1c9zpEjhxo3oyVZlWvXrmXpfeztU048k5Ke/KmVkrpHjx4ZTDLvZZFgJFdRN1w9nXpiICVr106FIV8J4ye00lhAytJkOa2Q2zFw87ZX69Nn72ijghtpSnDlRChK1ki58izL5SnmjpBgw5Kr0FvR8Mxh2S2itS2cg8+mBnqS8ZLHOYs/ud3m5b/vIylRgxKv58zUe8nvRdITan61vzZPWsbcaRIS1JgX91q1UmdaWcGtVk1EnThp9DVRJ0/Crabe8jIounZtRJ9Mv7zPe+8h+uxZ1QJanxp3ozYgTQZBo4FVFkprzU1CUiKO3QzEG8WrGFxseqN4VQQEnX3ia+MS41VwI22j361YHxvP7Eu3jAxklxNkTyc3vFmyBjaeTr8M/TcBV8+qzmn6GpeqjoCrZ1KP8Y1ANCxRzeAYy2PtMseuX0B8YgIalkxdprh/fhTwyYWAaxm3bzd3dna2KFk6Dw4fuqybJxdijxy6jPIVMhdkSInb5Ut3VRnak8g5QHx8xikH7XnXk5axVInM4NB/1bt3b9UhTZoR9O3bVzUBkIzJoEGDdONv3njjDSxcuBBvvfWWGj8jY3Ok9C0rChQooP4Ab9q0STUgcHJygqtr+qsl0slNJlMiJWLrp/yDPEXdkae4OwJ+u6EyL5Ua5VbPr5t8VrWCbtw1pXNTtWb5cHjTTWyZG4gaLfMh7HYM9q2+pv6tdfl4iDov883jooKh7QsuqvE42nWK194tiNXfnUaBsp4oVM5bvebi4RB0G5964mKpKrbMh10zz8OvsBv8irrj1B83kRiXhFL1U/aflI3JvWlqvV/E4HWSjSlUzReObnbpMjtH1wWhUFVf1QJa2kif2XpLtYLWtpK+e+kR7l+JRK6SHuoeOBH3HuPQymuqrbSlZm+07i1YiIITv1WBSMzp06pNtAQg0lVNyHPx9+7j9uTJ6vH9xUtQYsli+HXvjkd798C7eQs4ly2D68OHG6zX2sUFXk2bIHhi+ta6USdPqLE2Bb/9FndmzkRyXBx827VV99V5tGcPLNnU3SuwoPM3OHbjAg5fP6faRLs4OGLh35vU8ws/GIZb4Q/wv99nq8fVC5RGHs8cOBl8Sf1/eLOPYG1lhe93LtOtU4IZSf7LPXKK+ubFxFZ9VEvpBf+uU8jYkPxeOZHbI2UsVQn/lHuASEAkYz8suU100RwpFwxFIZ/cqJC3GMKiI9Q9bsa/00vt966LRqvnZ+9bh77138PE1n1Va+k3SlRVrb9bzPxct47Ju5ZjUddh6j5Eh4POYcAbKcd4QUDKvXIiYqPxy8HfMblNf4RFP1KPZ7T7HAevnFZtpS1Zpy51MPJ/q1G6TF6UKZsPvy7dj8eP4/FWq5Tv1uFDV8LPzwN9BzZVj+fO2oly5fMjb34fNQZn8YK/cPf2Q7RqU03XgGD+nD9Rt0FpFfSEP4xW99l5cD8CjZqklO6fPX0D/5wNRsXKBeHu7oTgm6GYNWOH6rrG7E165lSKlhGWqD1nefLkwR9//IEvvvhCNSeQZgAfffSRrkmBkAyKZGxatmwJDw8PjBkzJssZHHkfKT2TFtTSrrpLly4qaHoVyBgaGfQvN+aMehiHnIXd8MGoyroStUcPYtUXv5ZHDkd8MLoyts67iFn9/lbBT8238qNOm9Qyp9joROxcfBkRIbFwcrND6dr+aPhBEYNuXKVq+amW1BIcbZkTCN88zuomnwXKeMHSSQc0udnmoVVXVWmab0E3tPy6gq5ELTLE8JiIh7ejcefCIzV2Ji0ZSx1+OwZbJ51RN2aVAMiviDtaj6qsWkYLaQ1+9dB9dUNQaTAg7yXjdJoMLAibNJk5S/NwyxbYensjd/9+sMuRA4/Pn1eD/bWNB+xz5daNKxPRJ07g6uDByDNgAPIMGoi4oCBc6dNXdw8cLe8WLdSFkbBNhjc41HZRk/fIPWAAii9aBCs7Wzy+dBlX+vRJl+2xNKuO74KvqydGtuiBnG7eOHnrEpr/NAj3/70/Sj4vfyRrUrOO0gZ6dItPUNg3N6LiHqt20XK/G/3SJg8nF4x7qxfyeuZAWEwE1p3ag29+/xmJyalZ+bfLvY75nVO/O5Z3H6P+P+qPX1RbaUtVNX8p7Bn0k+6x3J9GSIvn7ovHIJeHL/J7p2aVg0LvqGBmStvP8FmDdggOv4+Pl07Q3QNHrDq2EzlcPTG6ZQ/VNECC06YzBhrcxHPg6qnqOK/9ZIIqP9x27hB6r/gOlu7NZhXw8GE0Zv+4A6EhkSheMjdmzP5Q1/r57p1wVZquFRHxGGNHrlPLSnAiGaBflvZSLaaFlJNLg4JNG5eq4MbD0xmly+bF3EWfqpbRQtpQ7955FnNm7lTBlARCMk7no0/fUDcNJcsLcKw0HL1LAFZc7Mv9YCI6FP8R00/1etmbQf/qX2EWjpUoyf1hQqoEXoBNv9ovezPoX0kzDsKqV03uDxOhmfU3IhPWv+zNoH+52bU2uX2x069Ehs81um8eF7AY1hIRERERWYhEC8jgWHbdBxERERGRBUnMxiYDM2fORMGCBeHo6IgaNWqo+zM+yerVq9U9IWV5ucWKDOPQJ4VlMhY9V65cajx5o0aNcClNeXVmMMAhIiIiIrIQ2RXgrFy5UjXNkuZZx48fV2PNmzRpkuE9Gg8ePKhuWC9j0U+cOIFWrVqp6ezZ1A6U3333nbqR/ezZs3Ho0CF130dZZ2xsbJa2jQEOEREREZEF3QcnMYMpKyZPnowePXqo5lalS5dWQYmzszPmz59vdPlp06ahadOmqvGW3EZFmmpVrlwZP/74oy57I52HpRHXO++8o25wv3jxYty+fRsbNmzI0rYxwCEiIiIishCJT8jgxMXFqfsx6k8yL634+HgcO3ZMlZBpye1P5LHc/9EYma+/vJDsjHZ56SB89+5dg2Wku7CUvmW0zoywyQARERERkYXokJRxp7SRI0eq247okxI0ma8vJCRE3Vze3z+lVbeWPL5w4YLRdUvwYmx5ma99Xjsvo2UyiwEOERERERFB7s0o42r0mdoN4jODAQ4REREREUGCmcwENL6+vrCxscG9e/cM5svjnDlTb6yrT+Y/aXnt/2WedFHTX6ZixfQ3EX8SjsEhIiIiIqJMs7e3R5UqVbBr1y7dvOTkZPW4Vq1aRl8j8/WXFzt27NAtX6hQIRXk6C8jY4Ckm1pG68wIMzhERERERJQlUsrWtWtXVK1aFdWrV1cd0KKjo1VXNdGlSxfkyZMHEyZMUI8/++wz1KtXD5MmTUKLFi2wYsUKHD16FHPmzFHPW1lZYcCAARg7diyKFSumAp5hw4Yhd+7cqp10VjDAISIiIiKiLGnfvj0ePHigbswpTQCkjGzr1q26JgE3btxQndW0ateujV9//VW1gf76669VECPtn8uWLatb5ssvv1RB0ieffILw8HDUqVNHrVNuDJoVDHCIiIiIiCjL+vbtqyZj9uzZk25e27Zt1ZQRyeKMHj1aTf8Fx+AQEREREZHZYIBDRERERERmgwEOERERERGZDQY4RERERERkNhjgEBERERGR2WCAQ0REREREZoMBDhERERERmQ0GOEREREREZDYY4BARERERkdmw0mg0mpe9EUT/VVxcHCZMmIChQ4fCwcGBO/Ql4/EwPTwmpoXHw7TweJgWHg/6rxjgkFmIiIiAh4cHHj16BHd395e9ORaPx8P08JiYFh4P08LjYVp4POi/YokaERERERGZDQY4RERERERkNhjgEBERERGR2WCAQ2ZBGguMGDGCDQZMBI+H6eExMS08HqaFx8O08HjQf8UmA0REREREZDaYwSEiIiIiIrPBAIeIiIiIiMwGAxwiIiKyaFZWVtiwYUO2L0svlv6xCQoKUo9PnjzJw2CBGOCQWfnggw8wfvz4bFtffHw8ChYsiKNHj2bbOs3FgwcP0KtXL+TPn18NCM2ZMyeaNGmCAwcO6JY5ePAgmjdvDi8vLzg6OqJcuXKYPHkykpKSXuq2myP5In/SNHLkSN0Xvnby8fHBm2++iRMnTrzszTdr3bp1Q6tWrXSP7969i379+qFw4cLqdydfvnx46623sGvXrpe6naa0v7SfUXt7exQtWhSjR49GYmLic3vPO3fuoFmzZtm+rKUeNzs7OxQqVAhffvklYmNjX/amkQVigEMv5Q+fv78/GjdujPnz5yM5Odnoa+Rk2cbGBkeOHMnUe5w6dQp//PEH+vfvr5tXv359oyd72i/KdevWqRM8OdEzdqVHvlwHDx6Mr7766j/9/OaoTZs26sR40aJFuHjxIjZu3Kj2d2hoqHp+/fr1qFevHvLmzYvdu3fjwoUL+OyzzzB27Fh06NABGo3mZf8IZkVOurTT1KlT4e7ubjBPPsdaO3fuVPO2bduGqKgodbIWHh7+UrffUkiQWaVKFfz555/4/vvvcebMGWzduhUNGjRAnz59XvbmmYymTZuqz+ilS5fw+eefqwBd9pexi1DZQS7QSLCZ3cta6nG7evUqpkyZgp9//ll1OCV64TREL0DXrl01TZs21dy5c0cTHBysOXbsmGbcuHEaV1dXTbNmzTQJCQkGy1+/fl09179/f03Pnj0z9R4fffSR5tNPPzWYV69ePU2PHj3U++pPWosXL9aMGjVKM3fuXDnb1pw4cSLdesPCwjT29vaas2fPPvPPb24ePnyo9teePXuMPh8VFaXx8fHRvPvuu+me27hxo3rtihUrXsCWWqYFCxZoPDw80s2/du1aus/5gQMH1LytW7e+4K20rL9/77zzjvq3/L3LkyeP+h0x9ntFhvtLq3HjxpqaNWvqnhs7dqwmV65cmoIFC6rnb9y4oWnbtq363Ht5eWnefvtt9XnX98svv2hKly6t/p7nzJlT06dPH91z8juwfv169e+4uDj1nCzj4OCgyZ8/v2b8+PFGlxWnT5/WNGjQQOPo6Kjx9vZW3zmRkZHpjv/333+v1inL9O7dWxMfH2/2x02+AypVqqT+nZSUpPajHDPZV+XLl9esXr3aYHn5nm3RooXGzc1NnQPUqVNHc/nyZfXc4cOHNY0aNVLfLe7u7pq6deuqcwl9+sfG2N87shzM4NALoy1jypMnDypXroyvv/4av/32G7Zs2YKFCxcaLLtgwQK0bNlSlUAtX74cjx8/fuK6peRpzZo1qswjLWdnZ/W++pN+Sdvw4cPRqFGjDNct5VWvvfYaVqxY8Uw/tzlydXVVk9Q6x8XFpXt++/btKpOjnzXQkmNUvHhxdVzp5XNycsrWK+GUsbCwMJWtkUyNi4tLuuc9PT25+57wOdV+RqWULzAwEDt27MCmTZuQkJCgMv5ubm7Yt2+fKpOVv0+STdC+ZtasWWq/f/LJJyprJhlnKX0zZvr06er5VatWqfdZtmyZKlU2Jjo6Wr23fE9ItcHq1atVhrRv374Gy0kW+8qVK+r/kvWW77y033vm5uzZs6pMWSohxIQJE7B48WLMnj0b//zzDwYOHIjOnTtj79696vlbt26hbt266lxBMpzHjh3Dhx9+qKu4iIyMRNeuXbF//378/fffKFasmCqBlvlEadmmm0P0Ar3xxhuoUKGCKhX7+OOP1Ty5CCMBzsyZM1GyZEn1JSTBiwQjGTl9+jQePXqEqlWrPpftrF69uvripBS2trbqy7lHjx7qy0oCVilHk9Kz8uXLq5I1UapUKaO7TI6rdhl6eaQsbcyYMepkUD7j9HxdvnxZ/X2Tzz9ljuwvCWiknFLGLcnYPwkO582bpztxXrp0qSp1lnlSaizkO0QCxj179qgyZCmNlVI3KZPVqlatmtH3vHHjhjp5rlOnjlpfgQIFMty+X3/9VY0xkRN3bdD6448/qgs5EydOVOXYQgIgmS9l13L8W7RooX4u+RtqTiTglL8nEpTIxS9ra2v1c8u/ZXysBH+1atVSy8oYNAlWpIxNvj/kO9/Dw0NdTJRSdiEXw/TPF/TNmTNHHWMJkOSCKJE+ZnDopZM/9lKXriV/AGNiYtRVMSFXeH755ZcnruP69evqi8PPzy/dcz/99JMu4yCTfMllVe7cudV7kOEYnNu3b6srnXKlVE4kJNDRvyrJcTamqXbt2up3QU66ZOzaypUrdSdi9Pzw9yHrJ8rSnETGiLVv316NwxHSrEQb3Aj5DEvwKBkc7d95b29vFXhI1uT+/fvqb1XDhg0zPWZUxmOWKFFCjemUjHRGzp8/ry7S6WfkJOMvAZdkf7TKlCmjvqO0cuXKpbbL3MhYMtl3hw4dUtmW7t27q+8KOT7yvS5jb/W/jyUwlGMk5HWvv/66LrhJ6969eyoglOBTAiEZZyhjCCUgJUqLGRwyiS997VU3IY0H5MtMsgSiY8eO+OKLL9QfwSJFihhdh5SwSVpbfz1anTp1wv/+97//VAYi5RHyx5kMycmHfGHJNGzYMJWFkwGlMshd++UvJ9NpyfzSpUtzd74kEtDI/pfmGiyLenHkxEz+RknDDXr6ibKUlUkgIxeYtN8HIm15n5zkSuMGKSVLK0eOHCqLkBVyoebatWuqfFouuLVr106VMUslwbNKe9Iun4OMGuy8yuTYaEv/5Ltcgj+5QFm2bFk1b/PmzapMXZ+2YYO2XDYjEjBJ6fO0adNUVk1eJ9kglteSMczg0EsnJ7vSTlJboy7dtyTrIl9oMskfQ0l3yx/LjPj6+qoAxNgfOrnSI39wtZMsm1WyXfJFSU8mJ81Sky4lIXIFddKkSemWkYyPdEaSwJVeDmlLLBcLGNy8WPI7IZlpKcWR35O02Mku/YmytKHXD24yCkjkb4pk8PX/1sskf/8lsyNjaLLShluyA3Khbe7cueqCwNq1a9X3QFpShisZJP3jKWOAJKiSDJAlk30gY22/+eYb9d0gAYlkW9IeI/l7JKS8WUrBZUyVMbJfJaMm424kIybrCwkJecE/Fb0qGODQSyUDCWXAp6SwhVyBk7bC8oUh6WrtJCfKUvqU0f1TKlasqP5/7ty55zZYslKlSs9l3a8iuYom9dBS+y7jn+Rqpwyu/e677/DOO++okxOpq5YmEjKoV5aRMkS5kiflH++99566KkpkaSS4kb9jMuZJTprlxFwu8sjAdu3YBMoaydLLhSv52yMnyPL3SEpm5WQ4ODhYLSPlbfI9IvtZ9vnx48cxY8YMo+uTe3VJExTJtMlYQfnbJs1pjF0QkPeWTLZkF+R7QpoIyFghGTPKsk+gbdu2qjRPvg+k6Yw0FpAmC1KRoT0G8lhIY4aIiAg1llPuPSfHacmSJbpSP8mAymP5fZESONn3T8v6kOViiRq9MDLIUG5wJ1/uUksr3YSkq4oMDuzSpYtaRk6A5eRXm87Wkis8Q4cOVa+RwZlpSXZFruLJgEVtsJMZckVOrihJfbbQ/iFN221NvjRlMDalkNrpGjVqqPscyBeVXHGTYyT10XLFTshxlC/7cePGqbpqqYeXLygpFxwwYIDRckIicycDq+XETn4vZDyg3DNE/n5JiZWUZFHWSafMv/76S92v7N1331VdtSTzL2NuJBMjJACRv0HyN0tOtCUgkr9RxkjGRy7WyAm2nJxLMwK5x5qxUjd5b2mAIM0LZDl5LBfsJEiilIY0ErjI/pTAUz7r8r0v98mRgFHbUVVIyaxc9JSSdGk6IPtevs9lTJP2/EAumMlr5PtGmhYY69RJJKykVzR3BT1vctVee5VG/uDJ4GapzX3//ffVF498cUhLSOmCdvjwYaPdbSQtLVfKpOOaMXJyIAMWAwICdPPkxpPyB1I7JiQtyQrJIMi0ZByJdkCrrE/eW4IgXi0iIiIiMm0McMhsSKMBqXmWeunsLPWQOmwJxrRXmYiIiIjIdHEMDpkNya5IBic7Bx1K0wJpSSp1w0RERERk+pjBISIiIiIis8EMDhERERERmQ0GOEREREREZDYY4BARERERkdlggENERERERGaDAQ4REREREZkNBjhERGSWrKyssGHDhpe9GURE9IIxwCEiouemW7duKtDo2bNnuuf69OmjnpNlMmPPnj1q+fDw8Ewtf+fOHTRr1izL20xERK82BjhERPRc5cuXDytWrMDjx49182JjY/Hrr78if/782f5+coNekTNnTjg4OGT7+omIyLQxwCEioueqcuXKKshZt26dbp78W4KbSpUq6eYlJydjwoQJKFSoEJycnFChQgWsWbNGPRcUFIQGDRqof3t5eRlkfurXr4++fftiwIAB8PX1RZMmTYyWqAUHB6Njx47w9vaGi4sLqlatikOHDvHoExGZGduXvQFERGT+PvzwQyxYsACdOnVSj+fPn4/u3bursjMtCW6WLl2K2bNno1ixYvjrr7/QuXNn5MiRA3Xq1MHatWvRpk0bBAYGwt3dXQVBWosWLUKvXr1w4MABo+8fFRWFevXqIU+ePNi4caPK7hw/flwFVUREZF4Y4BAR0XMngcrQoUNx/fp19VgCESlb0wY4cXFxGD9+PHbu3IlatWqpeYULF8b+/fvx888/q+BEMi/Cz88Pnp6eBuuXgOi7777L8P2lHO7Bgwc4cuSIbj1FixZ9bj8vERG9PAxwiIjouZMsTIsWLbBw4UJoNBr1bykn07p8+TJiYmLQuHHjdONp9MvYMlKlSpUnPn/y5Em1Hm1wQ0RE5osBDhERvbAyNRkrI2bOnJmuhExs3rxZlZHpy0yjABlT8yT65WxERGTeGOAQEdEL0bRpU5WRkcH/2kYAWqVLl1aBzI0bN1Q5mjH29vbq/0lJSVl+7/Lly2PevHkICwtjFoeIyMyxixoREb0QNjY2OH/+PM6dO6f+rc/NzQ2DBw/GwIEDVcOAK1euqCYAM2bMUI9FgQIFVHC0adMmNZ5Gm/XJDOmeJo0FWrVqpcb/XL16VTUtCAgIyPafk4iIXi4GOERE9MJI9zOZjBkzZgyGDRumuqmVKlVKZXykZE3aRgspXRs1ahSGDBkCf39/XblbZkj2Z/v27apBQfPmzVGuXDl8++236QItIiJ69VlpZLQnERERERGRGWAGh4iIiIiIzAYDHCIiIiIiMhsMcIiIiIiIyGwwwCEiIiIiIrPBAIeIiIiIiMwGAxwiIiIiIjIbDHCIiIiIiMhsMMAhIiIiIiKzwQCHiIiIiIjMBgMcIiIiIiIyGwxwiIiIiIjIbDDAISIiIiIimIv/A9zGph+3IgL6AAAAAElFTkSuQmCC",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzgAAADwCAYAAADM38HGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjgsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvwVt1zgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAbglJREFUeJzt3Qd4U2UXB/B/996lZe+99xLZyFRBZG8UPxBEwAUqG0FUtggCskGWgAiyl4zKngJlllUKtKUTuvM9560JSZtCiwVC8v89T6C5ubm5uTdt7rnnvOdaaTQaDYiIiIiIiMyA9cteASIiIiIiouzCAIeIiIiIiMwGAxwiIiIiIjIbDHCIiIiIiMhsMMAhIiIiIiKzwQCHiIiIiIjMBgMcIiIiIiIyGwxwiIiIiIjIbDDAISIiIiIis8EAh4joGVhZWWHAgAEvfNstXLhQvXZQUBAs1ahRo9Q2INMj+0X2jxY/r0T0MjDAIaLnTnuQo705OjqiePHiKkC4e/fuCzkY1t6sra2RK1cutGrVCn///fdzfe1XnXbbyTa7efNmusejoqLg5OT0n4K98ePHY/369bBEBQsWNPhsuri4oHr16li8ePHLXjWzcOfOHQwdOhQNGjSAm5ub2sZ79ux52atFRC8AAxwiemHGjBmDJUuW4Mcff0Tt2rUxa9Ys1KpVCw8fPnzury2vJa8twZYcjJ89exZ169bFyZMn8Srp1q0bHj16hAIFCryw13RwcMCvv/6abvratWv/87KfJcD5+uuv1TYwBxUrVlSfS7lJQBkZGYkePXpg7ty5MAcv4/OqFRgYiIkTJ+L27dsoV67cC399Inp5bF/iaxORhWnevDmqVq2qfn7//ffh4+ODyZMn4/fff0enTp3+07IlSHJ2ds7w8XfffRe+vr66+61bt0bZsmWxevVqdZD5qrCxsVG37PK07SZatGihApzPP//cYPry5cvRsmVL/Pbbb3gRYmNjVZbD1tZW3cxBnjx50LVrV939nj17onDhwpgyZQr69OkDU6PdBy/r85oVVapUQVhYGLy9vbFmzRq0a9fupawHEb14zOAQ0UvTsGFD9f+1a9d005YuXaoOTKT0SQ5MOnbsmK48qn79+io4OXbsmMrCyAH6l19+maXXzpkzp/o/7YFyfHw8Ro4ciaJFi6rMRb58+dSBvUw3RrIPsi4yb5kyZbBlyxaDx69fv44PP/wQJUqUUO9Jgjo50NIfQ3P06FFVPrNo0aJ0y9+6dat6bOPGjU8c0/DTTz+p15f1yJ07N/r374+IiIhs2W6dO3dWma4LFy7opoWEhGDXrl3qMWMysx3lfcgBs7xvbZmWHODrl8edO3dOvYaXlxfq1Klj8Fha8tmREi95XzK/vMdt27YZbOemTZuqQFf2RaFChdC7d2+Ykhw5cqBkyZK4cuWKwfSUlBRMnTpV7WMp8fT398f//vc/PHjwIN0yNm/ejHr16qmyLHd3d1SrVk0Fo/oksNf+nsn2kCBLMh36ZF+4urqqdZEgV5bXpUsX9Zjsx8GDB6v1lelvvfUWbt26lW5djH1epTRPSkT379+v9pe8HwnqjJXmnT59Wr0XWc+8efNi3LhxWLBgQabGocl6yd8QIrI85nEKjIheSdqDODnoF9988w2GDx+O9u3bqwzP/fv3MWPGDHWgeuLECXh6euqeK2dmJSMkAZAcnMkB35OEh4frDhTlQG7s2LHqwEpeS0sekwM1OfD64IMPUKpUKZw5c0adTb948WK6UiqZT8q0JICRg6np06ejbdu2uHHjhu49HTlyBAcPHlTrKQdoclAm5XISbMjBuxyMS1ZLDvBWrVqlypP0rVy5Uh2sy4F5RuSAf/To0WjcuDH69eunSnPkNeS1Dxw4ADs7u2febkK2v6y7HCRLmaF2veTgVzI4aWV2O0pZluxnOciV+USRIkUMliXBYLFixVQpm0ajyXAd5f3LdpDSR1lHe3t7HDp0SAVhb7zxBu7du6f+lwNyGZchnyXZF9lRZpedkpKSVKAg+1yfBDMSLPTq1QsDBw5UJwWk1FN+L/T3scwjQZsEQsOGDVPvU+aRwFsbjGqXI4HPhAkT1Di4adOmqeWk/T2T9ZHPngSXP/zwgy7bJ/tNAkpZpmxz2c7GPgsZuXz5ssqqvvfee+ozP3/+fBVQSdAl6y7k91TGz0gwI+9FMkfz5s1TATMR0RNpiIieswULFsiRqWbHjh2a+/fva27evKlZsWKFxsfHR+Pk5KS5deuWJigoSGNjY6P55ptvDJ575swZja2trcH0evXqqeXNnj37qa89cuRINW/am6enp2bLli0G8y5ZskRjbW2t2bdvn8F0eR15zoEDB3TT5L69vb3m8uXLummnTp1S02fMmKGb9vDhw3TrFBAQoOZbvHixbtqwYcM0dnZ2mvDwcN20+Ph4tZ69e/dOty2vXbum7t+7d0+txxtvvKFJTk7Wzffjjz+q+ebPn/9M201/28k++/TTTzVFixbVPVatWjVNr169dNuif//+z7QdXVxcND169MjwtTt16pThY1qXLl1Sr9emTRuDbSBSUlLU/+vWrVPPOXLkiMZUFChQQO032b5yk896t27d0m1P2Y4ybdmyZQbPl8+v/vSIiAiNm5ubpkaNGppHjx4Z3Q4JCQkaPz8/TdmyZQ3m2bhxo1rWiBEjdNNkv8i0oUOHGizr5MmTavqHH35oML1z585quuyfjD6v2vct0/766y/dNPkcOzg4aD755BPdtI8++khjZWWlOXHihG5aWFiYxtvbO90yn2b16tXqObt37870c4jo1cUSNSJ6YSTDIGfQpVxJMgiSAVi3bp0ahyBn0uXMv2RUQkNDdTcpJZMz+Lt37zZYlpzFlbPQmSXjRLZv365KlqTERbq4SbZFsiv6ZTuSbZASIf110JbSpV0HeT/6GYfy5curkqCrV6/qpklpjVZiYqLKoEjZlpwlP378uO6xDh06qMf1MwqyrlJmJo9lZMeOHUhISMCgQYNUtzMtGb8h67Jp06b/tN205Ey9nHWXrJD2/4zK07K6HZ+kb9++T51HMkLy2RkxYoTBNhDaUjZtVkJK/WQ7mwrZx/I7ITcZCC9ZLdk/33//vcH29PDwQJMmTQy2p2Q75HdIuz3l8x0dHa0yVJKdNLYdpExPslmSddSfR7Ivsr/Sfl6EZAX1/fnnn+p/ySTpk89gZpUuXRqvv/667r68fynj1P/dkayTNCHRHyMnJWfaMjkiooywRI2IXpiZM2eqwELGvUhplBzQaA9IL126pEqQJJgxRr/MSkhQJGVIWtJ9Sr+zljymX38vZVb6TQakPEZe66OPPlJjUrTrcP78eXWwZYwcGOrLnz9/unmktEh/XISsk5QBSVAlJTf6ZVayzloVKlRQB5hS+iVlO0J+lnXWBgbGyBgfIdtSn7x/KXvTPp7RdsusSpUqqfWTMjUJFiTwzGi9srodn0TGyWSm1FE+R3LQnBEZxyEBrZSySamclAhKowkJ0p5U8pT2c5UVEpToB7jG1KhRQ40rSU5OVp395Gf5/OjvI9mesh5+fn5P3J7akk8ZZ5XVz4uQ/Stlhfrkd1XKE9MuQ7Z32nJCY8vMSGZ+d+R1JMBJS04QEBE9CQMcInphZKyFtotaWnIGXs4yywBpY12X5Ey1vrQHjh9//LHBIH05oH3SNS9keXJwKR3ctJ2hZB3kLLp0djNGMk/6MuoOpR/ESAAlwY2c3ZaDNTnolfcpGSx5PX2SqZFxSHJ2Xsb0bNiwQXWXy86OYU874H4SCQZkbI+sm6xr2myJVla34/NaX32yzaWTllz76I8//lDNG2SsyqRJk9S0tJ+vjD5XWSH7Xds0ISMSwEomUMhYFwkyZAC+jIkZMmSIbntKcLNs2TKjy8gokMwOEvxltJ//i8z87hARPSsGOERkEuRssBzcyBl7yfJklXTo0m+3m3aQtjEygFrExMSoAEfW4dSpU2jUqJHRLl3PQg6qZRC1HEhrxcXFpetwJiRokAyDlNNJhksupCmB0JNory8ijQUkY6MlZWsyEF178JwdJMCRMjC5gKKUUmUkK9sxO7azvJ4EAdK04Wktv2vWrKluEkhKNkrKnVasWKEGzWfmc5UV2sHyWSGlYhKcS1MFaSyg/VxKKeJrr732xIBPm1GRTFBGWQ79z0vaDJxMy8z1amQe2d6SMdLP2sjzs5O8jpRDpmVsGhGRPo7BISKT8M4776izunKAn/YsrtyXsStPIuVJcjCvvcn4hKd1VZPxN1JqpS39kfE/UkZm7CKLUqYkmZ6skveU9v1IZzgpSUpLxq1I5kNK0+SWK1cuVVr3JPJepZxJOrjpv84vv/yiypqy0tnqaeQAWloVS8mdZOMykpXtKAfwxoK9rJBSM8kySPe0tFkx7TaR0qe0+0EbDGXUAtzY5yorN9l/z+KLL75Qn3ft9pPtKZ8X6fxnLEjXbj/pEifZNdk/EkQb2w6SQZXP++zZsw3et2ROpawwM58X6cIn5DOnTz4b2UkyWgEBAQYX45Xf24wyWUREWszgEJFJkINnGX8g7WClfa8ctMrBmmQhpBGBtBH+9NNP/1MmRcqQ5EAvODhYBQBy0CsHetosglx1XVo1y8B2GbgtZ8zlwFKu/yLTpawpoxK7jEi5kWQ7pDRNDpblgE3OxmvbSBvL4kiWRAaAy1icp5UHSXmSbDMJDJs1a6baM8uZdLkujrQBftbsQ0akZOtpsrIdJRCV7SHlbHL9HsngSelgVki24quvvlIBgAxcl2BZSqukEYIsUw74pcxMtkmbNm3UZ00G40sAIY0Y5BovpkQCCBlHI9tErmckGR3J5sj7kIN9CWRkTJqMzZEGBFLOJmPK5L3I+CLJRsm+114/SLJpckFX2QbyvIkTJ6pGBrJcKYHUtomW69PItW2eRgJDeZ5sTwmipU30zp07sz2zItkzaUUtzRWk1FPbJlrG70igk5nsn/xNEf/884/6X34XteOMvv7662xdXyIyIS+7jRsRmT9tq9jMtOj97bffNHXq1FHtg+VWsmRJ1TI3MDDQoN1xmTJlMvXaxtpEy3Jr1aqlWbVqVbr5pY3uxIkT1fKlba2Xl5emSpUqmtGjR2siIyN186Vt5avfAle/7fGDBw9UO2VfX1+Nq6urpmnTppoLFy6km0+/5bF2Pffv35/ucWNtd7VtoWVbSatpf39/Tb9+/dRr68vKdtPfdtLC+EmMbYvMbkfZFnXr1lXtwmU52m3ypNdO2yZaS1piV6pUSfd68n63b9+uHjt+/LhqOZ0/f371uLRKbtWqlebo0aOal0U+Ay1btjT62MKFC9V7lP2tNWfOHLUNZVtJO+hy5cppPv/8c01wcLDBczds2KCpXbu2ms/d3V1TvXp1za+//mowz8qVK3XbStoud+nSRbVr1yf7Qn5XjJEW0wMHDlSt3mWeN998U7V/z2ybaGPvW/aX3PRJi+jXX39drWfevHk1EyZM0EyfPl0tMyQkRPM0xlrEa29EZL6s5J+XHWQRERERZYY07Pj555/V2LmMmhUQkWXjGBwiIiIySWlbdMvYJCkzq1OnDoMbIsoQx+AQERGRSZLW6nLNImnAIWOFZOycdBccPnz4y141IjJhDHCIiIjIJEkDCGkQMmfOHNVUoHLlyirIeVp3QSKybByDQ0REREREZoNjcIiIiIiIyGwwwCEiIiIiIrPBAIeIiIiIiMwGAxwiIiIiIjIb7KJGilW/mtwSJkIz62/uDxPC/WF6uE9MC/eHaeH+ML39QS8eMzhERERERGQ2GOAQEREREZHZYIBDRERERERmgwEOERERERGZDQY4RERERERkNhjgEBERERGR2WCAQ0REREREZoMBDhERERERmQ0GOEREREREZDYY4BARERERUZb89ddfePPNN5E7d25YWVlh/fr1T33Onj17ULlyZTg4OKBo0aJYuHBhunlmzpyJggULwtHRETVq1MDhw4eztmIMcIiIiIiIKKtiY2NRoUIFFZBkxrVr19CyZUs0aNAAJ0+exKBBg/D+++9j69atunlWrlyJIUOGYOTIkTh+/LhaftOmTXHv3r0srZttlt8NERERERFZtObNm6tbZs2ePRuFChXCpEmT1P1SpUph//79mDJligpixOTJk9GnTx/06tVL95xNmzZh/vz5GDp0aKZfiyVqRERERET0XAUEBKBx48YG0ySwkekiISEBx44dM5jH2tpa3dfOk1nM4BAREREREeLj49VNn4yXkdt/FRISAn9/f4Npcj8qKgqPHj3CgwcPkJycbHSeCxcuZOm1GOAQEREREVkI6/61MnxsRI6mGD16tME0GQ8zatQovEoY4BARERERWQhr24xHqAwbNkwN8teXHdkbkTNnTty9e9dgmtx3d3eHk5MTbGxs1M3YPPLcrOAYHCIiIiIiC2FtY53hTYIZCTj0b9kV4NSqVQs7d+40mLZ9+3Y1Xdjb26NKlSoG86SkpKj72nky/R6zZY3ppZE+4VOnTuUeICIiIqJMZXAyumVFTEyMavcsN20baPn5xo0bumxQ9+7ddfP37dsXV69exeeff67G1Pz0009YtWoVBg8erJtHskdz587FokWLcP78efTr10+1o9Z2VcssBjjZLCgoSF3sSLuztXr27InWrVtn98uZhdeLVsSGfj/g9oQ/oJn1N96uUPepz6lXrDKODVuEuOl/4dLo1ehRs2W6eT6s1xbXxq3Do+l78ffnv6BagdIGjzvY2uPHjp8i9PutiJ6yC2s+mAA/N29YOu4P08L9YVq4P0wP94lp4f6wnADn6NGjqFSpkrppgxP5ecSIEer+nTt3dMGOkBbR0vJZsjZyfRtpFz1v3jxdi2jRoUMH/PDDD2oZFStWVMfTW7ZsSdd44KnvMUtzEz0HLg5OOHX7Evqv+CFT8xf0yYVN/Sdh98VjqDi+O6buWol5XYfhjVI1dPO0r9IYk9t+jNGb5qHy+B44desStg6cihxuXrp5prQbhDfL1UG7eV+i3pR+yO3hi7X/+9bi9zH3h2nh/jAt3B+mh/vEtHB/WE6AU79+fWg0mnS3hQsXqsfl/z179qR7zokTJ1SntitXrqgEQFoDBgzA9evX1TyHDh1CjRo1sv4es/wMUpFknTp14OnpCR8fH7Rq1UrtJG10KiSClUyO7EjpPCGptt9//11Nk5t2h3/xxRcoXrw4nJ2dUbhwYQwfPhyJiYkGW/mPP/5AtWrV4OjoCF9fX7Rp0ybDvSCRsKxX2hpHU7blnwAM3/Az1p/am6n5+77+Dq6FBePT36bjQkgQZu5dgzUndmNwo466eYY06oS5B37HwoBNOB8ShL6/TsTDhDj0rtVKPe7u6IL3ar+JIWumYXfgMRy/EYhei8fhtSLlUaNQGVgy7g/Twv1hWrg/TA/3iWnh/ni1x+CYC/N5Jy+Q1AJKGk5ScxJIyEWIJOiQgVCHDx9W8+zYsUOl5tauXYtPP/0U7du3R7NmzdQ0udWuXVvN5+bmpiLcc+fOYdq0aaruUK7oqiWpPFl2ixYtVMQrr1e9enWj6/Xdd9+pq7xu27YNjRo1grmqVbgsdlw4YjBt67m/UatwOfWznY0tquQvYTCPnFGQ+9p5qhQoCXtbO4N5Au9ex/WwO6hVKHUe4v54FfH3w7Rwf5ge7hPTwv3x4tnY22R4MxdsE/0M2rZta3B//vz5yJEjhwpS5H8hmR39lnbS/k5SbWnb3H399dcGDQMkGFqxYoUagCW++eYbdOzY0aAnudQtpiWZoCVLlmDv3r0oU8a8MxA53X1wNyrcYJrc93ByhaOdA7yc3WBrY2tkngco6V9Qt4z4xAREPooxnCc6XD1G3B+vKv5+mBbuD9PDfWJauD9ePOsslqK9ihjgPINLly6pwU9SFxgaGqoyN0IGUpUubTiQ/WlWrlyJ6dOnqxI36UaRlJSkWvJpyeCqPn36PHEZMkhLskqSUZIyt2e5Qi0RERERmT9rCwhwzP8dPgdvvvkmwsPDVTmZBDlyEwkJCVlaTkBAALp06aLKzzZu3KhK0L766iuD5Ujm52lef/11JCcnq1Z7TzNhwgR4eHgY3GTaqyQkKgz+7obdzuS+ZGPiEuMRGhOBpOQkI/N4qedql+FgZ6+yPgbzuHnr5iHuj1cRfz9MC/eH6eE+MS3cHy+eNcfgUFphYWEIDAxUpWUyzqVUqVJ48OCB7nG5SJGQgEOfTE877eDBgyhQoIAKaqpWrYpixYqprhH6ypcv/9SGATImZ/PmzRg/frxqrfck0pM8MjLS4CbTXiUBV8+iUYmqBtOalKqOgKtn1M+JyUk4diMQjUpU0z0ujR3kvnaeY9cvICEpEY1KPp6nuH9+FPDJhYBrqfMQ98eriL8fpoX7w/Rwn5gW7o9Xt4uaKWOJWhZ5eXmp8TVz5sxBrly5VFmaDOzX8vPzU1kX6bSWN29e1flMsiQyvmbr1q0qOJLnyzQJaOT5MuZGuqRJQ4F169YZvN7IkSNVIFWkSBE1FkdK2P7880815kafNC2Q6c2bN4etrS0GDRpkdP2lHM3UStKkpWTRHHl19wv55EaFvMUQHhuFmw/uYvzb/ZDHMwd6LBqjHp+9by0G1H8XE9sMwPyDf6BhiapoX7kRWs78RLeMyTt/xaIew3H0xnkcDjqHQQ07wMXBEQsCNqnHo+Ji8cvBPzC57UCEx0aq+zPaf4KDV07j0LV/YMm4P0wL94dp4f4wPdwnpoX7w/TZ2FrB3DHAySLpmCYBycCBA1G2bFmUKFFCjaGRdtBqg9raqvtjxoxR43SkfExaQss4GvlfMjUy1mb37t1466231NVbpd+3jItp2bKlahMtbaW1ZLmrV6/G2LFj8e2336rxOXXrGr8QprSuliBJSt5sbGzw0Ucf4VVQNX8p7Bnyk8H1aYS0eO61eCxyefgiv/fj5gxBYXdUMDOl3cf4uEF73Iq4h/eXTsC286mlgmLVsR3I4eqJMa36qAGMJ29dQrMZg3Ev+nHjgcGrpyJFk4LfPpigLvq59dwhfLjiO1g67g/Twv1hWrg/TA/3iWnh/jB99mbUDjojVhrpn0sWz6pfTYvfBqZCM+tv7g8Twv1herhPTAv3h2nh/jC9/WFqSi58N8PHLvRcA3PADA4RERERkYWwtzb/DA4DHCIiIiIiC2FvwzE4RERERERkJuwtYAwOMzhERERERBbCnhkcIiIiIiIyF/bM4BARERERkbmwZwaHiIiIiIjMhZOtDcwdx+AQEREREVkIe2ZwiIiIiIjIXNhzDA4REREREZkLe2ZwiIiIiIjIXNhZm/+FPs3/Sj9ERERERKTYWWd8y6qZM2eiYMGCcHR0RI0aNXD48OEM561fvz6srKzS3Vq2bKmbp2fPnukeb9asWZbXi00GiIiIiIgshF02laitXLkSQ4YMwezZs1VwM3XqVDRt2hSBgYHw8/NLN//atWuRkJCgux8WFoYKFSqgXbt2BvNJQLNgwQLdfQcHhyyvGzM4REREREQWwi6bMjiTJ09Gnz590KtXL5QuXVoFOs7Ozpg/f77R+b29vZEzZ07dbfv27Wr+tAGOBDT683l5eWX5PTLAISIiIiKyoDE4dhncMksyMceOHUPjxo1106ytrdX9gICATC3jl19+QceOHeHi4mIwfc+ePSoDVKJECfTr109lerKKJWpERERERBbC7gnpjfj4eHVLm1FJWyYWGhqK5ORk+Pv7G0yX+xcuXHjqOshYnbNnz6ogJ2152jvvvINChQrhypUr+PLLL9G8eXMVNNnYZP4CpczgEBERERFZCLsnZHAmTJgADw8Pg5tMy24S2JQrVw7Vq1c3mC4Znbfeeks91rp1a2zcuBFHjhxRWZ2sYIBDRERERGQh7Gwyvg0bNgyRkZEGN5mWlq+vr8qo3L1712C63JdxM08SGxuLFStW4L333nvquhYuXFi91uXLl59vgJOYmIgiRYrg/PnzWX0qERERERGZaAbHwcEB7u7uBjdjXczs7e1RpUoV7Ny5UzctJSVF3a9Vq9YTX3/16tWqDK5r165PXddbt26pMTi5cuV6vgGOnZ0d4uLisvo0IiIiIiIyky5qQ4YMwdy5c7Fo0SKV+JCGAJKdka5qonv37kazP1KeJuVnPj4+BtNjYmLw2Wef4e+//0ZQUJAKlt5++20ULVpUtZ9+7k0G+vfvj4kTJ2LevHmwtWWfAiIiIiKiV4FjNl0Hp0OHDrh//z5GjBiBkJAQVKxYEVu2bNE1Hrhx44bqrKZPrpGzf/9+bNu2Ld3ypOTt9OnTKmCKiIhA7ty58cYbb2Ds2LFZvhaOlUaj0WT1DbVp00ZFVa6urmoQUNr2bnIhHyIiIiIiMi3LA/tn+FjnEjNhDp4p/eLp6Ym2bdtm/9rQS/NVwAfc+ibim1pz8CB+5cteDfqXl0MHnChVktvDhFQ6fwF2g+u87NWgfyVO2Y//7Xqf28NE/NxwHqac7PuyV4P+NbjibJPbFrZpsirm6JkCnAULFmT/mhARERER0XNla53568m8qv7TABqpu5NaOiFXG82RI0d2rRcREREREWUzWwvI4DzTO5QOCb1791Yt2+rWratuMhBI+lk/fPgw+9eSiIiIiIj+M1srmwxvFh3gSFu4vXv34o8//lBdDuT2+++/q2mffPJJ9q8lERERERFlSwbHNoObRZeo/fbbb1izZg3q16+vm9aiRQs4OTmhffv2mDVrVnauIxERERERZQNbjsExTsrQtD2u9fn5+bFEjYiIiIjIRNmaUaYmI8/0DmvVqoWRI0ciLi5ON+3Ro0cYPXq0eoyIiIiIiEyPrQWMwXmmErVp06ahadOmyJs3LypUqKCmnTp1Co6Ojti6dWt2ryMREREREWUDW5aoGVe2bFlcunQJy5Ytw4ULF9S0Tp06oUuXLmocDhERERERmR5bCyhRe+br4Dg7O6NPnz7ZuzZERERERPTc2DKD89iGDRvQvHlz2NnZqZ+f5K233uLHkoiIiIjIxNhaMYOj07p1a4SEhKhOafJzRqysrJCcnPxi9hAREREREWWanY2d2W+tTJeopaSkGP2ZiIiIiIheDTZWzzxC5ZXxTDmqmzdvZv+aEBERERHRcw9wbDK4WXSAU7BgQdSrVw9z587FgwcPsn+tiIiIiIgo29kwwDHu6NGjqF69OsaMGYNcuXKpMTlr1qxBfHw8P4ZERERERCbKxto2w5tFZ3AqVaqE77//Hjdu3MDmzZuRI0cOfPDBB/D390fv3r2zfy2JiIiIiOg/s2EG58mkY1qDBg1UqdqOHTtQqFAhLFq0KNMbuH79+hg0aBBM2YtcRyn9mzp16gt5LSIiIiKyPDZWdhnezMV/ykXdunULy5cvV7ezZ8+iVq1amDlzZqafv3btWnVdHVOwZ88eFazJmCJPT8/nuo4LFy5UQVNERITB9CNHjsDFxQWW6PKO27i4+SbiIhPgkd8VlboWhXdhd6Pz7plwEqGBkemm5yzvjTpDyunuRwXH4syqa7gfGAFNsgbueVxQa0BpOPs46uYJuxyJs78FIfxKFKysreCZ3xWvf1oONvY2sHRrVhzC0oUHEB4ag6LF/fHJsJYoUy6v0Xk3/n4C44avM5hmb2+Lv46OMDr/xLEbsG71UQz6rBk6dqud7vGEhCS812UOLgWGYPGqfiheMhcsnW/nzvDr/R7sfH3x6MIF3PpmHB6eOZPh/J5NmyLXwI9hnycP4q9fR/CkHxD111+6x62dnZF7yCfwaNQItp6eiL91C/eXLkHYypW6eYouWgy36tUNlhu6YgVujh4FS9fvtXcwpGEn5HTzxungKxi0dgqO3Dif4UX1vmjcDd2qNUceD19cvHcTwzbOwrYLh3TzWFtZY0Sz3uhc5Q3kdPNBcFQoFh/+E+O3Pz5p2LpcXXzwWmtUzlsCPi4eqPp9T5wKvvxC3q+pq5+nAZrkbwoPew/cirmJFRd/RVD0tQznb5S3MermqQ9vR2/EJMbg+L1jWHf1NySlJGV6mbbWtmhXtD2q+leHrZUtzoX/g+WByxCdGAVLd3brLZz84yYeRSTAp4ALXutVHP5FjX+n/z76BO6cMzweEvkreaPF0Arq5yOrr+HKwXuICYuDta01chRyQ/WOheBfzEM3/9IBAYi5H2ewjBqdCqNS6wLZ/v5edTZm1EwgI8/0Dn/++WcV1Bw4cAAlS5ZEly5d8Pvvv6NAgax9iLy9vWHqXuQ6SqmfJbp56B5Or7iCyj2Kw7uwGy5tu419P5xB02+rwdHdPt38tT8qg5Qkje5+fGwidgw/irzVHm+/mHuPsOebkyhYNydKtykAOydbRN2OhbWdtUFws2/SGZRsmR8VuxaFtbUVIm7GSGoSlm77ljOY9v0WfDH8TRXUrFgagEF9F2PlhoHw9nE1+hwXVwes2jDw8YQMtuOenedw9vQt5PBzy/D1f5y8Db453FSAQ4Bn8+bI88VQ3Bw1Cg9Pn0KO7j1QZO48nG/RHEnh4en3RcVKKPjDJARPmYzIPXvg3aoVCs34EYHvtkXcpUtqHlmeW40auP7550i4fRtur72GfCNGIPHePUTt3q1bVuiqVbgzY7rufsqjRxa/S9pVbIjvWw9A/9U/4PD1cxhYrz02/W8yykzohPsx6Q/UxrT4QAUufVdNROC9G3ijRHWs6TUedaf3xcnbqfvjs0Zd8L/ardH7129w7s41VMlfEvM6fomouFj8uG9N6n51cMKBq6ex5sQu/NxxqMXvB62qftXwbrH2WB64FNcir6JRvsYYWHEQRv79NaITo9Ntp2r+1dGmSFssurAAVyOvwM/ZHz1LSXm9Bqsvr8r0MtsX7YhyvuUw5+xsPEp6hE7FO6NvuQ/x/fFvLXrfXD54FwcXX0bd90vAr5g7zvx5E5vGn0KnKTXg5JH+O73pJ2WRkvT48iNx0UlY/fkRFK7pp5vmmcsZdXoVg7u/E5ISUnB6001s+uYUOk2vCSe944Rq7QuhVKPHJ8TsHM3/QP5Z2GTjWBtJbMiwFblWZoUKFTBjxgw1Tj+jE/y9evUymObg4IC4uMeBqUajwciRI1V1mCQCXnvtNcyaNQvFihV7/mNwxo0bhxo1auDYsWMqczNs2LAsBzdpy7+kPGv8+PFqDI+bmxvy58+POXPm6OatXbs2vvjiC4Pn379/X2VX/vr3rKQ0Ofj000+RJ08elQmRdZTMjNb169fx5ptvwsvLSz1epkwZ/PnnnwgKClLZGyGPSeldz549062juHPnDlq2bAknJydVkieBXtrSssmTJ6NcuXLqNfLly4cPP/wQMTEx6jFZH9m5kZGR6nXkNmpU6tnQtMuRMU5vv/02XF1d4e7ujvbt2+Pu3bu6x+V5FStWxJIlS9RzPTw80LFjR0RHp/+Dbsoubr2FQvVyoeDrOVWWpXKPYrCxt0bQX8YPbu1d7eDoaa+73Tv7QGVc8lZ/HOCcXXNNZXTKdygCrwJucPVzQu5KvgYB06nlV1C0cR6UbJUfHnlc4JbLGfmq+8FGLwiyVL8uPoi321ZBq9aVUaiInwp0HJ3ssHH98QyfI59lH1+3xzcjgdC9u1GYNOFPjJ7wLmxsjWfJDu67iEMBlzHwk6bZ+p5eZX49eiJs9WqEr1uLuCtXcHPUSKTExcHnnbZG58/RvRui9u/HvfnzEX/1Ku5Mn45H588hR+cuunlcKlVE2O/rEXPkMBKCbyNs9So8CgyES/nyBstKiXuEpNBQ3S0lNhaWblD9jvgl4A8sOvwnzt8Nwoerv8fDhDj0rNHK6PxdqjbFxB1LsOX837gWFoyfD67H5vMBGFy/o26eWgXL4o+z+7H5XACuPwjB2lN7sD3wMKrlL6WbZ9nRrfhm20LsvHj0hbzPV0XjfE2wP3gfDt45gDsP72BZ4FIkpCSgdu46Rucv4lEUVyIv48jdwwiLC8P58HPq54LuhTK9TEcbJ7yWuw5WX1qFwAcXcCP6OhaeX4CinkVRyL0wLJkEH6Ua5UbJBrngnddFBTq29ta4sPuO0fkdXe3g7Omgu906HQ5bB2sU0QtwitXxR97y3irA8c7ngtrdiyLhUTLCrqceW2nZOdoYLEvu0/Mbg7Ny5UoMGTJEBSTHjx9XAU7Tpk1x7969DJ8jx7NyLK29ybG5vu+++w7Tp0/H7NmzcejQIXUsLcvUD4Iy45mO5OTAW1ZA3kh2mjRpEqpWrYoTJ06ooKBfv34IDAxUj0mWaMWKFSqy09+wuXPnxuuvv67uDxgwAAEBAWq+06dPo127dmjWrBku/XvGsn///ioIkoDozJkzmDhxogoeJAj57bff1DzyerLBp02bZnQdu3fvjuDgYBWoyHMkCEu7I62trdXO+eeff9SYpF27duHzzz/XBWoSxOjvYAnK0pKLqUpwEx4ejr1792L79u24evUqOnToYDDflStXsH79emzcuFHdZN5vv311zh7JWZuIoGj4lfbSTZNSMf8yXgi7krk0/7V9IchXww+2Dql/yDQpGoScDodrTmfs++E0/vjoIHaOOY7bx0J1z4mLSkD41WgV8OwadwJ/DDyYWvp2MX3pm6VJTExC4Pk7qFaziMFnulqNIjhz6laGz3v0MAGtm07CW01+wGcDl+Pq5XvpPtOjv/wNXXu+hsJFH39x6QsLi8GE0RswanxbODiaRvnqy2ZlZwfnMmUQHXDw8USNBtEBAXCuWNHoc1wqVDScX0o29x+Ai978sSdOwqNBQ9j5pe4L1+o14FCwIKIOHDB4nlerN1HuYABKbtiAXIOHwMrxcYmnJbKzsUXlvMUNggz5Xtp16ShqFihj9DkOtnaISzLsMhqXGI/ahR8HkwFBZ9GgeBUUy5FP3S+fuyheK1xeBUWUMRsrG+R3K6CCFN3+gAYXws+jcAaBhgQ38pyCbqkBja+jL8r6lMPZsDOZXmYB9wKqRO38g8fz3H0YogKmwh6P/3ZamuSkFNy/GoO85Qy/0/OW88bdS5n7TpdAqGhtvwyDE3mNczuDYe9sC58ChifSTvx+Awve24fVXxzByQ03kJLMC9M/zwBHTuj36dNHnbgvXbq0CkqcnZ0xf/78J54MzZkzp+4mDcr0/5bKMfLXX3+tjoHLly+PxYsXq+NuOdbNimfKUcnKiYcPH6pgJyEhweBxWaFn0aJFCxXYCMnWTJkyBbt370aJEiVU9kIyKfv379cFNJI96dSpk1ofWY8FCxao/yXoERI4bNmyRU2X7JA81rZtW5VdEYULF05Xiubn52cwBkffhQsXVDMFGSsjgZiYN29eurSZfsZHMiuS8erbty9++ukn2Nvbq0yLdgdnZOfOnSoIu3btmgrAhOxkyTrJ61erVk130CgpP8l6iW7duqnnfvPNN3gVxEcnQpMCOHoYHsw6uNsh6s7Dpz4//GoUom7Fomrv4o+XGZWIpLhkBG66gTJtC6Fcu8IIOROOgB//Qb0vKiBHSU/E3ks9E3BufRDKdywCj/wuuH7gLv767hSajKsKt5zOsFQRDx4iOTkF3j6G48G8fFwQdO2+0ecUKOiDr0a3VmN1YmLisGzhAfTpPhe/rh0Av5ypNdJL5u+Hja012nepaXQZ8odt7Ndr0aZ9VZQqkwfBt3mNLWHj6QUrW1skhoUZbK+ksFA4Fnp8xlmfra8vkkLTzy/TtW6NG4t8Y8ai7N6/oElMVNv/5ojhiD36+MD9wcaNSAgOVmVrTiWKI/cnn8KxUEFcG6hXimhhfF08YGtji3vRhqWBd6PDUcLPeCXDtguH8XH9jth35RSuhN1Gw2JV0Lp8PdhYPz7H+N3OpXB3dMHZocuQrEmBjZU1hv85B78e3/7c39OrzNXOFTbWNohOMDx4jkqIQk5n49+xkq1xtXPDZ1W+gNW/5Tp7b+3B5ut/ZnqZ7vbuSExJVKVpaefxsDc+1sQSxEXJd7omXSmak4cdIoKfnv29ezkK4TdjUa9vyXSPXT8Wiu3TziEpIRnOnvZo9VUFg/K0cs3ywLeQm8oIhVyMxKFfr+JhRDxqd89aaZMlsMmGMThy7C+VXFLFpX8ytHHjxirZkBGpaJKqLzl+rVy5sjo+l2NbIce8Uuomy9CSY2apyJJlSpVSZj3TO5TSMCnhkuDBmOTk5GdZrEFgpA0AtNkRGZ/yxhtvYNmyZSrAkY0gb1bGAwkJBuR1ixd/fKArJGPj4+Ojfh44cKDKCm3btk1tPAl2shKMSXbH1tZW7RCtokWLqrI2fRIETZgwQQVEUVFRSEpKUqk1CQglss2M8+fPq8BGG9wIiY4l+JLHtAGOBFDa4EbIdYmelBqU7ZH2ekVS//iquvZXCDzyuhg0JNBm+XJX9kXxpqmD4j0LuCLschSu7g5WAY52nkINUkvjhJSy3T8XgaB9ISoooswrVyG/ummVr5AfHVvPwLo1R/G/AY1w4VwwVi77G4tW9tWdIElr1fJDePgwAT3eq8tN/wLk6NoNLhUq4Eq/fqpEzbVqNeQdnjoGR7JDQsrWtOIuXUTi/fsotnAR7PPlQ8LNm9xPmTRk3TTM7vA5zg5bpv72XAkLVuVtPau3NBjX06lyE3RbOhrnQq6hQp5imNR6IO5EhWLJEePftfRsinuWQPMCLVRDgGtRV+Hn5IcOxTuiRUIr/Bm0kZv1Jbqw6w6887sYbUiQu4wX2n1XVQVR53fdwfap/+Cdb6rogqkKrR5/B0lmR06o/TU3EDU6FWHpeRbG4GR0nJj2WDE0NFQdd+tnYITcl+NfYyRhIdkdOfaWoRo//PCDqmySiqe8efOq4Ea7jLTL1D72XEvUJEMhKya1cTIWRQIdKcWSTMaGDRvwrNJ2K5MDIYnwtKRMTS4ompiYqLI3konRZmMkIrSxsVHR5MmTJ3U3CQa05Wbvv/++KvOSLIcERJKFkcFQ2UnG87Rq1UrtPClhk/XRdpZLm+nKDk/bZmlJ4CXRsP5Npr0sDm52sLIG4iITDaZLFsbRyGBEfUnxyapBgTQSSLdMGyu45zYMJt1yO+NhWOovrZNn6rLdc7tkOI+l8vRyho2NNcLDDM+2PQiLVWNrMsPWzkZ1Prt1I/Us98ljQXgQHovWTSfjtUqj1C0kOALTJ21F62aT1TzHDl/F2VM3UbfqGPV4u1apv7e9Ov2MMV+thaVKjngATVIS7P49UaNl6+OLxNDHZZf6ZKyMrW/6+WW6sHJwQK5Bg3B74reI2rMbcRcvInT5MjzY/Cf8emV8LbOHp0+r/x3yW25XotDYSCQlJ8HPzbABjb+bN0KiwjJ4TgTenf8lPL5ogiJj30XZCZ0RE/8IV8ODdfN8++aH+H7nMqw6sRNn71xV422m7V2Fzxt1e+7v6VUmHdCSU5LhliZrIhmWyATjJcdvFX4bh0ICcODOPgTH3sbJ0BNYf2UdmhdoDitYZWqZkqmxs7aDk62TkXkst4uao7t8p1vhUaTh8c6jyEQ1JuZJEuOSceXgXTV2xxgpWfPI6Qz/4h6o37ek+p6XQCcjfkXdkZKsQXSazmoEWCUnZXh7nseJ0m1ZhnrI+PF69eqpTsWSwNAmK7LTMwU4MqZE6u4kQJB0lKSaunbtqsblPM+DZanHk0yIBFQS4EjAo3/xUYkkJXshWRX9m34pmGREpFxMNuonn3yiujQIKR17WvZJIk/JxsgYIa3Lly+r1tJaEtBIgCHjiWrWrKkySlI7qE9e62lZrlKlSuHmzZvqpnXu3DnVUUIyOc9KUokSnOrf9NOLL5q0e/Qs6IZ75x5vQ0lvy32fIk9O8986fB8piSnIX9s/3TK9CrkhOk2JW0zIQzj7po4fkP+lQUH6eR7B2ffVzWhlBzs7W5QolQtHDl3VTZPPtNwvV8F4m+i0pMTtyqW78MmRWh/d/M2KWLrmQ9XyWXuTLmpder6GabO6q3mGDG2BJasfzzN5Zlc1fex37dD3o0awVFI+9vCff+BWs9bjiVZWcKtZEw9PnjT6nNhTJw3nl+C9dm3E/ju/lLxZy9+8tCdDpF5dr2wqLaeSqWUjifczzhKbu8TkJBy/dRENi1cxvCZcsSr4+/o/T3xufFICgiNDVdvoNuXr4Y8z+3SPOds7IkXqdfXIQba0j6aMJWuS1QD/Ul6PmzFIkFLSqySuRj3+G6bP3tpBjanRl4KULC3zetR11VK6pN48/s7+8HH0UZ3ZLJVkTXIUdsXtM4bf6bfPPoB/sSd/p1/5+x6SkzQo/m9VxVNpNGo8TkZCg6JVM08nd47nTEfaoWdwy+xxoq+vr0os6De/EnL/SUMw0p6kl+N3OZYW2uf9l2X+pxK12NhYNVZFSHmWlKzJgbxkU6SLwvMinRRat26N4cOHq8yMjL/RkteXgEciQwkuZIPJesl4FMmmSOczyTw1b95czStBiYzvkUBCSJAmX1IyUF/GAklmShoQ6JOW2FLa9sEHH6iWdbJjJEiSebVlNxJQSYZJMkPSsU1aacugK31SViYZJ1k3adQgZWtpS9fkdWR7ynuSAVcSWMn4JIl4teN/noWxNOPLJmVkR+ZeUEGJtk10UnyKrnTs8JwLcPKyT1c2dm3fHVWG5uCa/o9Xieb58PdP5+BbwhN+pTzVGJw7J8NQb2jqIGvZXzLPP+uD1LVv5Ba0P0SN+6k54NkDSHPRqXttjP16HUqVzo3S5fJi5dIAxD1KQMvWqeWZ0iwgh787Pvy4ibr/y+zdKFs+H/Lm90Z0VOoYnJA7EXj7ndSDQA9PZ3XTJ13UpNNagUKp40Jy5jIc++bknHrSIW8+b904Hkt1b9FCFJjwLR6ePYvYM6fh170HrJ2cELYuNbNV4NtvkXD3Hu5MSc2G3V+8BMUWL4Zfz16I3LsHXi1aqkYFN0emXpdIOqFFHz6M3J99hpS4+NQStWrV4f322yqrI6QMzatVK0Tt/QvJERFwLFEceYcOQ/SRIyrjY8mm7lmB+Z2/wrGbF3Dk+nnVJtrF3gmLDm1Sjy/o/DVuR97H15tSz0pWz18auT181TVr5P8RTXurk4M/7FquW+amfw5gaJPuuBFxV7WJrpi3OAbV74CFh1LHhQgvZzfk9/RHLo/U35nifqklOSHR4WoMkKXacXO7avMcFH0dQVHXVEtnexsHHAxObZghj0XER2D91dTfl9Nhp1SXtBvRN3At6poqUXurUGucDj2tC3yetsy45Ec4ELwf7Yp1QGxiLOKS49CxeCfVwEDK3ixZ+Zb5sPunC8hRxA1+Rdxx+s9bSIxPRon6qZmZXT+eg4u3A2p0LpKuuUDBqr5wdLNLl9k5vi4IBav4wtnLAXHRieo6O7HhCbpOazLm5t6lKFXGZu9ko+5Lq+pir+c0eoxg8ZIfX+8pLQfHzB0nysn6KlWqqGNZOTbXngyV+9L0KzPkZL9UVMlxt5DuxBLIyDIkyyNkqIdUjMkQk+ce4EgmQ8ajyIG6HKBLakl+lgN5GQPyPMkBv2yIunXrqlbS+qSZgAzol6Dj9u3bKrqULIqUjGk3pHRSkwuUShcz6bAmjQyEtJYePXo0hg4dqrpBSKAkg/fTkoH+7733nnp92QmSsZLaQcd/OwvJ9pDslnRok4hX5pN5ZHlaUm8oWSTpiBYWFqba62lbRWvJAbhcW+ijjz5Sy5AvQ1nf7C6pMwXSAU2aDZxbF6S70GedT8rpStQehsWlu6SKZF7CLkapi3Iak6eKr2o3HbjpJk4uuwy3nE6oNaAMfIs/PlAu1jQvkhNTcOrXK0iISVSvW/ez8qqltKVr0qycajYw96ddCAuNQbESOTFlVjdd6+eQkEhVhqAlQc2E0b+red3cnVCydC7MWdxHtZim/y5i82bYenkj18CPYOubA4/On8eVD/og6d/GA3a5cquzpFqxJ08g6LNPkevjQcg1eDDirwfh2kcDdNfAEUGfDEHuwUNQ4PvvYevhoZoJBE+dqi7kqc0cudWqrQumEkLuIGL7NoTMmmXxu3T1yV3I4eqJkc3eR053b5y6fRmtfv4E92JSz1rn8/I3yMY42NljdIs+KOyTW5WmSWe0nsvGIjLucYvbj9dOwejmfTCj7Sfwc/VSF/qce3ADxm1boJvnzTJ18Evnr3T3l/cYo/4fs2U+xm7NuGuRuTt674hqDCClZ1Iidiv6Jqafmqq74Ka3o49BxkaNs9Fo8HbhNvB08ERMYjROh57C+qvrMr1MseryCmiQoq59Ix3VzoX9g+UXl8LSFa3tr8bJHFl1DQ8jEuBb0BUth5VXjQFEtJSB631/iIjghwi5EImWX6XvzitJzIjbD7F171kV3EgAJIHT26MqqZbR2szR5YP3cHRNkPped/dzRPkW+VCh1eNxzKRH74K2/4W0iO7Ro4c68S7XvpET8pIE0V7rRo595fhaW901ZswYdVwuyQCpSJLr50ibaBlCoj32lWSEHMvLsBcJeCSpIc3DtEFUZllp9PsuZ9LSpUtVRkEaDUhJlhx4SztjieYkKEjbyticSbAkZW/SWKBRo1e3jOargA9e9irQv76pNQcP4h9fTZ5eLi+HDjhRKn1HH3p5Kp2/ALvBxq9xQi9e4pT9+N+u1AMUevl+bjgPU072fdmrQf8aXNGwisckRK/O+DG3dlla1I8//qi70KdkXeQyKdL1THstSUmAaBMGgwcPVkNEZF6pAJMMkAQzUnWV9kKfchkWCYLq1KmjuhCnbSL2XAKctKQ7mHRMkIyKZE3MmYw/kvIyKR+Ta9jI9W0kW3Tx4sV0A/5fJQxwTAcDHNPCAMf0MMAxLQxwTAsDHNNikgFO5K8ZP+bxePjHq+y/N8KWwZHOzgatk82ZjK/58ssvVTc2ac8s5WbSuvpVDm6IiIiIyDJokg071+ozfiEHCwlwZCyLpJtkEJB0LUvblliyHOaqadOm6kZEREREZKljcMwuwPn4449VgCOdycqWLZvhhfuIiIiIiMiEpDDAMWrFihVYtWqVrq0bERERERG92m2iLTqDI93SpMUbERERERG9QlLMP8B5pksky3Vmpk2bplq5ERERERHRK5TBSc7gZmkZnHfeeSddI4HNmzejTJky6TqISY9rIiIiIiIyMcnmE8j85wDHw+PxFeBFmzZtnsf6EBERERHR85KUbPbbNtMBzoIFC57vmhARERER0fOVkPF1cCx6DI5ISkrCjh078PPPPyM6OlpNCw4ORkxMTHauHxERERERZWcGJymDmyV3Ubt+/TqaNWuGGzduID4+Hk2aNIGbmxsmTpyo7s+ePTv715SIiIiIiP6bJPMJZLI1gyMX+qxatSoePHgAJycng3E5O3fuzM71IyIiIiKi7JKUlPHNkjM4+/btw8GDB9X1cPQVLFgQt2/fzq51IyIiIiKibKRJzjiDY2XJAU5KSgqSjWycW7duqVI1IiIiIiIyQUksUTPqjTfewNSpU3X3raysVHOBkSNHokWLFi9uBxERERERUeYlsUTNqEmTJqFp06YoXbo04uLi0LlzZ1y6dAk+Pj749ddf+REjIiIiIjJFSeafwXmmErW8efPi1KlTWLFiBU6fPq2yN++99x66dOli0HSAiIiIiIhMSJL5BzjP1EUtLCwMtra26Nq1Kz766CP4+voiMDAQR48ezf41JCIiIiKi7JFk/iVqWQpwzpw5ozql+fn5oWTJkjh58iSqVauGKVOmYM6cOWjQoAHWr1///NaWiIiIiIieXZL5X+jTSqPRaDI7c/PmzVXmZujQoViyZAk2btyoxuLMnTtXPS7ZnGPHjuHvv/9+nutMRERERETPIHlljwwfs+mwyPICHClF27VrF8qXL6/G3bi7u+PIkSOoUqWKevzChQuoWbMmIiIinuc603Ow4uIAblcT0bH4j5h+qt/LXg3618AKs3CsREluDxNSJfAC7AbXedmrQf9KnLIfVv1qcnuYCM2svxGduO5lrwb9y82ujclti6Sl3TJ8zLbrkiwta+bMmfj+++8REhKCChUqYMaMGahevbrReSUhsnjxYpw9e1bdl/hh/PjxBvP37NkTixYZBlmSTNmyZcvzK1ELDw9Hzpw51c+urq5wcXGBl5eX7nH5OTo6OksrQEREREREL0hCcsa3LFi5ciWGDBmiLhNz/PhxFeBIMHLv3j2j8+/ZswedOnXC7t27ERAQgHz58qlLz9y+fdtgvmbNmuHOnTu627N0aM5ykwG55s2T7hMRERERkWnSxCdneMuKyZMno0+fPujVq5e6dMzs2bPh7OyM+fPnG51/2bJl+PDDD1GxYkU1ln/evHlISUnBzp07DeZzcHBQCRXtTT+Z8tzaREvqSF5YyDVw+vbtqzI5Ij4+PssrQEREREREL4YmMeNARo7l0x7Py3G/9thfKyEhQY27HzZsmG6atbU1GjdurLIzmfHw4UMkJibC29s7XaZHGppJYNOwYUOMGzdOXWvzuWVwevTooV7Qw8ND3aRNdO7cuXX35bHu3btnaQWIiIiIiOjF0CSmZHibMGGC7rhee5NpaYWGhiI5ORn+/v4G0+W+jMfJjC+++ELFERIU6ZenyTgdyepMnDgRe/fuVU3O5LWeWwZnwYIFWVo4ERERERGZkISUDB+SjIyMq9GXNnuTHb799lusWLFCZWscHR110zt27Kj7uVy5cqqxWZEiRdR8jRo1en4lakREREREZH4lao5GytEy6qxsY2ODu3fvGkyX+9qGZBn54YcfVICzY8cOFcA8SeHChdVrXb58OUsBTpabDBARERERkfmVqGWWvb29avOs3yBA2zCgVq1aGT7vu+++w9ixY1Xb56pVqz71dW7duoWwsDDkypULWcEMDhERERGRhdBkIZB5Eillk/H5EqjItWymTp2K2NhY1VVNyLj8PHny6MbwyJiaESNGYPny5ShYsKBurI5cekZuco3N0aNHo23btioLdOXKFXz++ecoWrSoaj+dFQxwiIiIiIgsRWLWBuxnpEOHDrh//74KWiRYkfbPkpnRNh64ceOG6qymNWvWLNV97d133zVYjlxHZ9SoUark7fTp0+pCnxEREaoBgVwnRzI+WR0HxACHiIiIiMhCaLIpgyMGDBigbsZIYwB9QUFBT1yWk5MTtm7dmi3rxQCHiIiIiMhCpGRjgGOqGOAQEREREVmIZAY4RERERERkLlKSsmcMjiljBoeIiIiIyEKkMINDRERERETmIoUBDhERERERmYukeJaoERERERGRmUhhBoeIiIiIiMxFSjZd6NOUscmAmZCLKTVo0AAPHjyAp6cnXjWHNt3EwbVBiHmQAP9Crmjxv5LIW9wjw/kfxSRi55LLOB9wD4+iE+Hp54RmfYqjeNUc6vH4h0nYteyKejw2MgG5CruheZ8SyJNmmfdvxmD7wksIOhuBlOQU5Mjnig7DyqvlWbozW27hxB838DAiAT4FXFG3d3H4F3U3Ou+6UccRfC4i3fQClXzQalgF9fPhVVdx6eA9xITFwcbWGjkKu6FGx8LIWezxPtk08TRCg6LxKCoRDi62yFvOC7W7FIWLd9auYGyOcnTuDP/33oNdDl88unABN8aOw8MzZzKc37NZU+T5+GPY58mD+KDruPXDD4j66y/d41UCLxh93q3vvsPdX+arnx0KFkTezz+Da+XKsLKzw6PAQNyeNh0xhw7B0vV77R0MadgJOd28cTr4CgatnYIjN84bndfW2gZfNO6GbtWaI4+HLy7eu4lhG2dh24XH29HayhojmvVG5ypvIKebD4KjQrH48J8Yv32Rbp7hTXujfaVGyOfph4TkJBy/FYgRm+bg8I1zsGSvF62Iz5p0RZX8JZDbMwdaz/4cv596/Fk3pl6xypj87scok6sQbj64i3GbF2LR35sM5vmwXlu13Jzu3jh16zI+WjkJR64/3tYOtvaY9O5AdKzSBA62dth6/hA+/PV73IsOh6Vb9WsAlizYi7DQGBQrkQufffkWypbLZ3TeP9Yfxeiv1xhMs7e3xcHj43T3f565Hdu2nMbdkAjY2dmgVOm8+HDgGyhbPr9unsEDFuHihWA8CI+Fm7sTqtcsioFDmiOHn/HvLUuWnMTr4BA9d2f3hWDrvEC82b+UCkD+3nADS0Ycx0ezX4Orp326+ZMSU7B4+HG4eNqjw9AKcPNxQOS9R3B0tdPN8/uMc7h3PQbvDCkLN28HnN5zB4uGH8eAn2rB3cdRzRN+5yF++eIoKjfJjQadi8DB2Rb3bsTA1t7G4vf6pYN3sX/xJdTvUwL+xTxwatNN/PHNSXSeWhPOHun3SfNPyxn8wYyLTsTKz46gSC0/3TTP3M4qSHL3d0JSQnLqMsedRNcZteDknrrMPGU8UaVNATh72SM2PAEHl1zClsln0HZcVYveJ17NmyPvsKG4MXIUYk+dgl+PHij2yzz806w5ksLTH0y5VKqEwpMm4fbkyYjcvQfeb7ZCkZk/4vw7bRF36ZKa59RrdQye41G3Lgp8Mw4Ptm7TTSs6ezbirwfhYo8eSImLh1+P7ig6exbONnkDSaGhsFTtKjbE960HoP/qH3D4+jkMrNcem/43GWUmdML9mPSB/pgWH6jApe+qiQi8dwNvlKiONb3Go+70vjh5O3V/fNaoC/5XuzV6//oNzt25hir5S2Jexy8RFReLH/elHvxdun8TH6+dgmthwXCyc8DH9drjz76TUfKbjgiNTf+6lsLFwQmnbl/C/IN/YF3fiU+dv6BPLmzqPwmz961DlwUj0ahEVczrOgx3IkOx7Xxq0Nm+SmNMbvsx+v46EYeu/YNBDTti68CpKDGqA+5HP1DzTGk3CC3L1ka7eV8i8lEMfuzwKdb+71vU+eEDWLJtm09hyncbMWxEG5Qtnw+/LjmAj/73C37741N4+7gafY6LqwN+2/ip7r5VmscLFMyBz798C3nyeiM+PgnLF+9D/w9+wfo/P4OXd+oyq1YvjN59GsA3hxvu3Y3CtB824YvBSzF/2YfP9f2+ilIsoETN+mWvwKumfv36+OijjzBo0CB4eXnB398fc+fORWxsLHr16gU3NzcULVoUmzdv1j1n7969qF69OhwcHJArVy4MHToUSUlJuscLFiyIqVOnGrxOxYoVMWrUKN19KysrzJs3D23atIGzszOKFSuGDRs2qMeCgoJU9kbIOsm8PXv2xKvi4PrrqNI0Lyo1zgO//K5o9WEp2DnY4MT220bnP7HjtsrgdPqqAvKX9oSXvxMKlvNGzkJu6vHE+GScP3gPb/QqhoJlveCT21kFMN65nHDkz1u65UgGqFgVX7zRqzhyFXGHdy5nlKzhZzSosjQnN95EmUa5UapBbnjndVGBjq29Nc7vDjY6vwSXLp4Outut0w9g62CNojUfBzjF6+REvvLe8PB3gk8+V9TpXgwJj5IRej1GN0/FVvmRs7gH3HM4IVcJD1RuXQAhl6Is4mzTk/j36onQVasRtnYt4q5cwY2RI5ESFweftm2Nzu/XvRsi9+1XmZi4q1cRPG06Hp47B7+uXXTzSICif/Ns1BDRhw4h4Vbq74iNlyccCxVEyJy5eBR4EfHXr+P2pMmwcXaGU7FisGSD6nfELwF/YNHhP3H+bhA+XP09HibEoWeNVkbn71K1KSbuWIIt5/9WwcnPB9dj8/kADK7fUTdPrYJl8cfZ/dh8LgDXH4Rg7ak92B54GNXyl9LNs+L4duy6eFQt41zINXy6fgY8nFxRLncRWLIt/wRg+Iafsf7U3kzN3/f1d9Q2/PS36bgQEoSZe9dgzYndGNzo8f4Y0qgT5h74HQsDNuF8SJAKdGQf966Vuo/dHV3wXu03MWTNNOwOPIbjNwLRa/E4vFakPGoUKgNLtmzxfrR+tzrealMVhYv4Y9iI1nB0tMeGdUczfI4ct/j6uuluPr6p3+dazVpWRI1axZA3nw+KFPXH4M9bITYmHpcuhujm6dL9dZSrkB+5cnuhQqUC6PF+fZw5fRNJFlCO9SwBTkoGN3PBAOcZLFq0CL6+vjh8+LAKdvr164d27dqhdu3aOH78ON544w1069YNDx8+xO3bt9GiRQtUq1YNp06dwqxZs/DLL79g3LjHqdfMGj16NNq3b4/Tp0+rZXbp0gXh4eHIly8ffvvtNzVPYGAg7ty5g2nTpuFVINmYO5ejUbiCt26atbUVClf0xs3ASKPPCTx0H/lKemDT7Av4rttezOx/EH+tuoaUZI16XP5PSdGoA3J9dvY2uPFvGZU8fvFoKHzyOGPxiOP4rusezPnkkCpps3QSTNy/Go285R7vEytrK3U/5GJUppZxblcwitX2h52jTYav8c+OYNg728K3gPEzenExibi47y5yFfdQJW2WSkrDnMuUQdTBg48najSIPhgA10oVjT7HtWJFRAfozQ8gav8BuFQ0Pr+tjw886tVD6JrUvyMi+UGECo68W78NaycnwMYGOTp0QGJoKB7+8w8slZ2NLSrnLY6dFx8frGk0Guy6dBQ1Cxg/sJXypbikeINpcYnxqF24vO5+QNBZNCheBcVypJbxlM9dFK8VLq+CoozW4/1abyPiUTROB1/OpndnGWoVLosdF44YTNt67m/UKlxOt22l3E1/HtnHcl87T5UCJWFva2cwT+Dd67gedge1CqXOY4kSE5Nw4dxt1KhZVDfN2tpalYudPnU9w+c9epiAVk2+RctGEzDko0W4cvnuE19j3erDcHVzRPESuYzOExn5EFs2nkT5ivlha8eqDEsMcDgG5xlUqFABX3/9tfp52LBh+Pbbb1XA06dPHzVtxIgRKpCRQOSPP/5QAciPP/6ozlCULFkSwcHB+OKLL9R88oufWZKV6dSpk/p5/PjxmD59ugqymjVrBm/v1INRPz+/V2oMzsOoBBVsuHoZZk0kixJ6K9bocx6EPMK10w9Qrn5OdB1ZSZWabZx1AcnJKWjQKbXUTAKgvSuuwTevC1w9HXDmrxDcDIxQWRoh43Ike7B/zTU07FoUTXoWw+VjoVg54RR6flNFZYQsVVxUIjQpGjinyWTJ/QfBD5/6/LuXoxB+MxYN+z0+86wVdCwUW6f+o0rUpMTwra8r6srTtA4uvYwzW28hKT4F/sXc0Wpo6hgeS2UrWVlbWySFhRlMTwwLhWPhQsaf4+uLxFDD+ZPCQmHn62t0fp82rZEcG4uIbY/L08TFnr1Q5KeZqHj8mJwVQGJ4OC693wfJUZkLdM2Rr4sHbG1s042zuBsdjhJ+BYw+Z9uFw/i4fkfsu3IKV8Juo2GxKmhdvh5s9P7+f7dzqcoKnB26DMmaFNhYWWP4n3Pw6/HtBstqUbo2lnUfBWc7R9yJCkPzWYMRFmv8ZBAZl9PdB3ej0uy/qHCVDXO0c4CXs5vax+nneYCS/gV1y4hPTFClaWk/B/KYpYp48FB9F6ctRZP7QdfuG32OlJ8NH9NWjdWJiY7D0oV/oXfXn7Bq/RD453w8RnPfnvP48rNfEReXqMrQZs55D55eLgbLmj55M1b9ehBxjxJVNmfKzB7P6Z2+2pKSUk8ImzPLPS36H5Qv//ism42NDXx8fFCu3OMzNlK2Ju7du4fz58+jVq1aKrjReu211xATE4Nb/5aCPMvruri4wN3dXb1GVsTHxyMqKsrgJtNeJRoN4OJhj7f6l0buou4o+3pO1G1fCEc3P96eMvZGzrhN6rkPY9/ZiUN/3EC5ujl1+0EO4IWUpNVuXUA1IXi9XSEUr+aLI1uytl8offbGJ7+L0YYEecp4ocP31dB2bBXkr+iDrVPO4mFkgsE8ld7Kjw4Tq6vgR7J5O348p/YlPT++bdsi/I+N0CQY7ov8I0eowCqwSxecb9ceETt2qDE4tjlSm3lQ5gxZNw2X79/E2WHL8PD73ZjWdogqb5OTO/rjejpVboJuS0ej+qTeaizOkAad0K1aM4Nl7bl8HFV/6IW60/upJgXLe4xBDtdX56QWUVrlKxZAq7eroETJ3KhSrTC+n9oNXl6uWLvasJlJ1epFsPy3gZi/tB9qvVYcwz5djvAwwwCze6+6WLZ6IH6c8576/hg5bBW/P4yQURIZ3cwFA5xnYGf3eDC7kINm/Wnag+iUlMyl+iSLk/YALjExMVOvm9nX0JowYQI8PDwMbjLtZXF2t1d/hKR7mr6YiAS4ehnvnCXZHikts7Z5HDTmyOuiliElb0IyNb2/rYavVjfEkAWv44PJNZCcpIFXTqfHr2tjhRz5Dc/++OZzReT9OFgyR3c7VZIm3dP0yf20WZ20EuOScfnAXZRqmNvo41Ky5pnTWY2zkQyP7IPzuwzH9UhGRxoSyHidNwaVwfUTYbh7yXIzBkkPHkCTlKTKyPTZ+UiWxvhAfxlTY+drOL9tBvO7VqkCx8KFEbp6tcF0t5o14VG/Pq4OHoLY4yfw6Nw53Bw9JnXsT+vWsFShsZFISk6Cn5thltffzRshUWEZPCcC787/Eh5fNEGRse+i7ITOiIl/hKvhjz/73775Ib7fuQyrTuzE2TtXsezoVkzbuwqfN+pmsCwZB3Il9DYOXf8HH6z8FkkpyeiVwdgfMk72k797mv3n7q2yMVI6GBoTofZx+nm8dPtY/news1dZn8x+DiyBp5czbGys0wUect/H13g5clpSUlaiVG7cvGG4HZ2c7ZEvv6/KzIwY+656nd/XGpYaSkZHMkI1axfD+O8748C+QJw5dSMb3pl5SUrO+GYuGOA8Z6VKlUJAQIBBAHPgwAHVjCBv3rzqfo4cOdS4GS3Jqly7di1Lr2Nvn3rgmZz85E+nlNRFRkYa3GTay2JrZ41cRd1w9fTjUgA5q3ntVDjylTDeJloaC0hZmv7Zz9Dgh3DztlfL02fvaKO6qElTgisnwlCyRuqZZ5kvTzF3hN4yLLkKux0LzxyW3SJa28L51tnUTkHajJfcz1n8ye02L/99TwWSJV7PmanXkt+L5CfU/Gp/bZ40j7nTJCaqMS/utWo9nmhlBbdaNRFz4qTR58ScPAm3mnrzy6Do2rURezL9/D7vvovYs2dVC2h9atyNWoE02TONBlZZKK01N4mqPfNFNCxexeBkU4NiVfD39SePTYpPSkBwZKhqG92mfD38cWaf7jFne0ekaAw/58kpyap99JPI49KumDIv4OpZ1TlNX5NS1RFw9YxuHx+7EYhGJaoZ7GO5r53n2PULSEhKRKOSj+cp7p8fBXxyIeBaxu3bzZ2dnS1Kls6Dw4cejwuTE7FHDl1G+QrGSzjTkhK3y5dCVBnak8gxQEJCxikH7XHXk+axVEkWkMHhGJzn7MMPP1Qd0qQZwYABA1QTgJEjR2LIkCG68TcNGzbEwoUL8eabb6rxMzI2R0rfsqJAgQLqD/DGjRtVAwInJye4uqY/WyKd3ORmSqREbN2Uf5CnqDvyFHdHwO83kBCXjEqNU7MAayefVa2gm/RI7dxUrXk+HN54E5vnBqJGq3wID36IfauvqZ+1Lh8PVcdlvnlcVDC0bcFFNR5Hu0zx2jsFsfq70yhQ1hOFynmr51w8HIqe4x8fuFiqiq3yYefM8/Ar7Aa/ou449edNJMUno1T91O0nZWNybZpanQ27N0k2plA1Xzi62aXL7BxdG4RCVX1VC2hpI31my23VClrbSjrkUiTuXYlGrpIe6ho4UXcf4dDKa6qttGR8LNndBQtRcOK3KhB5ePq0ahMtAYh0VRPyWMLdewiePFndv7d4CUosWQy/Xr0QuXcPvFu0hHPZMrg+YoTBcq1dXODVrCluTUzfWjfm5Ak11qbgt9/izsyZSImPh2/7duq6OpF79sCSTd2zAvM7f4VjNy/gyPXzqk20i70TFh1KvY7Kgs5f43bkfXy96Wd1v3r+0sjt4YtTwZfV/yOa9lZ//3/YtVy3zE3/HMDQJt1xI+KuahNdMW9xDKrfAQsP/akLgIY17o6N/xzAnahQ+Lp4ol+dd9R1dX47tRuWTNpEF82ResJQFPLJjQp5iyE8Nkpd42b82/2QxzMHeiwaox6fvW8tBtR/FxPbDFCtpRuWqIr2lRuh5cxPdMuYvPNXLOoxHEdvnMfhoHMY1LADXBwcsSAgdR9L++5fDv6ByW0HIjw2Ut2f0f4THLxyWrWVtmRdutfBqK9Wo3SZvChTNh+WL92PR48S8Gbr1O/WEcNWws/PAwMGp5Zfzp21A+XK50fe/D5qDM7iBX8hJPgBWretpmtAMH/OLtRtUFoFPREPYtV1du7fi0Ljpqml+2dP38A/Z2+hYuWCcHd3wq2bYZg1Y7vquiYlcGTInAKZjDDAec7y5MmDP//8E5999plqTiDNAN577z1dkwIhGRTJ2LRq1UqVjI0dOzbLGRx5HemyJi2opV119+7dVdD0KpAxNDLoXy7MGfMgHjkLu6Hb6Mq6EjUpGdMbwgSPHI7oNqYytsy7iFkf/a2Cn5pv5kedtqmDP0VcbBJ2LL6MqNA4OLnZoXRtfzTqVsSgG1epWn6qJbUER5vnBMI3j7O6yGeBMl6wdNIBTS62eWjVVVWa5lvQDa2+rKArUYsONdwn4kFwLO5ciFRjZ9KSk9ARwQ+xZdIZdWFWCYD8irijzejKqmW0kNbgVw/dUxcElQYD8loyTqfp4IKwSZOZszQPNm+Grbc3cg/8CHY5cuDR+fNqsL+28YB9rty6cWUi9sQJXP30U+QZNAh5hgxGfFAQrvQfoLsGjpZ3y5bqxEj4RsMLHGq7qMlr5B40CMUXLYKVnS0eXbqMK/37p8v2WJrVJ3epcS8jm72fehHI25fR6udPcC8mNeuZz8vfIBsjpUyjW/RBYZ/cqjRNOqP1XDYWkXGPy3jk+jajm/fBjLafwM/VS13oc+7BDRi3bYF6PDklBSX8C6iLhfq6eiAsNkodfDeY0V+1jLZkVfOXwp4hP+nuy/VphLR47rV4LHJ5+CK/9+OsclDYHRXMTGn3MT5u0B63Iu7h/aUTdNfAEauO7VD7eEyrPqppwMlbl9BsxmCD5hKDV09V+/m3DyaoLNrWc4fw4YrvYOneaF4BDx7EYvaP2xEWGo3iJXNjxuzeutbPIXciVGm6VlTUI4wbtVbNK8GJZIB+WdpPtZgWUsosDQo2bliqghsPT2eULpsXcxf9T7WMFtKGeveOs5gzc4cKpiQQknE67/2vobpoKFlegGOl4ehdkusrXBzA7WAiOhb/EdNP9XvZq0H/GlhhFo6VKMntYUKqBF6A3WDDC5XSy5M4ZT+s+tXkLjARmll/Izpx3cteDfqXm10bk9sWO/xKZPhY43vmcQKLYS0RERERkYVIsoAMjmXXfRARERERWZCkbGwyMHPmTBQsWBCOjo6oUaOGuj7jk6xevVpdE1Lml0usyDAOfVJYJmPRc+XKpcaTN27cGJfSlFdnBgMcIiIiIiILkV0BzsqVK1XTLGmedfz4cTXWvGnTphleo/HgwYPqgvUyFv3EiRNo3bq1up09e1Y3z3fffacuZD979mwcOnRIXfdRlhkXl7VLeDDAISIiIiKyEEnZdB2cyZMno0+fPqq5VenSpVVQ4uzsjPnz5xudf9q0aWjWrJlqvCWXUZGmWpUrV8aPP/6oy95I52FpxPX222+rC9wvXrwYwcHBWL9+fZbWjQEOEREREZGFSHpCBic+Pl5dj1H/JtPSSkhIwLFjx1QJmZa0v5f7cv1HY2S6/vxCsjPa+aWDcEhIiME80l1YSt8yWmZG2GSAiIiIiMhCdEzOuFPaqFGj1GVH9EkJmkzXFxoaqi4u7++f2qpbS+5fuHDB6LIleDE2v0zXPq6dltE8mcUAh4iIiIiIINdmlHE1+kztAvGZwQCHiIiIiIggwUxmAhpfX1/Y2Njg7t27BtPlfs6cjy+sq0+mP2l+7f8yTbqo6c9TsWL6i4g/CcfgEBERERFRptnb26NKlSrYuXOnblpKSoq6X6tWLaPPken684vt27fr5i9UqJAKcvTnkTFA0k0to2VmhBkcIiIiIiLKEill69GjB6pWrYrq1aurDmixsbGqq5ro3r078uTJgwkTJqj7H3/8MerVq4dJkyahZcuWWLFiBY4ePYo5c+aox62srDBo0CCMGzcOxYoVUwHP8OHDkTt3btVOOisY4BARERERUZZ06NAB9+/fVxfmlCYAUka2ZcsWXZOAGzduqM5qWrVr18by5ctVG+gvv/xSBTHS/rls2bK6eT7//HMVJH3wwQeIiIhAnTp11DLlwqBZwQCHiIiIiIiybMCAAepmzJ49e9JNa9eunbplRLI4Y8aMUbf/gmNwiIiIiIjIbDDAISIiIiIis8EAh4iIiIiIzAYDHCIiIiIiMhsMcIiIiIiIyGwwwCEiIiIiIrPBAIeIiIiIiMwGAxwiIiIiIjIbDHCIiIiIiMhsWGk0Gs3LXgmi/yo+Ph4TJkzAsGHD4ODgwA36knF/mB7uE9PC/WFauD9MC/cH/VcMcMgsREVFwcPDA5GRkXB3d3/Zq2PxuD9MD/eJaeH+MC3cH6aF+4P+K5aoERERERGR2WCAQ0REREREZoMBDhERERERmQ0GOGQWpLHAyJEj2WDARHB/mB7uE9PC/WFauD9MC/cH/VdsMkBERERERGaDGRwiIiIiIjIbDHCIiIiIiMhsMMAhIiIii2ZlZYX169dn+7z0Yunvm6CgIHX/5MmT3A0WiAEOmZVu3bph/Pjx2ba8hIQEFCxYEEePHs22ZZqL+/fvo1+/fsifP78aEJozZ040bdoUBw4c0M1z8OBBtGjRAl5eXnB0dES5cuUwefJkJCcnv9R1N0fyRf6k26hRo3Rf+Nqbj48P3njjDZw4ceJlr75Z69mzJ1q3bq27HxISgo8++giFCxdWvzv58uXDm2++iZ07d77U9TSl7aX9jNrb26No0aIYM2YMkpKSnttr3rlzB82bN8/2eS11v9nZ2aFQoUL4/PPPERcX97JXjSwQAxx6KX/4/P390aRJE8yfPx8pKSlGnyMHyzY2Njhy5EimXuPUqVP4888/MXDgQN20+vXrGz3Y035Rrl27Vh3gyYGesTM98uX66aef4osvvvhP798ctW3bVh0YL1q0CBcvXsSGDRvU9g4LC1OPr1u3DvXq1UPevHmxe/duXLhwAR9//DHGjRuHjh07QqPRvOy3YFbkoEt7mzp1Ktzd3Q2myedYa8eOHWra1q1bERMTow7WIiIiXur6WwoJMqtUqYJdu3bh+++/x5kzZ7BlyxY0aNAA/fv3f9mrZzKaNWumPqOXLl3CJ598ogJ02V7GTkJlBzlBI8Fmds9rqfvt6tWrmDJlCn7++WfV4ZTohdMQvQA9evTQNGvWTHPnzh3NrVu3NMeOHdN88803GldXV03z5s01iYmJBvNfv35dPTZw4EBN3759M/Ua7733nuZ///ufwbR69epp+vTpo15X/6a1ePFizejRozVz586Vo23NiRMn0i03PDxcY29vrzl79uwzv39z8+DBA7W99uzZY/TxmJgYjY+Pj+add95J99iGDRvUc1esWPEC1tQyLViwQOPh4ZFu+rVr19J9zg8cOKCmbdmy5QWvpWX9/Xv77bfVz/L3Lk+ePOp3xNjvFRluL60mTZpoatasqXts3Lhxmly5cmkKFiyoHr9x44amXbt26nPv5eWleeutt9TnXd8vv/yiKV26tPp7njNnTk3//v11j8nvwLp169TP8fHx6jGZx8HBQZM/f37N+PHjjc4rTp8+rWnQoIHG0dFR4+3trb5zoqOj0+3/77//Xi1T5vnwww81CQkJZr/f5DugUqVK6ufk5GS1HWWfybYqX768ZvXq1Qbzy/dsy5YtNW5ubuoYoE6dOprLly+rxw4fPqxp3Lix+m5xd3fX1K1bVx1L6NPfN8b+3pHlYAaHXhhtGVOePHlQuXJlfPnll/j999+xefNmLFy40GDeBQsWoFWrVqoE6tdff8WjR4+euGwpeVqzZo0q80jL2dlZva7+Tb+kbcSIEWjcuHGGy5byqtdeew0rVqx4pvdtjlxdXdVNap3j4+PTPb5t2zaVydHPGmjJPipevLjar/TyOTk5ZeuZcMpYeHi4ytZIpsbFxSXd456entx8T/icaj+jUsoXGBiI7du3Y+PGjUhMTFQZfzc3N+zbt0+VycrfJ8kmaJ8za9Ystd0/+OADlTWTjLOUvhkzffp09fiqVavU6yxbtkyVKhsTGxurXlu+J6TaYPXq1SpDOmDAAIP5JIt95coV9b9kveU7L+33nrk5e/asKlOWSggxYcIELF68GLNnz8Y///yDwYMHo2vXrti7d696/Pbt26hbt646VpAM57Fjx9C7d29dxUV0dDR69OiB/fv34++//0axYsVUCbRMJ0rLNt0UoheoYcOGqFChgioVe//999U0OQkjAc7MmTNRsmRJ9SUkwYsEIxk5ffo0IiMjUbVq1eeyntWrV1dfnJTK1tZWfTn36dNHfVlJwCrlaFJ6Vr58eVWyJkqVKmV0k8l+1c5DL4+UpY0dO1YdDMpnnJ6vy5cvq79v8vmnzJHtJQGNlFPKuCUZ+yfB4bx583QHzkuXLlWlzjJNSo2FfIdIwLhnzx5VhiylsVLqJmWyWtWqVTP6mjdu3FAHz3Xq1FHLK1CgQIbrt3z5cjXGRA7ctUHrjz/+qE7kTJw4UZVjCwmAZLqUXcv+b9mypXpf8jfUnEjAKX9PJCiRk1/W1tbqfcvPMj5Wgr9atWqpeWUMmgQrUsYm3x/yne/h4aFOJkopu5CTYfrHC/rmzJmj9rEESHJClEgfMzj00skfe6lL15I/gA8fPlRnxYSc4fnll1+euIzr16+rLw4/P790j/3000+6jIPc5Esuq3Lnzq1egwzH4AQHB6sznXKmVA4kJNDRPyvJcTamqXbt2up3QQ66ZOzaypUrdQdi9Pzw9yHrB8rSnETGiHXo0EGNwxHSrEQb3Aj5DEvwKBkc7d95b29vFXhI1uTevXvqb1WjRo0yPWZUxmOWKFFCjemUjHRGzp8/r07S6WfkJOMvAZdkf7TKlCmjvqO0cuXKpdbL3MhYMtl2hw4dUtmWXr16qe8K2T/yvS5jb/W/jyUwlH0k5Hmvv/66LrhJ6+7duyoglOBTAiEZZyhjCCUgJUqLGRwyiS997Vk3IY0H5MtMsgSiU6dO+Oyzz9QfwSJFihhdhpSwSVpbfzlaXbp0wVdfffWfykCkPEL+OJMhOfiQLyy5DR8+XGXhZECpDHLXfvnLwXRaMr106dLcnC+JBDSy/aW5BsuiXhw5MJO/UdJwg55+oCxlZRLIyAkm7feBSFveJwe50rhBSsnSypEjh8oiZIWcqLl27Zoqn5YTbu3bt1dlzFJJ8KzSHrTL5yCjBjuvMtk32tI/+S6X4E9OUJYtW1ZN27RpkypT16dt2KAtl82IBExS+jxt2jSVVZPnSTaI5bVkDDM49NLJwa60k9TWqEv3Lcm6yBea3OSPoaS75Y9lRnx9fVUAYuwPnZzpkT+42pvMm1WyXvJFSU8mB81Sky4lIXIGddKkSenmkYyPdEaSwJVeDmlLLCcLGNy8WPI7IZlpKcWR35O02Mku/YGytKHXD24yCkjkb4pk8PX/1stN/v5LZkfG0GSlDbdkB+RE29y5c9UJgd9++019D6QlZbiSQdLfnzIGSIIqyQBZMtkGMtb266+/Vt8NEpBItiXtPpK/R0LKm6UUXMZUGSPbVTJqMu5GMmKyvNDQ0Bf8ruhVwQCHXioZSCgDPiWFLeQMnLQVli8MSVdrb3KgLKVPGV0/pWLFiur/c+fOPbfBkpUqVXouy34VyVk0qYeW2ncZ/yRnO2Vw7XfffYe3335bHZxIXbU0kZBBvTKPlCHKmTwp/3j33XfVWVEiSyPBjfwdkzFPctAsB+ZykkcGtmvHJlDWSJZeTlzJ3x45QJa/R1IyKwfDt27dUvNIeZt8j8h2lm1+/PhxzJgxw+jy5Fpd0gRFMm0yVlD+tklzGmMnBOS1JZMt2QX5npAmAjJWSMaMsuwTaNeunSrNk+8DaTojjQWkyYJUZGj3gdwX0pghKipKjeWUa8/JflqyZImu1E8yoHJffl+kBE62/dOyPmS5WKJGL4wMMpQL3MmXu9TSSjch6aoigwO7d++u5pEDYDn41aazteQMz7Bhw9RzZHBmWpJdkbN4MmBRG+xkhpyRkzNKUp8ttH9I03Zbky9NGYxNqaR2ukaNGuo6B/JFJWfcZB9JfbScsROyH+XL/ptvvlF11VIPL19QUi44aNAgo+WEROZOBlbLgZ38Xsh4QLlmiPz9khIrKcmirJNOmX/99Ze6Xtk777yjumpJ5l/G3EgmRkgAIn+D5G+WHGhLQCR/o4yRjI+crJEDbDk4l2YEco01Y6Vu8trSAEGaF8h8cl9O2EmQRKkNaSRwke0pgad81uV7X66TIwGjtqOqkJJZOekpJenSdEC2vXyfy5gm7fGBnDCT58j3jTQtMNapk0hYSa9obgp63uSsvfYsjfzBk8HNUpvbuXNn9cUjXxzSElK6oB0+fNhodxtJS8uZMum4ZowcHMiAxYCAAN00ufCk/IHUjglJS7JCMggyLRlHoh3QKsuT15YgiGeLiIiIiEwbAxwyG9JoQGqepV46O0s9pA5bgjHtWSYiIiIiMl0cg0NmQ7IrksHJzkGH0rRAWpJK3TARERERmT5mcIiIiIiIyGwwg0NERERERGaDAQ4REREREZkNBjhERERERGQ2GOAQEREREZHZYIBDRERERERmgwEOERGZJSsrK6xfv/5lrwYREb1gDHCIiOi56dmzpwo0+vbtm+6x/v37q8dknszYs2ePmj8iIiJT89+5cwfNmzfP8joTEdGrjQEOERE9V/ny5cOKFSvw6NEj3bS4uDgsX74c+fPnz/bXkwv0ipw5c8LBwSHbl09ERKaNAQ4RET1XlStXVkHO2rVrddPkZwluKlWqpJuWkpKCCRMmoFChQnByckKFChWwZs0a9VhQUBAaNGigfvby8jLI/NSvXx8DBgzAoEGD4Ovri6ZNmxotUbt16xY6deoEb29vuLi4oGrVqjh06BD3PhGRmbF92StARETmr3fv3liwYAG6dOmi7s+fPx+9evVSZWdaEtwsXboUs2fPRrFixfDXX3+ha9euyJEjB+rUqYPffvsNbdu2RWBgINzd3VUQpLVo0SL069cPBw4cMPr6MTExqFevHvLkyYMNGzao7M7x48dVUEVEROaFAQ4RET13EqgMGzYM169fV/clEJGyNW2AEx8fj/Hjx2PHjh2oVauWmla4cGHs378fP//8swpOJPMi/Pz84OnpabB8CYi+++67DF9fyuHu37+PI0eO6JZTtGjR5/Z+iYjo5WGAQ0REz51kYVq2bImFCxdCo9Gon6WcTOvy5ct4+PAhmjRpkm48jX4ZW0aqVKnyxMdPnjyplqMNboiIyHwxwCEiohdWpiZjZcTMmTPTlZCJTZs2qTIyfZlpFCBjap5Ev5yNiIjMGwMcIiJ6IZo1a6YyMjL4X9sIQKt06dIqkLlx44YqRzPG3t5e/Z+cnJzl1y5fvjzmzZuH8PBwZnGIiMwcu6gREdELYWNjg/Pnz+PcuXPqZ31ubm749NNPMXjwYNUw4MqVK6oJwIwZM9R9UaBAARUcbdy4UY2n0WZ9MkO6p0ljgdatW6vxP1evXlVNCwICArL9fRIR0cvFAIeIiF4Y6X4mN2PGjh2L4cOHq25qpUqVUhkfKVmTttFCStdGjx6NoUOHwt/fX1fulhmS/dm2bZtqUNCiRQuUK1cO3377bbpAi4iIXn1WGhntSUREREREZAaYwSEiIiIiIrPBAIeIiIiIiMwGAxwiIiIiIjIbDHCIiIiIiMhsMMAhIiIiIiKzwQCHiIiIiIjMBgMcIiIiIiIyGwxwiIiIiIjIbDDAISIiIiIis8EAh4iIiIiIzAYDHCIiIiIiMhsMcIiIiIiICObi/6GDqoy7NoNmAAAAAElFTkSuQmCC",
"text/plain": [
"
"
]
@@ -248,7 +248,7 @@
},
{
"cell_type": "markdown",
- "id": "d73a293f",
+ "id": "2a726be7",
"metadata": {},
"source": [
"### 1.4 Metric Explanations\n",
@@ -260,14 +260,14 @@
"| **TP** | Temporal Precision | Are bout boundaries (start/end frames) accurate? |\n",
"| **IC** | Intra-bout Continuity | Is each detected bout clean, without label fragmentation? |\n",
"\n",
- "**Absent-behavior rule (v0.2.0):**\n",
+ "**Absent-behavior rule (since v0.2.0; per-file scoring, the default):**\n",
"- GT has no bouts **and** machine predicts nothing → all metrics = **1.0** (correct absence) \n",
"- GT has no bouts **but** machine predicts bouts → all metrics = **0.0** (false detection)"
]
},
{
"cell_type": "markdown",
- "id": "723dc4da",
+ "id": "ccdbc5e0",
"metadata": {},
"source": [
"---\n",
@@ -276,7 +276,7 @@
},
{
"cell_type": "markdown",
- "id": "129853f9",
+ "id": "5a0759b7",
"metadata": {},
"source": [
"### 2a. Frame-based F1: our implementation\n",
@@ -291,7 +291,7 @@
{
"cell_type": "code",
"execution_count": 7,
- "id": "cc4eee5e",
+ "id": "01bf9d01",
"metadata": {},
"outputs": [
{
@@ -329,12 +329,12 @@
" per_rec_f1.append(np.nanmean(f1s))\n",
"\n",
"frame_f1_score = np.nanmean(per_rec_f1)\n",
- "print(f\"Frame-based F1: {frame_f1_score:.3f}\") "
+ "print(f\"Frame-based F1: {frame_f1_score:.3f}\")"
]
},
{
"cell_type": "markdown",
- "id": "6b969b09",
+ "id": "ecbcba3b",
"metadata": {},
"source": [
"### 2b. BANOS metrics: same recordings, same behaviors"
@@ -342,18 +342,18 @@
},
{
"cell_type": "code",
- "execution_count": 9,
- "id": "f1ba3a31",
+ "execution_count": 8,
+ "id": "85f77926",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "BANOS DA (F1): 0.774\n",
+ "BANOS DA (F1): 0.749\n",
"BANOS SO: 0.698\n",
"BANOS TP: 0.338\n",
- "BANOS IC: 0.995\n"
+ "BANOS IC: 0.992\n"
]
}
],
@@ -373,7 +373,7 @@
},
{
"cell_type": "markdown",
- "id": "88c1634e",
+ "id": "c07f61bc",
"metadata": {},
"source": [
"### 2c. Side-by-side comparison"
@@ -381,8 +381,8 @@
},
{
"cell_type": "code",
- "execution_count": 10,
- "id": "3415f1b7",
+ "execution_count": 9,
+ "id": "363063a5",
"metadata": {},
"outputs": [
{
@@ -392,10 +392,10 @@
"Metric Value What it captures\n",
"----------------------------------------------------------------------\n",
"Frame-based F1 0.815 Frame overlap; correct absence = 1\n",
- "BANOS DA (F1) 0.774 Bout-level detection (did system find right bouts?)\n",
+ "BANOS DA (F1) 0.749 Bout-level detection (did system find right bouts?)\n",
"BANOS SO 0.698 Temporal quality of overlapping bouts\n",
"BANOS TP 0.338 Boundary precision\n",
- "BANOS IC 0.995 Label stability within detected bouts\n"
+ "BANOS IC 0.992 Label stability within detected bouts\n"
]
}
],
@@ -417,7 +417,7 @@
},
{
"cell_type": "markdown",
- "id": "b52dbd23",
+ "id": "e501f21e",
"metadata": {},
"source": [
"### 2d. Why our approach differs from CalMS21 — and why intentionally\n",
@@ -433,7 +433,9 @@
"\n",
"2. **Explicit absent-behavior scoring**: we explicitly reward correct absence (score 1) and penalize false detection (score 0). CalMS21’s pooled approach implicitly handles this by having behaviors present in at least some videos.\n",
"\n",
- "We do **not** align with CalMS21’s pooled approach because BANOS’s purpose is to quantify annotation quality **per recording** also, not globally only. Collapsing all recordings into one global pool hides per-recording variability that is central to BANOS’s value.\n",
+ "By default, we do **not** align with CalMS21’s pooled approach because BANOS’s purpose is to quantify annotation quality **per recording** also, not globally only. Collapsing all recordings into one global pool hides per-recording variability that is central to BANOS’s value.\n",
+ "\n",
+ "To compare or rank models on a whole dataset, `banos.score(data, aggregation=\"pooled\")` scores each behavior once on the bout counts and terms of all recordings, without the absent-behavior rule. The per-file mean, by contrast, mixes detection quality with the share of recordings without the behavior (see the README).\n",
"\n",
"> For direct comparison with CalMS21 leaderboard numbers, use the CalMS21 dataset with their exact metric. \n",
"> For publications, state both the frame F1 methodology and the BANOS metrics explicitly."
@@ -441,7 +443,7 @@
},
{
"cell_type": "markdown",
- "id": "4a840d3e",
+ "id": "199e3622",
"metadata": {},
"source": [
"### 2e. Thought experiment — the shifted annotator\n",
@@ -451,8 +453,8 @@
},
{
"cell_type": "code",
- "execution_count": 11,
- "id": "534a6e39",
+ "execution_count": 10,
+ "id": "e4a47340",
"metadata": {},
"outputs": [
{
@@ -460,7 +462,7 @@
"output_type": "stream",
"text": [
"Pred A (identical): Frame F1=1.00 BANOS DA=1.000 SO=1.000 TP=1.000\n",
- "Pred B (shift 15f): Frame F1=0.00 BANOS DA=NaN SO=NaN TP=NaN\n",
+ "Pred B (shift 15f): Frame F1=0.00 BANOS DA=0.000 SO=NaN TP=NaN\n",
"Pred C (shift 2f): Frame F1=0.80 BANOS DA=1.000 SO=0.667 TP=0.200\n"
]
}
@@ -515,23 +517,23 @@
},
{
"cell_type": "markdown",
- "id": "6f8ac5d5",
+ "id": "3bd70b20",
"metadata": {},
"source": [
"**Interpretation:**\n",
- "- **Frame F1** tells you how many frames overlap — a 15-frame shift collapses the score to 0.\n",
- "- **BANOS DA** tells you whether the system found the right *bouts* (count-level matching). Both shifted predictions still detect 2 bouts matching 2 GT bouts, so DA=1.0.\n",
- "- **BANOS TP** quantifies the temporal shift magnitude — a small shift gives high TP (~1.0), a large shift gives low TP.\n",
- "- **BANOS SO** measures fractional temporal IoU of matched bouts.\n",
+ "- **Frame F1** tells you how many frames overlap — a 15-frame shift (B) collapses the score to 0, a 2-frame shift (C) gives 0.80.\n",
+ "- **BANOS DA** tells you whether the system found the right *bouts*: a predicted bout detects a GT bout when they share at least one frame. C detects both GT bouts, so DA = 1.0. B shares no frame with them: both GT bouts are missed and both predicted bouts are false, so DA = 0, and SO and TP are undefined (NaN), having no overlapping pair (with `missed_bouts=\"zero\"`, they would be 0).\n",
+ "- **BANOS TP** quantifies the boundary error, 1 / (1 + |Δstart| + |Δend|): C, 2 frames late at both ends, gets 1 / 5 = 0.2. It falls quickly with the shift.\n",
+ "- **BANOS SO** measures the temporal IoU of overlapping bouts: for C, 8 shared frames out of 12, 0.667.\n",
"\n",
- "Frame F1 says “shifted by 15 frames is equally wrong as completely missing”. \n",
- "BANOS tells you *how* the system is wrong — and by how much."
+ "Frame F1 gives C a single 0.80. \n",
+ "BANOS tells you *how* the system is wrong — every bout found (DA), but late (SO, TP)."
]
}
],
"metadata": {
"kernelspec": {
- "display_name": "banos",
+ "display_name": "Python 3",
"language": "python",
"name": "python3"
},
@@ -545,7 +547,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.12.9"
+ "version": "3.12.12"
}
},
"nbformat": 4,
diff --git a/matlab/BANOS/BANOS_score.m b/matlab/BANOS/BANOS_score.m
index 76341f1..3aaff3a 100644
--- a/matlab/BANOS/BANOS_score.m
+++ b/matlab/BANOS/BANOS_score.m
@@ -20,13 +20,39 @@
% overallMetrics (single average across all behaviors and recordings).
%
% NAME-VALUE OPTIONS:
-% 'matching' - 'greedy' (default). 'optimal' is not yet implemented in
-% MATLAB; use the Python package for optimal matching.
+% 'matching' - 'greedy' (default). 'optimal' is not yet implemented in
+% MATLAB; use the Python package for optimal matching.
+% 'missed_bouts' - how SO, TP and IC count a GT bout with no predicted
+% frame inside it: 'exclude' (default) leaves it out of
+% their averages, 'zero' adds a term equal to 0 to each.
+% Precision, recall and F1 do not depend on it.
+% 'aggregation' - how the metrics are combined: 'per_file' (default)
+% scores each recording on its own, then averages;
+% 'pooled' scores each behavior once on the bout counts
+% and terms of all recordings (see BANOSpoolComponents).
+% The outputs have the same form in both modes.
+% Option names and values are case-insensitive. An unknown name or value,
+% or a name without a value, raises an error (BANOS:unknownOption,
+% BANOS:invalidOption, BANOS:missingOptionValue).
+%
+% ERRORS:
+% BANOS:optimalNotImplemented - 'matching', 'optimal' (Python only).
+% BANOS:missingGroundTruth - a single prediction without its ground truth.
+% BANOS:invalidData - a first argument that is neither a cell
+% array nor a struct.
+% BANOSpreprocessData raises and warns about the data itself.
%
% NaN VALUES:
-% A behavior with no ground-truth bouts returns NaN for all metrics.
-% NaN values are correctly excluded from group and overall averages
-% (via nanmean), so they do not bias results.
+% With 'per_file', a behavior with no ground-truth bouts in a recording
+% scores 1.0 on all metrics if nothing is predicted there and 0.0
+% otherwise; groupMetrics and overallMetrics average with
+% mean(..., 'omitnan'), leaving NaN out.
+% With 'pooled', this 1.0/0.0 rule does not apply and overallMetrics
+% averages the pooled behaviors, leaving NaN out.
+% A metric is NaN when it is undefined: precision when nothing is
+% predicted, recall when there is no GT bout (pooled), F1 when there is no
+% bout at all (pooled), SO, TP and IC when they have no term (no overlap,
+% or no found GT bout).
%
% EXAMPLE — single recording:
% pred = [{'event1','event2'}; num2cell(predData)];
@@ -41,59 +67,58 @@
% disp(overallMet.f1_score)
%
% See also: BANOSpreprocessData, BANOScalculateBANOSForEachFile,
-% BANOSaggregateMetrics, Contents
+% BANOSaggregateMetrics, BANOSpoolComponents, Contents
% --- Parse name-value options ------------------------------------------
- matching = 'greedy';
- i = 1;
- while i <= length(varargin)
- if ischar(varargin{i}) || isstring(varargin{i})
- key = lower(char(varargin{i}));
- if strcmp(key, 'matching')
- if i + 1 > length(varargin)
- error('BANOS_score: ''matching'' option requires a value.');
- end
- matching = lower(char(varargin{i + 1}));
- i = i + 2;
- continue;
- end
+ % Single recording: the first cell array argument is the ground truth;
+ % the remaining arguments are name-value options.
+ gt = [];
+ if iscell(data)
+ gtIndex = find(cellfun(@iscell, varargin), 1);
+ if ~isempty(gtIndex)
+ gt = varargin{gtIndex};
+ varargin(gtIndex) = [];
end
- i = i + 1;
end
+ options = BANOSparseOptions(varargin, {'matching', 'missed_bouts', 'aggregation'});
+ pooled = strcmp(options.aggregation, 'pooled');
- if strcmp(matching, 'optimal')
- error('BANOS_score: optimal matching is not yet implemented in MATLAB. Use the Python package (banos.score(..., matching=''optimal'')).');
+ if strcmp(options.matching, 'optimal')
+ error('BANOS:optimalNotImplemented', 'BANOS_score: optimal matching is not yet implemented in MATLAB. Use the Python package (banos.score(..., matching=''optimal'')).');
end
% --- Detect single vs multi-recording mode ----------------------------
if isstruct(data)
% Multi-recording: data is already a properly formatted struct
[preprocessed, ~] = BANOSpreprocessData(data);
- fileMetrics = BANOScalculateBANOSForEachFile(preprocessed);
- [groupMetrics, overallMetrics] = BANOSaggregateMetrics(fileMetrics);
+ [fileMetrics, fileComponents] = BANOScalculateBANOSForEachFile(preprocessed, ...
+ 'missed_bouts', options.missed_bouts);
+ if pooled
+ [groupMetrics, overallMetrics] = BANOSpoolComponents(fileComponents);
+ else
+ [groupMetrics, overallMetrics] = BANOSaggregateMetrics(fileMetrics);
+ end
varargout{1} = groupMetrics;
varargout{2} = overallMetrics;
elseif iscell(data)
- % Single-recording: data = pred cell array, varargin{1} = gt cell array
- % (after matching option parsing, the first non-option arg is gt)
- gt = [];
- for j = 1:length(varargin)
- if iscell(varargin{j})
- gt = varargin{j};
- break;
- end
- end
+ % Single-recording: data = pred cell array, gt = first cell array in
+ % varargin (taken out above, before the options were parsed)
if isempty(gt)
- error('BANOS_score: single-recording mode requires two cell array arguments: BANOS_score(pred, gt).');
+ error('BANOS:missingGroundTruth', 'BANOS_score: single-recording mode requires two cell array arguments: BANOS_score(pred, gt).');
end
tmpData.single_recording = {data, gt};
[preprocessed, ~] = BANOSpreprocessData(tmpData);
- fileMetrics = BANOScalculateBANOSForEachFile(preprocessed);
- varargout{1} = fileMetrics.single_recording;
+ [fileMetrics, fileComponents] = BANOScalculateBANOSForEachFile(preprocessed, ...
+ 'missed_bouts', options.missed_bouts);
+ if pooled
+ varargout{1} = BANOSpoolComponents(fileComponents);
+ else
+ varargout{1} = fileMetrics.single_recording;
+ end
else
- error('BANOS_score: first argument must be a cell array (single recording) or a struct (multiple recordings).');
+ error('BANOS:invalidData', 'BANOS_score: first argument must be a cell array (single recording) or a struct (multiple recordings).');
end
end
diff --git a/matlab/BANOS/BANOSaggregateMetrics.m b/matlab/BANOS/BANOSaggregateMetrics.m
index 9956d8c..e3c079f 100644
--- a/matlab/BANOS/BANOSaggregateMetrics.m
+++ b/matlab/BANOS/BANOSaggregateMetrics.m
@@ -37,8 +37,8 @@
for j = 1:length(metricFields)
field = metricFields{j};
for behavior = fieldnames(groupMetrics)'
- groupMetrics.(behavior{1}).(field) = nanmean(groupMetrics.(behavior{1}).(field));
+ groupMetrics.(behavior{1}).(field) = mean(groupMetrics.(behavior{1}).(field), 'omitnan');
end
- overallMetrics.(field) = nanmean(overallMetrics.(field));
+ overallMetrics.(field) = mean(overallMetrics.(field), 'omitnan');
end
end
\ No newline at end of file
diff --git a/matlab/BANOS/BANOScalculateBANOSForEachFile.m b/matlab/BANOS/BANOScalculateBANOSForEachFile.m
index 35894cc..e6fda0f 100644
--- a/matlab/BANOS/BANOScalculateBANOSForEachFile.m
+++ b/matlab/BANOS/BANOScalculateBANOSForEachFile.m
@@ -1,21 +1,31 @@
-function banosMetrics = BANOScalculateBANOSForEachFile(preprocessedData)
+function [banosMetrics, fileComponents] = BANOScalculateBANOSForEachFile(preprocessedData, varargin)
% Calculates BANOS metrics for each behavior in each file.
%
% Arguments:
% preprocessedData - A struct with preprocessed prediction and ground truth data.
+ % 'missed_bouts' - (name-value) how SO, TP and IC count a GT bout that no
+ % predicted bout overlaps: 'exclude' (default) or 'zero'
+ % (see BANOScalculateSO).
%
% Returns:
% banosMetrics - A struct containing BANOS metrics for each behavior in each file.
+ % fileComponents - A struct containing the metric components (bout counts
+ % and terms) for each behavior in each file, to pool with
+ % BANOSpoolComponents (see BANOScomputeBehaviorMetrics).
+ options = BANOSparseOptions(varargin, {'missed_bouts'});
fileNames = fieldnames(preprocessedData);
banosMetrics = struct();
+ fileComponents = struct();
for i = 1:length(fileNames)
fileName = fileNames{i};
extractMatrix = preprocessedData.(fileName);
predMatrix = extractMatrix{1};
gtMatrix = extractMatrix{2};
- behaviorMetrics = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix);
+ [behaviorMetrics, behaviorComponents] = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix, ...
+ 'missed_bouts', options.missed_bouts);
banosMetrics.(fileName) = behaviorMetrics;
+ fileComponents.(fileName) = behaviorComponents;
end
end
diff --git a/matlab/BANOS/BANOScalculateIC.m b/matlab/BANOS/BANOScalculateIC.m
index e4cd775..b300ca2 100644
--- a/matlab/BANOS/BANOScalculateIC.m
+++ b/matlab/BANOS/BANOScalculateIC.m
@@ -1,35 +1,43 @@
-function ic = BANOScalculateIC(predSequence, gtBouts)
- % Calculates the Intra-bout Continuity (IC) for each ground truth bout.
+function [ic, icTerms] = BANOScalculateIC(predSequence, gtBouts, varargin)
+ % Calculates the Intra-bout Continuity (IC) over the found ground truth bouts.
%
- % IC = 1 - (switches / boutLength) averaged over all GT bouts.
- % Zero-length bouts are excluded from the average (contribute NaN).
- % Returns NaN if gtBouts is empty or all bouts are zero-length
- % (matches Python behaviour — NaN propagates through nanmean).
+ % IC = 1 - (switches / boutLength) averaged over the found GT bouts, where
+ % boutLength = end - start is the number of frame pairs of the bout.
+ % A GT bout is found when at least one of its frames is predicted (value 1).
+ % One-frame bouts (boutLength = 0) are always excluded from the average.
+ % Returns NaN if there is no term (matches Python behaviour — the means
+ % over recordings and behaviors leave NaN out).
%
% Arguments:
% predSequence - The sequence of predictions (1-based index vector).
% gtBouts - Cell array of ground truth bouts {[start, end], ...}.
+ % 'missed_bouts' - (name-value) how a missed GT bout is counted:
+ % 'exclude' (default) leaves it out, 'zero' adds a term
+ % equal to 0.
%
% Returns:
- % ic - The average IC across GT bouts, or NaN if undefined.
+ % ic - The average of the terms, or NaN if undefined.
+ % icTerms - Row vector of the terms, one per found GT bout longer than
+ % one frame, and 0 for each missed one with 'zero'.
- icScores = [];
+ options = BANOSparseOptions(varargin, {'missed_bouts'});
+ icTerms = [];
for i = 1:length(gtBouts)
boutLength = gtBouts{i}(2) - gtBouts{i}(1);
- switches = BANOScountSwitchesWithinBout(predSequence, gtBouts{i}(1), gtBouts{i}(2));
- if boutLength > 0
- icScores(end + 1) = 1 - (switches / boutLength);
- else
- icScores(end + 1) = NaN; % zero-length bout: undefined, excluded from mean
+ if boutLength == 0
+ continue; % zero-length bout: no frame pair, no term
+ end
+ if any(predSequence(gtBouts{i}(1):gtBouts{i}(2)) == 1)
+ switches = BANOScountSwitchesWithinBout(predSequence, gtBouts{i}(1), gtBouts{i}(2));
+ icTerms(end + 1) = 1 - (switches / boutLength);
+ elseif strcmp(options.missed_bouts, 'zero')
+ icTerms(end + 1) = 0; % missed bout
end
end
- % nanmean excludes NaN entries from both numerator and denominator,
- % matching Python's: sum(valid) / len(valid).
- % Returns NaN when icScores is empty or all values are NaN.
- if isempty(icScores)
+ if isempty(icTerms)
ic = NaN;
else
- ic = nanmean(icScores);
+ ic = mean(icTerms);
end
end
diff --git a/matlab/BANOS/BANOScalculatePrecisionRecallF1.m b/matlab/BANOS/BANOScalculatePrecisionRecallF1.m
index fcd2bc5..b87280e 100644
--- a/matlab/BANOS/BANOScalculatePrecisionRecallF1.m
+++ b/matlab/BANOS/BANOScalculatePrecisionRecallF1.m
@@ -1,8 +1,11 @@
function [precision, recall, f1] = BANOScalculatePrecisionRecallF1(tp, fp, fn)
% Calculates Precision, Recall, and F1 score at the bout level.
%
- % Returns NaN (not 0) when the denominator is zero, matching Python behaviour.
- % NaN values propagate correctly through BANOSaggregateMetrics (nanmean).
+ % Precision and recall are NaN (not 0) when their denominator is zero,
+ % matching Python behaviour. F1 is 0 when no bout is matched but some
+ % bouts were predicted or missed (TP = 0 and FP + FN > 0), and NaN only
+ % when all three counts are zero.
+ % BANOSaggregateMetrics leaves NaN values out of its means.
%
% Arguments:
% tp - Count of True Positives (matched bout pairs).
@@ -12,7 +15,8 @@
% Returns:
% precision - TP / (TP + FP), or NaN if TP + FP = 0.
% recall - TP / (TP + FN), or NaN if TP + FN = 0.
- % f1 - Harmonic mean of precision and recall, or NaN if undefined.
+ % f1 - Harmonic mean of precision and recall; 0 if TP = 0 and
+ % FP + FN > 0; NaN if TP = FP = FN = 0.
if (tp + fp) > 0
precision = tp / (tp + fp);
@@ -26,7 +30,9 @@
recall = NaN;
end
- if ~isnan(precision) && ~isnan(recall) && (precision + recall) > 0
+ if tp == 0 && (fp + fn) > 0
+ f1 = 0;
+ elseif ~isnan(precision) && ~isnan(recall) && (precision + recall) > 0
f1 = 2 * precision * recall / (precision + recall);
else
f1 = NaN;
diff --git a/matlab/BANOS/BANOScalculateSO.m b/matlab/BANOS/BANOScalculateSO.m
index ac41cb5..fdb620c 100644
--- a/matlab/BANOS/BANOScalculateSO.m
+++ b/matlab/BANOS/BANOScalculateSO.m
@@ -1,26 +1,39 @@
-function so = BANOScalculateSO(predBouts, gtBouts)
- % Calculates the Segment Overlap (SO) for each true positive bout.
+function [so, soTerms] = BANOScalculateSO(predBouts, gtBouts, varargin)
+ % Calculates the Segment Overlap (SO) over the overlapping bout pairs.
+ %
+ % SO is the average tIoU of every overlapping (predicted, GT) bout pair.
%
% Arguments:
% predBouts - Cell array of predicted bouts.
% gtBouts - Cell array of ground truth bouts.
+ % 'missed_bouts' - (name-value) how a GT bout that no predicted bout
+ % overlaps is counted: 'exclude' (default) leaves it
+ % out, 'zero' adds a term equal to 0.
%
% Returns:
- % so - The average SO for the true positive bouts.
+ % so - The average of the terms, or NaN if there is no term.
+ % soTerms - Row vector of the terms: the tIoU of each overlapping pair,
+ % GT bout by GT bout, and 0 for each missed GT bout with 'zero'.
- tiouScores = [];
+ options = BANOSparseOptions(varargin, {'missed_bouts'});
+ soTerms = [];
for i = 1:length(gtBouts)
+ found = false;
for j = 1:length(predBouts)
if BANOSoverlaps(predBouts{j}, gtBouts{i})
- tiouScores(end + 1) = BANOScalculateTiou(predBouts{j}, gtBouts{i});
+ soTerms(end + 1) = BANOScalculateTiou(predBouts{j}, gtBouts{i});
+ found = true;
end
end
+ if ~found && strcmp(options.missed_bouts, 'zero')
+ soTerms(end + 1) = 0;
+ end
end
- % Return NaN if no overlapping bouts were found (matches Python behaviour).
- % NaN propagates correctly through BANOSaggregateMetrics (nanmean).
- if isempty(tiouScores)
+ % Return NaN if there is no term (matches Python behaviour).
+ % BANOSaggregateMetrics leaves NaN out of its means.
+ if isempty(soTerms)
so = NaN;
else
- so = mean(tiouScores);
+ so = mean(soTerms);
end
end
diff --git a/matlab/BANOS/BANOScalculateTP.m b/matlab/BANOS/BANOScalculateTP.m
index 904827b..86ad21e 100644
--- a/matlab/BANOS/BANOScalculateTP.m
+++ b/matlab/BANOS/BANOScalculateTP.m
@@ -1,27 +1,41 @@
-function tp = BANOScalculateTP(predBouts, gtBouts)
- % Calculates the Temporal Precision (TP) for each true positive bout.
+function [tp, tpTerms] = BANOScalculateTP(predBouts, gtBouts, varargin)
+ % Calculates the Temporal Precision (TP) over the overlapping bout pairs.
+ %
+ % TP is the average of 1 / (1 + |start deviation| + |end deviation|)
+ % over every overlapping (predicted, GT) bout pair.
%
% Arguments:
% predBouts - Cell array of predicted bouts.
% gtBouts - Cell array of ground truth bouts.
+ % 'missed_bouts' - (name-value) how a GT bout that no predicted bout
+ % overlaps is counted: 'exclude' (default) leaves it
+ % out, 'zero' adds a term equal to 0.
%
% Returns:
- % tp - The average TP for the true positive bouts.
+ % tp - The average of the terms, or NaN if there is no term.
+ % tpTerms - Row vector of the terms, one per overlapping pair, GT bout
+ % by GT bout, and 0 for each missed GT bout with 'zero'.
- tpScores = [];
+ options = BANOSparseOptions(varargin, {'missed_bouts'});
+ tpTerms = [];
for i = 1:length(gtBouts)
+ found = false;
for j = 1:length(predBouts)
if BANOSoverlaps(predBouts{j}, gtBouts{i})
startDiff = abs(predBouts{j}(1) - gtBouts{i}(1));
endDiff = abs(predBouts{j}(2) - gtBouts{i}(2));
- tpScores(end + 1) = 1 / (1 + startDiff + endDiff);
+ tpTerms(end + 1) = 1 / (1 + startDiff + endDiff);
+ found = true;
end
end
+ if ~found && strcmp(options.missed_bouts, 'zero')
+ tpTerms(end + 1) = 0;
+ end
end
- % Return NaN if no overlapping bouts were found (matches Python behaviour).
- if isempty(tpScores)
+ % Return NaN if there is no term (matches Python behaviour).
+ if isempty(tpTerms)
tp = NaN;
else
- tp = mean(tpScores);
+ tp = mean(tpTerms);
end
end
diff --git a/matlab/BANOS/BANOScomputeBehaviorMetrics.m b/matlab/BANOS/BANOScomputeBehaviorMetrics.m
index a5ee719..3e30a48 100644
--- a/matlab/BANOS/BANOScomputeBehaviorMetrics.m
+++ b/matlab/BANOS/BANOScomputeBehaviorMetrics.m
@@ -1,18 +1,41 @@
-function behaviorMetrics = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix)
+function [behaviorMetrics, behaviorComponents] = BANOScomputeBehaviorMetrics(predMatrix, gtMatrix, varargin)
% Computes metrics for each behavior in the prediction and ground truth matrices.
+ %
+ % A behavior with no GT bout scores 1.0 on all six metrics if nothing is
+ % predicted (correct absence) and 0.0 otherwise (false detection).
+ %
+ % Name-value option:
+ % 'missed_bouts' - how SO, TP and IC count a GT bout that no predicted
+ % bout overlaps: 'exclude' (default) or 'zero'
+ % (see BANOScalculateSO).
+ %
+ % Returns:
+ % behaviorMetrics - Struct with one field per behavior holding .precision,
+ % .recall, .f1_score, .so, .tp and .ic.
+ % behaviorComponents - Struct with one field per behavior holding what the
+ % metrics are computed from: the bout counts .n_tp,
+ % .n_fp, .n_fn and the terms .so_terms, .tp_terms,
+ % .ic_terms averaged by SO, TP and IC. Summed over
+ % files by BANOSpoolComponents.
+ options = BANOSparseOptions(varargin, {'missed_bouts'});
behaviorMetrics = struct();
+ behaviorComponents = struct();
for behavior = 1:size(predMatrix, 2)
behaviorName = predMatrix{1,behavior};
- predBouts = BANOSidentifyBouts(cell2mat(predMatrix(2:end, behavior)));
+ predSequence = cell2mat(predMatrix(2:end, behavior));
+ predBouts = BANOSidentifyBouts(predSequence);
gtBouts = BANOSidentifyBouts(cell2mat(gtMatrix(2:end, behavior)));
+ [nTp, nFp, nFn] = BANOScalculateTpFpFn(predBouts, gtBouts);
+ [so, soTerms] = BANOScalculateSO(predBouts, gtBouts, 'missed_bouts', options.missed_bouts);
+ [tp, tpTerms] = BANOScalculateTP(predBouts, gtBouts, 'missed_bouts', options.missed_bouts);
+ [ic, icTerms] = BANOScalculateIC(predSequence, gtBouts, 'missed_bouts', options.missed_bouts);
+ behaviorComponents.(behaviorName) = struct('n_tp', nTp, 'n_fp', nFp, 'n_fn', nFn, ...
+ 'so_terms', soTerms, 'tp_terms', tpTerms, 'ic_terms', icTerms);
+
if ~isempty(gtBouts)
- [tp, fp, fn] = BANOScalculateTpFpFn(predBouts, gtBouts);
- [precision, recall, f1] = BANOScalculatePrecisionRecallF1(tp, fp, fn);
- so = BANOScalculateSO(predBouts, gtBouts);
- tp = BANOScalculateTP(predBouts, gtBouts);
- ic = BANOScalculateIC(cell2mat(predMatrix(2:end, behavior)), gtBouts);
+ [precision, recall, f1] = BANOScalculatePrecisionRecallF1(nTp, nFp, nFn);
elseif isempty(predBouts)
% Correct absence: system correctly detected nothing
precision = 1.0; recall = 1.0; f1 = 1.0;
diff --git a/matlab/BANOS/BANOSidentifyBouts.m b/matlab/BANOS/BANOSidentifyBouts.m
index ce659bf..0ad54fc 100644
--- a/matlab/BANOS/BANOSidentifyBouts.m
+++ b/matlab/BANOS/BANOSidentifyBouts.m
@@ -5,7 +5,8 @@
% sequence - A binary vector representing predictions or ground truth.
%
% Returns:
- % bouts - A cell array of tuples, each tuple contains start and end indices of a bout.
+ % bouts - A cell array with one [start, end] row vector per bout (1-based
+ % indices, both included).
bouts = {};
start = NaN;
diff --git a/matlab/BANOS/BANOSparseOptions.m b/matlab/BANOS/BANOSparseOptions.m
new file mode 100644
index 0000000..82a8d6a
--- /dev/null
+++ b/matlab/BANOS/BANOSparseOptions.m
@@ -0,0 +1,58 @@
+function options = BANOSparseOptions(args, names)
+ % Reads and validates BANOS name-value options.
+ %
+ % Arguments:
+ % args - Cell array of name-value pairs, e.g. {'missed_bouts', 'zero'}.
+ % Names and values are case-insensitive.
+ % names - Cell array of the option names the caller accepts.
+ %
+ % Returns:
+ % options - Struct with one field per accepted name, holding the given
+ % value in lowercase, or the default value.
+ %
+ % Options and their values (the first value is the default):
+ % 'matching' - 'greedy', 'optimal'
+ % 'missed_bouts' - 'exclude', 'zero'
+ % 'aggregation' - 'per_file', 'pooled'
+ %
+ % Errors:
+ % BANOS:unknownOption - a name the caller does not accept.
+ % BANOS:missingOptionValue - a name without a value.
+ % BANOS:invalidOption - a value that is not allowed for its name.
+
+ allowedValues = struct( ...
+ 'matching', {{'greedy', 'optimal'}}, ...
+ 'missed_bouts', {{'exclude', 'zero'}}, ...
+ 'aggregation', {{'per_file', 'pooled'}});
+
+ options = struct();
+ for k = 1:numel(names)
+ values = allowedValues.(names{k});
+ options.(names{k}) = values{1};
+ end
+ accepted = strjoin(strcat('''', names, ''''), ', ');
+
+ for i = 1:2:numel(args)
+ name = args{i};
+ if ~(ischar(name) || (isstring(name) && isscalar(name)))
+ error('BANOS:unknownOption', ...
+ 'Option names must be text. Accepted options: %s.', accepted);
+ end
+ name = lower(char(name));
+ if ~any(strcmp(name, names))
+ error('BANOS:unknownOption', ...
+ 'Unknown option ''%s''. Accepted options: %s.', name, accepted);
+ end
+ if i + 1 > numel(args)
+ error('BANOS:missingOptionValue', 'Option ''%s'' requires a value.', name);
+ end
+ value = args{i + 1};
+ allowed = allowedValues.(name);
+ if ~(ischar(value) || (isstring(value) && isscalar(value))) ...
+ || ~any(strcmpi(char(value), allowed))
+ error('BANOS:invalidOption', 'Option ''%s'' must be %s.', ...
+ name, strjoin(strcat('''', allowed, ''''), ' or '));
+ end
+ options.(name) = lower(char(value));
+ end
+end
diff --git a/matlab/BANOS/BANOSpoolComponents.m b/matlab/BANOS/BANOSpoolComponents.m
new file mode 100644
index 0000000..25fef46
--- /dev/null
+++ b/matlab/BANOS/BANOSpoolComponents.m
@@ -0,0 +1,70 @@
+function [groupMetrics, overallMetrics] = BANOSpoolComponents(fileComponents)
+ % Pools the metric components of all files and scores each behavior once.
+ %
+ % For each behavior, the TP, FP and FN counts are summed over the files and
+ % give precision, recall and F1 (BANOScalculatePrecisionRecallF1); SO, TP
+ % and IC are the means of all their terms over all files. A file without
+ % the behavior adds its false bouts to FP and no SO, TP or IC term: the
+ % 1.0/0.0 absent-behavior rule of the per-file metrics does not apply.
+ % A behavior with no GT bout and no predicted bout in any file is NaN on
+ % all six metrics.
+ %
+ % Arguments:
+ % fileComponents - Struct with one field per file, each holding the
+ % behavior components returned as second output by
+ % BANOScalculateBANOSForEachFile.
+ %
+ % Returns:
+ % groupMetrics - Pooled metrics for each behavior.
+ % overallMetrics - Mean of the pooled metrics over the behaviors, NaN left
+ % out (mean with 'omitnan').
+
+ pooled = struct();
+ fileNames = fieldnames(fileComponents);
+ for i = 1:length(fileNames)
+ components = fileComponents.(fileNames{i});
+ behaviors = fieldnames(components);
+ for j = 1:length(behaviors)
+ behavior = behaviors{j};
+ c = components.(behavior);
+ if ~isfield(pooled, behavior)
+ pooled.(behavior) = struct('n_tp', 0, 'n_fp', 0, 'n_fn', 0, ...
+ 'so_terms', [], 'tp_terms', [], 'ic_terms', []);
+ end
+ p = pooled.(behavior);
+ p.n_tp = p.n_tp + c.n_tp;
+ p.n_fp = p.n_fp + c.n_fp;
+ p.n_fn = p.n_fn + c.n_fn;
+ p.so_terms = [p.so_terms, c.so_terms];
+ p.tp_terms = [p.tp_terms, c.tp_terms];
+ p.ic_terms = [p.ic_terms, c.ic_terms];
+ pooled.(behavior) = p;
+ end
+ end
+
+ groupMetrics = struct();
+ overallMetrics = struct('precision', [], 'recall', [], 'f1_score', [], 'so', [], 'tp', [], 'ic', []);
+ metricFields = fieldnames(overallMetrics);
+ behaviors = fieldnames(pooled);
+ for j = 1:length(behaviors)
+ p = pooled.(behaviors{j});
+ [precision, recall, f1] = BANOScalculatePrecisionRecallF1(p.n_tp, p.n_fp, p.n_fn);
+ % Mean of each term list; NaN when a list is empty (no term)
+ terms = {p.so_terms, p.tp_terms, p.ic_terms};
+ termMeans = NaN(1, 3);
+ for k = 1:3
+ if ~isempty(terms{k})
+ termMeans(k) = mean(terms{k});
+ end
+ end
+ groupMetrics.(behaviors{j}) = struct('precision', precision, 'recall', recall, ...
+ 'f1_score', f1, 'so', termMeans(1), 'tp', termMeans(2), 'ic', termMeans(3));
+ for k = 1:length(metricFields)
+ overallMetrics.(metricFields{k})(end + 1) = groupMetrics.(behaviors{j}).(metricFields{k});
+ end
+ end
+
+ for k = 1:length(metricFields)
+ overallMetrics.(metricFields{k}) = mean(overallMetrics.(metricFields{k}), 'omitnan');
+ end
+end
diff --git a/matlab/BANOS/BANOSpreprocessData.m b/matlab/BANOS/BANOSpreprocessData.m
index d94a72c..23920fa 100644
--- a/matlab/BANOS/BANOSpreprocessData.m
+++ b/matlab/BANOS/BANOSpreprocessData.m
@@ -2,9 +2,19 @@
% Preprocesses prediction and ground truth data for BANOS computation.
%
% Steps performed for each file:
- % 1. Replace NaN values with 0.
- % 2. Drop columns whose values are not strictly binary (0/1).
- % 3. Retain only behavior columns whose names appear in BOTH pred and gt.
+ % 1. Check that pred and gt have the same number of frames (rows 2+),
+ % and at least one.
+ % 2. Replace NaN values with 0.
+ % 3. Drop columns whose values are not strictly binary (0/1).
+ % 4. Retain only behavior columns whose names appear in BOTH pred and gt.
+ %
+ % Errors and warnings:
+ % BANOS:lengthMismatch (error) - pred and gt differ in number of frames.
+ % BANOS:noFrame (error) - a file has no frame.
+ % BANOS:nanReplaced (warning) - NaN values of a column replaced by 0.
+ % BANOS:columnDropped (warning) - a column dropped (values other than
+ % 0 and 1, or missing or invalid on the other side).
+ % Each message names the file and the column.
%
% Arguments:
% dataDict - struct; each field is a recording name whose value is
@@ -27,6 +37,18 @@
predMatrix = extractMatrix{1};
gtMatrix = extractMatrix{2};
+ % --- Check the number of frames (rows 2+) ---
+ nPredFrames = max(size(predMatrix, 1) - 1, 0);
+ nGtFrames = max(size(gtMatrix, 1) - 1, 0);
+ if nPredFrames ~= nGtFrames
+ error('BANOS:lengthMismatch', ...
+ 'File ''%s'': the prediction has %d frames but the ground truth has %d.', ...
+ fileName, nPredFrames, nGtFrames);
+ end
+ if nPredFrames == 0
+ error('BANOS:noFrame', 'File ''%s'' has no frame.', fileName);
+ end
+
% --- Separate headers (row 1) from data (rows 2+) ---
predHeaders = predMatrix(1, :);
gtHeaders = gtMatrix(1, :);
@@ -34,12 +56,30 @@
gtData = cell2mat(gtMatrix(2:end, :));
% --- Replace NaNs with zeros ---
+ sides = {'prediction', 'ground truth'};
+ nanCounts = {sum(isnan(predData), 1), sum(isnan(gtData), 1)};
+ sideHeaders = {predHeaders, gtHeaders};
+ for s = 1:2
+ for k = find(nanCounts{s} > 0)
+ warning('BANOS:nanReplaced', ...
+ 'File ''%s'': %d NaN value(s) in %s column ''%s'' replaced by 0.', ...
+ fileName, nanCounts{s}(k), sides{s}, sideHeaders{s}{k});
+ end
+ end
predData(isnan(predData)) = 0;
gtData(isnan(gtData)) = 0;
% --- Drop non-logical columns from data, track which indices ---
[predData, predDropped] = BANOSdropNonLogicalColumns(predData);
[gtData, gtDropped] = BANOSdropNonLogicalColumns(gtData);
+ sideDropped = {predDropped, gtDropped};
+ for s = 1:2
+ for k = sideDropped{s}
+ warning('BANOS:columnDropped', ...
+ 'File ''%s'': %s column ''%s'' dropped (values other than 0 and 1).', ...
+ fileName, sides{s}, sideHeaders{s}{k});
+ end
+ end
% Update header arrays to reflect dropped columns
predKeepIdx = setdiff(1:length(predHeaders), predDropped);
@@ -47,6 +87,16 @@
predHeaders = predHeaders(predKeepIdx);
gtHeaders = gtHeaders(gtKeepIdx);
+ % --- Warn about columns present on one side only ---
+ oneSide = {setdiff(predHeaders, gtHeaders, 'stable'), setdiff(gtHeaders, predHeaders, 'stable')};
+ for s = 1:2
+ for k = 1:numel(oneSide{s})
+ warning('BANOS:columnDropped', ...
+ 'File ''%s'': %s column ''%s'' dropped (missing or invalid in the %s).', ...
+ fileName, sides{s}, oneSide{s}{k}, sides{3 - s});
+ end
+ end
+
% --- Retain only columns present in both pred and gt ---
commonHeaders = intersect(predHeaders, gtHeaders, 'stable');
[~, predIdx] = ismember(commonHeaders, predHeaders);
diff --git a/matlab/BANOS/Contents.m b/matlab/BANOS/Contents.m
index 3f2cb56..cb7b6b2 100644
--- a/matlab/BANOS/Contents.m
+++ b/matlab/BANOS/Contents.m
@@ -1,5 +1,5 @@
% BANOS Behavior ANnOtation Score toolbox.
-% Version 0.2.0 23-Jan-2025
+% Version 1.1.0 30-Oct-2026
%
% High-level entry point (mirrors Python banos.score):
% BANOS_score - Score one or multiple annotation pairs.
@@ -7,7 +7,8 @@
% Pipeline functions:
% BANOSpreprocessData - Fill NaN, drop non-binary columns, align headers.
% BANOScalculateBANOSForEachFile - Compute metrics for each recording.
-% BANOSaggregateMetrics - Average metrics across recordings (nanmean).
+% BANOSaggregateMetrics - Average metrics across recordings (NaN left out).
+% BANOSpoolComponents - Pool bout counts and terms across recordings.
% BANOScomputeBehaviorMetrics - Per-behavior metric computation for one recording.
%
% Metric functions:
@@ -25,6 +26,7 @@
% BANOSisLogicalColumn - Test whether a column contains only 0 and 1.
% BANOSmatchHeaders - Retain only common behavior columns in pred and GT.
% BANOSoverlaps - Test whether two bout intervals overlap.
+% BANOSparseOptions - Read and validate name-value options.
%
% Setup:
% setup - Add this toolbox to the MATLAB path.
diff --git a/pyproject.toml b/pyproject.toml
index d8f9017..de37896 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,15 +1,32 @@
[project]
name = "banos"
-version = "1.0.0"
+version = "1.1.0"
description = "Behavior ANnOtation Score: ethological metrics for evaluating behavior annotation quality"
readme = "README.md"
-license = { text = "MIT" }
+license = "MIT"
+license-files = ["LICENSE"]
requires-python = ">=3.11"
-keywords = ["behavior", "annotation", "ethology", "video", "scoring", "metrics"]
+authors = [{ name = "Benoît J. T. Girard" }]
+maintainers = [{ name = "Benoît J. T. Girard" }]
+keywords = [
+ "behavior",
+ "annotation",
+ "ethology",
+ "video",
+ "scoring",
+ "metrics",
+ "computational ethology",
+ "animal behavior",
+ "behavioral neuroscience",
+ "action segmentation",
+ "bout detection",
+ "evaluation metrics",
+ "inter-annotator agreement",
+ "rodent",
+]
classifiers = [
"Development Status :: 5 - Production/Stable",
"Intended Audience :: Science/Research",
- "License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
@@ -21,21 +38,14 @@ dependencies = [
"numpy>=1.24",
"pandas>=2.0",
"scipy>=1.10",
- "matplotlib>=3.7",
- "seaborn>=0.12",
]
[project.optional-dependencies]
-dev = [
- "pytest>=7.4",
- "pytest-cov>=4.1",
- "pytest-html>=4.0",
- "pre-commit>=3.5",
- "ruff>=0.4",
-]
notebook = [
"jupyter>=1.0",
"nbformat>=5.9",
+ "matplotlib>=3.7",
+ "seaborn>=0.12",
]
calms21 = [
"tensorflow>=2.12",
@@ -49,12 +59,26 @@ Repository = "https://github.com/BelloneLab/BANOS"
Issues = "https://github.com/BelloneLab/BANOS/issues"
[build-system]
-requires = ["hatchling"]
+requires = ["hatchling>=1.27,<2"] # 1.27 writes the SPDX license (License-Expression)
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/banos"]
+[tool.hatch.build.targets.sdist]
+# The package, its tests and the data they read; not the MATLAB code,
+# notebooks, Keras model, reports or images of the repository.
+include = [
+ "/src",
+ "/tests",
+ "/data/dataset_human_vs_human",
+ "/demo/quickstart.py",
+ "/README.md",
+ "/LICENSE",
+ "/CHANGELOG.md",
+]
+exclude = ["/tests/fixtures/matlab"]
+
[tool.ruff]
line-length = 100
@@ -77,7 +101,10 @@ dev = [
"nbformat>=5.10.4",
"pytest>=7.4",
"pytest-cov>=4.1",
- "pytest-html>=4.2.0",
"pre-commit>=3.5",
"ruff>=0.4",
+ "mypy>=2.4.0",
+ # Stubs of the pandas and scipy versions in uv.lock (3.0 and 1.17)
+ "pandas-stubs>=3.0.5.260914,<3.1",
+ "scipy-stubs>=1.17.1.5,<1.18",
]
diff --git a/reports/fixture_comparison.html b/reports/fixture_comparison.html
index 9f6dcef..5749f35 100644
--- a/reports/fixture_comparison.html
+++ b/reports/fixture_comparison.html
@@ -3,18 +3,28 @@
SAME = match NULL->VALUE = Unexpected divergence (should not occur in v0.2.0) CHANGED = Unexpected numeric mismatch VALUE->NULL = unexpected
+CHANGED: 0
+
SAME = match VALUE->NULL = Unexpected divergence (should not occur in v0.2.0) NULL->VALUE = Unexpected divergence (should not occur in v0.2.0) CHANGED = Unexpected numeric mismatch
Location
Metric
Python v0.2.0
Matlab v0.2.0
Status
Recording_1/overall
precision
0.928261
0.928261
SAME
Recording_1/overall
recall
0.781293
0.781293
SAME
Recording_1/overall
f1_score
0.836455
0.836455
SAME
Recording_1/overall
so
0.761228
0.761228
SAME
Recording_1/overall
tp
0.306230
0.306230
SAME
Recording_1/overall
ic
0.991718
0.991718
SAME
Recording_1/group/attack
precision
1.000000
1.000000
SAME
Recording_1/group/attack
recall
1.000000
1.000000
SAME
Recording_1/group/attack
f1_score
1.000000
1.000000
SAME
Recording_1/group/attack
so
1.000000
1.000000
SAME
Recording_1/group/attack
tp
1.000000
1.000000
SAME
Recording_1/group/attack
ic
1.000000
1.000000
SAME
Recording_1/group/investigation
precision
0.800000
0.800000
SAME
Recording_1/group/investigation
recall
0.734694
0.734694
SAME
Recording_1/group/investigation
f1_score
0.765957
0.765957
SAME
Recording_1/group/investigation
so
0.544410
0.544410
SAME
Recording_1/group/investigation
tp
0.084636
0.084636
SAME
Recording_1/group/investigation
ic
0.982941
0.982941
SAME
Recording_1/group/mount
precision
1.000000
1.000000
SAME
Recording_1/group/mount
recall
0.533333
0.533333
SAME
Recording_1/group/mount
f1_score
0.695652
0.695652
SAME
Recording_1/group/mount
so
0.733277
0.733277
SAME
Recording_1/group/mount
tp
0.078471
0.078471
SAME
Recording_1/group/mount
ic
0.990861
0.990861
SAME
Recording_1/group/other
precision
0.913043
0.913043
SAME
Recording_1/group/other
recall
0.857143
0.857143
SAME
Recording_1/group/other
f1_score
0.884211
0.884211
SAME
Recording_1/group/other
so
0.767227
0.767227
SAME
Recording_1/group/other
tp
0.061814
0.061814
SAME
Recording_1/group/other
ic
0.993069
0.993069
SAME
Recording_2/overall
precision
0.967105
0.967105
SAME
Recording_2/overall
recall
0.839924
0.839924
SAME
Recording_2/overall
f1_score
0.897301
0.897301
SAME
Recording_2/overall
so
0.825946
0.825946
SAME
Recording_2/overall
tp
0.309082
0.309082
SAME
Recording_2/overall
ic
0.993907
0.993907
SAME
Recording_2/group/attack
precision
1.000000
1.000000
SAME
Recording_2/group/attack
recall
1.000000
1.000000
SAME
Recording_2/group/attack
f1_score
1.000000
1.000000
SAME
Recording_2/group/attack
so
1.000000
1.000000
SAME
Recording_2/group/attack
tp
1.000000
1.000000
SAME
Recording_2/group/attack
ic
1.000000
1.000000
SAME
Recording_2/group/investigation
precision
0.894737
0.894737
SAME
Recording_2/group/investigation
recall
0.739130
0.739130
SAME
Recording_2/group/investigation
f1_score
0.809524
0.809524
SAME
Recording_2/group/investigation
so
0.660942
0.660942
SAME
Recording_2/group/investigation
tp
0.046151
0.046151
SAME
Recording_2/group/investigation
ic
0.989051
0.989051
SAME
Recording_2/group/mount
precision
1.000000
1.000000
SAME
Recording_2/group/mount
recall
0.833333
0.833333
SAME
Recording_2/group/mount
f1_score
0.909091
0.909091
SAME
Recording_2/group/mount
so
0.931754
0.931754
SAME
Recording_2/group/mount
tp
0.145170
0.145170
SAME
Recording_2/group/mount
ic
0.994648
0.994648
SAME
Recording_2/group/other
precision
0.973684
0.973684
SAME
Recording_2/group/other
recall
0.787234
0.787234
SAME
Recording_2/group/other
f1_score
0.870588
0.870588
SAME
Recording_2/group/other
so
0.711086
0.711086
SAME
Recording_2/group/other
tp
0.045008
0.045008
SAME
Recording_2/group/other
ic
0.991930
0.991930
SAME
Recording_3/overall
precision
0.959481
0.959481
SAME
Recording_3/overall
recall
0.850967
0.850967
SAME
Recording_3/overall
f1_score
0.898627
0.898627
SAME
Recording_3/overall
so
0.723716
0.723716
SAME
Recording_3/overall
tp
0.268876
0.268876
SAME
Recording_3/overall
ic
0.993223
0.993223
SAME
Recording_3/group/attack
precision
1.000000
1.000000
SAME
Recording_3/group/attack
recall
1.000000
1.000000
SAME
Recording_3/group/attack
f1_score
1.000000
1.000000
SAME
Recording_3/group/attack
so
1.000000
1.000000
SAME
Recording_3/group/attack
tp
1.000000
1.000000
SAME
Recording_3/group/attack
ic
1.000000
1.000000
SAME
Recording_3/group/investigation
precision
0.935484
0.935484
SAME
Recording_3/group/investigation
recall
0.783784
0.783784
SAME
Recording_3/group/investigation
f1_score
0.852941
0.852941
SAME
Recording_3/group/investigation
so
0.456310
0.456310
SAME
Recording_3/group/investigation
tp
0.022322
0.022322
SAME
Recording_3/group/investigation
ic
0.984966
0.984966
SAME
Recording_3/group/mount
precision
1.000000
1.000000
SAME
Recording_3/group/mount
recall
0.739130
0.739130
SAME
Recording_3/group/mount
f1_score
0.850000
0.850000
SAME
Recording_3/group/mount
so
0.674504
0.674504
SAME
Recording_3/group/mount
tp
0.028795
0.028795
SAME
Recording_3/group/mount
ic
0.991713
0.991713
SAME
Recording_3/group/other
precision
0.902439
0.902439
SAME
Recording_3/group/other
recall
0.880952
0.880952
SAME
Recording_3/group/other
f1_score
0.891566
0.891566
SAME
Recording_3/group/other
so
0.764048
0.764048
SAME
Recording_3/group/other
tp
0.024386
0.024386
SAME
Recording_3/group/other
ic
0.996212
0.996212
SAME
Recording_4/overall
precision
0.975000
0.975000
SAME
Recording_4/overall
recall
0.615686
0.615686
SAME
Recording_4/overall
f1_score
0.729167
0.729167
SAME
Recording_4/overall
so
0.707087
0.707087
SAME
Recording_4/overall
tp
0.275715
0.275715
SAME
Recording_4/overall
ic
0.996530
0.996530
SAME
Recording_4/group/attack
precision
1.000000
1.000000
SAME
Recording_4/group/attack
recall
1.000000
1.000000
SAME
Recording_4/group/attack
f1_score
1.000000
1.000000
SAME
Recording_4/group/attack
so
1.000000
1.000000
SAME
Recording_4/group/attack
tp
1.000000
1.000000
SAME
Recording_4/group/attack
ic
1.000000
1.000000
SAME
Recording_4/group/investigation
precision
0.900000
0.900000
SAME
Recording_4/group/investigation
recall
0.529412
0.529412
SAME
Recording_4/group/investigation
f1_score
0.666667
0.666667
SAME
Recording_4/group/investigation
so
0.391471
0.391471
SAME
Recording_4/group/investigation
tp
0.014025
0.014025
SAME
Recording_4/group/investigation
ic
0.996882
0.996882
SAME
Recording_4/group/mount
precision
1.000000
1.000000
SAME
Recording_4/group/mount
recall
0.333333
0.333333
SAME
Recording_4/group/mount
f1_score
0.500000
0.500000
SAME
Recording_4/group/mount
so
0.701119
0.701119
SAME
Recording_4/group/mount
tp
0.040087
0.040087
SAME
Recording_4/group/mount
ic
0.995289
0.995289
SAME
Recording_4/group/other
precision
1.000000
1.000000
SAME
Recording_4/group/other
recall
0.600000
0.600000
SAME
Recording_4/group/other
f1_score
0.750000
0.750000
SAME
Recording_4/group/other
so
0.735760
0.735760
SAME
Recording_4/group/other
tp
0.048749
0.048749
SAME
Recording_4/group/other
ic
0.993949
0.993949
SAME
Recording_5/overall
precision
1.000000
1.000000
SAME
Recording_5/overall
recall
0.892105
0.892105
SAME
Recording_5/overall
f1_score
0.941459
0.941459
SAME
Recording_5/overall
so
0.836730
0.836730
SAME
Recording_5/overall
tp
0.279289
0.279289
SAME
Recording_5/overall
ic
0.995120
0.995120
SAME
Recording_5/group/attack
precision
1.000000
1.000000
SAME
Recording_5/group/attack
recall
1.000000
1.000000
SAME
Recording_5/group/attack
f1_score
1.000000
1.000000
SAME
Recording_5/group/attack
so
1.000000
1.000000
SAME
Recording_5/group/attack
tp
1.000000
1.000000
SAME
Recording_5/group/attack
ic
1.000000
1.000000
SAME
Recording_5/group/investigation
precision
1.000000
1.000000
SAME
Recording_5/group/investigation
recall
0.800000
0.800000
SAME
Recording_5/group/investigation
f1_score
0.888889
0.888889
SAME
Recording_5/group/investigation
so
0.658959
0.658959
SAME
Recording_5/group/investigation
tp
0.035333
0.035333
SAME
Recording_5/group/investigation
ic
0.989914
0.989914
SAME
Recording_5/group/mount
precision
1.000000
1.000000
SAME
Recording_5/group/mount
recall
0.900000
0.900000
SAME
Recording_5/group/mount
f1_score
0.947368
0.947368
SAME
Recording_5/group/mount
so
0.910024
0.910024
SAME
Recording_5/group/mount
tp
0.035960
0.035960
SAME
Recording_5/group/mount
ic
0.996541
0.996541
SAME
Recording_5/group/other
precision
1.000000
1.000000
SAME
Recording_5/group/other
recall
0.868421
0.868421
SAME
Recording_5/group/other
f1_score
0.929577
0.929577
SAME
Recording_5/group/other
so
0.777939
0.777939
SAME
Recording_5/group/other
tp
0.045862
0.045862
SAME
Recording_5/group/other
ic
0.994024
0.994024
SAME
Recording_6/overall
precision
0.914083
0.914083
SAME
Recording_6/overall
recall
0.745671
0.745671
SAME
Recording_6/overall
f1_score
0.792505
0.792505
SAME
Recording_6/overall
so
0.624464
0.624464
SAME
Recording_6/overall
tp
0.264068
0.264068
SAME
Recording_6/overall
ic
0.993987
0.993987
SAME
Recording_6/group/attack
precision
1.000000
1.000000
SAME
Recording_6/group/attack
recall
0.363636
0.363636
SAME
Recording_6/group/attack
f1_score
0.533333
0.533333
SAME
Recording_6/group/attack
so
0.355488
0.355488
SAME
Recording_6/group/attack
tp
0.014143
0.014143
SAME
Recording_6/group/attack
ic
0.993666
0.993666
SAME
Recording_6/group/investigation
precision
0.767442
0.767442
SAME
Recording_6/group/investigation
recall
0.785714
0.785714
SAME
Recording_6/group/investigation
f1_score
0.776471
0.776471
SAME
Recording_6/group/investigation
so
0.539580
0.539580
SAME
Recording_6/group/investigation
tp
0.020317
0.020317
SAME
Recording_6/group/investigation
ic
0.990392
0.990392
SAME
Recording_6/group/mount
precision
1.000000
1.000000
SAME
Recording_6/group/mount
recall
1.000000
1.000000
SAME
Recording_6/group/mount
f1_score
1.000000
1.000000
SAME
Recording_6/group/mount
so
1.000000
1.000000
SAME
Recording_6/group/mount
tp
1.000000
1.000000
SAME
Recording_6/group/mount
ic
1.000000
1.000000
SAME
Recording_6/group/other
precision
0.888889
0.888889
SAME
Recording_6/group/other
recall
0.833333
0.833333
SAME
Recording_6/group/other
f1_score
0.860215
0.860215
SAME
Recording_6/group/other
so
0.602789
0.602789
SAME
Recording_6/group/other
tp
0.021811
0.021811
SAME
Recording_6/group/other
ic
0.991888
0.991888
SAME
Recording_7/overall
precision
1.000000
1.000000
SAME
Recording_7/overall
recall
0.583910
0.583910
SAME
Recording_7/overall
f1_score
0.706530
0.706530
SAME
Recording_7/overall
so
0.664495
0.664495
SAME
Recording_7/overall
tp
0.276530
0.276530
SAME
Recording_7/overall
ic
0.994944
0.994944
SAME
Recording_7/group/attack
precision
1.000000
1.000000
SAME
Recording_7/group/attack
recall
0.525000
0.525000
SAME
Recording_7/group/attack
f1_score
0.688525
0.688525
SAME
Recording_7/group/attack
so
0.390695
0.390695
SAME
Recording_7/group/attack
tp
0.030229
0.030229
SAME
Recording_7/group/attack
ic
0.989471
0.989471
SAME
Recording_7/group/investigation
precision
1.000000
1.000000
SAME
Recording_7/group/investigation
recall
0.300000
0.300000
SAME
Recording_7/group/investigation
f1_score
0.461538
0.461538
SAME
Recording_7/group/investigation
so
0.536495
0.536495
SAME
Recording_7/group/investigation
tp
0.031331
0.031331
SAME
Recording_7/group/investigation
ic
0.994544
0.994544
SAME
Recording_7/group/mount
precision
1.000000
1.000000
SAME
Recording_7/group/mount
recall
1.000000
1.000000
SAME
Recording_7/group/mount
f1_score
1.000000
1.000000
SAME
Recording_7/group/mount
so
1.000000
1.000000
SAME
Recording_7/group/mount
tp
1.000000
1.000000
SAME
Recording_7/group/mount
ic
1.000000
1.000000
SAME
Recording_7/group/other
precision
1.000000
1.000000
SAME
Recording_7/group/other
recall
0.510638
0.510638
SAME
Recording_7/group/other
f1_score
0.676056
0.676056
SAME
Recording_7/group/other
so
0.730790
0.730790
SAME
Recording_7/group/other
tp
0.044561
0.044561
SAME
Recording_7/group/other
ic
0.995760
0.995760
SAME
Recording_8/overall
precision
0.891667
0.891667
SAME
Recording_8/overall
recall
0.346721
0.346721
SAME
Recording_8/overall
f1_score
0.606173
0.606173
SAME
Recording_8/overall
so
0.530562
0.530562
SAME
Recording_8/overall
tp
0.038138
0.038138
SAME
Recording_8/overall
ic
0.995577
0.995577
SAME
Recording_8/group/attack
precision
0.875000
0.875000
SAME
Recording_8/group/attack
recall
0.368421
0.368421
SAME
Recording_8/group/attack
f1_score
0.518519
0.518519
SAME
Recording_8/group/attack
so
0.268364
0.268364
SAME
Recording_8/group/attack
tp
0.020423
0.020423
SAME
Recording_8/group/attack
ic
0.991919
0.991919
SAME
Recording_8/group/investigation
precision
1.000000
1.000000
SAME
Recording_8/group/investigation
recall
0.538462
0.538462
SAME
Recording_8/group/investigation
f1_score
0.700000
0.700000
SAME
Recording_8/group/investigation
so
0.591051
0.591051
SAME
Recording_8/group/investigation
tp
0.025271
0.025271
SAME
Recording_8/group/investigation
ic
0.995282
0.995282
SAME
Recording_8/group/mount
precision
NaN
NaN
SAME
Recording_8/group/mount
recall
0.000000
0.000000
SAME
Recording_8/group/mount
f1_score
NaN
NaN
SAME
Recording_8/group/mount
so
NaN
NaN
SAME
Recording_8/group/mount
tp
NaN
NaN
SAME
Recording_8/group/mount
ic
1.000000
1.000000
SAME
Recording_8/group/other
precision
0.800000
0.800000
SAME
Recording_8/group/other
recall
0.480000
0.480000
SAME
Recording_8/group/other
f1_score
0.600000
0.600000
SAME
Recording_8/group/other
so
0.732271
0.732271
SAME
Recording_8/group/other
tp
0.068721
0.068721
SAME
Recording_8/group/other
ic
0.995105
0.995105
SAME
Recording_9/overall
precision
0.798398
0.798398
SAME
Recording_9/overall
recall
0.653819
0.653819
SAME
Recording_9/overall
f1_score
0.712355
0.712355
SAME
Recording_9/overall
so
0.682730
0.682730
SAME
Recording_9/overall
tp
0.277809
0.277809
SAME
Recording_9/overall
ic
0.995671
0.995671
SAME
Recording_9/group/attack
precision
0.956522
0.956522
SAME
Recording_9/group/attack
recall
0.733333
0.733333
SAME
Recording_9/group/attack
f1_score
0.830189
0.830189
SAME
Recording_9/group/attack
so
0.424100
0.424100
SAME
Recording_9/group/attack
tp
0.020590
0.020590
SAME
Recording_9/group/attack
ic
0.989386
0.989386
SAME
Recording_9/group/investigation
precision
0.375000
0.375000
SAME
Recording_9/group/investigation
recall
0.187500
0.187500
SAME
Recording_9/group/investigation
f1_score
0.250000
0.250000
SAME
Recording_9/group/investigation
so
0.631678
0.631678
SAME
Recording_9/group/investigation
tp
0.034810
0.034810
SAME
Recording_9/group/investigation
ic
0.998630
0.998630
SAME
Recording_9/group/mount
precision
1.000000
1.000000
SAME
Recording_9/group/mount
recall
1.000000
1.000000
SAME
Recording_9/group/mount
f1_score
1.000000
1.000000
SAME
Recording_9/group/mount
so
1.000000
1.000000
SAME
Recording_9/group/mount
tp
1.000000
1.000000
SAME
Recording_9/group/mount
ic
1.000000
1.000000
SAME
Recording_9/group/other
precision
0.862069
0.862069
SAME
Recording_9/group/other
recall
0.694444
0.694444
SAME
Recording_9/group/other
f1_score
0.769231
0.769231
SAME
Recording_9/group/other
so
0.675140
0.675140
SAME
Recording_9/group/other
tp
0.055838
0.055838
SAME
Recording_9/group/other
ic
0.994669
0.994669
SAME
Recording_10/overall
precision
0.899107
0.899107
SAME
Recording_10/overall
recall
0.475760
0.475760
SAME
Recording_10/overall
f1_score
0.583475
0.583475
SAME
Recording_10/overall
so
0.666806
0.666806
SAME
Recording_10/overall
tp
0.285923
0.285923
SAME
Recording_10/overall
ic
0.996191
0.996191
SAME
Recording_10/group/attack
precision
1.000000
1.000000
SAME
Recording_10/group/attack
recall
0.285714
0.285714
SAME
Recording_10/group/attack
f1_score
0.444444
0.444444
SAME
Recording_10/group/attack
so
0.184903
0.184903
SAME
Recording_10/group/attack
tp
0.011189
0.011189
SAME
Recording_10/group/attack
ic
0.991412
0.991412
SAME
Recording_10/group/investigation
precision
0.625000
0.625000
SAME
Recording_10/group/investigation
recall
0.263158
0.263158
SAME
Recording_10/group/investigation
f1_score
0.370370
0.370370
SAME
Recording_10/group/investigation
so
0.657207
0.657207
SAME
Recording_10/group/investigation
tp
0.064683
0.064683
SAME
Recording_10/group/investigation
ic
0.995061
0.995061
SAME
Recording_10/group/mount
precision
1.000000
1.000000
SAME
Recording_10/group/mount
recall
1.000000
1.000000
SAME
Recording_10/group/mount
f1_score
1.000000
1.000000
SAME
Recording_10/group/mount
so
1.000000
1.000000
SAME
Recording_10/group/mount
tp
1.000000
1.000000
SAME
Recording_10/group/mount
ic
1.000000
1.000000
SAME
Recording_10/group/other
precision
0.971429
0.971429
SAME
Recording_10/group/other
recall
0.354167
0.354167
SAME
Recording_10/group/other
f1_score
0.519084
0.519084
SAME
Recording_10/group/other
so
0.825115
0.825115
SAME
Recording_10/group/other
tp
0.067820
0.067820
SAME
Recording_10/group/other
ic
0.998293
0.998293
SAME
Mode 2: Python v0.1.5 vs Python v0.2.0
-
Shows ALL changes: absent-behavior fix (0→1 correct absence) + IC bug fix
+
Shows the changes of v0.2.0: undefined metrics NaN instead of 0, correct absence 1.0 instead of 0, IC fixes