feat(dynaclr): read per-FOV tracks from in-plate tracks.geff

applications/airtable/README.md

@@ -125,7 +125,19 @@ The `prepare` CLI automates the full pipeline to rechunk and preprocess a datase
 2. Discover positions and channels from the NFS zarr
 3. Generate `crop_concat.yml` for `biahub concatenate` (zarr v3 with sharding)
 4. Generate `qc_config.yml` for focus-slice QC
-5. Generate and submit SLURM jobs (concatenation + tracking copy, then QC + preprocessing)
+5. Generate and submit SLURM jobs (concatenation + tracks copy, then QC + preprocessing)
+
+Tracks are copied according to where the NFS dataset keeps them:
+
+- **In-plate** (datasets tracked with current `biahub track`): each FOV's GEFF
+  (`tracks.geff`, or `tracks_<name>.geff` for a run that named its tracked label),
+  `tracks_<row>_<col>_<fov>.csv` and `labels/` (the segmentation `labels/<model>` and
+  the tracked `labels/tracks_<model>` the GEFF links to) are copied into the same FOV
+  of the VAST zarr, since `biahub concatenate` copies image arrays only. No
+  `tracking.zarr` is created and `tracks_path` can stay empty in Airtable (it
+  defaults to `data_path`).
+- **Separate tracking zarr** (older datasets): `1-preprocess/label-free/3-track/<name>_cropped.zarr`
+  is rsynced to `tracking.zarr`, as before.
 
 ### Usage
 
@@ -152,10 +164,10 @@ uv run --package airtable-utils \
 ```
 /hpc/projects/organelle_phenotyping/datasets/{dataset_name}/
   {dataset_name}.zarr       # zarr v3 rechunked (OME-Zarr 0.5)
-  tracking.zarr              # copied from NFS
+  tracking.zarr              # copied from NFS (older datasets only; newer ones keep tracks.geff and labels/ in the FOVs)
   crop_concat.yml            # generated config for biahub
   qc_config.yml              # generated config for QC
-  01_concatenate.sh          # bash: biahub concatenate (submits SLURM via submitit) + tracking copy
+  01_concatenate.sh          # bash: biahub concatenate (submits SLURM via submitit) + tracks copy
   02_qc_preprocess.sh        # SLURM: QC + preprocess (parallel, GPU)
 ```
 
@@ -248,6 +260,7 @@ registry = db.get_marker_registry()
 
 # Register positions programmatically
 from pathlib import Path
+
 positions = list(Path("/path/to/dataset.zarr").glob("*/*/*"))
 result = register_fovs(positions, db=db)
 print(f"created={len(result.created)} updated={len(result.updated)}")

applications/airtable/src/airtable_utils/prepare.py

@@ -130,6 +130,27 @@ def resolve_nfs_paths(dataset_name: str, nfs_root: Path) -> dict[str, Path]:
     return {"zarr": zarr_path, "tracking": tracking_path}
 
 
+def has_in_plate_tracks(zarr_path: Path) -> bool:
+    """Return whether any FOV of an HCS plate stores its tracks in-plate.
+
+    ``biahub track`` writes ``<plate>/<row>/<col>/<fov>/tracks.geff`` (plus a
+    tracks CSV and the tracked label image under ``labels/``) into the plate
+    itself; a run that names its tracked label writes ``tracks_<name>.geff``.
+    Older datasets keep tracks in a separate tracking zarr instead.
+
+    Parameters
+    ----------
+    zarr_path : Path
+        Path to the HCS plate root.
+
+    Returns
+    -------
+    bool
+        True if at least one FOV has a ``tracks.geff`` or ``tracks_*.geff`` group.
+    """
+    return next(Path(zarr_path).glob("*/*/*/tracks*.geff"), None) is not None
+
+
 def resolve_vast_paths(dataset_name: str, vast_root: Path) -> dict[str, Path]:
     """Return expected VAST output paths for a dataset.
 
@@ -438,31 +459,44 @@ def generate_sbatch_override_file(overrides: dict[str, str]) -> str:
 def generate_concatenate_script(
     crop_concat_path: Path,
     vast_zarr_path: Path,
-    nfs_tracking_path: Path,
-    vast_tracking_path: Path,
+    nfs_tracking_path: Path | None,
+    vast_tracking_path: Path | None,
     conda_env: str,
     sbatch_override_path: Path | None = None,
+    nfs_zarr_path: Path | None = None,
 ) -> str:
-    """Generate a bash script for biahub concatenate + tracking copy.
+    """Generate a bash script for biahub concatenate + tracks copy.
 
     This is NOT a SLURM script. ``biahub concatenate`` submits its own
     SLURM jobs internally via submitit. The ``-m`` flag makes it block
-    until those jobs complete. After concatenation, tracking is rsynced.
+    until those jobs complete. After concatenation, tracks are copied:
+
+    * in-plate tracks (``nfs_zarr_path`` set): each FOV's GEFF(s) (``tracks.geff``
+      or ``tracks_*.geff``), ``tracks_*.csv`` and ``labels/`` (the segmentation
+      ``labels/<model>`` and the tracked ``labels/tracks_<model>`` the GEFF links
+      to) are copied from the NFS plate into the same FOV of the VAST plate, so
+      ``tracks_path`` can stay unset (defaults to ``data_path``);
+    * otherwise the separate tracking zarr is rsynced, if
+      ``nfs_tracking_path`` is given.
 
     Parameters
     ----------
     crop_concat_path : Path
         Path to the generated crop_concat.yml.
     vast_zarr_path : Path
         Target zarr output path.
-    nfs_tracking_path : Path
-        Source tracking zarr on NFS.
-    vast_tracking_path : Path
-        Target tracking zarr on VAST.
+    nfs_tracking_path : Path or None
+        Source tracking zarr on NFS (legacy layout). Ignored when
+        ``nfs_zarr_path`` is set.
+    vast_tracking_path : Path or None
+        Target tracking zarr on VAST (legacy layout).
     conda_env : str
         Conda environment name for biahub.
     sbatch_override_path : Path or None
         Path to sbatch override file for biahub's internal SLURM jobs.
+    nfs_zarr_path : Path or None
+        Source plate on NFS whose FOVs hold the tracks (see
+        :func:`has_in_plate_tracks`). None for the legacy layout.
 
     Returns
     -------
@@ -481,23 +515,51 @@ def generate_concatenate_script(
         cmd_parts.append(f'-sb "{sbatch_override_path}"')
     biahub_cmd = " ".join(cmd_parts)
 
-    return dedent(f"""\
+    script = dedent(f"""\
         #!/bin/bash
         set -euo pipefail
 
         echo "=== Step 1: biahub concatenate (submits SLURM jobs via submitit) ==="
         {biahub_cmd}
         echo "Concatenation complete."
-
-        echo "=== Step 2: Copy tracking zarr ==="
-        if [ -d "{nfs_tracking_path}" ]; then
-            rsync -a --copy-links "{nfs_tracking_path}/" "{vast_tracking_path}/"
-            echo "Tracking copy complete."
-        else
-            echo "WARNING: NFS tracking zarr not found at {nfs_tracking_path}, skipping."
-        fi
     """)
 
+    if nfs_zarr_path is not None:
+        # biahub concatenate copies image arrays only; carry each FOV's tracks and
+        # label images (segment's labels/<model>, track's labels/tracks_<model>) over.
+        script += dedent(f"""\
+
+            echo "=== Step 2: Copy in-plate tracks (tracks*.geff + tracks CSV + labels/) ==="
+            src_root="{nfs_zarr_path}"
+            dst_root="{vast_zarr_path}"
+            n_copied=0
+            for src_fov in "$src_root"/*/*/*/; do
+                ls -d "${{src_fov}}"tracks*.geff > /dev/null 2>&1 || continue
+                fov_rel="${{src_fov#"$src_root"/}}"
+                dst_fov="$dst_root/$fov_rel"
+                if [ ! -d "$dst_fov" ]; then
+                    echo "WARNING: $fov_rel not in $dst_root, skipping its tracks."
+                    continue
+                fi
+                rsync -a --include='tracks*.geff/***' --include='tracks_*.csv' --include='labels/***' \\
+                    --exclude='*' "$src_fov" "$dst_fov"
+                n_copied=$((n_copied + 1))
+            done
+            echo "Copied tracks for $n_copied FOVs."
+        """)
+    elif nfs_tracking_path is not None and vast_tracking_path is not None:
+        script += dedent(f"""\
+
+            echo "=== Step 2: Copy tracking zarr ==="
+            if [ -d "{nfs_tracking_path}" ]; then
+                rsync -a --copy-links "{nfs_tracking_path}/" "{vast_tracking_path}/"
+                echo "Tracking copy complete."
+            else
+                echo "WARNING: NFS tracking zarr not found at {nfs_tracking_path}, skipping."
+            fi
+        """)
+    return script
+
 
 def generate_qc_slurm(
     dataset_name: str,
@@ -619,7 +681,12 @@ def check_dataset_status(dataset_name: str, nfs_root: Path, vast_root: Path) ->
 
     nfs_exists = nfs_zarr.exists()
     vast_zarr_exists = vast["zarr"].exists()
-    vast_tracking_exists = vast["tracking"].exists()
+    if vast["tracking"].exists():
+        tracking = "yes"
+    elif vast_zarr_exists and has_in_plate_tracks(vast["zarr"]):
+        tracking = "in-plate"
+    else:
+        tracking = "no"
 
     zarr_fmt: str = "-"
     ome_ver: str = "-"
@@ -637,7 +704,7 @@ def check_dataset_status(dataset_name: str, nfs_root: Path, vast_root: Path) ->
         "vast_zarr": "yes" if vast_zarr_exists else "no",
         "zarr_version": zarr_fmt,
         "ome_version": ome_ver,
-        "tracking": "yes" if vast_tracking_exists else "no",
+        "tracking": tracking,
         "preprocessed": preprocessed,
     }
 

applications/airtable/src/airtable_utils/prepare_cli.py

@@ -23,6 +23,7 @@
     generate_qc_config,
     generate_qc_slurm,
     generate_sbatch_override_file,
+    has_in_plate_tracks,
     resolve_nfs_paths,
     resolve_vast_paths,
     write_yaml,
@@ -162,13 +163,18 @@ def run(dataset_name: str, config_path: str, dry_run: bool, force: bool) -> None
         sbatch_override_path.write_text(sbatch_content)
         click.echo(f"  Wrote: {sbatch_override_path}")
 
+    # New datasets carry tracks inside the plate (<fov>/tracks.geff); copy those
+    # into the VAST plate. Older datasets have a separate tracking zarr.
+    in_plate_tracks = has_in_plate_tracks(nfs["zarr"])
+    click.echo(f"  Tracks: {'in-plate (tracks.geff)' if in_plate_tracks else nfs['tracking']}")
     concat_script = generate_concatenate_script(
         crop_concat_path=crop_concat_path,
         vast_zarr_path=vast["zarr"],
-        nfs_tracking_path=nfs["tracking"],
-        vast_tracking_path=vast["tracking"],
+        nfs_tracking_path=None if in_plate_tracks else nfs["tracking"],
+        vast_tracking_path=None if in_plate_tracks else vast["tracking"],
         conda_env=cfg.concatenate.conda_env,
         sbatch_override_path=sbatch_override_path,
+        nfs_zarr_path=nfs["zarr"] if in_plate_tracks else None,
     )
     concat_script_path = vast["output_dir"] / "01_concatenate.sh"
     concat_script_path.write_text(concat_script)

applications/airtable/src/airtable_utils/registration.py

@@ -289,9 +289,10 @@ def format_register_summary(result: RegisterResult, dry_run: bool = False) -> st
 
 # Fields required for a complete flat parquet cell index.
 # "zarr" = written by register, "platemap" = biologist fills in Airtable.
+# tracks_path is optional: when empty, tracks are read from inside the plate
+# (data_path/<row>/<col>/<fov>/tracks.geff), as written by ``biahub track``.
 PARQUET_REQUIRED_FIELDS: list[tuple[str, str]] = [
     ("data_path", "zarr"),
-    ("tracks_path", "platemap"),
     ("channel_0_name", "zarr"),
     ("channel_0_marker", "zarr"),
     ("pixel_size_xy_um", "zarr"),

applications/airtable/tests/test_prepare.py

@@ -0,0 +1,58 @@
+"""Tests for the tracks-copy step of airtable_utils.prepare."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from airtable_utils.prepare import generate_concatenate_script, has_in_plate_tracks
+
+
+def _script(tmp_path: Path, **kwargs) -> str:
+    defaults = {
+        "crop_concat_path": tmp_path / "crop_concat.yml",
+        "vast_zarr_path": tmp_path / "vast" / "ds.zarr",
+        "nfs_tracking_path": tmp_path / "nfs" / "tracking.zarr",
+        "vast_tracking_path": tmp_path / "vast" / "tracking.zarr",
+        "conda_env": "biahub",
+    }
+    return generate_concatenate_script(**{**defaults, **kwargs})
+
+
+def test_has_in_plate_tracks(tmp_path):
+    plate = tmp_path / "plate.zarr"
+    (plate / "A" / "1" / "0").mkdir(parents=True)
+    assert not has_in_plate_tracks(plate)
+    (plate / "A" / "1" / "0" / "tracks.geff").mkdir()
+    assert has_in_plate_tracks(plate)
+
+
+def test_has_in_plate_tracks_named_geff(tmp_path):
+    """A track run with output_label writes tracks_<name>.geff instead of tracks.geff."""
+    plate = tmp_path / "plate.zarr"
+    (plate / "A" / "1" / "0" / "tracks_nuclei_run2.geff").mkdir(parents=True)
+    assert has_in_plate_tracks(plate)
+
+
+def test_legacy_tracking_zarr_rsync(tmp_path):
+    script = _script(tmp_path)
+    assert "biahub concatenate" in script
+    assert "Copy tracking zarr" in script
+    assert str(tmp_path / "nfs" / "tracking.zarr") in script
+    assert "tracks.geff" not in script
+
+
+def test_in_plate_tracks_copy(tmp_path):
+    nfs_zarr = tmp_path / "nfs" / "ds.zarr"
+    script = _script(tmp_path, nfs_tracking_path=None, vast_tracking_path=None, nfs_zarr_path=nfs_zarr)
+    assert "Copy in-plate tracks" in script
+    assert f'src_root="{nfs_zarr}"' in script
+    # The GEFF(s), the CSV and the label images the GEFF links to (concatenate copies
+    # image arrays only).
+    assert "--include='tracks*.geff/***'" in script
+    assert "--include='labels/***'" in script
+    assert "tracking.zarr" not in script
+
+
+def test_no_tracks_step(tmp_path):
+    script = _script(tmp_path, nfs_tracking_path=None, vast_tracking_path=None)
+    assert "Step 2" not in script

applications/dynaclr/docs/recipes/build-cell-index.md

@@ -5,13 +5,49 @@
 Pre-build a **cell index parquet** once, then point the training config at it.
 The parquet contains one row per cell observation per timepoint with all
 metadata already computed (lineage, conditions, HPI). Training startup drops
-from minutes (opening every zarr + reading every CSV) to a single
+from minutes (opening every zarr + reading every FOV's tracks) to a single
 `read_parquet` call.
 
 ## Prerequisites
 
 - DynaCLR installed (`uv pip install -e applications/dynaclr`)
 - A collection YAML (see `train-multi-experiment.md` Step 1)
+- Per-FOV tracks for every experiment (see "Where tracks are read from" below)
+
+## Where tracks are read from
+
+Each experiment's `tracks_path` is a root holding one directory per FOV,
+`<tracks_path>/<row>/<col>/<fov>/`. In that directory DynaCLR reads:
+
+1. `tracks.geff` if present — the GEFF track graph written by `biahub track`
+   (needs the `geff` package, installed with DynaCLR via `viscy-data[tracks]`);
+2. otherwise exactly one `tracks_<name>.geff` — what a track run that names its
+   tracked label image (`output_label`) writes;
+3. otherwise exactly one `*.csv` (the ultrack tracks table).
+
+All give the same table: `track_id, t, [z], y, x, id, parent_track_id, parent_id`.
+The GEFF's `seg_id` is the `track_id` painted into the tracked label image beside
+it in the same FOV, `labels/tracks_<name>` (`labels/tracks_nuclei` for the A549
+pipeline), a `(T, 1, Z, Y, X)` uint32 OME-NGFF label image; the segmentation it was
+tracked from is `labels/<name>`.
+
+`tracks_path` is optional and defaults to `data_path`. Datasets tracked with
+current `biahub track` store the tracks inside the image plate
+(`<plate>/<row>/<col>/<fov>/tracks.geff`), so omit `tracks_path`:
+
+```yaml
+experiments:
+  - name: 2026_10_01_A549_H2B_CAAX
+    data_path: ${datasets_root}/2026_10_01_A549_H2B_CAAX/2026_10_01_A549_H2B_CAAX.zarr
+    # tracks_path defaults to data_path (tracks.geff inside each FOV)
+```
+
+Older datasets keep their tracks in a separate tracking zarr; keep setting
+`tracks_path` for those:
+
+```yaml
+    tracks_path: ${datasets_root}/2025_01_28_A549_G3BP1_ZIKV_DENV/tracking.zarr
+```
 
 ## Step 1: Build the parquet
 
@@ -70,7 +106,7 @@ data:
 
 ```
 Without parquet (slow — minutes):
-  collection.yml → open every zarr → read every tracking CSV
+  collection.yml → open every zarr → read every FOV's tracks (GEFF or CSV)
                → reconstruct lineage → enrich metadata
 
 With parquet (fast — seconds):

applications/dynaclr/pyproject.toml

@@ -44,7 +44,7 @@ dependencies = [
   "seaborn",
   "tabulate",
   "torchvision",
-  "viscy-data[triplet]",
+  "viscy-data[tracks,triplet]",
   "viscy-models",
   "viscy-transforms",
   "viscy-utils[eval]",

applications/dynaclr/src/dynaclr/data/experiment.py

@@ -387,7 +387,6 @@ def from_cell_index(
                 ExperimentEntry(
                     name=exp_name,
                     data_path=store_path,
-                    tracks_path="",
                     channels=exp_channels.get(exp_name, []),
                     channel_names=channel_names,
                     perturbation_wells=dict(perturbation_wells),