diff --git a/IIVision/__init__.py b/IIVision/__init__.py index 915bcb4..1681b1f 100644 --- a/IIVision/__init__.py +++ b/IIVision/__init__.py @@ -6,7 +6,8 @@ run_pipeline(camera, camera_transform, calibration, stabilizer_config, pre_stabilize_filter=) -> Iterator[list[TrackedBlob]] - Calibration classes: Calibration, Calibrator + Background models: Calibration + Calibrator (fixed, FlowerBeds), + AdaptiveBackground (self-learning, FundingCAPTCHA — ADR-0023) Camera classes: MockCamera, OrbbecCamera Config types: StabilizerConfig, Transform, Rotator Output type: TrackedBlob @@ -14,7 +15,8 @@ from .pipeline import PreStabilizeFilter, build_calibration, filter_positions_near_points, run_pipeline from .blob_stabilizer import BlobStabilizer, StabilizerConfig, TrackedBlob, TrackState -from .blob_tracker import BlobTracker, Calibration, Calibrator, DetectionConfig, FramePacket +from .blob_tracker import BackgroundModel, BlobTracker, Calibration, Calibrator, DetectionConfig, FramePacket +from .adaptive_background import AdaptiveBackground, AdaptiveBackgroundConfig from .camera import MockCamera, OrbbecCamera, OrbbecRGBCamera, RGBFrame from .transforms import Rotator, Transform, look_yaw_degrees, orbbec_to_world, rotator_to_matrix, transform_position from .silhouette_warp import reproject_silhouette @@ -28,6 +30,9 @@ "StabilizerConfig", "TrackedBlob", "TrackState", + "AdaptiveBackground", + "AdaptiveBackgroundConfig", + "BackgroundModel", "BlobTracker", "Calibration", "Calibrator", diff --git a/IIVision/adaptive_background.py b/IIVision/adaptive_background.py new file mode 100644 index 0000000..5ec22ce --- /dev/null +++ b/IIVision/adaptive_background.py @@ -0,0 +1,278 @@ +""" +Self-learning depth background: "the farthest steady depth wins" (ADR-0023). + +A person can only make a depth pixel nearer, never farther. So each pixel's +background is the farthest depth it has held steadily, learned continuously +from the first frame — with or without people in view. No standback, no +calibration pass, nothing saved to disk. + +The first warmup_s of a stream are ignored: the Orbbec Gemini 336 reports +depth up to ~400 mm too far for its first frames (1 frame warm, ~5 cold), and +seeding from those leaves the true surface looking nearer — foreground until +absorb_s. + +Per frame, for each pixel with a valid sample d (thr = max(depth_delta_mm, +noise_k × noise)): + + seed bg and σ are the mean and spread of the first min_valid_frames + valid samples; the pixel is valid after them + at bg |d − bg| ≤ thr → slow EMA of bg and of the noise σ estimate + farther d > bg + thr for reveal_frames in a row → bg = mean of those samples + nearer d < bg − thr held steady (within thr) for absorb_s → bg = d + bump > reseed_fg_fraction of valid pixels nearer for reseed_after_s + → reseed bg from the current frame + +Exposes the same background / valid_mask / threshold() interface as +Calibration, so BlobTracker works with either. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass, fields +from typing import Callable + +import numpy as np + +from .blob_tracker import FramePacket + +log = logging.getLogger("IIVision.adaptive_background") + +# Mean absolute deviation of a Gaussian ≈ 0.8σ. +_MAD_TO_SIGMA = 1.0 / 0.8 + + +@dataclass +class AdaptiveBackgroundConfig: + depth_delta_mm: float = 80 # lower bound on thr (DetectionConfig.depth_delta_mm) + min_depth_mm: int = 50 # samples below this are invalid (0 = no return) + warmup_s: float = 1.0 # ignore frames this soon after the first push + min_valid_frames: int = 10 # valid samples before a pixel's bg is trusted + reveal_frames: int = 5 # consecutive farther frames to move bg out + noise_k: float = 4.0 # thr = max(depth_delta_mm, noise_k × σ) + bg_alpha: float = 0.01 # per-frame EMA of bg at background + noise_alpha: float = 0.05 # per-frame EMA of σ at background + absorb_s: float = 300.0 # steady nearer reading becomes bg after this + reseed_fg_fraction: float = 0.4 # bump guard: fraction of valid pixels in fg … + reseed_after_s: float = 30.0 # … for this long → reseed + stats_log_s: float = 30.0 # how often to log update timing (0 = never) + + @classmethod + def from_dict(cls, d: dict, **overrides) -> "AdaptiveBackgroundConfig": + """Build from a settings block; unknown keys (e.g. _comment) are ignored.""" + names = {f.name for f in fields(cls)} + kw = {k: v for k, v in d.items() if k in names} + kw.update(overrides) + return cls(**kw) + + +class AdaptiveBackground: + """ + Continuously learned background model. + + Usage: + model = AdaptiveBackground(AdaptiveBackgroundConfig(depth_delta_mm=30)) + tracker = BlobTracker(model, detection_config) + for frame in camera.frames(): + model.push(frame) + fg = tracker.detect_foreground(frame) + + Before the first push() width/height are 0; BlobTracker needs one push first. + reseed() may be called from another thread: it takes effect on the next push. + """ + + def __init__(self, config: AdaptiveBackgroundConfig | None = None, + clock: Callable[[], float] = time.monotonic) -> None: + self.config = config or AdaptiveBackgroundConfig() + self._clock = clock + self.width = 0 + self.height = 0 + self._reseed_pending = False + self._t_first: float | None = None + self._crowd_since: float | None = None + self.reseed_count = 0 + # Timing stats (wall-clock, independent of the injected clock). + self.last_update_ms = 0.0 + self._stats_n = 0 + self._stats_sum_ms = 0.0 + self._stats_max_ms = 0.0 + self._stats_t0 = time.monotonic() + + # ------------------------------------------------------------------ + # Interface read by BlobTracker + # ------------------------------------------------------------------ + + @property + def background(self) -> np.ndarray: + """uint16 (H*W,): background depth in mm, 0 where not valid.""" + bg = np.rint(self._bg).astype(np.uint16) + bg[~self.valid_mask] = 0 + return bg + + @property + def valid_mask(self) -> np.ndarray: + """bool (H*W,): pixel has had ≥ min_valid_frames valid samples.""" + return self._n_valid >= self.config.min_valid_frames + + @property + def noise(self) -> np.ndarray: + """float32 (H*W,): running σ estimate at background, mm.""" + return self._noise + + def threshold(self, depth_delta_mm: float) -> np.ndarray: + """int32 (H*W,): per-pixel foreground threshold, mm.""" + return np.maximum(depth_delta_mm, + self.config.noise_k * self._noise).astype(np.int32) + + # ------------------------------------------------------------------ + # Updates + # ------------------------------------------------------------------ + + def reseed(self) -> None: + """Reseed the background from the next pushed frame.""" + self._reseed_pending = True + + def push(self, frame: FramePacket) -> None: + """Update the model with one depth frame.""" + t_start = time.perf_counter() + if self.width == 0: + self._init_state(frame.width, frame.height) + elif frame.width != self.width or frame.height != self.height: + raise ValueError( + f"Frame size {frame.width}x{frame.height} != model {self.width}x{self.height}") + + now = self._clock() + if self._t_first is None: + self._t_first = now + if now - self._t_first < self.config.warmup_s: + return # camera still settling; nothing valid, nothing foreground + + d = np.frombuffer(frame.data, dtype=np.uint16).astype(np.float32) + self._update(d, now) + + self.last_update_ms = (time.perf_counter() - t_start) * 1000.0 + self._record_stats() + + def _init_state(self, width: int, height: int) -> None: + n = width * height + self.width, self.height = width, height + self._bg = np.zeros(n, dtype=np.float32) + self._noise = np.zeros(n, dtype=np.float32) + self._seed_m2 = np.zeros(n, dtype=np.float32) # Welford sum of squares while seeding + self._n_valid = np.zeros(n, dtype=np.uint16) + self._reveal_n = np.zeros(n, dtype=np.uint16) + self._reveal_sum = np.zeros(n, dtype=np.float32) + self._absorb_ref = np.zeros(n, dtype=np.float32) + self._absorb_t0 = np.zeros(n, dtype=np.float64) + self._absorb_on = np.zeros(n, dtype=bool) + + def _update(self, d: np.ndarray, now: float) -> None: + cfg = self.config + ok = d >= cfg.min_depth_mm + + if self._reseed_pending: + self._reseed_pending = False + self._reseed(d, ok, "requested") + return + + # 1. Seed: bg and σ from the mean and spread of the first min_valid_frames + # valid samples (Welford). One sample alone is fragile: a far outlier on a + # noisy pixel would leave every later reading "nearer" — stuck foreground. + # The other rules start once the pixel is valid. + seeding = ok & (self._n_valid < cfg.min_valid_frames) + if seeding.any(): + n = self._n_valid[seeding].astype(np.float32) + 1.0 + ds = d[seeding] + mean = self._bg[seeding] + step = ds - mean + mean = mean + step / n + self._seed_m2[seeding] += step * (ds - mean) + self._bg[seeding] = mean + self._n_valid[seeding] += 1 + done = seeding & (self._n_valid >= cfg.min_valid_frames) + k = cfg.min_valid_frames + self._noise[done] = np.sqrt(self._seed_m2[done] / max(1, k - 1)) + self._seed_m2[done] = 0.0 + + live = ok & ~seeding & (self._n_valid >= cfg.min_valid_frames) + thr = np.maximum(cfg.depth_delta_mm, cfg.noise_k * self._noise) + diff = d - self._bg + at_bg = live & (np.abs(diff) <= thr) + farther = live & (diff > thr) + nearer = live & (diff < -thr) + + # 2. At background: track slow drift and the noise level. + self._bg[at_bg] += cfg.bg_alpha * diff[at_bg] + sigma_sample = np.abs(diff[at_bg]) * _MAD_TO_SIGMA + self._noise[at_bg] += cfg.noise_alpha * (sigma_sample - self._noise[at_bg]) + + # Any valid sample that isn't farther breaks a reveal run; any that isn't + # nearer breaks an absorb run. + not_farther = live & ~farther + self._reveal_n[not_farther] = 0 + self._reveal_sum[not_farther] = 0.0 + self._absorb_on[live & ~nearer] = False + + # 3. Farther: background revealed once it holds for reveal_frames. + self._reveal_n[farther] += 1 + self._reveal_sum[farther] += d[farther] + revealed = farther & (self._reveal_n >= cfg.reveal_frames) + if revealed.any(): + self._bg[revealed] = self._reveal_sum[revealed] / self._reveal_n[revealed] + self._reveal_n[revealed] = 0 + self._reveal_sum[revealed] = 0.0 + + # 4. Nearer: steady for absorb_s → absorbed (a dropped bag, a moved table). + steady = nearer & self._absorb_on & (np.abs(d - self._absorb_ref) <= thr) + restart = nearer & ~steady + self._absorb_ref[restart] = d[restart] + self._absorb_t0[restart] = now + self._absorb_on[restart] = True + absorbed = steady & (now - self._absorb_t0 >= cfg.absorb_s) + if absorbed.any(): + self._bg[absorbed] = d[absorbed] + self._absorb_on[absorbed] = False + + # 5. Bump guard: most of the scene suddenly nearer and staying there. + valid = self._n_valid >= cfg.min_valid_frames + n_valid = int(np.count_nonzero(valid)) + fg_frac = (np.count_nonzero(nearer & valid) / n_valid) if n_valid else 0.0 + if fg_frac > cfg.reseed_fg_fraction: + if self._crowd_since is None: + self._crowd_since = now + elif now - self._crowd_since >= cfg.reseed_after_s: + log.warning("Background reseeded: %.0f%% of the scene has been foreground " + "for %.0fs (camera bumped?)", fg_frac * 100, + now - self._crowd_since) + self._reseed(d, ok, "bump guard") + else: + self._crowd_since = None + + def _reseed(self, d: np.ndarray, ok: np.ndarray, reason: str) -> None: + self._bg[ok] = d[ok] + self._reveal_n[:] = 0 + self._reveal_sum[:] = 0.0 + self._absorb_on[:] = False + self._crowd_since = None + self.reseed_count += 1 + log.info("Background reseeded from current frame (%s)", reason) + + def _record_stats(self) -> None: + ms = self.last_update_ms + self._stats_n += 1 + self._stats_sum_ms += ms + self._stats_max_ms = max(self._stats_max_ms, ms) + if self.config.stats_log_s <= 0: + return + now = time.monotonic() + if now - self._stats_t0 >= self.config.stats_log_s: + log.info("Background update %dx%d: mean %.1f ms, max %.1f ms over %d frames; " + "%.0f%% of pixels valid", + self.width, self.height, self._stats_sum_ms / self._stats_n, + self._stats_max_ms, self._stats_n, + 100.0 * np.count_nonzero(self.valid_mask) / max(1, self._n_valid.size)) + self._stats_t0 = now + self._stats_n = 0 + self._stats_sum_ms = 0.0 + self._stats_max_ms = 0.0 diff --git a/IIVision/blob_tracker.py b/IIVision/blob_tracker.py index 4746864..ef7308d 100644 --- a/IIVision/blob_tracker.py +++ b/IIVision/blob_tracker.py @@ -12,6 +12,7 @@ from dataclasses import dataclass, field from pathlib import Path +from typing import Protocol import numpy as np from scipy.ndimage import label as nd_label @@ -107,6 +108,14 @@ class Calibration: valid_mask: np.ndarray # bool, shape (H*W,) noise_floor: np.ndarray # uint16, shape (H*W,) — per-pixel IQR from calibration frames + def threshold(self, depth_delta_mm: float) -> np.ndarray: + """int32 (H*W,): per-pixel foreground threshold, mm. + + At least depth_delta_mm, raised to 3×IQR where sensor noise is high — + prevents noisy pixels from flickering as foreground. + """ + return np.maximum(depth_delta_mm, self.noise_floor.astype(np.int32) * _NOISE_SIGMA) + def save(self, path: Path | str) -> None: """Save to a .npz file.""" np.savez( @@ -220,16 +229,35 @@ def build(self) -> Calibration: # BlobTracker # --------------------------------------------------------------------------- +class BackgroundModel(Protocol): + """What BlobTracker reads from a background model. + + Implemented by Calibration (fixed, from Calibrator) and AdaptiveBackground + (continuously learned, ADR-0023). + """ + width: int + height: int + + @property + def background(self) -> np.ndarray: ... # uint16 (H*W,), mm + + @property + def valid_mask(self) -> np.ndarray: ... # bool (H*W,) + + def threshold(self, depth_delta_mm: float) -> np.ndarray: ... # int32 (H*W,), mm + + class BlobTracker: """ - Per-frame blob detector. Requires a ready Calibration — use Calibrator to build one. + Per-frame blob detector over a background model: a ready Calibration (use + Calibrator to build one) or an AdaptiveBackground that has had a frame pushed. Usage: tracker = BlobTracker(calibration) result = tracker.detect(live_frame) """ - def __init__(self, calibration: Calibration, config: DetectionConfig | None = None) -> None: + def __init__(self, calibration: BackgroundModel, config: DetectionConfig | None = None) -> None: self._calibration = calibration self.detection_config = config or DetectionConfig() @@ -238,21 +266,19 @@ def __init__(self, calibration: Calibration, config: DetectionConfig | None = No # ------------------------------------------------------------------ @property - def calibration(self) -> Calibration: + def calibration(self) -> BackgroundModel: return self._calibration @property def threshold_map(self) -> np.ndarray: """Per-pixel effective foreground threshold (mm), shape (H, W), dtype uint16. - max(depth_delta_mm, noise_floor * NOISE_SIGMA). Zero where calibration invalid. + The model's threshold(depth_delta_mm). Zero where the background is invalid. """ cal = self._calibration - t = np.maximum( - self.detection_config.depth_delta_mm, - cal.noise_floor.astype(np.int32) * _NOISE_SIGMA, - ) - return np.where(cal.valid_mask, t, 0).reshape(cal.height, cal.width).astype(np.uint16) + t = cal.threshold(self.detection_config.depth_delta_mm) + t = np.where(cal.valid_mask, t, 0).clip(0, 65535) + return t.reshape(cal.height, cal.width).astype(np.uint16) def detect_foreground(self, frame: FramePacket) -> np.ndarray: """uint16 (H, W): foreground pixels = depth_mm, background pixels = 0. @@ -301,15 +327,13 @@ def _subtract_background(self, frame: FramePacket) -> np.ndarray: depth <= self.detection_config.max_depth_mm ) + background = cal.background bg_valid = cal.valid_mask & in_range - bg_closer = cal.background > depth - delta = cal.background.astype(np.int32) - depth.astype(np.int32) - # Per-pixel threshold: at least depth_delta_mm, raised to 3×IQR where - # sensor noise is high — prevents noisy pixels from flickering as foreground. - threshold = np.maximum( - self.detection_config.depth_delta_mm, - cal.noise_floor.astype(np.int32) * _NOISE_SIGMA, - ) + bg_closer = background > depth + delta = background.astype(np.int32) - depth.astype(np.int32) + # Per-pixel threshold: at least depth_delta_mm, raised where the model + # has measured high sensor noise — keeps noisy pixels from flickering. + threshold = cal.threshold(self.detection_config.depth_delta_mm) fg_from_valid = bg_valid & bg_closer & (delta > threshold) fg[fg_from_valid] = 255 diff --git a/IIVision/test_adaptive_background.py b/IIVision/test_adaptive_background.py new file mode 100644 index 0000000..59ea284 --- /dev/null +++ b/IIVision/test_adaptive_background.py @@ -0,0 +1,347 @@ +"""Tests for AdaptiveBackground: synthetic depth arrays plus an injected clock.""" +from __future__ import annotations + +import numpy as np + +from IIVision.adaptive_background import AdaptiveBackground, AdaptiveBackgroundConfig +from IIVision.blob_tracker import ( + BlobTracker, + Calibration, + CameraIntrinsics, + DetectionConfig, + FramePacket, +) + +W, H = 40, 30 +FPS = 15.0 +FLOOR = 3000 # empty-scene depth, mm +PERSON = 1800 # a visitor, mm +DELTA = 30 # detection.depth_delta_mm +DET = DetectionConfig(min_depth_mm=500, max_depth_mm=6000, depth_delta_mm=DELTA) +BODY = (slice(8, 22), slice(12, 26)) # a visitor-sized patch +INSIDE = (slice(10, 20), slice(14, 24)) # its interior, clear of despeckle edges + + +class Clock: + def __init__(self) -> None: + self.t = 0.0 + + def __call__(self) -> float: + return self.t + + +def _frame(depth: np.ndarray) -> FramePacket: + h, w = depth.shape + return FramePacket(width=w, height=h, data=depth.astype(np.uint16).tobytes(), + intrinsics=CameraIntrinsics(fx=100, fy=100, cx=w / 2, cy=h / 2)) + + +def _scene(person: bool = False, floor: int = FLOOR) -> np.ndarray: + d = np.full((H, W), floor, dtype=np.uint16) + if person: + d[BODY] = PERSON + return d + + +class Rig: + """Model + tracker + clock, stepping at FPS.""" + + def __init__(self, **cfg) -> None: + self.clock = Clock() + cfg.setdefault("stats_log_s", 0) + cfg.setdefault("warmup_s", 0) + self.model = AdaptiveBackground( + AdaptiveBackgroundConfig(depth_delta_mm=DELTA, **cfg), clock=self.clock) + self.tracker = BlobTracker(self.model, DET) + + def step(self, depth: np.ndarray, n: int = 1) -> np.ndarray: + """Push n frames of depth; return the last frame's foreground mask (H, W).""" + for _ in range(n): + frame = _frame(depth) + self.model.push(frame) + fg = self.tracker.detect_foreground(frame) > 0 + self.clock.t += 1.0 / FPS + return fg + + def run_for(self, depth: np.ndarray, seconds: float) -> np.ndarray: + return self.step(depth, int(round(seconds * FPS))) + + def bg(self) -> np.ndarray: + return self.model.background.reshape(H, W) + + +# ── Boot ────────────────────────────────────────────────────────────────────── + +def test_empty_floor_is_live_within_a_second(): + rig = Rig() + fg = rig.run_for(_scene(), 0.7) + assert rig.model.valid_mask.all() + assert not fg.any() + # A visitor walking on now is foreground straight away. + assert rig.step(_scene(person=True))[INSIDE].all() + + +def test_person_present_from_frame_0_is_revealed_when_they_walk_off(): + cfg = AdaptiveBackgroundConfig() + rig = Rig() + fg = rig.run_for(_scene(person=True), 5.0) + assert not fg.any(), "invisible until they move: they are the only depth seen there" + assert (rig.bg()[INSIDE] == PERSON).all() + + rig.step(_scene(), cfg.reveal_frames - 1) + assert (rig.bg()[INSIDE] == PERSON).all(), "not before reveal_frames" + rig.step(_scene()) + assert (rig.bg()[INSIDE] == FLOOR).all(), "revealed within reveal_frames" + + # From then on they're tracked. + assert rig.step(_scene(person=True))[INSIDE].all() + + +def test_unseen_pixels_are_never_foreground(): + rig = Rig() + depth = _scene() + depth[:, :10] = 0 # no return there, ever + rig.run_for(depth, 1.0) + depth[BODY] = PERSON + depth[:, :10] = PERSON # something now in front of the unknown region + fg = rig.step(depth) + assert not fg[:, :10].any() + assert fg[INSIDE].all() + + +def test_startup_frames_reading_too_far_leave_no_ghosts(): + # Measured on the Gemini 336: the first frames of a stream read up to ~400 mm + # too far (1 frame warm, ~5 cold), then step to the true depth. Seeding from + # them made the true floor "nearer" — little blobs that lasted absorb_s. + rig = Rig(warmup_s=1.0) + too_far = _scene(floor=FLOOR + 400) + fg = rig.step(too_far, 5) + assert not fg.any() and not rig.model.valid_mask.any(), "warm-up frames are ignored" + fg = rig.run_for(_scene(), 2.0) + assert not fg.any() + assert (rig.bg() == FLOOR).all() + + +def test_far_outlier_first_sample_on_a_noisy_pixel_does_not_stick(): + # Measured on the Gemini 336: a pixel at ~3.7 m with σ ≈ 80 mm whose first + # sample was a far outlier. Seeding bg from that one sample left every later + # reading "nearer" — foreground, never at bg, too jumpy to be absorbed. + rng = np.random.default_rng(1) + rig = Rig() + first = _scene() + first[INSIDE] = FLOOR + 400 + rig.step(first) + fgs = [] + for _ in range(int(20 * FPS)): + d = _scene() + d[INSIDE] = rng.normal(FLOOR, 80, d[INSIDE].shape).round() + fgs.append(rig.step(d)[INSIDE].mean()) + assert np.mean(fgs[int(5 * FPS):]) < 0.001 + + +def test_seed_noise_comes_from_the_seeding_samples(): + rng = np.random.default_rng(2) + rig = Rig() + for _ in range(AdaptiveBackgroundConfig().min_valid_frames): + rig.step(rng.normal(FLOOR, 50, (H, W)).round().astype(np.uint16)) + assert 30 < float(np.median(rig.model.noise)) < 70 + assert abs(float(np.median(rig.model.background.astype(float))) - FLOOR) < 20 + + +# ── Standing still ─────────────────────────────────────────────────────────── + +def test_person_still_for_120s_stays_foreground(): + rig = Rig() + rig.run_for(_scene(), 2.0) + fg = rig.run_for(_scene(person=True), 120.0) + assert fg[INSIDE].all() + + +def test_static_object_absorbed_after_absorb_s(): + cfg = AdaptiveBackgroundConfig() + rig = Rig() + rig.run_for(_scene(), 2.0) + bag = _scene(person=True) + fg = rig.run_for(bag, cfg.absorb_s - 1.0) + assert fg[INSIDE].all(), "foreground until absorb_s" + fg = rig.run_for(bag, 2.0) + assert not fg.any(), "background after absorb_s" + assert (rig.bg()[INSIDE] == PERSON).all() + + # Someone walks in front of the absorbed object: still detected. + nearer = bag.copy() + nearer[INSIDE] = 1000 + assert rig.step(nearer)[12:18, 16:22].all() + # The object is taken away: the floor behind it is revealed again. + rig.step(_scene(), cfg.reveal_frames) + assert (rig.bg()[INSIDE] == FLOOR).all() + + +def test_fidgeting_restarts_the_absorb_timer(): + cfg = AdaptiveBackgroundConfig() + rig = Rig() + rig.run_for(_scene(), 2.0) + a, b = _scene(person=True), _scene(person=True) + b[BODY] = PERSON + 200 # shifts by far more than thr + for _ in range(int(cfg.absorb_s / 50) + 2): + rig.run_for(a, 25.0) + fg = rig.run_for(b, 25.0) + assert rig.clock.t > cfg.absorb_s + 2.0 + assert fg[INSIDE].all() + + +# ── Noise and spikes ───────────────────────────────────────────────────────── + +def test_single_frame_far_spike_leaves_bg_unchanged(): + rig = Rig() + rig.run_for(_scene(), 1.0) + before = rig.bg().copy() + spike = _scene() + spike[15, 20] = 5500 # flying pixel / multipath + spike[5:8, 5:30] = 4500 + rig.step(spike) + rig.step(_scene()) + np.testing.assert_array_equal(rig.bg(), before) + + +def test_far_spikes_short_of_reveal_frames_do_not_move_bg(): + cfg = AdaptiveBackgroundConfig() + rig = Rig() + rig.run_for(_scene(), 1.0) + spike = _scene(floor=FLOOR + 500) + for _ in range(20): + rig.step(spike, cfg.reveal_frames - 1) + rig.step(_scene()) + assert (rig.bg() == FLOOR).all() + + +def test_gaussian_noise_on_empty_scene_has_under_0_1_percent_foreground(): + # Orbbec structured light at 3 m: σ on the order of 10–20 mm. Use the top end. + w, h, sigma = 160, 120, 20.0 + rng = np.random.default_rng(0) + clock = Clock() + model = AdaptiveBackground(AdaptiveBackgroundConfig(depth_delta_mm=DELTA, stats_log_s=0, + warmup_s=0), clock=clock) + tracker = BlobTracker(model, DET) + + def noisy() -> FramePacket: + d = rng.normal(FLOOR, sigma, (h, w)).round().clip(0, 65535) + return _frame(d) + + for _ in range(int(10 * FPS)): # warm-up + model.push(noisy()) + clock.t += 1 / FPS + + raw = despeckled = 0 + n = int(20 * FPS) + for _ in range(n): + frame = noisy() + model.push(frame) + clock.t += 1 / FPS + raw += np.count_nonzero(tracker._subtract_background(frame)) + despeckled += np.count_nonzero(tracker.detect_foreground(frame)) + assert raw / (n * w * h) < 0.001 + assert despeckled == 0 + # The noise estimate settles near the real σ. + assert 0.7 * sigma < float(np.median(model.noise)) < 1.3 * sigma + assert abs(float(np.median(model.background.astype(float))) - FLOOR) < 5 + + +def test_threshold_has_depth_delta_floor(): + rig = Rig() + rig.run_for(_scene(), 2.0) # noiseless: σ estimate decays toward 0 + t = rig.model.threshold(DELTA) + assert (t == DELTA).all() + assert (rig.tracker.threshold_map == DELTA).all() + + +# ── Bump guard ─────────────────────────────────────────────────────────────── + +def test_bump_nearer_reseeds_after_reseed_after_s(): + cfg = AdaptiveBackgroundConfig() + rig = Rig() + rig.run_for(_scene(), 2.0) + bumped = _scene(floor=FLOOR - 400) + fg = rig.run_for(bumped, cfg.reseed_after_s - 1.0) + assert rig.model.reseed_count == 0 + assert fg.mean() > 0.5 + fg = rig.run_for(bumped, 2.0) + assert rig.model.reseed_count == 1 + assert not fg.any() + assert (rig.bg() == FLOOR - 400).all() + # After the reseed, people are tracked against the new background. + assert rig.step(_scene(person=True, floor=FLOOR - 400))[INSIDE].all() + + +def test_bump_farther_heals_through_reveal_without_reseed(): + cfg = AdaptiveBackgroundConfig() + rig = Rig() + rig.run_for(_scene(), 2.0) + fg = rig.step(_scene(floor=FLOOR + 400), cfg.reveal_frames) + assert not fg.any() + assert (rig.bg() == FLOOR + 400).all() + rig.run_for(_scene(floor=FLOOR + 400), cfg.reseed_after_s + 5) + assert rig.model.reseed_count == 0 + + +def test_small_crowd_does_not_trigger_bump_guard(): + cfg = AdaptiveBackgroundConfig() + rig = Rig() + rig.run_for(_scene(), 2.0) + fg = rig.run_for(_scene(person=True), cfg.reseed_after_s + 5) # ~16% of the frame + assert rig.model.reseed_count == 0 + assert fg[INSIDE].all() + + +def test_reseed_request_takes_effect_on_next_push(): + rig = Rig() + rig.run_for(_scene(), 1.0) + rig.model.reseed() + fg = rig.step(_scene(person=True)) + assert rig.model.reseed_count == 1 + assert not fg.any() + assert (rig.bg()[INSIDE] == PERSON).all() + rig.step(_scene(), AdaptiveBackgroundConfig().reveal_frames) + assert (rig.bg()[INSIDE] == FLOOR).all() + + +# ── Interface ──────────────────────────────────────────────────────────────── + +def test_config_from_dict_ignores_comments_and_takes_overrides(): + cfg = AdaptiveBackgroundConfig.from_dict( + {"_comment": "x", "absorb_s": 200, "reveal_frames": 3}, depth_delta_mm=25) + assert cfg.absorb_s == 200 and cfg.reveal_frames == 3 and cfg.depth_delta_mm == 25 + + +def test_frame_size_change_raises(): + rig = Rig() + rig.step(_scene()) + try: + rig.model.push(_frame(np.full((10, 10), FLOOR, dtype=np.uint16))) + except ValueError: + return + raise AssertionError("expected ValueError") + + +def test_update_time_is_recorded(): + rig = Rig() + rig.step(_scene()) + assert rig.model.last_update_ms > 0.0 + + +def test_update_time_is_logged(caplog): + import logging + rig = Rig(stats_log_s=1e-9) + with caplog.at_level(logging.INFO, logger="IIVision.adaptive_background"): + rig.step(_scene(), 2) + assert any("Background update" in r.getMessage() and " ms" in r.getMessage() + for r in caplog.records) + + +def test_calibration_threshold_matches_previous_formula(): + n = 4 + cal = Calibration(width=2, height=2, + background=np.full(n, 3000, dtype=np.uint16), + valid_mask=np.ones(n, dtype=bool), + noise_floor=np.array([0, 5, 20, 100], dtype=np.uint16)) + np.testing.assert_array_equal(cal.threshold(30), [30, 30, 60, 300]) diff --git a/ShowControl/FundingCAPTCHA/app.py b/ShowControl/FundingCAPTCHA/app.py index 31c510c..9e0a068 100644 --- a/ShowControl/FundingCAPTCHA/app.py +++ b/ShowControl/FundingCAPTCHA/app.py @@ -1,17 +1,20 @@ #!/usr/bin/env python3 """ -FundingCAPTCHA unified pygame app (ADR-0012, ADR-0013, ADR-0016). +FundingCAPTCHA unified pygame app (ADR-0012, ADR-0013, ADR-0016, ADR-0023). Usage: python3 app.py [--camera | --mock-camera | --test-input [--test-depth N]] [--port 8080] States: - BG_CAL → "stand back / calibrating" countdown; camera builds depth model SCREENSAVER → idle; detects player and counts down attract dwell GAME → active Arc +There is no calibration screen. The depth background is learned continuously +from the first frame, with people in view (ADR-0023): the floor is live in +under a second, and anyone standing there at launch shows up after one step. + Keys: - R → restart from BG_CAL + R → reseed the depth background from the next camera frame Q / Escape → quit C → clear paint canvas (--test-input only) """ @@ -48,7 +51,8 @@ # ── Optional imports ────────────────────────────────────────────────────────── try: - from IIVision import MockCamera, OrbbecCamera, Calibrator, DetectionConfig, BlobTracker + from IIVision import (MockCamera, OrbbecCamera, AdaptiveBackground, + AdaptiveBackgroundConfig, DetectionConfig, BlobTracker) _CV_AVAILABLE = True except ImportError: _CV_AVAILABLE = False @@ -94,7 +98,6 @@ def _save_local(data: dict) -> None: # ── App state ───────────────────────────────────────────────────────────────── class AppState(Enum): - BG_CAL = auto() SCREENSAVER = auto() GAME = auto() @@ -215,11 +218,32 @@ async def _serve() -> None: # ── Camera thread ───────────────────────────────────────────────────────────── +def build_detection_config(settings: dict) -> "DetectionConfig": + det_cfg = settings.get("detection", {}) + return DetectionConfig( + depth_delta_mm = det_cfg.get("depth_delta_mm", 25), + min_blob_pixels = det_cfg.get("min_blob_pixels", 500), + min_depth_mm = det_cfg.get("min_depth_mm", 500), + max_depth_mm = det_cfg.get("max_depth_mm", 2500), + ) + + +def build_background(settings: dict, detection_config: "DetectionConfig") -> "AdaptiveBackground": + """The self-learning depth background (ADR-0023), from settings["background"]. + + detection.depth_delta_mm is the lower bound on its per-pixel threshold. + """ + cfg = AdaptiveBackgroundConfig.from_dict( + settings.get("background", {}), + depth_delta_mm=detection_config.depth_delta_mm) + return AdaptiveBackground(cfg) + + def _camera_thread(camera: Any, settings: dict, cam_q: queue.Queue, stop: threading.Event, - collect: threading.Event) -> None: + reseed: threading.Event | None = None) -> None: try: - _camera_inner(camera, settings, cam_q, stop, collect) + _camera_inner(camera, settings, cam_q, stop, reseed) except Exception as exc: log.error("Camera thread crashed: %s", exc, exc_info=True) cam_q.put({"type": "error", "msg": str(exc)}) @@ -227,52 +251,25 @@ def _camera_thread(camera: Any, settings: dict, def _camera_inner(camera: Any, settings: dict, cam_q: queue.Queue, stop: threading.Event, - collect: threading.Event) -> None: - det_cfg = settings.get("detection", {}) - calib_frames = settings.get("calibration_frames", 60) + reseed: threading.Event | None = None) -> None: + """Every frame updates the background model and emits foreground — from frame 1. - calibrator = Calibrator(calib_frames) - cam_transform = build_cam_transform(settings) - sil_dilation_px = int(settings.get("silhouette_dilation_px", 1)) - frame_idx = 0 + Setting `reseed` makes the model relearn the background from the next frame. + """ + detection_config = build_detection_config(settings) + background = build_background(settings, detection_config) + tracker = BlobTracker(background, detection_config) + cam_transform = build_cam_transform(settings) + sil_dilation_px = int(settings.get("silhouette_dilation_px", 1)) with camera: - collecting = False - for frame in camera.frames(): - if stop.is_set(): - return - # Keep the stream warm but discard frames until the main loop's - # standback countdown clears, so the background model is only built - # from frames taken after people have had time to leave the ROI. - if not collecting: - if not collect.is_set(): - continue - collecting = True - log.info("BG_CAL: collecting %d frames", calib_frames) - frame_idx += 1 - cam_q.put({"type": "cal_progress", - "frame": frame_idx, "total": calib_frames}) - if calibrator.push_frame(frame): - break - - if stop.is_set(): - return - - calibration = calibrator.build() - log.info("BG_CAL complete") - cam_q.put({"type": "cal_done"}) - - detection_config = DetectionConfig( - depth_delta_mm = det_cfg.get("depth_delta_mm", 25), - min_blob_pixels = det_cfg.get("min_blob_pixels", 500), - min_depth_mm = det_cfg.get("min_depth_mm", 500), - max_depth_mm = det_cfg.get("max_depth_mm", 2500), - ) - tracker = BlobTracker(calibration, detection_config) - for frame in camera.frames(): if stop.is_set(): break + if reseed is not None and reseed.is_set(): + reseed.clear() + background.reseed() + background.push(frame) foreground = tracker.detect_foreground(frame) corrected = apply_cam_transform( foreground, getattr(frame, "intrinsics", None), cam_transform, @@ -444,16 +441,6 @@ def main() -> None: attract_dwell = float(settings.get("attract_dwell_s", 3.0)) min_fg_px = int(settings.get("min_foreground_pixels", 2000)) - # Calibration countdown (ADR-0013): show a "stand back / calibrating" screen - # for calibration_countdown_s total, holding off frame collection for the - # first calibration_standback_s so people can clear the ROI. - cal_countdown_s = float(settings.get("calibration_countdown_s", 10.0)) - cal_standback_s = float(settings.get("calibration_standback_s", 5.0)) - if cal_standback_s >= cal_countdown_s: - log.warning("calibration_standback_s (%.1f) >= calibration_countdown_s (%.1f); " - "clamping standback to countdown - 1", cal_standback_s, cal_countdown_s) - cal_standback_s = max(0.0, cal_countdown_s - 1.0) - # ── Test input handler ──────────────────────────────────────────────────── test_handler: TestInputHandler | None = None if use_test_input: @@ -522,27 +509,26 @@ def _make_camera() -> Any: # ── Camera thread state ─────────────────────────────────────────────────── cam: dict[str, Any] = { - "q": queue.Queue(maxsize=32), - "stop": threading.Event(), - "collect": threading.Event(), - "thread": None, + "q": queue.Queue(maxsize=32), + "stop": threading.Event(), + "reseed": threading.Event(), + "thread": None, } def _start_camera() -> None: - cam["q"] = queue.Queue(maxsize=32) - cam["stop"] = threading.Event() - cam["collect"] = threading.Event() + cam["q"] = queue.Queue(maxsize=32) + cam["stop"] = threading.Event() + cam["reseed"] = threading.Event() if args.camera or use_mock: t = threading.Thread( target=_camera_thread, - args=(_make_camera(), settings, cam["q"], cam["stop"], cam["collect"]), + args=(_make_camera(), settings, cam["q"], cam["stop"], cam["reseed"]), daemon=True, ) t.start() cam["thread"] = t - log.info("Camera thread started (%s)", "mock" if use_mock else "Orbbec") - else: - cam["q"].put({"type": "cal_done"}) + log.info("Camera thread started (%s) — background learns from the first frame", + "mock" if use_mock else "Orbbec") def _do_restart() -> None: cam["stop"].set() @@ -550,24 +536,20 @@ def _do_restart() -> None: cam["thread"].join(timeout=5.0) _save_local({}) _start_camera() - log.info("Camera restarted — entering BG_CAL") + log.info("Camera restarted — background relearning from the first frame") # ── State ───────────────────────────────────────────────────────────────── - state: AppState = AppState.BG_CAL + state: AppState = AppState.SCREENSAVER current_foreground: np.ndarray | None = None sil_cache: pygame.Surface | None = None # scaled silhouette, rebuilt on new frames fg_pixel_count = 0 attract_elapsed = 0.0 saver_elapsed = 0.0 SAVER_DWELL = 120.0 - cal_frame = 0 - cal_total = settings.get("calibration_frames", 60) - cal_elapsed = 0.0 - cal_started = False # have we signalled the camera to collect yet? - cal_done_seen = False # has the camera finished building the model? t_last_osc = time.monotonic() _start_camera() + broadcast({"state": "screensaver"}) # ── Main loop ───────────────────────────────────────────────────────────── while True: @@ -581,26 +563,22 @@ def _do_restart() -> None: if event.key in (pygame.K_q, pygame.K_ESCAPE): cam["stop"].set(); pygame.quit(); sys.exit(0) elif event.key == pygame.K_r: - state = AppState.BG_CAL - current_foreground = None - sil_cache = None - attract_elapsed = 0.0 - cal_elapsed = 0.0 - cal_started = False - cal_done_seen = False - threading.Thread(target=_do_restart, daemon=True).start() + cam["reseed"].set() + log.info("R: reseeding the depth background from the next frame") elif event.key == pygame.K_c and test_handler: test_handler.clear() # HTTP restart if _http_restart.is_set(): _http_restart.clear() - state = AppState.BG_CAL + if state == AppState.GAME: + cur_saver = saver_bag.next() or no_saver + saver_elapsed = 0.0 + state = AppState.SCREENSAVER current_foreground = None + sil_cache = None + fg_pixel_count = 0 attract_elapsed = 0.0 - cal_elapsed = 0.0 - cal_started = False - cal_done_seen = False threading.Thread(target=_do_restart, daemon=True).start() # Test-input: update paint canvas and push as foreground frame @@ -615,21 +593,7 @@ def _do_restart() -> None: msg = cam["q"].get_nowait() mtype = msg["type"] - if mtype == "cal_progress": - cal_frame = msg["frame"] - cal_total = msg["total"] - broadcast({"state": "bg_cal", - "progress": cal_frame / max(cal_total, 1)}) - - elif mtype == "cal_done": - # Model built. Don't advance yet — the BG_CAL branch holds the - # screen until the countdown also elapses (gates a fast camera), - # and keeps holding past the countdown if this never arrives - # (gates a slow camera). - cal_done_seen = True - log.info("BG_CAL frames collected") - - elif mtype == "foreground": + if mtype == "foreground": current_foreground = msg["frame"] fg_new = True fg_pixel_count = int(np.count_nonzero(current_foreground)) @@ -646,47 +610,7 @@ def _do_restart() -> None: # ── Draw ────────────────────────────────────────────────────────────── screen.fill(BLACK) - if state == AppState.BG_CAL: - cal_elapsed += dt - - # Once the standback window passes, tell the camera to start - # collecting frames for the background model. - if not cal_started and cal_elapsed >= cal_standback_s: - cam["collect"].set() - cal_started = True - log.info("BG_CAL: standback elapsed — signalling camera to collect") - - in_standback = cal_elapsed < cal_standback_s - if in_standback: - title = "PLEASE STAND BACK" - remaining = cal_standback_s - cal_elapsed - else: - title = "CALIBRATING…" - remaining = cal_countdown_s - cal_elapsed - - t_title = font_big.render(title, True, WHITE) - screen.blit(t_title, t_title.get_rect(center=(game_w // 2, WH // 2 - 40))) - - # Count 5→1, then drop the number. If the clock runs out before the - # camera reports cal_done, the screen freezes on "CALIBRATING…" with - # no number rather than advancing with a dirty model. - if remaining > 0.0: - t_num = font_big.render(f"{int(remaining) + 1}", True, WHITE) - screen.blit(t_num, t_num.get_rect(center=(game_w // 2, WH // 2 + 20))) - - broadcast({"state": "bg_cal", - "phase": "standback" if in_standback else "collect", - "remaining_s": round(max(0.0, remaining), 1), - "progress": cal_frame / max(cal_total, 1)}) - - if cal_elapsed >= cal_countdown_s and cal_done_seen: - state = AppState.SCREENSAVER - cur_saver = saver_bag.next() or no_saver - saver_elapsed = 0.0 - log.info("BG_CAL complete — entering SCREENSAVER") - broadcast({"state": "screensaver"}) - - elif state == AppState.SCREENSAVER: + if state == AppState.SCREENSAVER: saver_elapsed += dt if saver_elapsed >= SAVER_DWELL: saver_elapsed = 0.0 diff --git a/ShowControl/FundingCAPTCHA/body_grid_tester.py b/ShowControl/FundingCAPTCHA/body_grid_tester.py index 5590eb5..3b62c4f 100644 --- a/ShowControl/FundingCAPTCHA/body_grid_tester.py +++ b/ShowControl/FundingCAPTCHA/body_grid_tester.py @@ -5,14 +5,13 @@ Displays silhouettes detected in configurable depth slabs on a configurable grid. Used to tune camera ROI, slab depths, and grid size for Body Grid. -States: - BG_CAL → solid black; camera builds background depth model - LIVE → grid + silhouette display +Live from the first frame: the depth background is learned continuously, +with people in view (ADR-0023). No calibration screen. Keys: D → toggle debug overlay (mini thumbnails + stats) - C → cycle calibration views: background depth / valid mask / noise floor / threshold / delta - R → restart from BG_CAL + C → cycle background views: background depth / valid mask / noise σ / threshold / delta + R → reseed the depth background from the next frame Q → quit Config keys (captcha-settings.json / captcha-settings.local.json): @@ -22,7 +21,7 @@ slab_styles {"": {"color": [R,G,B]}} cell_activation_threshold float default 0.30 activation_rule "additive"|"max" default "additive" - calibration_frames int default 60 + background {...} self-learning background tunables (ADR-0023) """ from __future__ import annotations @@ -34,7 +33,6 @@ import sys import threading import time -from enum import Enum, auto from pathlib import Path import numpy as np @@ -48,7 +46,8 @@ sys.path.insert(0, str(DIR)) try: - from IIVision import (MockCamera, OrbbecCamera, Calibrator, BlobTracker, DetectionConfig) + from IIVision import (MockCamera, OrbbecCamera, AdaptiveBackground, + AdaptiveBackgroundConfig, BlobTracker, DetectionConfig) _CV_AVAILABLE = True except ImportError: _CV_AVAILABLE = False @@ -187,17 +186,21 @@ async def _serve() -> None: # ── Camera thread ────────────────────────────────────────────────────────────── +# Background-model views are re-sent this often: the model keeps learning. +MODEL_VIEW_S = 0.5 + + def _camera_thread(camera, settings: dict, cam_q: queue.Queue, - stop: threading.Event) -> None: + stop: threading.Event, reseed: threading.Event) -> None: try: - _camera_inner(camera, settings, cam_q, stop) + _camera_inner(camera, settings, cam_q, stop, reseed) except Exception as exc: log.error("Camera thread crashed: %s", exc, exc_info=True) cam_q.put({"type": "error", "msg": str(exc)}) def _camera_inner(camera, settings: dict, cam_q: queue.Queue, - stop: threading.Event) -> None: + stop: threading.Event, reseed: threading.Event) -> None: det_cfg = settings.get("detection", {}) detection_config = DetectionConfig( depth_delta_mm = det_cfg.get("depth_delta_mm", 25), @@ -205,46 +208,40 @@ def _camera_inner(camera, settings: dict, cam_q: queue.Queue, min_depth_mm = det_cfg.get("min_depth_mm", 500), max_depth_mm = det_cfg.get("max_depth_mm", 4000), ) + background = AdaptiveBackground(AdaptiveBackgroundConfig.from_dict( + settings.get("background", {}), depth_delta_mm=detection_config.depth_delta_mm)) + tracker = BlobTracker(background, detection_config) - calib_frames = settings.get("calibration_frames", 60) cam_transform = build_cam_transform(settings) sil_dilation_px = int(settings.get("silhouette_dilation_px", 1)) - log.info("BG_CAL: collecting %d frames", calib_frames) - calibrator = Calibrator(calib_frames) - frame_idx = 0 + t_views = 0.0 with camera: for frame in camera.frames(): if stop.is_set(): - return - frame_idx += 1 - cam_q.put({"type": "cal_progress", "frame": frame_idx, "total": calib_frames}) - if calibrator.push_frame(frame): break + if reseed.is_set(): + reseed.clear() + background.reseed() + background.push(frame) + H, W = background.height, background.width + bg_2d = background.background.reshape(H, W) + + now = time.monotonic() + if now - t_views >= MODEL_VIEW_S: + t_views = now + cam_q.put({ + "type": "model", + "cal_background": bg_2d.copy(), + "cal_valid_mask": background.valid_mask.reshape(H, W).astype(np.uint8) * 255, + "cal_noise_floor": background.noise.reshape(H, W).clip(0, 65535).astype(np.uint16), + "cal_threshold": tracker.threshold_map.copy(), + }) - if stop.is_set(): - return - - calibration = calibrator.build() - log.info("BG_CAL complete") - H, W = calibration.height, calibration.width - tracker = BlobTracker(calibration, detection_config) - cam_q.put({ - "type": "cal_done", - "cal_background": calibration.background.reshape(H, W).copy(), - "cal_valid_mask": (calibration.valid_mask.reshape(H, W).astype(np.uint8) * 255), - "cal_noise_floor": calibration.noise_floor.reshape(H, W).copy(), - "cal_threshold": tracker.threshold_map.copy(), - }) - - bg_2d = calibration.background.reshape(H, W).astype(np.int32) - - for frame in camera.frames(): - if stop.is_set(): - break fg = tracker.detect_foreground(frame) raw_arr = np.frombuffer(frame.data, dtype=np.uint16).reshape(H, W) - delta = np.clip(bg_2d - raw_arr.astype(np.int32), 0, 32767).astype(np.uint16) + delta = np.clip(bg_2d.astype(np.int32) - raw_arr.astype(np.int32), + 0, 32767).astype(np.uint16) corrected = apply_cam_transform( fg, getattr(frame, "intrinsics", None), cam_transform, dilation_px=sil_dilation_px, @@ -263,18 +260,6 @@ def _letterbox(win_w: int, win_h: int, aspect: float) -> tuple[int, int, int, in return (win_w - gw) // 2, (win_h - gh) // 2, gw, gh -def _draw_cal(surf: pygame.Surface, fonts: dict, - WW: int, WH: int, frame: int, total: int) -> None: - pct = frame / max(total, 1) - msg = fonts["big"].render(f"BG_CAL {frame}/{total}", True, YELLOW) - surf.blit(msg, (WW // 2 - msg.get_width() // 2, WH // 2 - 30)) - bw, bh = 400, 20 - bx, by = WW // 2 - bw // 2, WH // 2 + 20 - pygame.draw.rect(surf, MID_GREY, (bx, by, bw, bh)) - pygame.draw.rect(surf, GREEN, (bx, by, int(bw * pct), bh)) - pygame.draw.rect(surf, WHITE, (bx, by, bw, bh), 1) - - def _draw_grid(surf: pygame.Surface, cfg: BodyGridConfig, activations: CellActivations, slab_styles: dict, gx: int, gy: int, gw: int, gh: int) -> None: @@ -326,7 +311,7 @@ def _draw_silhouette(surf: pygame.Surface, foreground_roi: np.ndarray, "", "background depth (mm)", "valid mask [white=trusted]", - "noise floor IQR (mm)", + "noise σ (mm)", "effective threshold (mm)", "delta: bg − current (mm)", ] @@ -391,7 +376,7 @@ def _line(text: str, color: tuple = LT_GREY) -> None: _line(f" delta: {det_cfg.get('depth_delta_mm', 25)}mm") _line(f" range: {det_cfg.get('min_depth_mm', 500)}–{det_cfg.get('max_depth_mm', 4000)}mm") _line(f" blobs: ≥{det_cfg.get('min_blob_pixels', 500)}px") - _line("C: cycle cal view", LT_GREY) + _line("C: cycle bg view R: reseed", LT_GREY) # Per-cell coverage in debug: draw on grid cells cell_w = gw / cfg.cols @@ -419,9 +404,9 @@ def _mini_surf(arr: np.ndarray, mini_h: int) -> tuple[pygame.Surface, int]: for arr, label in [ (foreground_roi, "foreground ROI"), (raw_frame, "raw depth"), - (cal_background, "cal: background"), - (cal_valid_mask, "cal: valid mask"), - (cal_noise_floor, "cal: noise floor"), + (cal_background, "bg: background"), + (cal_valid_mask, "bg: valid mask"), + (cal_noise_floor, "bg: noise σ"), ]: if arr is None or arr.size == 0: continue @@ -435,13 +420,6 @@ def _mini_surf(arr: np.ndarray, mini_h: int) -> tuple[pygame.Surface, int]: bottom_y -= lbl.get_height() + 5 -# ── App state ────────────────────────────────────────────────────────────────── - -class AppState(Enum): - BG_CAL = auto() - LIVE = auto() - - # ── Main ─────────────────────────────────────────────────────────────────────── def main() -> None: @@ -496,23 +474,19 @@ def _make_camera(): height=cam_cfg.get("height", 400), fps=cam_cfg.get("fps", 10)) - cam: dict = {"q": queue.Queue(maxsize=32), "stop": threading.Event(), "thread": None} + cam: dict = {"q": queue.Queue(maxsize=32), "stop": threading.Event(), + "reseed": threading.Event(), "thread": None} def _start_camera() -> None: - cam["q"] = queue.Queue(maxsize=32) - cam["stop"] = threading.Event() if args.camera or use_mock: t = threading.Thread( target=_camera_thread, - args=(_make_camera(), settings, cam["q"], cam["stop"]), + args=(_make_camera(), settings, cam["q"], cam["stop"], cam["reseed"]), daemon=True, ) t.start() cam["thread"] = t - else: - cam["q"].put({"type": "cal_done"}) - state: AppState = AppState.BG_CAL activations: CellActivations = {} foreground_roi: np.ndarray | None = None raw_frame: np.ndarray | None = None @@ -521,8 +495,7 @@ def _start_camera() -> None: cal_valid_mask: np.ndarray | None = None cal_noise_floor: np.ndarray | None = None cal_threshold: np.ndarray | None = None - cal_view: int = 0 # 0=off, 1-5 cycle through cal views - cal_frame, cal_total = 0, settings.get("calibration_frames", 60) + cal_view: int = 0 # 0=off, 1-5 cycle through background views fps_display = 0.0 _start_camera() @@ -540,22 +513,12 @@ def _start_camera() -> None: elif event.key == pygame.K_d: debug = not debug log.info("Debug overlay %s", "on" if debug else "off") - elif event.key == pygame.K_c and state == AppState.LIVE: + elif event.key == pygame.K_c: cal_view = (cal_view + 1) % len(_CAL_VIEW_LABELS) log.info("Cal view: %s", _CAL_VIEW_LABELS[cal_view] or "off") elif event.key == pygame.K_r: - state = AppState.BG_CAL - activations = {} - foreground_roi = None - raw_frame = None - raw_delta = None - cal_background = None - cal_valid_mask = None - cal_noise_floor = None - cal_threshold = None - cal_view = 0 - cam["stop"].set() - threading.Thread(target=_start_camera, daemon=True).start() + cam["reseed"].set() + log.info("R: reseeding the depth background from the next frame") if _debug_toggle_event.is_set(): _debug_toggle_event.clear() @@ -567,19 +530,11 @@ def _start_camera() -> None: msg = cam["q"].get_nowait() mtype = msg["type"] - if mtype == "cal_progress": - cal_frame = msg["frame"] - cal_total = msg["total"] - broadcast({"state": "bg_cal", "progress": cal_frame / max(cal_total, 1)}) - - elif mtype == "cal_done": - state = AppState.LIVE - cal_background = msg.get("cal_background") - cal_valid_mask = msg.get("cal_valid_mask") - cal_noise_floor = msg.get("cal_noise_floor") - cal_threshold = msg.get("cal_threshold") - log.info("BG_CAL complete — LIVE") - broadcast({"state": "live"}) + if mtype == "model": + cal_background = msg["cal_background"] + cal_valid_mask = msg["cal_valid_mask"] + cal_noise_floor = msg["cal_noise_floor"] + cal_threshold = msg["cal_threshold"] elif mtype == "foreground": fg: np.ndarray = msg["frame"] # already corrected + mirrored @@ -598,26 +553,22 @@ def _start_camera() -> None: screen.fill(BLACK) - if state == AppState.BG_CAL: - _draw_cal(screen, fonts, WW, WH, cal_frame, cal_total) - - elif state == AppState.LIVE: - if cal_view > 0: - _cal_arrs = [None, cal_background, cal_valid_mask, - cal_noise_floor, cal_threshold, raw_delta] - arr = _cal_arrs[cal_view] if cal_view < len(_cal_arrs) else None - _draw_cal_view(screen, fonts, arr, _CAL_VIEW_LABELS[cal_view], - gx, gy, gw, gh) - else: - _draw_grid(screen, bg_cfg, activations, slab_styles, gx, gy, gw, gh) - if foreground_roi is not None: - _draw_silhouette(screen, foreground_roi, bg_cfg, slab_styles, gx, gy, gw, gh) - if debug: - _draw_debug(screen, fonts, foreground_roi, raw_frame, activations, - bg_cfg, fps_display, - cal_background, cal_valid_mask, cal_noise_floor, - settings.get("detection", {}), - gx, gy, gw, gh) + if cal_view > 0: + _cal_arrs = [None, cal_background, cal_valid_mask, + cal_noise_floor, cal_threshold, raw_delta] + arr = _cal_arrs[cal_view] if cal_view < len(_cal_arrs) else None + _draw_cal_view(screen, fonts, arr, _CAL_VIEW_LABELS[cal_view], + gx, gy, gw, gh) + else: + _draw_grid(screen, bg_cfg, activations, slab_styles, gx, gy, gw, gh) + if foreground_roi is not None: + _draw_silhouette(screen, foreground_roi, bg_cfg, slab_styles, gx, gy, gw, gh) + if debug: + _draw_debug(screen, fonts, foreground_roi, raw_frame, activations, + bg_cfg, fps_display, + cal_background, cal_valid_mask, cal_noise_floor, + settings.get("detection", {}), + gx, gy, gw, gh) pygame.display.flip() diff --git a/ShowControl/FundingCAPTCHA/captcha-settings.json b/ShowControl/FundingCAPTCHA/captcha-settings.json index 148a233..99c67a4 100644 --- a/ShowControl/FundingCAPTCHA/captcha-settings.json +++ b/ShowControl/FundingCAPTCHA/captcha-settings.json @@ -27,9 +27,19 @@ "min_foreground_pixels": 2000, "silhouette_opacity": 0.55, - "calibration_frames": 60, - "calibration_countdown_s": 10.0, - "calibration_standback_s": 5.0, + "background": { + "_comment": "Self-learning depth background (ADR-0023): the farthest steady depth wins. detection.depth_delta_mm is the lower bound on the threshold. absorb_s must stay above the longest deliberate stand-still (Watering can: 120 s). warmup_s skips the camera's first frames, which read too far.", + "warmup_s": 1.0, + "min_valid_frames": 10, + "reveal_frames": 5, + "noise_k": 4, + "bg_alpha": 0.01, + "noise_alpha": 0.05, + "absorb_s": 300, + "reseed_fg_fraction": 0.4, + "reseed_after_s": 30, + "stats_log_s": 30 + }, "bodycaptcha": { "timer_s": 35, diff --git a/ShowControl/FundingCAPTCHA/playlist_app.py b/ShowControl/FundingCAPTCHA/playlist_app.py index 1cfec29..bd25b00 100644 --- a/ShowControl/FundingCAPTCHA/playlist_app.py +++ b/ShowControl/FundingCAPTCHA/playlist_app.py @@ -6,8 +6,8 @@ setup_duration_s (overriding its own duration_s); a video plays to the end of its file, silent and letterboxed. -The camera runs throughout, so calibration happens once and Setups find the -Body live whenever they come round. +The camera runs throughout, so the depth background keeps learning (ADR-0023) +and Setups find the Body live whenever they come round. python3 playlist_app.py --camera # show python3 playlist_app.py --mock-camera --windowed # drifting fake bodies diff --git a/ShowControl/FundingCAPTCHA/shader_app.py b/ShowControl/FundingCAPTCHA/shader_app.py index 71f405d..1ffa122 100644 --- a/ShowControl/FundingCAPTCHA/shader_app.py +++ b/ShowControl/FundingCAPTCHA/shader_app.py @@ -2,8 +2,9 @@ Cycles through the Setups listed in shaders/setups.json, each for its duration_s, feeding the Orbbec silhouette in as Buffer A (the Body). -Shares the camera, calibration, and settings of app.py; replaces its -game loop with an OpenGL one. +Shares the camera, self-learning background (ADR-0023), and settings of +app.py; replaces its game loop with an OpenGL one. Bodies show from launch: +there is no standback or calibration wait. python3 shader_app.py --camera # show python3 shader_app.py --mock-camera # drifting fake bodies @@ -195,20 +196,14 @@ def start_camera(args, settings: dict) -> tuple[queue.Queue, threading.Event]: if not _CV_AVAILABLE: sys.exit("ERROR: CV libraries not available — pip install -r requirements.txt") cam_cfg = settings.get("camera", {}) - collect = threading.Event() cam_args = dict(width=cam_cfg.get("width", 640), height=cam_cfg.get("height", 400), fps=cam_cfg.get("fps", 10)) camera = (MockCamera(**cam_args) if args.mock_camera else OrbbecCamera(serial=cam_cfg.get("serial") or None, **cam_args)) threading.Thread(target=_camera_thread, - args=(camera, settings, cam_q, stop, collect), daemon=True).start() - # Hold off background collection so people can clear the ROI (ADR-0013). - standback = float(settings.get("calibration_standback_s", 5.0)) - timer = threading.Timer(standback, collect.set) - timer.daemon = True - timer.start() - log.info("Camera started (%s) — calibrating in %.0fs, keep the floor clear", - "mock" if args.mock_camera else "Orbbec", standback) + args=(camera, settings, cam_q, stop), daemon=True).start() + log.info("Camera started (%s) — background learns from the first frame", + "mock" if args.mock_camera else "Orbbec") return cam_q, stop @@ -222,8 +217,6 @@ def drain_camera(cam_q: queue.Queue): return fg if msg["type"] == "foreground": fg = msg["frame"] - elif msg["type"] == "cal_done": - log.info("Calibration done — Body live") elif msg["type"] == "error": log.error("Camera: %s", msg["msg"]) diff --git a/ShowControl/FundingCAPTCHA/test_live_background.py b/ShowControl/FundingCAPTCHA/test_live_background.py new file mode 100644 index 0000000..3f90779 --- /dev/null +++ b/ShowControl/FundingCAPTCHA/test_live_background.py @@ -0,0 +1,58 @@ +"""Entry points go live from the first frame: no standback, no calibration (ADR-0023).""" +import argparse +import json +import time +from pathlib import Path + +import numpy as np + +import app +import body_grid_tester +from shader_app import drain_camera, start_camera +from shadertoy import pack_body + +SETTINGS = json.loads((Path(__file__).parent / "captcha-settings.json").read_text()) + + +def test_mock_camera_shows_bodies_within_about_two_seconds(): + # MockCamera has its bodies in view from frame 0. They must not wait for a + # calibration pass, and must not be baked into the background for good. + args = argparse.Namespace(camera=False, mock_camera=True, test_input=False) + cam_q, stop = start_camera(args, SETTINGS) + t0 = time.monotonic() + body_px = 0 + try: + while time.monotonic() - t0 < 5.0: + fg = drain_camera(cam_q) + if fg is not None: + body = pack_body(fg, SETTINGS["camera_roi"], SETTINGS["depth_slabs"]) + body_px = int(np.count_nonzero(body[..., 0])) + if body_px > 200: + break + time.sleep(0.01) + elapsed = time.monotonic() - t0 + finally: + stop.set() + assert body_px > 200, "no body in the Body texture after 5 s" + # background.warmup_s (1 s, the Orbbec's bad first frames) + min_valid_frames. + warmup = SETTINGS["background"]["warmup_s"] + assert elapsed < warmup + 1.5, f"bodies took {elapsed:.2f}s to appear" + + +def test_no_calibration_settings_or_states_remain(): + for key in ("calibration_frames", "calibration_countdown_s", "calibration_standback_s"): + assert key not in SETTINGS + assert "background" in SETTINGS + assert {s.name for s in app.AppState} == {"SCREENSAVER", "GAME"} + assert not hasattr(body_grid_tester, "AppState") + + +def test_background_settings_build_the_model(): + det = app.build_detection_config(SETTINGS) + model = app.build_background(SETTINGS, det) + cfg = model.config + assert cfg.depth_delta_mm == SETTINGS["detection"]["depth_delta_mm"] + assert cfg.absorb_s > 120, "must outlast Watering can's 120 s stand-still" + for key, value in SETTINGS["background"].items(): + if not key.startswith("_"): + assert getattr(cfg, key) == value, key diff --git a/docs/FundingCAPTCHA.md b/docs/FundingCAPTCHA.md index dd0322e..37e6707 100644 --- a/docs/FundingCAPTCHA.md +++ b/docs/FundingCAPTCHA.md @@ -8,14 +8,13 @@ The 7-foot CRT-styled kiosk Element. Players activate a projected grid using the ## Architecture summary -`app.py` is a single pygame process (ADR-0012) — no browser, no kiosk service. It owns the projector display, depth camera pipeline, BG calibration, **Game** rotation, and a lightweight HTTP/WebSocket monitoring server on port 8080. +`app.py` is a single pygame process (ADR-0012) — no browser, no kiosk service. It owns the projector display, depth camera pipeline, **Game** rotation, and a lightweight HTTP/WebSocket monitoring server on port 8080. -State machine: -- **BG_CAL** — solid black; depth camera builds a `calibration_frames` background model +State machine (no calibration screen — the depth background is learned continuously from the first frame, with people in view; ADR-0023): - **SCREENSAVER** — idle animation; detects a Player and counts down `attract_dwell_s` - **GAME** — active Arc; ends with a Blow-Up, returns to SCREENSAVER -Pipeline — Orbbec depth camera → IIVision background subtraction → despeckle → silhouette warp → **Body Grid** activator (ADR-0013). The Body Grid maps depth pixels inside configured **Depth Slabs** to a boolean grid of cells; cells go active when `≥ cell_activation_threshold` of their pixels are covered by foreground inside any slab. +Pipeline — Orbbec depth camera → IIVision self-learning background (`AdaptiveBackground`, ADR-0023) → background subtraction → despeckle → silhouette warp → **Body Grid** activator (ADR-0013). The Body Grid maps depth pixels inside configured **Depth Slabs** to a boolean grid of cells; cells go active when `≥ cell_activation_threshold` of their pixels are covered by foreground inside any slab. Display: pygame `FULLSCREEN | NOFRAME`. Direct DRM (`SDL_VIDEODRIVER=kmsdrm`) — Pi OS Lite, no compositor (ADR-0011). @@ -31,6 +30,7 @@ Key references: - ADR-0019 — game UI shell - ADR-0021 — shader player (`shader_app.py`, Shadertoy Setups) - ADR-0022 — shader player blob records (`blobs` channel source) +- ADR-0023 — self-learning depth background (no boot-time standback) - `app.py`, `body_grid.py`, `silhouette.py`, `games/`, `ScreenSavers/` --- @@ -48,7 +48,7 @@ Key references: The first three are mutually exclusive. Keys at runtime: -- `R` — restart from BG_CAL +- `R` — reseed the depth background from the next frame (e.g. after moving furniture) - `Q` / Escape — quit - `C` — clear paint canvas (`--test-input` only) @@ -56,7 +56,7 @@ Keys at runtime: ## Shader player — `shader_app.py` -Alternative show mode: instead of the Game Arc, cycles Shadertoy **Setups** from `shaders/` with the depth silhouette as Buffer A (ADR-0021). Same camera, calibration, and settings as `app.py`. Prototyping guide: [`shaders/README.md`](../ShowControl/FundingCAPTCHA/shaders/README.md). +Alternative show mode: instead of the Game Arc, cycles Shadertoy **Setups** from `shaders/` with the depth silhouette as Buffer A (ADR-0021). Same camera, background model, and settings as `app.py`. Prototyping guide: [`shaders/README.md`](../ShowControl/FundingCAPTCHA/shaders/README.md). Setups in rotation: `ripples` (example), `endless_sunset` ([design](superpowers/specs/2026-10-30-endless-sunset-design.md); tunables at the top of its `common.glsl`), `hunter_bird` ([design](superpowers/specs/2026-11-04-hunter-bird-design.md); a bird's-eye debug view of the blob tracker — tunables, including the motion noise floor, at the top of its `common.glsl`), `watering_can` ([design](superpowers/specs/2026-11-06-watering-can-design.md); a visitor standing still grows vines, leaves and flowers over their silhouette in ~20 s, and stepping sideways rewinds them — tunables, including `WALK_DIST`, `GROW_S` and `REWIND_S`, at the top of its `common.glsl`), `rainbow_poop` ([design](superpowers/specs/2026-11-08-rainbow-poop-design.md); about every 5 s a rainbow poop drifts down from the sky on to each visitor, and the splats stick to their silhouette, drip and fade over 20 s — step back the other way after it appears and it lands on the floor; tunables, including `BOMB_EVERY_S`, `FALL_SPEED`, `SPLAT_R` and `SPLAT_LIFE_S`, at the top of its `common.glsl`). @@ -82,7 +82,7 @@ Shader Setups and full-screen videos in one shuffle bag: every item plays once b - `setups`: Setup folder names from `shaders/`. Each plays for `setup_duration_s` (300 s), which overrides the Setup's own `duration_s`. - `videos`: file names in `ShowControl/FundingCAPTCHA/videos/`. Each plays to the end of its file, silent and letterboxed. The folder is gitignored, so copy the files onto the show PC. Decoding uses PyAV (`av` in `requirements.txt`). -A missing or unreadable item is logged and skipped. The camera runs throughout, so calibration happens once at startup. +A missing or unreadable item is logged and skipped. The camera runs throughout, so the background keeps learning across items. Flags: `--camera` / `--mock-camera` / `--test-input` / `--windowed` as `shader_app.py`, and `--playlist PATH` to use another playlist file. Keys: `N` / `→` next item · `R` reload the current Setup from disk · Escape quit. @@ -104,7 +104,7 @@ Top-level fields: - `min_foreground_pixels` — anti-noise threshold for Player presence - `silhouette_opacity` — 0–1, silhouette overlay opacity in-game - `silhouette_dilation_px` — closes the scatter holes reprojection leaves in the silhouette (default 1). Raise if the silhouette looks holey at this camera angle; each step also fattens it by a pixel, shifting which Body Grid cells activate at the edges. -- `calibration_frames` — default 60 +- `background` — self-learning depth background (ADR-0023): `warmup_s` (ignore the camera's first, too-far frames), `min_valid_frames`, `reveal_frames`, `noise_k`, `bg_alpha`, `noise_alpha`, `absorb_s` (keep > 120 for Watering can), `reseed_fg_fraction`, `reseed_after_s`, `stats_log_s` (how often per-frame update time is logged). `detection.depth_delta_mm` is the lower bound on its threshold. - Per-game blocks: `bodycaptcha`, `keepaway_body` (timer_s, hold_s, hold_decay_rate, grace_s, abandon_s, weights for **Intensity** computation) --- @@ -162,7 +162,7 @@ FundingCAPTCHA sends to the TreeHouse Hub each frame: ## Monitoring server `http://:8080/` — small HTML page with state + live log stream over WebSocket. Includes: -- Current `AppState` (BG_CAL / SCREENSAVER / GAME) +- Current `AppState` (SCREENSAVER / GAME) - Current Game + Level - Live `intensity` value - Last 100 log lines @@ -246,7 +246,7 @@ If the install uses private show content, also re-run the sync — see ## Body Grid tester -`body_grid_tester.py` — standalone tool to exercise the Body Grid in isolation (no Game). Useful for verifying camera calibration + slab config visually. +`body_grid_tester.py` — standalone tool to exercise the Body Grid in isolation (no Game). Useful for verifying the background model + slab config visually (`C` cycles background views, `R` reseeds). ```bash cd ShowControl/FundingCAPTCHA @@ -259,8 +259,9 @@ python3 body_grid_tester.py | Symptom | Likely cause | Fix | |---|---|---| -| Black screen, stuck in BG_CAL | Camera not producing frames | Check Orbbec USB; check udev rule (`install.sh` writes it — re-run if missing); `journalctl -u captcha -f` | +| No silhouette ever appears | Camera not producing frames | Check Orbbec USB; check udev rule (`install.sh` writes it — re-run if missing); `journalctl -u captcha -f` | | Players ignored, screensaver loops | `min_foreground_pixels` too high, or slab depths wrong | Body grid tester; tune `depth_slabs[]` near/far per Play Zone | +| A ghost silhouette that won't go away, or a still Player fading out | Something moved into the scene (absorbed after `background.absorb_s`), or a Player stood still longer than that | Wait it out, or press `R` to reseed the background; tune `background.absorb_s` (keep > 120) | | Cells flicker on/off | `cell_activation_threshold` too low | Raise it (try 0.40) | | Game never escalates | Level set thin or shuffle bag favoring easy | Add Levels at higher `difficulty`; check `intensity_weights` | | Game refuses to end | Game lacks self-termination | Audit Game code against ADR-0003 | diff --git a/docs/adr/0007-osc-fabric-schema.md b/docs/adr/0007-osc-fabric-schema.md index a033147..e25e19d 100644 --- a/docs/adr/0007-osc-fabric-schema.md +++ b/docs/adr/0007-osc-fabric-schema.md @@ -22,7 +22,7 @@ The TreeHouse OSC receive side had four addresses sketched in `osc_server.py`; t | `/treehouse/mode` | Dashboard → TreeHouse | string | `"active"` \| `"dim"` \| `"inactive"` | | `/flowerbeds/mode` | Dashboard → FlowerBeds | string | `"active"` \| `"dim"` \| `"inactive"` | | `/captcha/mode` | Dashboard → CAPTCHA | string | `"active"` \| `"dim"` \| `"inactive"` | -| `/captcha/restart` | Any → CAPTCHA | (no args) | Hard-restart CV pipeline: close camera, reopen, redo background calibration | +| `/captcha/restart` | Any → CAPTCHA | (no args) | Hard-restart CV pipeline: close camera, reopen, relearn the depth background (ADR-0023) | | `/pipes/mode` | Dashboard → Pipes | string | `"active"` \| `"dim"` \| `"inactive"` | | `/treehouse/brightness` | TouchOSC → TreeHouse | float 0.0–1.0 | Debug/operator override only; not part of the mode system | diff --git a/docs/adr/0013-fundingcaptcha-silhouette-body-interaction.md b/docs/adr/0013-fundingcaptcha-silhouette-body-interaction.md index 015f9e6..889bf2a 100644 --- a/docs/adr/0013-fundingcaptcha-silhouette-body-interaction.md +++ b/docs/adr/0013-fundingcaptcha-silhouette-body-interaction.md @@ -1,6 +1,7 @@ # FundingCAPTCHA uses body silhouette interaction, not touch detection **Supersedes:** ADR-0004 (depth-primary touch detection), ADR-0011 (screen calibration and projection) +**Partly superseded by:** [ADR-0023](0023-fundingcaptcha-self-learning-depth-background.md) — BG_CAL is gone; the depth background is learned continuously from the first frame ## Context @@ -14,9 +15,9 @@ Key consequences: **ScreenProjector is removed.** There is no screen plane to project onto. Camera pixels map to display pixels via a camera-mounting correction: the camera thread calls `apply_cam_transform()` (see `silhouette.py`) which reprojects the depth frame to appear as though the camera is centred on the screen — compensating for physical mounting position and orientation — then mirrors horizontally so players see a natural mirror view. ROI crop is applied downstream in `BodyGridActivator`. Games and screensavers receive a fully corrected, display-space frame and must not flip. -**CORNER_CAL is removed.** The per-installation corner-touch calibration step (operator touching three screen corners) is eliminated. BG_CAL (background model collection at startup) remains. +**CORNER_CAL is removed.** The per-installation corner-touch calibration step (operator touching three screen corners) is eliminated. BG_CAL (background model collection at startup) remains. *Superseded by ADR-0023: there is no BG_CAL; the background is learned continuously.* -**Blob detection is not used by FundingCAPTCHA.** The camera thread calls IIVision's `Calibrator` and background subtraction, then emits raw foreground depth frames. The `run_pipeline()` call (blob detection + stabilisation) is skipped. Grid cell activation is computed directly from per-pixel depth values. +**Blob detection is not used by FundingCAPTCHA.** The camera thread calls IIVision's background model (`AdaptiveBackground` since ADR-0023; `Calibrator` before) and background subtraction, then emits raw foreground depth frames. The `run_pipeline()` call (blob detection + stabilisation) is skipped. Grid cell activation is computed directly from per-pixel depth values. **Grid activation is per-pixel, not per-blob.** A cell activates when ≥ `cell_activation_threshold` (configurable, default 0.30) of its pixels are covered by a given Depth Slab. This produces smooth, intuitive silhouette-fills rather than point-contact events. diff --git a/docs/adr/0021-fundingcaptcha-shader-player.md b/docs/adr/0021-fundingcaptcha-shader-player.md index 4ffb78a..f54c524 100644 --- a/docs/adr/0021-fundingcaptcha-shader-player.md +++ b/docs/adr/0021-fundingcaptcha-shader-player.md @@ -1,6 +1,6 @@ # FundingCAPTCHA shader player — Shadertoy Setups driven by the depth camera -**Amended by:** [ADR-0022](0022-shader-player-blob-records.md) (a `blobs` channel source: boxes, stable IDs, lifecycle) +**Amended by:** [ADR-0022](0022-shader-player-blob-records.md) (a `blobs` channel source: boxes, stable IDs, lifecycle), [ADR-0023](0023-fundingcaptcha-self-learning-depth-background.md) (no calibration wait at startup) ## Context @@ -26,5 +26,5 @@ The existing app draws with pygame surfaces in software. Per-pixel procedural wo - Buffers are RGBA16F, not Shadertoy's 32-bit float: V3D can't linear-filter 32-bit float. `HW_PERFORMANCE` is defined as 0; GLSL ES packing built-ins come from `ARB_shading_language_packing`. - A Setup that throws while rendering is dropped from the rotation for the run, not fatal. - Unsupported Shadertoy features: cubemaps, sound, video/texture inputs, per-channel filter/wrap settings (always linear + clamp), Buffer A code. -- Calibration has no on-screen countdown: shaders play with an empty Body for `calibration_standback_s` plus collection. Keep people out of frame at startup. +- ~~Calibration has no on-screen countdown: shaders play with an empty Body for `calibration_standback_s` plus collection. Keep people out of frame at startup.~~ Superseded by ADR-0023: the background is learned from the first frame, so the Body is live in under a second and people in frame at startup appear after their first step. - Not yet measured on the Pi 5: frame rate per Setup. `shader_app.py` logs fps every 10 s. diff --git a/docs/adr/0023-fundingcaptcha-self-learning-depth-background.md b/docs/adr/0023-fundingcaptcha-self-learning-depth-background.md new file mode 100644 index 0000000..3201de5 --- /dev/null +++ b/docs/adr/0023-fundingcaptcha-self-learning-depth-background.md @@ -0,0 +1,59 @@ +# FundingCAPTCHA learns its depth background continuously — no boot-time standback + +**Supersedes:** the BG_CAL section of [ADR-0013](0013-fundingcaptcha-silhouette-body-interaction.md) +**Amends:** [ADR-0021](0021-fundingcaptcha-shader-player.md) (startup consequence) + +## Context + +Every launch of `app.py`, `shader_app.py` (and `playlist_app.py`, which shares its camera), and `body_grid_tester.py` ran BG_CAL: a `calibration_standback_s` hold-off, then `calibration_frames` frames into IIVision's `Calibrator`, which built a fixed per-pixel median background. The floor had to be clear for that whole time. Anyone standing in front of the camera was baked into the background until the next restart. That cost hit every time we switched between the game app and the shader player, or restarted either. + +**A saved background (`Calibration.save()` / `.load()`) was rejected.** It goes stale whenever things get moved or the camera gets bumped, and then someone has to remember to recalibrate. + +**Motion detection was rejected.** It loses anyone who stands still, and Watering can rewards exactly that. + +## Decision + +FundingCAPTCHA uses `IIVision.AdaptiveBackground`: **the farthest steady depth wins.** A person can only make a depth pixel nearer, never farther, so each pixel's background is the farthest depth it has held steadily. It is learned continuously from the first frame, with people in view. + +Per pixel, all vectorised numpy, with `thr = max(depth_delta_mm, noise_k × σ)`. Samples below 50 mm, including 0 (no return), are ignored. + +0. **Warm-up.** The first `warmup_s` of a stream are ignored. Measured on the Gemini 336: the first frames read up to ~400 mm too far (1 frame when warm, ~5 from cold). Seeding from them made the true surface look nearer, leaving little blobs that lasted until `absorb_s`. +1. **Seed.** `bg` and σ are the mean and standard deviation of the first `min_valid_frames` valid samples, and the pixel is trusted (`valid`) after them. Seeding from a single sample is fragile: on a noisy far pixel (σ ≈ 80 mm at 3.7 m), one far outlier left every later reading "nearer", so the pixel was always foreground, never at background, and too jumpy to be absorbed. +2. **At background** (`|d − bg| ≤ thr`). A slow EMA moves `bg` toward `d` (`bg_alpha`). An EMA of `|d − bg| / 0.8` tracks σ (`noise_alpha`). Mean absolute deviation is about 0.8σ. +3. **Farther** (`d > bg + thr`) for `reveal_frames` frames in a row. `bg` becomes the mean of those samples. One frame that isn't farther resets the run, so flying pixels and multipath spikes can't push `bg` out. +4. **Nearer** (`d < bg − thr`, which is foreground), holding steady within `thr`, for `absorb_s`. Then `bg = d`. This covers a dropped bag or a moved table. Any jump resets the timer. +5. **Bump guard.** If more than `reseed_fg_fraction` of valid pixels stay foreground for `reseed_after_s`, `bg` is reseeded from the current frame and a warning is logged. Shifts farther heal through rule 3 with no reseed. + +Foreground is computed exactly as before, by `BlobTracker._subtract_background`: the pixel is in range, `valid`, and nearer than `bg` by more than the model's threshold. Despeckling is unchanged. `BlobTracker` now reads a small `BackgroundModel` protocol (`background`, `valid_mask`, `threshold(depth_delta_mm)`). Both `Calibration` and `AdaptiveBackground` implement it. `Calibrator` stays, and FlowerBeds (`IIVision.pipeline`) still uses it. + +Tunables live in `captcha-settings.json` → `background`. `detection.depth_delta_mm` remains the lower bound on `thr`. + +| setting | default | note | +|---|---|---| +| `warmup_s` | 1.0 | covers the Gemini 336's cold-start frames plus settling | +| `min_valid_frames` | 10 | about 0.7 s at 15 fps | +| `reveal_frames` | 5 | | +| `noise_k` | 4 | about the old 3×IQR threshold | +| `bg_alpha` | 0.01 | per frame | +| `noise_alpha` | 0.05 | per frame | +| `absorb_s` | 300 | **must stay above 120** (Watering can) | +| `reseed_fg_fraction` | 0.4 | of valid pixels | +| `reseed_after_s` | 30 | | +| `stats_log_s` | 30 | how often per-frame update time is logged | + +## Behaviour + +- **Floor empty at boot.** Live as soon as pixels become valid: `warmup_s` plus `min_valid_frames`, about 2 s. There is no countdown or standback screen in any entry point. +- **Someone standing at boot.** They're invisible until they move. One step reveals the background behind them, and from then on they're tracked. +- **A spot nobody ever uncovers.** It's never learned, and it's never falsely marked foreground either. +- **Someone still for longer than `absorb_s`.** They're absorbed into the background, and they reappear when they move (rule 3). +- **`R` in `app.py` and `body_grid_tester.py`** reseeds the background from the next frame. `/api/restart` still restarts the camera thread, which starts a fresh model. + +## Consequences + +- `calibration_frames`, `calibration_countdown_s` and `calibration_standback_s` are removed from `captcha-settings.json`. So are the BG_CAL state and the `cal_progress` / `cal_done` camera messages. +- The model is never saved to disk. Each boot learns again from the first frame. +- Per-frame update time is logged every `stats_log_s`: 12–14 ms mean (24 ms max) at 640×480 with the live Orbbec on the show PC, against a 66 ms frame budget at 15 fps. +- Pixels with no valid background (open space beyond range) stay never-foreground, as before. Treating "no return" as infinitely far would be a separate change. +- The model learns while the projector shows content. ADR-0004's black-frame-during-calibration rule came from the touch-screen setup that ADR-0013 removed; the camera now faces visitors, not the screen. +- To check on hardware: the update time stays within the frame budget on the show machine; `absorb_s` suits real dwell times; and a dense crowd doesn't set off the bump guard. After a reseed the crowd becomes background, then heals as people leave. If that's not acceptable, require a scene-wide depth shift as well. diff --git a/docs/agents/architecture.md b/docs/agents/architecture.md index 549ce66..50de9c7 100644 --- a/docs/agents/architecture.md +++ b/docs/agents/architecture.md @@ -87,7 +87,7 @@ Runs per frame once calibrated: 3. **Connected components** — 8-connected blobs via `scipy.ndimage.label` 4. **3D unproject** — pinhole model, median-Z windowing, camera → world coordinates -Calibration collects `calibration_frames` frames (from `settings.json`) to build a per-pixel median background and validity mask. +`BlobTracker` reads any background model with `background`, `valid_mask` and `threshold()`. FlowerBeds uses `Calibration`: `Calibrator` collects `calibration_frames` frames (from `settings.json`) to build a per-pixel median background and validity mask. FundingCAPTCHA uses `AdaptiveBackground` (`adaptive_background.py`, ADR-0023), learned continuously from the first frame — the farthest steady depth wins. ## Blob stabilisation (BlobStabilizer)