diff --git a/experiments/temporal-models/tube-builder-lab/docs/specs/2026-06-05-stable-tube-crop-window-design.md b/experiments/temporal-models/tube-builder-lab/docs/specs/2026-06-05-stable-tube-crop-window-design.md new file mode 100644 index 00000000..242848fc --- /dev/null +++ b/experiments/temporal-models/tube-builder-lab/docs/specs/2026-06-05-stable-tube-crop-window-design.md @@ -0,0 +1,116 @@ +# Per-tube stable crop window + comparison viz + +Date: 2026-06-05 +Status: implemented + +## Problem + +The crop the temporal head sees is computed **per-frame**. In +`bbox_tube_temporal` (`crop_tube_patches` / `model_input.process_tube`), each +tube entry's own bbox is expanded by `context_factor`, squared, and resized to +224. As a plume grows and drifts across the 30s-apart frames, the crop +**recenters and rescales every frame**: the smoke stays roughly centered and +same-size while the *background* slides and zooms. The result is a "jumpy" +sequence — the temporal head never sees the smoke actually move. + +We want the opposite: a **single fixed crop window per tube**, applied to every +frame, so the background stays static and the smoke visibly moves and grows +inside it. + +## Key decisions + +- **What is fixed:** position *and* size — one fixed window for the whole tube + (not size-only, not a smoothed/EMA window). Strongest "stable image, moving + smoke" effect. +- **How the window is derived:** the **union (enclosing) box** of all the tube's + observed detections. Guarantees the smoke is always inside the crop. Early + frames show a small plume in a large window — acceptable and expected. +- **No separate margin.** An earlier draft expanded the union by its own + `MARGIN` knob. Dropped: the crop step already adds context (`context_factor`, + `viz.CROP_CONTEXT`), and a second margin both double-counted and made the + stabilized crop tighter than the per-frame crops. The stabilized crop is now + built **identically** to a per-frame crop — same `CROP_CONTEXT`, same squaring — + just with the union box instead of a per-frame box. `CROP_CONTEXT` is the single + shared context knob; it currently sits at **1.6**, tightened by eye in the lab + below the model's training `context_factor ≈ 2.0` (a deliberate exploration + choice — note it now lowers the non-stabilized reference crop too, so that crop + no longer exactly mirrors the model's per-frame crop). +- **Placement (approach A → B):** stabilization is a **crop-window decision**, + not a tube-*building* step. It does not change which detections link into which + tube, so it never mutates the `Tube`. + - **A (this spec):** lab-only pure function + comparison viz; no changes to + `bbox_tube_temporal`. + - **B (later follow-up):** lift `tube_window` into the lib crop path as the + real temporal-head crop. Out of scope here. +- **Viz:** compare **non-stabilized vs stabilized** for the committed builder. + The former (current) builder was removed from the app (the candidate is now the + only baseline). + +### Note on the eventual production step (B) + +When stable crops are eventually fed to the temporal head, that is a different +input distribution than the head was trained on, so a **retrain** comes with it. +Not a concern for the lab viz; flagged so it is not forgotten. + +## Components & data flow + +``` +candidate tubes (unchanged) ──▶ stabilize.tube_window(tube) ──▶ (cx,cy,w,h) union window + │ +full frames ──▶ viz.crop_box_px (expand by CROP_CONTEXT, square) ─────┴──▶ crop / drawn box +``` + +### `src/tube_builder_lab/stabilize.py` (pure, unit-tested) + +- `tube_window(tube: Tube) -> tuple[float, float, float, float]` — the union + (enclosing) box of all **observed** detections, normalized. Raises + `ValueError` if the tube has no observed detection. Gap entries are ignored + (interpolated gap boxes are lerps of observed boxes, so the observed union + already encloses them). No margin applied. + +### `src/tube_builder_lab/viz.py` + +- `crop_box_px(bbox, img_w, img_h)` — **single source of truth** for the crop + region: expand by `CROP_CONTEXT`, then `norm_bbox_to_pixel_square`. Used by both + the crop and the drawn box, so *the box you see equals the crop you get*, for + per-frame boxes and union windows alike. +- `crop_tube_at_frame(image_path, bbox)` — square 224 crop via `crop_box_px`. + Pass a per-frame detection box for the non-stabilized crop, or a tube's + `tube_window` for the stabilized crop — same code path. +- `draw_tube_windows(image_path, windows)` — draws each `(window, colour)` as its + `crop_box_px` region; the stabilized-side mirror of `draw_tube_bboxes`, in the + tube colour. + +### `app.py` + +- **Former builder removed everywhere:** no current-vs-candidate frame view, + metric, timeline, or `current`/`Δ` columns; the summary tables show a single + `tubes` count and grey out missing-cache rows. +- **Main two-up viewer** (driven by the play/slider): left = non-stabilized full + frame with the per-frame box (`draw_tube_bboxes`); right = full frame with the + fixed union window (`draw_tube_windows`). Both in the tube colour, both shown + only for tubes active at the current frame; the window is identical every frame. +- **"candidate crops @ this frame"** expander: per tube, non-stabilized crop vs + stabilized crop side by side, shown only when the tube is active at the frame. + +## Edge cases + +- **Gap / no-detection frame:** the tube is treated as inactive there — neither + box nor crop is drawn (matches `draw_tube_bboxes` / `bboxes_at_frame`). +- **Single-box tube:** union = that box. +- **Image edges:** `norm_bbox_to_pixel_square` clamps to the image; `crop_and_resize` + re-squares by padding the missing side. + +## Tests + +- `tests/test_stabilize.py`: `tube_window` — two-box union, single-box, + axis-independent (w≠h) union, gap entries ignored, empty tube raises. +- `tests/test_viz.py`: `crop_box_px` — square + `CROP_CONTEXT`-scaled + centred, + and edge clamping. +- `make lint` + `make test` green. + +## Out of scope + +- Lifting `tube_window` into `bbox_tube_temporal` (approach B). +- Retraining the temporal head on the stabilized-crop distribution. +- Smoothed/EMA windows and size-only variants (rejected during brainstorm). diff --git a/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/app.py b/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/app.py index 1bda4caa..5969aa97 100644 --- a/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/app.py +++ b/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/app.py @@ -1,8 +1,9 @@ -"""Tube Builder Lab — current vs candidate tube linking, Layout A. +"""Tube Builder Lab — the builder's tubes + per-tube crop stabilization. Reads only data/05_model_input/detections/ + data/03_primary/sequences/**. Run with `streamlit run app.py` (or `make app`). Iterate by editing -src/tube_builder_lab/candidate.py and clicking "Re-run candidate". +src/tube_builder_lab/candidate.py (linking) or stabilize.py (crop window) and +saving — Streamlit reloads. """ from __future__ import annotations @@ -19,14 +20,15 @@ from tube_builder_lab import candidate as candidate_mod from tube_builder_lab.cache import detections_present, load_cached from tube_builder_lab.pipeline import ( - current_builder, detections_to_display_tubes, load_pipeline_config, ) +from tube_builder_lab.stabilize import tube_window from tube_builder_lab.store import build_frames, read_meta, seq_dir_for_key from tube_builder_lab.viz import ( crop_tube_at_frame, draw_tube_bboxes, + draw_tube_windows, tube_color, tube_count, tube_timeline_df, @@ -41,14 +43,12 @@ PLAY_FPS = 2 -def _both_tube_sets(key: str, cfg): - """(current_tubes, candidate_tubes) for a key, over the full sequence.""" +def _candidate_tubes(key: str, cfg): + """Candidate (the committed builder) tubes for a key, over the full sequence.""" fds = load_cached(DETECTIONS, key) - cur = detections_to_display_tubes(fds, current_builder(cfg), cfg, truncate=False) - cand = detections_to_display_tubes( + return detections_to_display_tubes( fds, candidate_mod.build_tubes_candidate, cfg, truncate=False ) - return cur, cand def _timeline_chart(tubes, n, current, color_map): # pragma: no cover @@ -110,12 +110,8 @@ def _viewer(key: str, cfg): # pragma: no cover st.info("no frames") return - cur, cand = _both_tube_sets(key, cfg) - c1, c2 = st.columns(2) - c1.metric("current tubes", tube_count(cur)) - c2.metric( - "candidate tubes", tube_count(cand), delta=tube_count(cand) - tube_count(cur) - ) + cand = _candidate_tubes(key, cfg) + st.metric("tubes", tube_count(cand)) frame_key = f"frame_{key}" st.session_state.setdefault(frame_key, 0) @@ -123,27 +119,29 @@ def _viewer(key: str, cfg): # pragma: no cover st.session_state[frame_key] = (st.session_state[frame_key] + 1) % n i = st.slider("frame", 0, n - 1, key=frame_key) if n > 1 else 0 - # Two synced frame views (same frame index i): current (left) vs candidate - # (right). One slider/play tick drives both, so they advance in lockstep. + # Two synced full-frame views for the current frame i (driven by the + # play/slider above): left = non-stabilized per-frame box (moves frame to + # frame); right = the FIXED stabilized crop window (identical every frame). img_path = seq_dir / meta.frames[i].file left, right = st.columns(2) left.image( - draw_tube_bboxes(img_path, cur, i), - caption=f"current — frame {i + 1}/{n}, {tube_count(cur)} tube(s)", + draw_tube_bboxes(img_path, cand, i), + caption=f"non-stabilized — frame {i + 1}/{n}, {tube_count(cand)} tube(s)", width="stretch", ) + active_windows = [ + (tube_window(t), tube_color(t.tube_id)) + for t in cand + if any(e.frame_idx == i and e.detection for e in t.entries) + ] right.image( - draw_tube_bboxes(img_path, cand, i), - caption=f"candidate — frame {i + 1}/{n}, {tube_count(cand)} tube(s)", + draw_tube_windows(img_path, active_windows), + caption=f"stabilized window — frame {i + 1}/{n}, {tube_count(cand)} tube(s)", width="stretch", ) - cur_colors = {f"T{t.tube_id}": tube_color(t.tube_id) for t in cur} cand_colors = {f"T{t.tube_id}": tube_color(t.tube_id) for t in cand} - st.caption(f"current — {tube_count(cur)} tube(s)") - if cur: - st.altair_chart(_timeline_chart(cur, n, i, cur_colors), width="stretch") - st.caption(f"candidate — {tube_count(cand)} tube(s)") + st.caption(f"tubes — {tube_count(cand)}") if cand: st.altair_chart(_timeline_chart(cand, n, i, cand_colors), width="stretch") @@ -157,64 +155,49 @@ def _viewer(key: str, cfg): # pragma: no cover f"T{t.tube_id}", unsafe_allow_html=True, ) + c_ns, c_st = col.columns(2) if entry: d = entry.detection - col.image( - crop_tube_at_frame( - seq_dir / meta.frames[i].file, (d.cx, d.cy, d.w, d.h) - ), - width=180, + c_ns.image( + crop_tube_at_frame(img_path, (d.cx, d.cy, d.w, d.h)), + caption="non-stabilized", + width="stretch", + ) + c_st.image( + crop_tube_at_frame(img_path, tube_window(t)), + caption="stabilized", + width="stretch", ) else: - col.caption("inactive") + c_ns.caption("inactive") def _summary_for(items, cfg) -> pd.DataFrame: # pragma: no cover rows = [] for item in items: - if not detections_present(DETECTIONS, item.key): - rows.append( - { - "key": item.key, - "current": None, - "candidate": None, - "Δ": None, - "note": item.note or "", - } - ) - continue - cur, cand = _both_tube_sets(item.key, cfg) + present = detections_present(DETECTIONS, item.key) rows.append( { "key": item.key, - "current": len(cur), - "candidate": len(cand), - "Δ": len(cand) - len(cur), + "tubes": len(_candidate_tubes(item.key, cfg)) if present else None, "note": item.note or "", } ) return pd.DataFrame(rows) -def _style_changes(row): # pragma: no cover - """Amber where the candidate changes the tube count, grey if data is missing, - white when current == candidate.""" - d = row["Δ"] - if pd.isna(d): - bg = "#eeeeee" - elif d != 0: - bg = "#fff3cd" - else: - bg = "white" +def _style_missing(row): # pragma: no cover + """Grey out rows whose detections cache is missing; white otherwise.""" + bg = "#eeeeee" if pd.isna(row["tubes"]) else "white" return [f"background-color: {bg}; color: #111"] * len(row) def _pick_from_table(df, key) -> str | None: # pragma: no cover - """Render a clickable single-row table (rows with a count change highlighted); + """Render a clickable single-row table (rows with missing data greyed out); return the key of a row clicked THIS rerun (else None) so the two tables don't fight over the active selection.""" event = st.dataframe( - df.style.apply(_style_changes, axis=1), + df.style.apply(_style_missing, axis=1), width="stretch", hide_index=True, on_select="rerun", @@ -244,8 +227,8 @@ def main() -> None: # pragma: no cover notes = {i.key: i.note for i in ws.all()} # Two clickable tables drive the viewer below; the row clicked most recently - # (this rerun) wins. Counts are current -> candidate per sequence, over the - # full (untruncated) sequence; rows where the count changes are highlighted. + # (this rerun) wins. Counts are the builder's tube count per sequence, over + # the full (untruncated) sequence; rows with a missing cache are greyed out. st.subheader("Targets — click a row to inspect") picked_target = _pick_from_table(_summary_for(ws.targets, cfg), "tbl_targets") st.subheader("Control (random sequences — regression watch)") diff --git a/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/stabilize.py b/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/stabilize.py new file mode 100644 index 00000000..a8fb4210 --- /dev/null +++ b/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/stabilize.py @@ -0,0 +1,34 @@ +"""Derive one fixed crop window per tube (pure, no Streamlit, no I/O). + +Stabilization is a crop-window decision, not a tube-building step: it does not +change which detections link into which tube, so it never mutates the Tube. It +returns the union (enclosing) box of all the tube's observed detections, so the +same region can be cropped from every frame and the temporal head sees a static +background with the smoke moving inside it. + +No context margin is applied here: the crop step expands this box by the same +``context_factor`` (``viz.CROP_CONTEXT``) it uses for per-frame crops, so the +stabilized crop stays in the model's training distribution. Returned coordinates +are normalized; clamping to the image happens at crop time in +``norm_bbox_to_pixel_square``. +""" + +from __future__ import annotations + +from bbox_tube_temporal.types import Tube + + +def tube_window(tube: Tube) -> tuple[float, float, float, float]: + """Union (enclosing) box of the tube's observed detections, normalized. + + Entries without a detection (gaps) are ignored: interpolated gap boxes are + lerps of observed boxes, so the union of observed boxes already encloses them. + """ + dets = [e.detection for e in tube.entries if e.detection is not None] + if not dets: + raise ValueError("tube_window requires at least one observed detection") + x0 = min(d.cx - d.w / 2 for d in dets) + y0 = min(d.cy - d.h / 2 for d in dets) + x1 = max(d.cx + d.w / 2 for d in dets) + y1 = max(d.cy + d.h / 2 for d in dets) + return (x0 + x1) / 2, (y0 + y1) / 2, x1 - x0, y1 - y0 diff --git a/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/viz.py b/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/viz.py index 9bf4fe2b..2c83e024 100644 --- a/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/viz.py +++ b/experiments/temporal-models/tube-builder-lab/src/tube_builder_lab/viz.py @@ -18,7 +18,7 @@ from bbox_tube_temporal.types import Tube from PIL import Image, ImageDraw, ImageFont -CROP_CONTEXT = 2.0 +CROP_CONTEXT = 1.6 CROP_SIZE = 224 TUBE_PALETTE = [ @@ -113,11 +113,47 @@ def draw_tube_bboxes( return img +def crop_box_px( + bbox: tuple[float, float, float, float], img_w: int, img_h: int +) -> tuple[int, int, int, int]: + """Square pixel crop box for a normalized bbox, expanded by ``CROP_CONTEXT``. + + The single source of truth for the crop region: ``crop_tube_at_frame`` + extracts it and ``draw_tube_windows`` draws it, so the box you see equals the + crop you get — for both per-frame boxes and stabilized union windows.""" + cx, cy, w, h = expand_bbox(*bbox, CROP_CONTEXT) + return norm_bbox_to_pixel_square(cx, cy, w, h, img_w, img_h) + + def crop_tube_at_frame(image_path: Path, bbox: tuple[float, float, float, float]): - """Square context crop centred on a normalized bbox (matches the explorer).""" + """Square context crop centred on a normalized bbox (matches the explorer). + + Pass a per-frame detection box for the non-stabilized crop, or a tube's union + window (``stabilize.tube_window``) for the stabilized crop — same code path, + same ``CROP_CONTEXT``.""" img = np.array(Image.open(image_path).convert("RGB")) img_h, img_w = img.shape[:2] - cx, cy, bw, bh = bbox - ecx, ecy, ew, eh = expand_bbox(cx, cy, bw, bh, CROP_CONTEXT) - box = norm_bbox_to_pixel_square(ecx, ecy, ew, eh, img_w, img_h) - return Image.fromarray(crop_and_resize(img, box, CROP_SIZE)) + return Image.fromarray( + crop_and_resize(img, crop_box_px(bbox, img_w, img_h), CROP_SIZE) + ) + + +def draw_tube_windows( + image_path: Path, + windows: list[tuple[tuple[float, float, float, float], str]], + width: int = 5, +): + """Frame image with each ``(window, colour)`` drawn as its FIXED stabilized + crop region (the exact region ``crop_tube_at_frame`` extracts for that union + window). The stabilized-side mirror of ``draw_tube_bboxes``: each box is in + its tube colour and identical on every frame the tube is active.""" + img = Image.open(image_path).convert("RGB") + w_img, h_img = img.size + draw = ImageDraw.Draw(img) + for window, color in windows: + draw.rectangle( + list(crop_box_px(window, w_img, h_img)), + outline=color, + width=width, + ) + return img diff --git a/experiments/temporal-models/tube-builder-lab/tests/test_stabilize.py b/experiments/temporal-models/tube-builder-lab/tests/test_stabilize.py new file mode 100644 index 00000000..fa853097 --- /dev/null +++ b/experiments/temporal-models/tube-builder-lab/tests/test_stabilize.py @@ -0,0 +1,64 @@ +"""Unit tests for the pure stable-crop-window function.""" + +from __future__ import annotations + +import pytest +from bbox_tube_temporal.types import Detection, Tube, TubeEntry + +from tube_builder_lab.stabilize import tube_window + + +def _det(cx: float, cy: float, w: float, h: float) -> Detection: + return Detection(class_id=0, cx=cx, cy=cy, w=w, h=h, confidence=1.0) + + +def _tube(*entries: TubeEntry) -> Tube: + return Tube( + tube_id=0, + entries=list(entries), + start_frame=entries[0].frame_idx, + end_frame=entries[-1].frame_idx, + ) + + +def test_union_of_two_boxes(): + # A: x[0.15,0.25] y[0.15,0.25]; B: x[0.35,0.45] y[0.35,0.45] + # union: x[0.15,0.45] y[0.15,0.45] -> center 0.3,0.3 size 0.3,0.3 + tube = _tube( + TubeEntry(frame_idx=0, detection=_det(0.2, 0.2, 0.1, 0.1)), + TubeEntry(frame_idx=1, detection=_det(0.4, 0.4, 0.1, 0.1)), + ) + assert tube_window(tube) == pytest.approx((0.3, 0.3, 0.3, 0.3)) + + +def test_single_box_tube_returns_that_box(): + tube = _tube(TubeEntry(frame_idx=0, detection=_det(0.5, 0.5, 0.2, 0.2))) + assert tube_window(tube) == pytest.approx((0.5, 0.5, 0.2, 0.2)) + + +def test_union_is_axis_independent(): + # x extent 0.3, y extent 0.1 -> width and height are computed separately. + tube = _tube( + TubeEntry(frame_idx=0, detection=_det(0.2, 0.5, 0.1, 0.1)), + TubeEntry(frame_idx=1, detection=_det(0.4, 0.5, 0.1, 0.1)), + ) + assert tube_window(tube) == pytest.approx((0.3, 0.5, 0.3, 0.1)) + + +def test_empty_tube_raises(): + tube = _tube(TubeEntry(frame_idx=0, detection=None, is_gap=True)) + with pytest.raises(ValueError): + tube_window(tube) + + +def test_gap_entries_without_detection_are_ignored(): + with_gap = _tube( + TubeEntry(frame_idx=0, detection=_det(0.2, 0.2, 0.1, 0.1)), + TubeEntry(frame_idx=1, detection=None, is_gap=True), + TubeEntry(frame_idx=2, detection=_det(0.4, 0.4, 0.1, 0.1)), + ) + without_gap = _tube( + TubeEntry(frame_idx=0, detection=_det(0.2, 0.2, 0.1, 0.1)), + TubeEntry(frame_idx=1, detection=_det(0.4, 0.4, 0.1, 0.1)), + ) + assert tube_window(with_gap) == pytest.approx(tube_window(without_gap)) diff --git a/experiments/temporal-models/tube-builder-lab/tests/test_viz.py b/experiments/temporal-models/tube-builder-lab/tests/test_viz.py index 4aa66b63..5d669d16 100644 --- a/experiments/temporal-models/tube-builder-lab/tests/test_viz.py +++ b/experiments/temporal-models/tube-builder-lab/tests/test_viz.py @@ -1,7 +1,9 @@ from bbox_tube_temporal.types import Detection, Tube, TubeEntry from tube_builder_lab.viz import ( + CROP_CONTEXT, bboxes_at_frame, + crop_box_px, norm_bbox_to_pixel, tube_color, tube_timeline_df, @@ -37,6 +39,24 @@ def test_norm_bbox_to_pixel(): ) +def test_crop_box_px_is_square_and_context_scaled(): + # Window wider than tall: the crop region squares to the larger side, scaled + # by CROP_CONTEXT, and stays centred on the window. (boxed == cropped.) + x0, y0, x1, y1 = crop_box_px((0.5, 0.5, 0.1, 0.05), 1000, 1000) + assert (x1 - x0) == (y1 - y0) # square + assert (x1 - x0) == round(0.1 * CROP_CONTEXT * 1000) # sized to max(w,h)*context + assert (x0 + x1) / 2 == 500 + assert (y0 + y1) / 2 == 500 + + +def test_crop_box_px_clamps_to_image_bounds(): + # Window in the corner: the box clamps to the image (crop_and_resize re-squares + # by padding); never negative or past the edge. + x0, y0, x1, y1 = crop_box_px((0.0, 0.0, 0.1, 0.1), 1000, 1000) + assert (x0, y0) == (0, 0) + assert x1 <= 1000 and y1 <= 1000 + + def test_timeline_df_one_row_per_entry(): t0 = _tube(0, [_entry(0, 0.5, 0.5), _entry(1, 0.5, 0.5, gap=True)]) t1 = _tube(1, [_entry(2, 0.2, 0.2)])