From 4250798c52c764a9c1d20a13e24b01c31daca176 Mon Sep 17 00:00:00 2001 From: P S Kesavan Date: Sat, 18 Jul 2026 12:45:20 +0530 Subject: [PATCH 01/13] fix(imaging): stop discarding the right half of every SPIM frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six call sites carried `if width > height * 2: take the left half`, a guess at splitting a stitched dual-camera frame. This rig is single-camera and its native SPIM frames are 2048x512 (4:1), so the heuristic fired on every real frame and silently threw away half the image. An embryo right of centre produced an empty or garbage projection, one straddling the midline got sliced, and the same path feeds the perceiver. A genuine dual-view volume is 4D (Views, Z, Y, X) and every one of these sites already handles that case explicitly on the preceding line. So the aspect-ratio branch was never selecting a view — only truncating one. Removed at all six. tests/test_view_splitting_regression.py puts a marker in the right half of a synthetic 2048x512 volume and asserts it survives. Five of its six cases fail against the previous code and pass now; the images.py case passes either way and is kept as a guard rather than a proof. Claude-Session: https://claude.ai/code/session_01DNw6x8E3Kbd3yNXsgZxxrF --- gently/app/agent.py | 5 +- gently/app/orchestration/timelapse.py | 5 +- gently/core/imaging.py | 11 +- gently/dataset/embryo_dataset.py | 11 +- gently/ui/web/routes/images.py | 6 +- gently/ui/web/server.py | 7 +- tests/test_view_splitting_regression.py | 213 ++++++++++++++++++++++++ 7 files changed, 230 insertions(+), 28 deletions(-) create mode 100644 tests/test_view_splitting_regression.py diff --git a/gently/app/agent.py b/gently/app/agent.py index 918903f6..607ca9bb 100644 --- a/gently/app/agent.py +++ b/gently/app/agent.py @@ -1616,9 +1616,8 @@ async def on_volume_acquired( view_a = volume[0] if volume.ndim == 4 else volume if view_a.ndim == 3: - z_depth, height, width = view_a.shape - if width > height * 2: - view_a = view_a[:, :, : width // 2] + # view_a is already one view (4D handled on the line above). + # No aspect-ratio splitting: native frames are 4:1. bounds = compute_crop_bounds(view_a) cropped = apply_crop_bounds(view_a, bounds) three_view_img, _ = projection_three_view(cropped) diff --git a/gently/app/orchestration/timelapse.py b/gently/app/orchestration/timelapse.py index 7762fac6..23e795ac 100644 --- a/gently/app/orchestration/timelapse.py +++ b/gently/app/orchestration/timelapse.py @@ -2577,9 +2577,8 @@ def _volume_to_b64(self, volume) -> tuple: view_a = volume[0] if volume.ndim == 4 else volume if view_a.ndim == 3: - z_depth, height, width = view_a.shape - if width > height * 2: - view_a = view_a[:, :, : width // 2] + # view_a is already one view (4D handled above). No aspect-ratio + # split — this feeds the perceiver; halving it corrupts the input. bounds = compute_crop_bounds(view_a) cropped = apply_crop_bounds(view_a, bounds) three_view_img, _ = projection_three_view(cropped) diff --git a/gently/core/imaging.py b/gently/core/imaging.py index 5f67d894..17a9a9a9 100644 --- a/gently/core/imaging.py +++ b/gently/core/imaging.py @@ -350,11 +350,12 @@ def load_volume(path: Path) -> np.ndarray: raise ImportError("tifffile is required for load_volume") vol = _tifffile.imread(str(path)) vol = np.squeeze(vol) - if vol.ndim == 3: - z_depth, height, width = vol.shape - # Extract View A (left half) if dual-view format - if width > height * 2: - vol = vol[:, :, : width // 2] + # A 3D volume is a single view — do NOT split by aspect ratio. Native SPIM + # frames are 2048x512 (4:1), so a width-based "dual-view" guess fires on + # every real frame and discards half the image. Explicit 4D (Views, Z, Y, X) + # volumes are the only real dual-view form; squeeze leaves those at ndim 4. + if vol.ndim == 4: + vol = vol[0] return vol diff --git a/gently/dataset/embryo_dataset.py b/gently/dataset/embryo_dataset.py index 48828692..ea2d097e 100644 --- a/gently/dataset/embryo_dataset.py +++ b/gently/dataset/embryo_dataset.py @@ -1058,18 +1058,11 @@ def _load_projection_from_volume(self, volume_path: str) -> str: volume = volume[0] if volume.ndim == 3: - z_depth, height, width = volume.shape - # Extract View A (left half) if dual-view format - if width > height * 1.5: - volume = volume[:, :, : width // 2] - # Max projection + # View A already selected by the 4D branch above. No aspect-ratio + # split: native SPIM frames are 2048x512 and would be halved. projection = np.max(volume, axis=0) else: projection = volume - # Extract View A if 2D dual-view - height, width = projection.shape - if width > height * 1.5: - projection = projection[:, : width // 2] # Normalize and encode from gently.core.imaging import image_to_base64, normalize_to_uint8 diff --git a/gently/ui/web/routes/images.py b/gently/ui/web/routes/images.py index a41cfcc7..a9225d9b 100644 --- a/gently/ui/web/routes/images.py +++ b/gently/ui/web/routes/images.py @@ -93,10 +93,8 @@ async def get_image_png(uid: str): data = data[0] # Handle 3D volumes - generate three-view projection if data.ndim == 3: - z_depth, height, width = data.shape - # Handle dual-view format - if width > height * 2: - data = data[:, :, : width // 2] + # View A already selected by the 4D branch above; + # never split a 3D volume by aspect ratio. # Auto-crop and project bounds = compute_crop_bounds(data) data = apply_crop_bounds(data, bounds) diff --git a/gently/ui/web/server.py b/gently/ui/web/server.py index 19a1cbe0..153a598b 100644 --- a/gently/ui/web/server.py +++ b/gently/ui/web/server.py @@ -480,10 +480,9 @@ def _array_to_image_data( pass else: # It's a volume (Z, H, W) - generate three-view projection - z_depth, height, width = array.shape - # Handle dual-view format (width > 2*height) - if width > height * 2: - array = array[:, :, : width // 2] + # View selection already happened via the 4D branch above; a 3D + # array is one view. Do not split by aspect ratio (2048x512 + # native frames would be halved). # Auto-crop to embryo region bounds = compute_crop_bounds(array) array = apply_crop_bounds(array, bounds) diff --git a/tests/test_view_splitting_regression.py b/tests/test_view_splitting_regression.py new file mode 100644 index 00000000..152eeb59 --- /dev/null +++ b/tests/test_view_splitting_regression.py @@ -0,0 +1,213 @@ +""" +Regression tests for the aspect-ratio view-splitting bug (XY-rendered-halfway). + +Several code paths used to guess "dual-view format" from the frame aspect ratio +(``width > height * 2``, or ``* 1.5``) and keep only the left half. Native SPIM +frames on this rig are 2048x512 (4:1), so the guess fired on *every* real frame +and silently discarded the right half of the image. + +View selection is now driven by array rank alone: an explicit 4D +(Views, Z, Y, X) volume selects View A via ``[0]``; a 3D volume is by +construction a single view and is kept whole. + +Tests cover: +- load_volume keeps full width for a 4:1 single-view TIFF +- load_volume still selects View A for a genuine 4D volume +- generate_jpeg_projection (the already-correct reference path) +- VisualizationServer._array_to_image_data preserves right-half signal +- TimelapseOrchestrator._volume_to_b64 returns an un-truncated view_a +- EmbryoDataset._load_projection_from_volume preserves right-half signal + +Strategy: build a 2048x512 volume whose ONLY signal is a bright block in the +RIGHT half (x=1600..1700). If a path truncates to the left half, what remains +is a uniform background, so the rendered projection collapses to near-zero +variance. Surviving signal => the marker was preserved. +""" + +import base64 +import io + +import numpy as np +import pytest + +BACKGROUND = 100 +MARKER = 60000 +MARKER_X0, MARKER_X1 = 1600, 1700 +MARKER_Y0, MARKER_Y1 = 200, 300 +WIDTH, HEIGHT, DEPTH = 2048, 512, 8 + + +def make_right_half_marked_volume() -> np.ndarray: + """(Z, Y, X) = (8, 512, 2048) uint16 volume, bright block only in right half.""" + vol = np.full((DEPTH, HEIGHT, WIDTH), BACKGROUND, dtype=np.uint16) + vol[:, MARKER_Y0:MARKER_Y1, MARKER_X0:MARKER_X1] = MARKER + return vol + + +def decode_to_array(b64: str) -> np.ndarray: + """Decode a base64 image string (data-URI prefix tolerated) to a numpy array.""" + from PIL import Image + + if "," in b64 and b64.strip().startswith("data:"): + b64 = b64.split(",", 1)[1] + return np.asarray(Image.open(io.BytesIO(base64.b64decode(b64)))) + + +def assert_signal_survived(arr: np.ndarray, path_name: str) -> None: + """A rendered projection must retain contrast from the right-half marker.""" + assert arr.size > 0, f"{path_name}: produced an empty image" + spread = float(arr.max()) - float(arr.min()) + assert spread > 10, ( + f"{path_name}: rendered projection is nearly uniform " + f"(min={arr.min()}, max={arr.max()}). The right-half marker was " + f"discarded — aspect-ratio view splitting has regressed." + ) + + +@pytest.fixture +def stub_tifffile(monkeypatch): + """Install a stub TIFF reader returning a caller-chosen array. + + tifffile is not a hard dependency of the test environment, and the real + library is irrelevant to what is under test here (the post-read view + handling). The fixture yields a setter for the array `imread` returns. + """ + import sys + import types + + holder: dict[str, np.ndarray] = {} + + stub = types.ModuleType("tifffile") + stub.imread = lambda _path: holder["array"] # type: ignore[attr-defined] + stub.imwrite = lambda *_a, **_k: None # type: ignore[attr-defined] + + monkeypatch.setitem(sys.modules, "tifffile", stub) + + import gently.core.imaging as imaging_mod + + monkeypatch.setattr(imaging_mod, "_tifffile", stub, raising=False) + + def set_array(arr: np.ndarray) -> None: + holder["array"] = arr + + return set_array + + +class TestLoadVolume: + def test_full_width_preserved_for_4to1_single_view(self, tmp_path, stub_tifffile): + """A 4:1 3D volume must load at full width, not halved.""" + stub_tifffile(make_right_half_marked_volume()) + + from gently.core.imaging import load_volume + + vol = load_volume(tmp_path / "vol.tif") + + assert vol.shape[-1] == WIDTH, ( + f"load_volume truncated width to {vol.shape[-1]} (expected {WIDTH})" + ) + assert vol[:, MARKER_Y0:MARKER_Y1, MARKER_X0:MARKER_X1].max() == MARKER, ( + "right-half marker missing after load_volume" + ) + + def test_view_a_selected_for_genuine_4d_volume(self, tmp_path, stub_tifffile): + """An explicit (Views, Z, Y, X) volume still resolves to View A.""" + view_a = make_right_half_marked_volume() + view_b = np.full_like(view_a, 7) + stub_tifffile(np.stack([view_a, view_b])) + + from gently.core.imaging import load_volume + + vol = load_volume(tmp_path / "dual.tif") + + assert vol.shape == (DEPTH, HEIGHT, WIDTH), f"unexpected shape {vol.shape}" + assert vol.max() == MARKER, "View A was not the selected view" + + +class TestGenerateJpegProjection: + """The already-correct reference path — guards against re-introduction.""" + + def test_right_half_marker_survives(self, tmp_path): + pytest.importorskip("PIL") + from gently.core.imaging import generate_jpeg_projection + + out = tmp_path / "proj.jpg" + result = generate_jpeg_projection(make_right_half_marked_volume(), out) + + assert result is not None, "generate_jpeg_projection returned None" + from PIL import Image + + assert_signal_survived(np.asarray(Image.open(result)), "generate_jpeg_projection") + + +@pytest.fixture +def crop_bounds_spy(monkeypatch): + """Record the array handed to compute_crop_bounds. + + Rendering-level assertions are unreliable for the three-view paths: the + layout composes separator/padding regions that create contrast even when + the image content is uniform, so a truncated frame can still yield a + high-contrast PNG. Observing the array at the crop step is direct. + """ + import gently.core.imaging as imaging_mod + + seen: list[np.ndarray] = [] + real = imaging_mod.compute_crop_bounds + + def spy(volume, *args, **kwargs): + seen.append(volume) + return real(volume, *args, **kwargs) + + monkeypatch.setattr(imaging_mod, "compute_crop_bounds", spy) + return seen + + +class TestVisualizationServerArrayToImageData: + def test_full_width_reaches_crop_step(self, crop_bounds_spy): + pytest.importorskip("PIL") + from gently.ui.web.server import VisualizationServer + + # _array_to_image_data does not touch `self`; call it unbound. + image_data = VisualizationServer._array_to_image_data( + None, make_right_half_marked_volume(), "test-uid", "volume_projection" + ) + + assert image_data.base64_png, "_array_to_image_data produced no PNG" + assert crop_bounds_spy, "compute_crop_bounds was never called" + assert crop_bounds_spy[0].shape[-1] == WIDTH, ( + f"_array_to_image_data truncated the volume to width " + f"{crop_bounds_spy[0].shape[-1]} (expected {WIDTH})" + ) + assert crop_bounds_spy[0][:, MARKER_Y0:MARKER_Y1, MARKER_X0:MARKER_X1].max() == MARKER, ( + "right-half marker missing before crop" + ) + + +class TestTimelapseVolumeToB64: + def test_view_a_not_truncated(self): + pytest.importorskip("PIL") + from gently.app.orchestration.timelapse import TimelapseOrchestrator + + # _volume_to_b64 does not touch `self`; call it unbound. + view_a, image_b64 = TimelapseOrchestrator._volume_to_b64( + None, make_right_half_marked_volume() + ) + + assert view_a is not None and image_b64 is not None + assert view_a.shape[-1] == WIDTH, ( + f"_volume_to_b64 returned a truncated view_a (width {view_a.shape[-1]}); " + "this value feeds the perceiver" + ) + assert_signal_survived(decode_to_array(image_b64), "TimelapseOrchestrator._volume_to_b64") + + +class TestEmbryoDatasetProjection: + def test_right_half_marker_survives(self, tmp_path, stub_tifffile): + pytest.importorskip("PIL") + from gently.dataset.embryo_dataset import EmbryoDataset + + stub_tifffile(make_right_half_marked_volume()) + + # _load_projection_from_volume does not touch `self`; call it unbound. + b64 = EmbryoDataset._load_projection_from_volume(None, str(tmp_path / "vol.tif")) + + assert_signal_survived(decode_to_array(b64), "EmbryoDataset._load_projection_from_volume") From 2fd5eed3ca81bc39f1c501023e42a1af566f8507 Mon Sep 17 00:00:00 2001 From: P S Kesavan Date: Sat, 18 Jul 2026 12:46:03 +0530 Subject: [PATCH 02/13] feat(operate): navigable steps, real Z instruments, one surface per stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ryan tests on the rig tomorrow. Three things stood between him and the workflow. He never would have seen it. The devices tab opened on Map and the finished guided flow sat behind an unlabelled fifth button. Operate is now the landing view. It could not be left or reversed. Progressive disclosure had been implemented as an exclusive CSS selector list (one .op-group at a time) with every control inside a group, so no control could outlive a step and setStep was only ever called forward; the header stepper was styled as a breadcrumb but answered clicks on one of its eight nodes. Every node is now a real button and the cursor moves in both directions. This is safe because the gates that matter are evaluated against live hardware state, not step position — revisiting Center while the head is down still refuses to move. Progress (_states) stays monotonic and is never rewound by navigation. Escape and an always-visible Cancel leave any step. Z was text, not an instrument. The device layer already returned {position, min, max, distance_to_floor} on every axis call and the frontend dropped min/max on the floor; a 1 Hz fdrive/bottom_z stream existed with a comment saying it was for this view and no subscribers. Both axes now render as travel gauges — real range, current position marked on it, hard-limit line on the F-drive — living on a shelf outside the step system, so raising the head is one click from anywhere. The F-drive gains a fine 1 um step. Also: - The head-down interlock survives a reload. It was a client-only flag starting false, so refreshing mid-embryo re-enabled Center and would have commanded an absolute XY move with the objective down while the strip read "up". Retract is a relative +100, so there is no absolute safe height to derive it from — it is persisted instead. A retract that fails no longer reports the head as up. - One surface per camera stream. Deleted the Map's embedded bottom-cam panel and guarded the frame handlers and renderStep on visibility, so a hidden Operate stops decoding frames and racing the visible view for stream ownership. - The Run chooser defaults to Manual. It defaulted to Adaptive, so the default path launched a timelapse and never reached Center. - The grid responds to its container, not the window. With the agent panel open the container is ~560px on a 1792px display, which the fixed 264/248px side columns overflowed — collapsing the viewport to a 24px strip and letting the rail overlap it, while the stacking breakpoint never fired because it keyed off viewport width. Deliberately untouched: the stream encoder. Image quality was never reduced — only frame rate (commit 6797e0a) — and the 15fps cap is what stands between a lowered head and a repeat of the 2026-06-29 Video-TDR freeze. Verified in-browser: landing view, forward and backward jumps, Escape from three targets, Cancel, gauge geometry against known positions, the reload interlock, and both layouts in light and dark. Spec: docs/superpowers/specs/2026-07-18-operate-navigable-instruments-design.md Claude-Session: https://claude.ai/code/session_01DNw6x8E3Kbd3yNXsgZxxrF --- ...18-operate-navigable-instruments-design.md | 129 ++++++++++ gently/ui/web/static/css/operate.css | 170 ++++++++++++- gently/ui/web/static/js/devices.js | 2 +- gently/ui/web/static/js/operate.js | 224 +++++++++++++++--- gently/ui/web/templates/index.html | 179 +++++++------- 5 files changed, 574 insertions(+), 130 deletions(-) create mode 100644 docs/superpowers/specs/2026-07-18-operate-navigable-instruments-design.md diff --git a/docs/superpowers/specs/2026-07-18-operate-navigable-instruments-design.md b/docs/superpowers/specs/2026-07-18-operate-navigable-instruments-design.md new file mode 100644 index 00000000..d36b725d --- /dev/null +++ b/docs/superpowers/specs/2026-07-18-operate-navigable-instruments-design.md @@ -0,0 +1,129 @@ +# Operate: navigable steps + real instruments + +**Date:** 2026-07-18 +**Context:** v1 release prep. Ryan is the first external user; he tests on the diSPIM with live samples tomorrow morning. +**Status:** approved shape, scoped for one night. + +## The problem + +Keshu's verdict after driving the Operate view: *"Operate itself is the rigid one."* His acceptance +criteria for this work, in his words: **you can go back and forth, the views are not cluttered, and it +looks professional.** + +Three findings from the 2026-07-18 audit set the scope. + +**Rigidity has one cause.** Progressive disclosure was implemented as an exclusive CSS selector list +(`operate.css:172-182` renders exactly one `.op-group`), and *every* control lives inside a +`.op-group`. The design has no concept of a control that outlives a step. `setStep()` is only ever +called forward; the header stepper is styled as a navigable breadcrumb but its click handler responds +to one of eight nodes (`operate.js:887-892`). Disclosure became imprisonment. + +The data model is already correct: progress (`_states`, monotonic) is separate from the view cursor +(`_step`, free). Nothing structurally prevents backward navigation — no code path does it. So this is +a small change, not a rewrite. + +**Ryan lands on the wrong surface.** The devices tab defaults to Map (`devices.js:149`); the finished +guided flow hides behind an unlabeled fifth button. + +**Z is rendered as text, not as an instrument.** The device layer already returns +`{position, min, max, distance_to_floor}` on every axis call (`device_layer.py:2615-2636`) and the +frontend discards `min`/`max`. A 1 Hz `fdrive`/`bottom_z` position stream exists with zero +subscribers. The floor gauge fakes its scale with a hardcoded `MAXD=500` against real travel of +30–25000 µm. + +## The shape + +Keep the paged model. Make it reversible, decluttered, and instrumented. + +### 1. The stepper becomes a real control + +- Every node is clickable and moves `_step` in either direction. +- `_states` stays monotonic — progress is a record of what has happened, not a permission system. +- **Gates key off live hardware state, never step position.** Centering stays blocked while + `_headLowered`; down-nudges stay greyed near the floor. Revisiting a step never re-arms a motion + the hardware state forbids. This is what makes free navigation safe. +- Nodes are real ` - + + @@ -524,30 +524,34 @@

Device -