Read hover and click values at the frame on screen

cortex/tests/test_webgl_headless.py

@@ -319,6 +319,39 @@ def test_overlay_visibility_changes_image(tmp_path):
 # ---------------------------------------------------------------------------
 
 
+@pytest.mark.parametrize("dtype_name", ["Vertex", "Volume"])
+def test_picked_value_follows_movie_frame(dtype_name):
+    """The click readout shows the value at the frame on screen.
+
+    Regression test for #636: the readout always indexed frame 0, so after
+    advancing a movie it kept showing the first frame's values.
+    """
+    from cortex.webgl.serve import JSProxy
+
+    nframes = 3
+    frames = np.arange(nframes, dtype=np.float32)
+    if dtype_name == "Vertex":
+        data = np.repeat(frames[:, None], nverts, axis=1)
+        view = cortex.Vertex(data, subj, vmin=0, vmax=nframes - 1)
+    else:
+        data = np.broadcast_to(frames[:, None, None, None], (nframes, *volshape))
+        view = cortex.Volume(data.copy(), subj, xfmname, vmin=0, vmax=nframes - 1)
+
+    with cortex.export.headless_viewer(view, viewer_params={}) as handle:
+        readout = JSProxy(handle.send, "window.picked_value")
+        # Every vertex or voxel in frame k holds k, so any point on the brain works.
+        for frame in [0, 2, 1]:
+            handle.setFrame(frame)
+            time.sleep(0.5)
+            handle.pick({"x": 640, "y": 360})
+            time.sleep(0.5)
+            text = readout.attrs["textContent"][1]
+            assert text, "the pick at the canvas center missed the brain"
+            # Volume floats are decoded through a 2D canvas, whose premultiplied
+            # alpha rounds their low bytes: 1.0 reads back as 1.015625.
+            assert float(text) == pytest.approx(frame, abs=0.05), (frame, text)
+
+
 def _count_red_pixels(png_path):
     """Count strongly red-dominant pixels (R - max(G, B) > 50)."""
     from PIL import Image

cortex/webgl/resources/js/dataset.js

@@ -273,6 +273,12 @@ var dataset = (function(module) {
         }
         this.setFrame(0);
     };
+    // Index of the frame on screen, as setFrame computes it. The shaders blend
+    // toward the next frame by framemix, but the NaN mask comes from this one,
+    // so value readouts report this frame's sample.
+    module.DataView.prototype.frameIndex = function() {
+        return Math.floor(((this.frame + this.delay) * this.rate).mod(this.frames));
+    };
     module.DataView.prototype.setFrame = function(time) {
         this.frame = time;
         var frame = ((time + this.delay) * this.rate).mod(this.frames);

cortex/webgl/resources/js/mriview.js

@@ -56,6 +56,20 @@ var mriview = (function(module) {
         return true;
     };
 
+    // Values at the frame the viewer is showing, one per dim, read from each
+    // data object by `read(data, frame)`. Returns null while that frame's
+    // buffer is still loading: movie mosaics arrive one frame at a time.
+    module.frameValues = function (dataview, read) {
+        var frame = dataview.frameIndex();
+        var values = [];
+        for (var i = 0; i < dataview.data.length; i++) {
+            var v = read(dataview.data[i], frame);
+            if (v === undefined) return null;
+            values.push(v);
+        }
+        return values;
+    };
+
     module.Viewer = function(figure) {
         jsplot.Axes.call(this, figure);
 
@@ -796,9 +810,9 @@ var mriview = (function(module) {
                         let subject = this.active.data[0].subject
                         let indexMap = subjects[subject].hemis[coords.hemi].indexMap
                         let vertex = indexMap[coords.vertex]
-                        // Now access the data for each channel (1 for 1D, 2 for 2D)
-                        values = this.active.data.map(function (d) {
-                            return d.verts[0][hemiIdx].array[vertex]
+                        // One value per channel (1 for 1D, 2 for 2D), at the frame on screen
+                        values = module.frameValues(this.active, function (d, frame) {
+                            return d.verts[frame] && d.verts[frame][hemiIdx].array[vertex]
                         })
                     }
                 } else {
@@ -815,8 +829,8 @@ var mriview = (function(module) {
                     }
                     let mouse_index = this.getMouseIndex(event)
                     if (mouse_index !== -1) {
-                        values = this.active.data.map(function (d) {
-                            return d.textures[0].image.data[mouse_index]
+                        values = module.frameValues(this.active, function (d, frame) {
+                            return d.textures[frame] && d.textures[frame].image.data[mouse_index]
                         })
                     }
                 }
@@ -995,9 +1009,9 @@ var mriview = (function(module) {
                 let subject = this.active.data[0].subject
                 let indexMap = subjects[subject].hemis[coords.hemi].indexMap
                 let vertex = indexMap[coords.vertex]
-                // Now access the data for each channel (1 for 1D, 2 for 2D)
-                values = this.active.data.map(function (d) {
-                    return d.verts[0][hemiIdx].array[vertex]
+                // One value per channel (1 for 1D, 2 for 2D), at the frame on screen
+                values = module.frameValues(this.active, function (d, frame) {
+                    return d.verts[frame] && d.verts[frame][hemiIdx].array[vertex]
                 })
             }
         } else {
@@ -1010,8 +1024,8 @@ var mriview = (function(module) {
                 }
                 let mouse_index = this.xyxToI(coords.voxel.x, coords.voxel.y, coords.voxel.z)
                 if (mouse_index !== -1) {
-                    values = this.active.data.map(function (d) {
-                        return d.textures[0].image.data[mouse_index]
+                    values = module.frameValues(this.active, function (d, frame) {
+                        return d.textures[frame] && d.textures[frame].image.data[mouse_index]
                     })
                 }
             }