NF optimize 4D timeseries performance and add interactive timeseries panel

cortex/dataset/braindata.py

@@ -101,10 +101,11 @@ def to_json(self, simple=False):
         """
         sdict = super().to_json(simple=simple)
         if simple:
+            vmin, vmax = _nan_to_num_bounds(self.data)
             sdict.update(dict(name=self.name,
                 subject=self.subject,
-                min=float(np.nan_to_num(self.data).min()), 
-                max=float(np.nan_to_num(self.data).max()),
+                min=vmin,
+                max=vmax,
                 ))
         return sdict
 
@@ -739,10 +740,37 @@ def __getitem__(self, masktype: str) -> T_masker:
         mask = db.get_mask(self.dv.subject, self.dv.xfmname, masktype)
         return self.dv.copy(self.dv.volume[:,mask].squeeze())
 
+def _nan_to_num_bounds(array: npt.ArrayLike) -> tuple[float, float]:
+    """(min, max) of ``np.nan_to_num(array)``, without copying it whole.
+
+    ``np.nan_to_num`` allocates a copy of its input, so taking the display
+    bounds of a 4D movie that way (once for the min, once for the max)
+    allocates several times the size of the data and dominated webgl viewer
+    startup for long runs. Reducing one slice at a time gives identical
+    values in bounded memory.
+    """
+    array = np.asarray(array)
+    if array.size == 0:
+        # let numpy raise the same error it always has for empty data
+        filled = np.nan_to_num(array)
+        return float(filled.min()), float(filled.max())
+    # 1D data has no useful slices to reduce over; anything higher is
+    # reduced along its first axis (time for movies, slices for volumes)
+    chunks = array if array.ndim > 1 else (array,)
+    vmin, vmax = np.inf, -np.inf
+    for chunk in chunks:
+        chunk = np.nan_to_num(chunk)
+        vmin = min(vmin, chunk.min())
+        vmax = max(vmax, chunk.max())
+    return float(vmin), float(vmax)
+
+
 def _hash(array: npt.ArrayLike) -> str:
     '''A simple numpy hash function'''
     array = np.asarray(array)
-    return hashlib.sha1(array.tobytes()).hexdigest()
+    if not array.flags["C_CONTIGUOUS"]:
+        array = np.ascontiguousarray(array)
+    return hashlib.sha1(memoryview(array).cast("B")).hexdigest()
 
 def _hdf_write(h5: Union[h5py.File, h5py.Group], data: npt.NDArray, name: str="data", group: str="/data") -> h5py.Dataset:
     try:

cortex/webgl/data.py

@@ -23,7 +23,8 @@
 class Package(object):
     """Package the data into a form usable by javascript"""
 
-    def __init__(self, data):
+    def __init__(self, data, lazy=False):
+        # lazy=True defers volume mosaic/PNG packing to get_image(). Only the first frame is packed up front.
         self.dataset = dataset.normalize(data)
         # Dataset.uniques() is a set of BrainData objects, but equality is
         # identity: two distinct objects with byte-identical data (e.g.
@@ -53,6 +54,7 @@ def __init__(self, data):
 
         self.brains = dict()
         self.images = dict()
+        self._pending = dict()
         for brain in self.uniques:
             name = brain.name
             self.subjects.add(brain.subject)
@@ -87,12 +89,21 @@ def __init__(self, data):
                     )
                 self.brains[name]["raw"] = True
             else:
-                encdata = encdata.astype(np.float32)
                 self.brains[name]["raw"] = False
+                if isinstance(brain, dataset.Vertex) or not lazy:
+                    encdata = encdata.astype(np.float32, copy=False)
 
             # VertexData requires reordering, only save normalized version for now
             if isinstance(brain, (dataset.Vertex, dataset.VertexRGB)):
                 self.images[name] = [encdata]
+            elif lazy:
+                # The mosaic grid depends only on the volume shape, 
+                # so the first frame determines the metadata for every frame.
+                mosaic0, shape = volume.mosaic(self._pack_dtype(name, encdata[0]),
+                                               show=False)
+                self.brains[name]["mosaic"] = shape
+                self._pending[name] = encdata
+                self.images[name] = [_pack_png(mosaic0)] + [None] * (len(encdata) - 1)
             else:
                 self.images[name] = [volume.mosaic(vol, show=False) for vol in encdata]
                 if len(set([shape for m, shape in self.images[name]])) != 1:
@@ -137,6 +148,23 @@ def metadata(self, submap=None, **kwargs):
             views=self.views, data=self.brains, images=self.image_names(**kwargs)
         )
 
+    def _pack_dtype(self, name, framedata):
+        """One frame in the dtype _pack_png accepts: raw data stays uint8,
+        everything else becomes float32 (a per-frame no-op-sized copy)."""
+        if self.brains[name]["raw"]:
+            return framedata
+        return np.asarray(framedata, dtype=np.float32)
+
+    def get_image(self, name, frame):
+        """Return the packed PNG (or npy blob) for one frame, packing it on
+        first request when the package was built with lazy=True."""
+        img = self.images[name][frame]
+        if img is None:
+            mosaic, _ = volume.mosaic(
+                self._pack_dtype(name, self._pending[name][frame]), show=False)
+            img = self.images[name][frame] = _pack_png(mosaic)
+        return img
+
     def image_names(self, fmt="/data/{name}/{frame}/"):
         names = dict()
         for name, imgs in self.images.items():

cortex/webgl/resources/js/dataset.js

@@ -146,8 +146,9 @@ var dataset = (function(module) {
         for (var i = 0; i < children.length; i++) {
             (function(idx) {
                 children[idx].loaded
-                    .progress(function(available) {
+                    .progress(function(available, frameIdx) {
                         if (available > this.delay) markReady(idx);
+                        if (frameIdx !== undefined) this._frameArrived(frameIdx);
                     }.bind(this))
                     .done(function() { markReady(idx); });
             }.bind(this))(i);
@@ -293,6 +294,16 @@ var dataset = (function(module) {
         this.frame = time;
         var frame = ((time + this.delay) * this.rate).mod(this.frames);
         var fframe = Math.floor(frame);
+        // Vertex movies stream in as one array, frame by frame in order, so a
+        // seek can ask for a frame that has not arrived yet. Keep showing the
+        // current frame (data and nanmask together) until it lands;
+        // _frameArrived then re-applies it. this.frame already holds the
+        // requested time, so that check knows what to wait for.
+        if (this.vertex) {
+            for (var i = 0; i < this.data.length; i++)
+                if (this.data[i].verts && this.data[i].verts[fframe] === undefined)
+                    return;
+        }
         this.uniforms.framemix.value = frame - fframe;
         for (var i = 0; i < this.data.length; i++) {
             this.data[i].set(this.uniforms, i, fframe, this._dispatch);
@@ -302,9 +313,18 @@ var dataset = (function(module) {
         // of a movie (both slots point at the same texture).
         var alpha = this.alphaData;
         if (alpha !== null && !this.vertex && alpha.textures.length > 0) {
+            // textures is a sparse Array(frames): a movie's frames stream in
+            // after the first one, so an alpha frame may not have arrived yet.
+            // Binding an undefined texture renders garbage, so keep the current
+            // alpha until it lands (_frameArrived re-applies it), and never
+            // blend into a frame that is not loaded.
             var at = alpha.textures;
-            this.uniforms.dataalpha.value[0] = at[fframe.mod(at.length)];
-            this.uniforms.dataalpha.value[1] = at[(fframe+1).mod(at.length)];
+            var a0 = at[fframe.mod(at.length)];
+            var a1 = at[(fframe+1).mod(at.length)];
+            if (a0 !== undefined) {
+                this.uniforms.dataalpha.value[0] = a0;
+                this.uniforms.dataalpha.value[1] = (a1 !== undefined) ? a1 : a0;
+            }
         }
         // Vertex data: build the single shared "nanmask" attribute, which the
         // shader multiplies into the color. It is 0 wherever any dim (or the
@@ -359,6 +379,25 @@ var dataset = (function(module) {
             this._dispatch({type:"attribute", name:"nanmask", value:combined});
         }
     }
+    // A streamed movie frame just arrived. If it is the frame on screen (or
+    // its blend partner), re-apply the textures: a seek to a not-yet-loaded
+    // frame otherwise stays blank until the next setFrame (e.g. playback).
+    module.DataView.prototype._frameArrived = function(frameIdx) {
+        if (this.frames <= 1)
+            return;
+        var fframe = Math.floor(((this.frame + this.delay) * this.rate).mod(this.frames));
+        if (frameIdx === fframe || frameIdx === (fframe + 1).mod(this.frames)) {
+            this.setFrame(this.frame);
+            this.dispatchEvent({type: "frameloaded", frame: frameIdx});
+        }
+    };
+    module.DataView.prototype.loadRest = function() {
+        for (var i = 0; i < this.data.length; i++)
+            if (this.data[i].loadRest)
+                this.data[i].loadRest();
+        if (this.alphaData !== null && this.alphaData.loadRest)
+            this.alphaData.loadRest();
+    };
     module.DataView.prototype.setFilter = function(interp) {
         this.filter = interp;
         for (var i = 0; i < this.data.length; i++)
@@ -382,8 +421,20 @@ var dataset = (function(module) {
         this.frames = images[json.name].length;
 
         this._interp = "nearest";
-        this.textures = [];
+        this.textures = new Array(this.frames);
+        this._nloaded = 0;
+        this._priority = 0;
+        this._inflight = false;
+        this._deferred = this.movie;
+        var nextFrame = function() {
+            for (var i = this._priority; i < this.frames; i++)
+                if (this.textures[i] === undefined) return i;
+            for (var j = 0; j < this._priority; j++)
+                if (this.textures[j] === undefined) return j;
+            return -1;
+        }.bind(this);
         var loadmosaic = function(idx) {
+            this._inflight = true;
             var img = new Image();
             img.addEventListener("load", function() {
                 this._width = img.width;
@@ -409,22 +460,57 @@ var dataset = (function(module) {
                 tex.needsUpdate = true;
                 tex.flipY = false;
                 this.shape = [((img.width-1) / this.mosaic[0])-1, ((img.height-1) / this.mosaic[1])-1];
-                this.textures.push(tex);
-
-                if (this.textures.length < this.frames) {
-                    this.loaded.notify(this.textures.length);
-                    loadmosaic(this.textures.length);
+                this.textures[idx] = tex;
+                this._nloaded += 1;
+                this._inflight = false;
+
+                this.loaded.notify(this._nloaded, idx);
+                if (this._nloaded < this.frames) {
+                    if (this._deferred) {
+                        this._paused = true;
+                        setTimeout(this.loadRest.bind(this), 2000);
+                    } else {
+                        var nxt = nextFrame();
+                        if (nxt >= 0)
+                            loadmosaic(nxt);
+                    }
                 } else {
                     this.loaded.resolve();
                 }
             }.bind(this));
-            img.src = this.data[this.textures.length];
+            img.src = this.data[idx];
         }.bind(this);
+        this._loadmosaic = loadmosaic;
+        this._nextFrame = nextFrame;
 
         loadmosaic(0);
     };
+    module.VolumeData.prototype.loadRest = function() {
+        this._deferred = false;
+        if (this._paused) {
+            this._paused = false;
+            var nxt = this._nextFrame();
+            if (nxt >= 0 && !this._inflight)
+                this._loadmosaic(nxt);
+        }
+    };
+
+    module.VolumeData.prototype.setPriority = function(frame) {
+        if (!this.movie || this._nloaded >= this.frames)
+            return;
+        this._priority = Math.max(0, Math.min(this.frames - 1, Math.round(frame)));
+        this._deferred = false;
+        if (!this._inflight) {
+            this._paused = false;
+            var nxt = this._nextFrame();
+            if (nxt >= 0)
+                this._loadmosaic(nxt);
+        }
+    };
     module.VolumeData.prototype.setFilter = function(interp) {
         for (var i = 0, il = this.textures.length; i < il; i++) {
+            if (this.textures[i] === undefined)
+                continue;
             this.textures[i].minFilter = module.filtertypes[interp];
             this.textures[i].magFilter = module.filtertypes[interp];
             this.textures[i].needsUpdate = true;
@@ -440,13 +526,20 @@ var dataset = (function(module) {
     };
 
     module.VolumeData.prototype.set = function(uniforms, dim, fframe) {
-        if (uniforms.data.value[dim*2] !== this.textures[fframe]) {
-            uniforms.data.value[dim*2] = this.textures[fframe];
-            if (this.frames > 1) {
-                uniforms.data.value[dim*2+1] = this.textures[(fframe+1).mod(this.frames)];
-            } else {
-                uniforms.data.value[dim*2+1] = null;
-            }
+        var tex = this.textures[fframe];
+        if (tex === undefined) {
+            return;
+        }
+        var next = tex;
+        if (this.frames > 1) {
+            next = this.textures[(fframe+1).mod(this.frames)];
+            if (next === undefined)
+                next = tex;
+        }
+        if (uniforms.data.value[dim*2] !== tex ||
+            uniforms.data.value[dim*2+1] !== next) {
+            uniforms.data.value[dim*2] = tex;
+            uniforms.data.value[dim*2+1] = this.frames > 1 ? next : null;
         }
     }
     module.VolumeData.prototype._setData = function(fframe, data) {
@@ -540,7 +633,9 @@ var dataset = (function(module) {
                     lattr.needsUpdate = true;
                     rattr.needsUpdate = true;
                     this.verts.push([lattr, rattr]);
-                    this.loaded.notify(available);
+                    // (count, index of the frame that just arrived), as for
+                    // volumes, so a frame held by a seek refreshes on arrival
+                    this.loaded.notify(available, available - 1);
                 }.bind(this));
             }.bind(this)).done(function(){
                 this.loaded.resolve();
@@ -555,8 +650,13 @@ var dataset = (function(module) {
     }
     module.VertexData.prototype.set = function(uniforms, dim, fframe, dispatch) {
         var name = dim == 0 ? "data0":"data2";
+        // The next frame may not have streamed in yet: blend with the current
+        // one rather than wrapping (by the loaded count) round to frame 0.
+        var next = this.verts[(fframe+1).mod(this.frames)];
+        if (next === undefined)
+            next = this.verts[fframe];
         dispatch({type:"attribute", name:"data"+(2*dim), value:this.verts[fframe]});
-        dispatch({type:"attribute", name:"data"+(2*dim+1), value:this.verts[(fframe+1).mod(this.verts.length)]});
+        dispatch({type:"attribute", name:"data"+(2*dim+1), value:next});
         // The combined nanmask is dispatched by DataView.setFrame after
         // every dim's data has been set, so we don't dispatch it here.
     }