From 621ddb1466619095003df1e6fab8e19d3312f7eb Mon Sep 17 00:00:00 2001 From: Jiwoong Park Date: Sat, 19 Sep 2026 21:21:13 -0700 Subject: [PATCH 1/2] Adds a timeseries panel to the viewer --- cortex/dataset/braindata.py | 34 +- cortex/webgl/data.py | 32 +- cortex/webgl/resources/js/dataset.js | 102 ++++- cortex/webgl/resources/js/figure.js | 423 ++++++++++++++++++ cortex/webgl/resources/js/mriview.js | 131 +++++- cortex/webgl/view.py | 173 +++++-- examples/timeseries/README.txt | 5 + .../timeseries/encoding_model_timeseries.py | 111 +++++ examples/timeseries/rgb_timeseries.py | 50 +++ 9 files changed, 1002 insertions(+), 59 deletions(-) create mode 100644 examples/timeseries/README.txt create mode 100644 examples/timeseries/encoding_model_timeseries.py create mode 100644 examples/timeseries/rgb_timeseries.py diff --git a/cortex/dataset/braindata.py b/cortex/dataset/braindata.py index 96ce4ab06..e0d35b049 100644 --- a/cortex/dataset/braindata.py +++ b/cortex/dataset/braindata.py @@ -86,10 +86,11 @@ def to_json(self, simple=False): """ sdict = super().to_json(simple=simple) if simple: + vmin, vmax = _nan_to_num_bounds(self.data) sdict.update(dict(name=self.name, subject=self.subject, - min=float(np.nan_to_num(self.data).min()), - max=float(np.nan_to_num(self.data).max()), + min=vmin, + max=vmax, )) return sdict @@ -710,10 +711,37 @@ def __getitem__(self, masktype: str) -> T_masker: mask = db.get_mask(self.dv.subject, self.dv.xfmname, masktype) return self.dv.copy(self.dv.volume[:,mask].squeeze()) +def _nan_to_num_bounds(array: npt.ArrayLike) -> tuple[float, float]: + """(min, max) of ``np.nan_to_num(array)``, without copying it whole. + + ``np.nan_to_num`` allocates a copy of its input, so taking the display + bounds of a 4D movie that way (once for the min, once for the max) + allocates several times the size of the data and dominated webgl viewer + startup for long runs. Reducing one slice at a time gives identical + values in bounded memory. + """ + array = np.asarray(array) + if array.size == 0: + # let numpy raise the same error it always has for empty data + filled = np.nan_to_num(array) + return float(filled.min()), float(filled.max()) + # 1D data has no useful slices to reduce over; anything higher is + # reduced along its first axis (time for movies, slices for volumes) + chunks = array if array.ndim > 1 else (array,) + vmin, vmax = np.inf, -np.inf + for chunk in chunks: + chunk = np.nan_to_num(chunk) + vmin = min(vmin, chunk.min()) + vmax = max(vmax, chunk.max()) + return float(vmin), float(vmax) + + def _hash(array: npt.ArrayLike) -> str: '''A simple numpy hash function''' array = np.asarray(array) - return hashlib.sha1(array.tobytes()).hexdigest() + if not array.flags["C_CONTIGUOUS"]: + array = np.ascontiguousarray(array) + return hashlib.sha1(memoryview(array).cast("B")).hexdigest() def _hdf_write(h5: Union[h5py.File, h5py.Group], data: npt.NDArray, name: str="data", group: str="/data") -> h5py.Dataset: try: diff --git a/cortex/webgl/data.py b/cortex/webgl/data.py index 0dfed7d4c..b3ff02644 100644 --- a/cortex/webgl/data.py +++ b/cortex/webgl/data.py @@ -21,13 +21,15 @@ class Package(object): """Package the data into a form usable by javascript""" - def __init__(self, data): + def __init__(self, data, lazy=False): + # lazy=True defers volume mosaic/PNG packing to get_image(). Only the first frame is packed up front. self.dataset = dataset.normalize(data) self.uniques = list(data.uniques(collapse=True)) self.subjects = set() self.brains = dict() self.images = dict() + self._pending = dict() for brain in self.uniques: name = brain.name self.subjects.add(brain.subject) @@ -62,12 +64,21 @@ def __init__(self, data): ) self.brains[name]["raw"] = True else: - encdata = encdata.astype(np.float32) self.brains[name]["raw"] = False + if isinstance(brain, dataset.Vertex) or not lazy: + encdata = encdata.astype(np.float32, copy=False) # VertexData requires reordering, only save normalized version for now if isinstance(brain, (dataset.Vertex, dataset.VertexRGB)): self.images[name] = [encdata] + elif lazy: + # The mosaic grid depends only on the volume shape, + # so the first frame determines the metadata for every frame. + mosaic0, shape = volume.mosaic(self._pack_dtype(name, encdata[0]), + show=False) + self.brains[name]["mosaic"] = shape + self._pending[name] = encdata + self.images[name] = [_pack_png(mosaic0)] + [None] * (len(encdata) - 1) else: self.images[name] = [volume.mosaic(vol, show=False) for vol in encdata] if len(set([shape for m, shape in self.images[name]])) != 1: @@ -112,6 +123,23 @@ def metadata(self, submap=None, **kwargs): views=self.views, data=self.brains, images=self.image_names(**kwargs) ) + def _pack_dtype(self, name, framedata): + """One frame in the dtype _pack_png accepts: raw data stays uint8, + everything else becomes float32 (a per-frame no-op-sized copy).""" + if self.brains[name]["raw"]: + return framedata + return np.asarray(framedata, dtype=np.float32) + + def get_image(self, name, frame): + """Return the packed PNG (or npy blob) for one frame, packing it on + first request when the package was built with lazy=True.""" + img = self.images[name][frame] + if img is None: + mosaic, _ = volume.mosaic( + self._pack_dtype(name, self._pending[name][frame]), show=False) + img = self.images[name][frame] = _pack_png(mosaic) + return img + def image_names(self, fmt="/data/{name}/{frame}/"): names = dict() for name, imgs in self.images.items(): diff --git a/cortex/webgl/resources/js/dataset.js b/cortex/webgl/resources/js/dataset.js index 9a0a71643..9affd732c 100644 --- a/cortex/webgl/resources/js/dataset.js +++ b/cortex/webgl/resources/js/dataset.js @@ -134,8 +134,9 @@ var dataset = (function(module) { for (var i = 0; i < this.data.length; i++) { (function(idx) { this.data[idx].loaded - .progress(function(available) { + .progress(function(available, frameIdx) { if (available > this.delay) markReady(idx); + if (frameIdx !== undefined) this._frameArrived(frameIdx); }.bind(this)) .done(function() { markReady(idx); }); }.bind(this))(i); @@ -307,6 +308,23 @@ var dataset = (function(module) { this._dispatch({type:"attribute", name:"nanmask", value:combined}); } } + // A streamed movie frame just arrived. If it is the frame on screen (or + // its blend partner), re-apply the textures: a seek to a not-yet-loaded + // frame otherwise stays blank until the next setFrame (e.g. playback). + module.DataView.prototype._frameArrived = function(frameIdx) { + if (this.frames <= 1) + return; + var fframe = Math.floor(((this.frame + this.delay) * this.rate).mod(this.frames)); + if (frameIdx === fframe || frameIdx === (fframe + 1).mod(this.frames)) { + this.setFrame(this.frame); + this.dispatchEvent({type: "frameloaded", frame: frameIdx}); + } + }; + module.DataView.prototype.loadRest = function() { + for (var i = 0; i < this.data.length; i++) + if (this.data[i].loadRest) + this.data[i].loadRest(); + }; module.DataView.prototype.setFilter = function(interp) { this.filter = interp; for (var i = 0; i < this.data.length; i++) @@ -330,8 +348,20 @@ var dataset = (function(module) { this.frames = images[json.name].length; this._interp = "nearest"; - this.textures = []; + this.textures = new Array(this.frames); + this._nloaded = 0; + this._priority = 0; + this._inflight = false; + this._deferred = this.movie; + var nextFrame = function() { + for (var i = this._priority; i < this.frames; i++) + if (this.textures[i] === undefined) return i; + for (var j = 0; j < this._priority; j++) + if (this.textures[j] === undefined) return j; + return -1; + }.bind(this); var loadmosaic = function(idx) { + this._inflight = true; var img = new Image(); img.addEventListener("load", function() { this._width = img.width; @@ -357,22 +387,57 @@ var dataset = (function(module) { tex.needsUpdate = true; tex.flipY = false; this.shape = [((img.width-1) / this.mosaic[0])-1, ((img.height-1) / this.mosaic[1])-1]; - this.textures.push(tex); - - if (this.textures.length < this.frames) { - this.loaded.notify(this.textures.length); - loadmosaic(this.textures.length); + this.textures[idx] = tex; + this._nloaded += 1; + this._inflight = false; + + this.loaded.notify(this._nloaded, idx); + if (this._nloaded < this.frames) { + if (this._deferred) { + this._paused = true; + setTimeout(this.loadRest.bind(this), 2000); + } else { + var nxt = nextFrame(); + if (nxt >= 0) + loadmosaic(nxt); + } } else { this.loaded.resolve(); } }.bind(this)); - img.src = this.data[this.textures.length]; + img.src = this.data[idx]; }.bind(this); + this._loadmosaic = loadmosaic; + this._nextFrame = nextFrame; loadmosaic(0); }; + module.VolumeData.prototype.loadRest = function() { + this._deferred = false; + if (this._paused) { + this._paused = false; + var nxt = this._nextFrame(); + if (nxt >= 0 && !this._inflight) + this._loadmosaic(nxt); + } + }; + + module.VolumeData.prototype.setPriority = function(frame) { + if (!this.movie || this._nloaded >= this.frames) + return; + this._priority = Math.max(0, Math.min(this.frames - 1, Math.round(frame))); + this._deferred = false; + if (!this._inflight) { + this._paused = false; + var nxt = this._nextFrame(); + if (nxt >= 0) + this._loadmosaic(nxt); + } + }; module.VolumeData.prototype.setFilter = function(interp) { for (var i = 0, il = this.textures.length; i < il; i++) { + if (this.textures[i] === undefined) + continue; this.textures[i].minFilter = module.filtertypes[interp]; this.textures[i].magFilter = module.filtertypes[interp]; this.textures[i].needsUpdate = true; @@ -388,13 +453,20 @@ var dataset = (function(module) { }; module.VolumeData.prototype.set = function(uniforms, dim, fframe) { - if (uniforms.data.value[dim*2] !== this.textures[fframe]) { - uniforms.data.value[dim*2] = this.textures[fframe]; - if (this.frames > 1) { - uniforms.data.value[dim*2+1] = this.textures[(fframe+1).mod(this.frames)]; - } else { - uniforms.data.value[dim*2+1] = null; - } + var tex = this.textures[fframe]; + if (tex === undefined) { + return; + } + var next = tex; + if (this.frames > 1) { + next = this.textures[(fframe+1).mod(this.frames)]; + if (next === undefined) + next = tex; + } + if (uniforms.data.value[dim*2] !== tex || + uniforms.data.value[dim*2+1] !== next) { + uniforms.data.value[dim*2] = tex; + uniforms.data.value[dim*2+1] = this.frames > 1 ? next : null; } } module.VolumeData.prototype._setData = function(fframe, data) { diff --git a/cortex/webgl/resources/js/figure.js b/cortex/webgl/resources/js/figure.js index a2e05b226..5ec673631 100644 --- a/cortex/webgl/resources/js/figure.js +++ b/cortex/webgl/resources/js/figure.js @@ -315,6 +315,429 @@ var jsplot = (function (module) { this.figure.notify("playtoggle", this, [this.movie.paused?"pause":"play"]); } + // Timeseries panel: overlays the picked voxel's timecourse for every + // checked dataset channel fetched on demand from the /timeseries handler. + module.TimeseriesAxes = function(figure, viewer) { + module.Axes.call(this, figure); + this.viewer = viewer || null; + this.object.style.backgroundColor = this.style.bg; + this.object.style.width = "100%"; + this.object.style.height = "100%"; + this.object.style.display = "flex"; + this.object.style.flexDirection = "column"; + + // control strip: one group per trace + this.controls = document.createElement("div"); + var cs = this.controls.style; + cs.display = "flex"; + cs.flexWrap = "wrap"; + cs.alignItems = "center"; + cs.gap = "14px"; + cs.padding = "3px 10px"; + cs.minHeight = "20px"; + cs.font = this.style.font; + cs.backgroundColor = this.style.bg; + cs.flex = "0 0 auto"; + this.object.appendChild(this.controls); + + // display mode: raw values or per-trace z-scores + this.mode = "raw"; + var sel = document.createElement("select"); + sel.style.background = this.style.bg; + sel.style.color = this.style.text; + sel.style.border = "1px solid " + this.style.spine; + sel.style.borderRadius = "3px"; + sel.style.font = this.style.font; + ["raw", "z-scored"].forEach(function(mname) { + var o = document.createElement("option"); + o.value = mname; + o.textContent = mname; + sel.appendChild(o); + }); + sel.addEventListener("change", function() { + this.mode = sel.value === "z-scored" ? "z" : "raw"; + this.draw(); + }.bind(this)); + this.controls.appendChild(sel); + + this.canvas = document.createElement("canvas"); + this.canvas.style.display = "block"; + this.canvas.style.width = "100%"; + this.canvas.style.flex = "1 1 auto"; + this.canvas.style.minHeight = "0"; + this.object.appendChild(this.canvas); + + // name -> {type: 'data'|'ref', resp, ref, channels: [{on, color}]} + this.traces = {}; + this.order = []; + this.label = ""; + this.message = "Click a voxel to plot its timeseries"; + this.frame = null; // playhead position, in data frames + this._xmap = null; // plot x-geometry of the last draw, for seeking + + // click a timepoint -> seek the brain to that volume + this.canvas.addEventListener("click", function(evt) { + if (!this._xmap || !this.viewer) + return; + var rect = this.canvas.getBoundingClientRect(); + var fx = (evt.clientX - rect.left - this._xmap.x0) / this._xmap.w; + if (fx < 0 || fx > 1) + return; + this.viewer.seekFrame(Math.round(fx * (this._xmap.n - 1))); + }.bind(this)); + + // one control group per movie dataset (3D views have no + // timecourse); the active one starts checked, or the first movie + // if the active view is a plain 3D volume + if (viewer && viewer.dataviews) { + var names = Object.keys(viewer.dataviews); + var movies = names.filter(function(nm) { + return viewer.dataviews[nm].frames > 1; + }); + var def = (viewer.active && viewer.active.frames > 1) + ? viewer.active.name : movies[0]; + for (var i = 0; i < movies.length; i++) { + var nch = viewer.dataviews[movies[i]].data[0].raw ? 3 : 1; + this.addTrace(movies[i], "data", movies[i] === def, nch); + } + } + setTimeout(this.resize.bind(this), 0); + } + module.TimeseriesAxes.prototype = Object.create(module.Axes.prototype); + module.TimeseriesAxes.prototype.constructor = module.TimeseriesAxes; + module.TimeseriesAxes.prototype.style = { + bg: "#0D1117", text: "#E8ECF5", muted: "#9AA3B5", + spine: "#3A4250", play: "#FFB454", + font: "11px sans-serif", + dataColors: ["#6FA8FF", "#FF6B6B", "#5DD97C", "#FFB454", "#B48EAD", "#66D9E8"], + refColors: ["#C8A96E", "#B48EAD", "#8FBCBB", "#D08770"], + // per-channel defaults for RGB datasets: the first gets true + // R/G/B, later ones get shifted triads so overlays stay readable + rgbTriads: [["#FF6B6B", "#5DD97C", "#6FA8FF"], + ["#FFA94D", "#3BC9DB", "#B197FC"], + ["#F783AC", "#A9E34B", "#748FFC"]], + }; + module.TimeseriesAxes.prototype._anyOn = function(t) { + for (var i = 0; i < t.channels.length; i++) + if (t.channels[i].on) + return true; + return false; + } + module.TimeseriesAxes.prototype.addTrace = function(name, type, on, nchan) { + if (this.traces[name]) + return this.traces[name]; + var S = this.style; + nchan = nchan || 1; + var nScalar = 0, nRGB = 0, nRef = 0; + for (var i = 0; i < this.order.length; i++) { + var o = this.traces[this.order[i]]; + if (o.type === "ref") nRef++; + else if (o.channels.length === 3) nRGB++; + else nScalar++; + } + var colors; + if (type === "ref") + colors = [S.refColors[nRef % S.refColors.length]]; + else if (nchan === 3) + colors = S.rgbTriads[nRGB % S.rgbTriads.length]; + else + colors = [S.dataColors[nScalar % S.dataColors.length]]; + + var t = {type: type, resp: null, ref: null, channels: []}; + for (var c = 0; c < nchan; c++) + t.channels.push({on: !!on, color: colors[c]}); + this.traces[name] = t; + this.order.push(name); + + var group = document.createElement("span"); + group.style.display = "flex"; + group.style.alignItems = "center"; + group.style.gap = "5px"; + group.style.font = S.font; + group.style.color = type === "ref" ? S.muted : S.text; + var txt = document.createElement("span"); + txt.textContent = name; + group.appendChild(txt); + + var chanNames = nchan === 3 ? ["R", "G", "B"] : [""]; + var self = this; + t.channels.forEach(function(ch, ci) { + var pair = document.createElement("label"); + pair.style.display = "flex"; + pair.style.alignItems = "center"; + pair.style.gap = "2px"; + pair.style.cursor = "pointer"; + var cb = document.createElement("input"); + cb.type = "checkbox"; + cb.checked = ch.on; + cb.style.margin = "0"; + cb.addEventListener("change", function() { + ch.on = cb.checked; + if (ch.on && t.type === "data" && !t.resp) + self.refetch(); + self.draw(); + }); + var col = document.createElement("input"); + col.type = "color"; + col.value = ch.color; + col.style.width = "15px"; + col.style.height = "15px"; + col.style.padding = "0"; + col.style.border = "none"; + col.style.background = "none"; + col.style.cursor = "pointer"; + col.addEventListener("input", function() { + ch.color = col.value; + self.draw(); + }); + pair.appendChild(cb); + pair.appendChild(col); + if (chanNames[ci]) + pair.appendChild(document.createTextNode(chanNames[ci])); + group.appendChild(pair); + }); + this.controls.appendChild(group); + return t; + } + module.TimeseriesAxes.prototype.refetch = function() { + if (this.viewer && this.viewer._tsCoords) + this.viewer.fetchTimeseries(this.viewer._tsCoords); + } + module.TimeseriesAxes.prototype.resize = function() { + var dpr = window.devicePixelRatio || 1; + this.canvas.width = this.canvas.clientWidth * dpr; + this.canvas.height = this.canvas.clientHeight * dpr; + this.draw(); + } + module.TimeseriesAxes.prototype.setMessage = function(msg) { + this.message = msg; + this.draw(); + } + module.TimeseriesAxes.prototype.update = function(name, resp, label) { + var t = this.addTrace(name, "data", true, resp.data.length); + t.resp = resp; + this.label = label; + // register reference traces (design-matrix regressors); they start + // unchecked so QC stays uncluttered by default + if (resp.refs) { + for (var rname in resp.refs) { + var rt = this.addTrace(rname, "ref", false, 1); + rt.ref = resp.refs[rname]; + } + } + this.draw(); + } + module.TimeseriesAxes.prototype.setFrame = function(frame) { + this.frame = frame; + this.draw(); + } + module.TimeseriesAxes.prototype._zscore = function(series) { + var i, m = 0; + for (i = 0; i < series.length; i++) + m += series[i]; + m /= series.length; + var sd = 0; + for (i = 0; i < series.length; i++) + sd += (series[i] - m) * (series[i] - m); + sd = Math.sqrt(sd / series.length) || 1; + var out = new Array(series.length); + for (i = 0; i < series.length; i++) + out[i] = (series[i] - m) / sd; + return out; + } + // matplotlib-style "nice" tick locations: steps of 1/2/5 x 10^k + module.TimeseriesAxes.prototype._ticks = function(lo, hi, target) { + var span = hi - lo; + if (!(span > 0)) + return [lo]; + var step = Math.pow(10, Math.floor(Math.log(span / target) / Math.LN10)); + var err = target / (span / step); + if (err <= 0.15) step *= 10; + else if (err <= 0.35) step *= 5; + else if (err <= 0.75) step *= 2; + var out = []; + for (var v = Math.ceil(lo / step) * step; v <= hi + 1e-9 * span; v += step) + out.push(Math.abs(v) < 1e-12 ? 0 : parseFloat(v.toPrecision(6))); + return out; + } + module.TimeseriesAxes.prototype.draw = function() { + var S = this.style; + var dpr = window.devicePixelRatio || 1; + var W = this.canvas.width / dpr, H = this.canvas.height / dpr; + if (W < 10 || H < 10) + return; + var ctx = this.canvas.getContext("2d"); + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.fillStyle = S.bg; + ctx.fillRect(0, 0, W, H); + ctx.font = S.font; + this._xmap = null; + + var dataAct = [], refAct = []; + for (var i = 0; i < this.order.length; i++) { + var name = this.order[i], t = this.traces[name]; + if (!this._anyOn(t)) + continue; + if (t.type === "data" && t.resp) + dataAct.push({name: name, t: t}); + else if (t.type === "ref" && t.ref) + refAct.push({name: name, t: t}); + } + if (dataAct.length === 0) { + ctx.fillStyle = S.muted; + ctx.textAlign = "center"; + ctx.fillText(this.message, W / 2, H / 2); + return; + } + + var lead = dataAct[0].t.resp; + var n = lead.data[0].length; + var zmode = this.mode === "z"; + var pad = {l: 52, r: 14, t: 20, b: 28}; + + // one labeled color strip per checked RGB dataset, stacked + var strips = []; + for (var di = 0; di < dataAct.length; di++) + if (dataAct[di].t.resp.data.length === 3) + strips.push(dataAct[di]); + var stripH = 12, stripGap = 2; + var y0 = pad.t + (strips.length ? strips.length * (stripH + stripGap) + 3 : 0); + var plotH = H - y0 - pad.b; + var x0 = pad.l, w = W - pad.l - pad.r; + + // assemble the visible channel lines, applying the mode transform + var lines = []; // {series, color, lw, alpha, own} + for (var di = 0; di < dataAct.length; di++) { + var t = dataAct[di].t; + for (var c = 0; c < t.channels.length; c++) { + if (!t.channels[c].on) + continue; + lines.push({series: zmode ? this._zscore(t.resp.data[c]) + : t.resp.data[c], + color: t.channels[c].color, + lw: 1.5, alpha: 1, own: false}); + } + } + for (var ri = 0; ri < refAct.length; ri++) + lines.push({series: refAct[ri].t.ref, + color: refAct[ri].t.channels[0].color, + lw: 1.1, alpha: 0.7, own: true}); + + // shared y-range across all data channels: raw mode shows true + // values, z mode shows z units (reference traces stay min-max + // scaled — their units are arbitrary) + var mn = Infinity, mx = -Infinity; + for (var li = 0; li < lines.length; li++) { + if (lines[li].own) + continue; + mn = Math.min(mn, Math.min.apply(null, lines[li].series)); + mx = Math.max(mx, Math.max.apply(null, lines[li].series)); + } + if (mn === mx) { mn -= 1; mx += 1; } + var shared = [mn, mx]; + + ctx.fillStyle = S.text; + ctx.textAlign = "left"; + ctx.fillText(this.label, x0, 13); + + // strips: always blend all three channels — they show the color + // actually painted on the brain, independent of line visibility + for (var si = 0; si < strips.length; si++) { + var sr = strips[si].t.resp; + var sy = pad.t + si * (stripH + stripGap); + var sn = sr.data[0].length; + var segW = w / sn; + for (var i = 0; i < sn; i++) { + ctx.fillStyle = "rgb(" + Math.round(sr.data[0][i] * 255) + "," + + Math.round(sr.data[1][i] * 255) + "," + + Math.round(sr.data[2][i] * 255) + ")"; + ctx.fillRect(x0 + i * segW, sy, segW + 1, stripH); + } + ctx.fillStyle = S.muted; + ctx.textAlign = "right"; + ctx.fillText(strips[si].name, x0 - 6, sy + stripH - 2); + } + + ctx.strokeStyle = S.spine; + ctx.strokeRect(x0 + .5, y0 + .5, w, plotH); + + // x axis is the volume index — QC thinks in frames, not seconds + var xticks = this._ticks(0, n - 1, 6); + ctx.fillStyle = S.muted; + ctx.textAlign = "center"; + ctx.strokeStyle = S.spine; + for (var xi = 0; xi < xticks.length; xi++) { + var tx = x0 + (xticks[xi] / (n - 1)) * w; + ctx.beginPath(); + ctx.moveTo(tx, y0 + plotH); + ctx.lineTo(tx, y0 + plotH + 4); + ctx.stroke(); + ctx.fillText(xticks[xi], tx, y0 + plotH + 14); + } + ctx.fillText("volume", x0 + w / 2, H - 3); + + // y ticks in real units (raw) or z units (z-scored), plus a rotated + // axis label naming the mode + var yticks = this._ticks(shared[0], shared[1], 4); + ctx.textAlign = "right"; + for (var yi = 0; yi < yticks.length; yi++) { + var ty = y0 + (1 - (yticks[yi] - shared[0]) / (shared[1] - shared[0])) * plotH; + ctx.beginPath(); + ctx.moveTo(x0 - 4, ty); + ctx.lineTo(x0, ty); + ctx.stroke(); + ctx.fillText(yticks[yi], x0 - 6, ty + 3.5); + } + ctx.save(); + ctx.translate(11, y0 + plotH / 2); + ctx.rotate(-Math.PI / 2); + ctx.textAlign = "center"; + ctx.fillText(zmode ? "z-scored" : "raw", 0, 0); + ctx.restore(); + + for (var li = 0; li < lines.length; li++) { + var L = lines[li]; + var py; + if (L.own) { + var lmn = Math.min.apply(null, L.series); + var lmx = Math.max.apply(null, L.series); + if (lmn === lmx) { lmn -= 1; lmx += 1; } + py = (function(a, b) { + return function(v) { return y0 + (1 - (v - a) / (b - a)) * plotH; }; + })(lmn, lmx); + } else { + py = (function(a, b) { + return function(v) { return y0 + (1 - (v - a) / (b - a)) * plotH; }; + })(shared[0], shared[1]); + } + ctx.strokeStyle = L.color; + ctx.globalAlpha = L.alpha; + ctx.lineWidth = L.lw; + ctx.lineJoin = "round"; + ctx.beginPath(); + for (var i = 0; i < L.series.length; i++) { + var lx = x0 + (i / (L.series.length - 1)) * w; + i ? ctx.lineTo(lx, py(L.series[i])) : ctx.moveTo(lx, py(L.series[i])); + } + ctx.stroke(); + ctx.globalAlpha = 1; + ctx.lineWidth = 1; + } + + if (this.frame !== null) { + var fx = x0 + (Math.min(this.frame, n - 1) / (n - 1)) * w; + ctx.strokeStyle = S.play; + ctx.setLineDash([4, 3]); + ctx.beginPath(); + ctx.moveTo(fx, pad.t); + ctx.lineTo(fx, H - pad.b); + ctx.stroke(); + ctx.setLineDash([]); + } + + this._xmap = {x0: x0, w: w, n: n}; + } + module.ImageAxes = function(figure) { module.Axes.call(this, figure); } diff --git a/cortex/webgl/resources/js/mriview.js b/cortex/webgl/resources/js/mriview.js index 790bb4135..2f2316358 100644 --- a/cortex/webgl/resources/js/mriview.js +++ b/cortex/webgl/resources/js/mriview.js @@ -48,7 +48,9 @@ var mriview = (function(module) { for (var i = 0; i < data.length; i++) { var d = data[i]; if (d.mosaic !== undefined) { - if (!d.textures || d.textures.length === 0) return false; + // textures is a sparse Array(frames): length is fixed, so + // check that frame 0 (what hover/click read) has arrived + if (!d.textures || d.textures[0] === undefined) return false; } else { if (!d.verts || d.verts.length === 0) return false; } @@ -367,6 +369,7 @@ var mriview = (function(module) { for (var i = 0; i < this.surfs.length; i++) { this.active.removeEventListener("update", this.surfs[i]._update); this.active.removeEventListener("attribute", this.surfs[i]._attrib); + this.active.removeEventListener("frameloaded", this._schedule_frame); } //set the new active data and update shaders for all surfaces this.active = this.dataviews[name]; @@ -383,6 +386,9 @@ var mriview = (function(module) { this.surfs[i].update(this.active); this.active.addEventListener("update", this.surfs[i]._update); this.active.addEventListener("attribute", this.surfs[i]._attrib); + if (!this._schedule_frame) + this._schedule_frame = this.schedule.bind(this); + this.active.addEventListener("frameloaded", this._schedule_frame); } } this.active.loaded.done(function() { @@ -975,6 +981,12 @@ var mriview = (function(module) { if (this.surfs[i].pick) coords = this.surfs[i].pick(this.renderer, this.camera, evt.x, evt.y); } + // On-demand timeseries fetch. + if (coords && coords !== -1) { + this._lastPickCoords = coords; + if (this.tsplot && this.tsplot_visible) + this.fetchTimeseries(coords); + } // set the picked value display // Length check first so we don't index data[0] on an empty array. // Skip RGB, then ensure all child buffers have populated. @@ -1030,16 +1042,103 @@ var mriview = (function(module) { } } + module.Viewer.prototype.toggleTimeseries = function() { + if (this.tsplot_visible) { + this.figure.hide("bottom"); + this.tsplot_visible = false; + } else { + if (!this.tsplot) { + // pixel size: setSize's "%" form is relative to figure width, + // which oversizes a bottom (height) panel + this.figure.setSize("bottom", 280); + this.tsplot = this.figure.add(jsplot.TimeseriesAxes, "bottom", true, this); + } else { + this.figure.show("bottom"); + } + this.tsplot_visible = true; + // a voxel picked before the panel was opened: plot it right away + if (this._lastPickCoords) + this.fetchTimeseries(this._lastPickCoords); + } + setTimeout(this.resize.bind(this), 500); + } + + module.Viewer.prototype.fetchTimeseries = function(coords) { + this._tsCoords = coords; + var tsplot = this.tsplot; + var voxel = null, label = ""; + if (coords.voxel) { + voxel = Math.round(coords.voxel.x) + "," + + Math.round(coords.voxel.y) + "," + + Math.round(coords.voxel.z); + label = "voxel (" + voxel.split(",").join(", ") + ")"; + } + // fetch every dataset with at least one checked channel + var names = []; + for (var i = 0; i < tsplot.order.length; i++) { + var t = tsplot.traces[tsplot.order[i]]; + if (t.type === "data" && tsplot._anyOn(t)) + names.push(tsplot.order[i]); + } + if (names.length === 0) + return; + names.forEach(function(name) { + // Send the raw CTM-order pick index; the server maps it back to + // the original vertex numbering via the ctmpack index array. + var params = {name: name, hemi: coords.hemi || "", + vertex: coords.vertex}; + if (voxel !== null) + params.voxel = voxel; + $.getJSON("/timeseries", params) + .done(function(resp) { + var lbl = label; + if (resp.vertex !== undefined && resp.vertex !== null) + lbl = "vertex " + resp.vertex + + (params.hemi ? " (" + params.hemi + ")" : ""); + tsplot.update(name, resp, lbl); + }) + .fail(function(xhr) { + var msg = "timeseries unavailable"; + try { msg = JSON.parse(xhr.responseText).error || msg; } + catch (e) {} + tsplot.setMessage(name + ": " + msg); + }); + }); + } + + // Jump the brain to one timepoint + module.Viewer.prototype.seekFrame = function(dataIdx) { + for (var i = 0; i < this.active.data.length; i++) + if (this.active.data[i].setPriority) + this.active.data[i].setPriority(dataIdx); + if (this.tsplot && this.tsplot_visible) + this.tsplot.setFrame(dataIdx); + if (!this.active.data[0].movie) + return; + var t = dataIdx / this.active.rate - this.active.delay; + if (this.state == "play") + this.playpause(); + this.frame = t; + this.active.setFrame(t); + if (this.movie) + this.movie.setFrame(t); + this.schedule(); + } + var movie_ui; module.Viewer.prototype.setupStim = function() { + // controls for a movie view + if ("movie" in this.ui._folders) + this.ui.remove("movie"); + var anyMovie = false; + for (var dname in this.dataviews) + if (this.dataviews[dname].frames > 1) + anyMovie = true; if (this.active.data[0].movie) { - if ("movie" in this.ui._folders) { - // nothing? - } else { - movie_ui = this.ui.addFolder("movie", true); - movie_ui.add({play_pause: {action: this.playpause.bind(this), key:' '}}); - movie_ui.add({frame: {action:[this, "setFrame", 0, this.active.frames-1]}}); - } + movie_ui = this.ui.addFolder("movie", true); + movie_ui.add({play_pause: {action: this.playpause.bind(this), key:' '}}); + movie_ui.add({frame: {action:[this, "setFrame", 0, this.active.frames-1]}}); + movie_ui.add({timeseries: {action: this.toggleTimeseries.bind(this)}}); if (this.movie) { this.movie.destroy(); @@ -1056,7 +1155,10 @@ var mriview = (function(module) { if (this.state == "play") { this.playpause(); } - this.ui.remove("movie"); + if (anyMovie) { + movie_ui = this.ui.addFolder("movie", true); + movie_ui.add({timeseries: {action: this.toggleTimeseries.bind(this)}}); + } } this.schedule(); if (this.movie) { @@ -1139,6 +1241,11 @@ var mriview = (function(module) { module.Viewer.prototype.setFrame = function(frame) { if (frame === undefined) return this.frame; + // First playback / slider interaction resumes the deferred movie + // frame download (movies only load frame 0 up front — see + // VolumeData in dataset.js). + if (this.active.loadRest) + this.active.loadRest(); if (frame >= this.active.frames) { frame %= this.active.frames; this._startplay += this.active.frames; @@ -1154,6 +1261,12 @@ var mriview = (function(module) { if (this.movie) { this.movie.setFrame(frame); } + if (this.tsplot && this.tsplot_visible) { + // convert playback-clock frame to a data-frame index, matching + // DataView.setFrame's (time + delay) * rate convention + this.tsplot.setFrame( + ((frame + this.active.delay) * this.active.rate).mod(this.active.frames)); + } // $(this.object).find("#movieprogress div").slider("value", frame); // $(this.object).find("#movieframe").attr("value", frame); if (replay == true) { diff --git a/cortex/webgl/view.py b/cortex/webgl/view.py index da506235f..275425c96 100644 --- a/cortex/webgl/view.py +++ b/cortex/webgl/view.py @@ -390,36 +390,56 @@ def show( if open_browser is None: open_browser = options.config.get('webshow', 'open_browser', fallback='true') == 'true' + # Plain 1D numeric arrays in a data dict (e.g. design-matrix regressors) + ref_traces: dict[str, list[float]] = {} + if isinstance(data, dict): + data = dict(data) + for key in list(data.keys()): + val = data[key] + if (isinstance(val, np.ndarray) and val.ndim == 1 + and np.issubdtype(val.dtype, np.number)): + ref_traces[key] = np.nan_to_num( + val.astype(np.float64)).tolist() + del data[key] + data = dataset.normalize(data) if not isinstance(data, dataset.Dataset): data = dataset.Dataset(data=data) html = FallbackLoader([os.path.split(os.path.abspath(template))[0], serve.cwd]).load(template) - db.auxfile = data - #Extract the list of stimuli, for special-casing stims: dict[str, str] = dict() - for name, view in data: - if 'stim' in view.attrs and os.path.exists(view.attrs['stim']): - sname = os.path.split(view.attrs['stim'])[1] - stims[sname] = view.attrs['stim'] - - package = Package(data) - # Keep the metadata as a plain dict (rather than a JSON string) so that - # JSMixer.addData can merge newly added dataviews into it at runtime. It - # is serialized to JSON on demand, when the mixer page is generated. - metadata = package.metadata() - images = package.images - subjects = list(package.subjects) - - ctmargs = dict(method='mg2', level=9, recache=recache, - external_svg=overlay_file, overlays_available=overlays_available) - ctms = dict((subj, utils.get_ctmpack(subj, types, **ctmargs)) - for subj in subjects) - package.reorder(ctms) - - subjectjs = json.dumps(dict((subj, "ctm/%s/"%subj) for subj in subjects)) - db.auxfile = None + package = None + metadata: dict[str, Any] = {} + images: dict[str, list] = dict() + subjects: list[str] = [] + ctms: dict[str, str] = dict() + subjectjs = "" + _ready = threading.Event() + + def _prepare(): + nonlocal package, metadata, subjects, subjectjs + db.auxfile = data + + #Extract the list of stimuli, for special-casing + for name, view in data: + if 'stim' in view.attrs and os.path.exists(view.attrs['stim']): + sname = os.path.split(view.attrs['stim'])[1] + stims[sname] = view.attrs['stim'] + + package = Package(data, lazy=True) + metadata = package.metadata() + subjects = list(package.subjects) + + ctmargs = dict(method='mg2', level=9, recache=recache, + external_svg=overlay_file, overlays_available=overlays_available) + ctms.update((subj, utils.get_ctmpack(subj, types, **ctmargs)) + for subj in subjects) + package.reorder(ctms) + images.update(package.images) + + subjectjs = json.dumps(dict((subj, "ctm/%s/"%subj) for subj in subjects)) + db.auxfile = None linear = lambda x, y, m: (1.-m)*x + m*y @@ -465,6 +485,8 @@ def show( class CTMHandler(web.RequestHandler): def get(self, path: str): + _ready.wait() + self.set_header("Cache-Control", "public, max-age=86400") subj, path = path.split('/') if path == '': self.set_header("Content-Type", "application/json") @@ -479,6 +501,7 @@ def get(self, path: str): class DataHandler(web.RequestHandler): def get(self, path: str): + _ready.wait() path = path.strip("/") frame: Union[int, str] try: @@ -488,7 +511,10 @@ def get(self, path: str): frame = 0 if dataname in images: + self.set_header("Cache-Control", "public, max-age=86400") dataimg = images[dataname][int(frame)] + if dataimg is None: + dataimg = package.get_image(dataname, int(frame)) if dataimg[1:6] == "NUMPY": self.set_header("Content-Type", "application/octet-stream") else: @@ -514,6 +540,7 @@ def initialize(self): pass def get(self, path: str): + _ready.wait() if path not in stims: self.set_status(404) self.write_error(404) @@ -527,6 +554,7 @@ def initialize(self): class MixerHandler(web.RequestHandler): def get(self): + _ready.wait() self.set_header("Content-Type", "text/html") generated = html.generate(data=json.dumps(metadata), colormaps=colormaps, @@ -986,6 +1014,7 @@ def make_movie_views(self, animation, filename="brainmovie%07d.png", class PickerHandler(web.RequestHandler): def get(self): + _ready.wait() voxel_arg = self.get_argument("voxel", None) if voxel_arg is None: self.set_status(400) @@ -1006,6 +1035,82 @@ def get(self): hemi: str = self.get_argument("hemi") pickerfun(voxel, vertex, hemi) + # The browser's pick reports vertices in CTM order. + # The ctmpack's index array maps a concatenated (left-then-right) + # CTM position back to the original vertex numbering that Vertex data uses. + ctm_vertex_index: dict[str, np.ndarray] = {} + def _get_ctm_index(subj): + if subj not in ctm_vertex_index: + npz = np.load(os.path.splitext(ctms[subj])[0] + ".npz") + ctm_vertex_index[subj] = npz["index"] + return ctm_vertex_index[subj] + + class TimeseriesHandler(web.RequestHandler): + """Return one voxel's (or vertex's) timecourse as JSON, on demand. + """ + def get(self): + _ready.wait() + views = dict(data) + name = self.get_argument("name", None) + if name is None: + name = next(iter(views), None) + view = views.get(name) + if view is None: + self.set_status(404) + self.finish({"error": "no dataview named %r" % name}) + return + if not getattr(view, "movie", False): + self.set_status(422) + self.finish({"error": "dataview %r has no time axis" % name}) + return + + vertex = None + try: + if isinstance(view, (dataset.Vertex, dataset.VertexRGB)): + base = view.red if isinstance(view, dataset.VertexRGB) else view + vertex = int(self.get_argument("vertex")) + if self.get_argument("hemi") == "right": + vertex += base.llen + vertex = int(_get_ctm_index(base.subject)[vertex]) + if isinstance(view, dataset.VertexRGB): + # slice each channel view directly: assembling the + # full RGBA array via .vertices copies the whole + # movie per request + chans = [c.vertices[:, vertex] + for c in (view.red, view.green, view.blue)] + else: + chans = [view.vertices[:, vertex]] + elif isinstance(view, (dataset.Volume, dataset.VolumeRGB)): + x, y, z = (int(i) for i in + self.get_argument("voxel").split(",")) + if isinstance(view, dataset.VolumeRGB): + chans = [c.volume[:, z, y, x] + for c in (view.red, view.green, view.blue)] + else: + chans = [view.volume[:, z, y, x]] + else: + self.set_status(422) + self.finish({"error": "unsupported dataview type %s" + % type(view).__name__}) + return + except (ValueError, IndexError, web.MissingArgumentError) as exc: + self.set_status(400) + self.finish({"error": str(exc)}) + return + + def norm(ts): + ts = np.asarray(ts) + if ts.dtype == np.uint8: # RGB channels: bytes -> [0, 1] + ts = ts / 255.0 + return np.nan_to_num(ts.astype(np.float64)) + chans = [norm(ts) for ts in chans] + channels = ["R", "G", "B"] if len(chans) == 3 else ["bold"] + resp = dict(name=name, channels=channels, + data=[ts.tolist() for ts in chans], + vertex=vertex, refs=ref_traces or None) + self.set_header("Content-Type", "application/json") + self.finish(json.dumps(resp)) + class WebApp(serve.WebApp): disconnect_on_close = autoclose def get_client(self): @@ -1027,6 +1132,7 @@ def get_local_client(self): (r'/stim/(.*)', StimHandler), (r'/mixer.html', MixerHandler), (r'/picker', PickerHandler), + (r'/timeseries', TimeseriesHandler), (r'/', MixerHandler), (r'/static/(.*)', StaticHandler)], port) @@ -1034,16 +1140,23 @@ def get_local_client(self): server.start() print("Started server on port %d"%server.port) url = "http://%s%s:%d/mixer.html"%(serve.hostname, domain_name, server.port) + + if display_url and not open_browser: + try: + from IPython.display import HTML, display + display(HTML('Open viewer: {0}'.format(url))) + except Exception: + print("Open viewer: %s" % url) + try: + _prepare() + except Exception: + server.stop() + raise + _ready.set() + if open_browser: webbrowser.open(url) client = server.get_client() client.server = server return client - elif display_url: - try: - from IPython.display import HTML, display - display(HTML('Open viewer: {0}'.format(url))) - except: - pass - return server diff --git a/examples/timeseries/README.txt b/examples/timeseries/README.txt new file mode 100644 index 000000000..4cbf981c6 --- /dev/null +++ b/examples/timeseries/README.txt @@ -0,0 +1,5 @@ +Timeseries Examples +------------------------------ + +Examples for the click-a-voxel timeseries panel in the WebGL viewer +(Open Controls -> movie -> timeseries). diff --git a/examples/timeseries/encoding_model_timeseries.py b/examples/timeseries/encoding_model_timeseries.py new file mode 100644 index 000000000..280ee9ddb --- /dev/null +++ b/examples/timeseries/encoding_model_timeseries.py @@ -0,0 +1,111 @@ +""" +================================================ +Inspect Encoding-Model Predictions and Timeseries +================================================ + +Load a static prediction-performance map (3D) together with the actual and +predicted BOLD responses it was computed from (both 4D) into one WebGL +viewer. Browse the R² map on the surface, open the timeseries panel +(Open Controls -> movie -> timeseries) and click a voxel: the panel plots +the actual and predicted timecourses at that location, so you can see *what* +a voxel responds to and how well the model captures it. Plain 1D arrays in +the data dict (the block design here) appear as optional reference traces. +Clicking a timepoint in the panel seeks the 4D datasets to that volume; +switch to the "actual" dataset (+ / - keys) to watch the brain follow. + +The data is synthetic and built for the example subject (S1), so the example +runs without any download. The experiment is a block design of 240 volumes: +20-volume blocks of stimulus 1, stimulus 2 and rest, repeated four times. +Six Gaussian blobs are placed at well-separated random locations inside +the cortical mask, with different tuning at a high and a low signal-to-noise level. +The "actual" response is the tuned, HRF-convolved block design plus noise; the +"predicted" response is the model's version of it, and the R² map is +computed from the two. Everything outside the blobs is noise. +""" + +import numpy as np +from scipy.stats import gamma +import cortex + +def pick_centers(candidates, n, min_dist): + """Random candidate voxels at least `min_dist` apart; + the spacing is relaxed if the candidate set cannot host n such points.""" + chosen = [] + while len(chosen) < n: + for i in np.random.permutation(len(candidates)): + c = candidates[i] + if all(np.linalg.norm(c - k) >= min_dist for k in chosen): + chosen.append(c) + if len(chosen) == n: + break + else: + min_dist *= 0.8 + return np.array(chosen) + +def blob(center, radius): + d2 = (zz - center[0]) ** 2 + (yy - center[1]) ** 2 + (xx - center[2]) ** 2 + return np.exp(-d2 / (2 * radius ** 2)) + + +np.random.seed(42) +subject, xfmname = "S1", "fullhead" +xfm = cortex.db.get_xfm(subject, xfmname) +shape = xfm.shape # (z, y, x) = (31, 100, 100) + +# --- block design: stim1 (20) - stim2 (20) - rest (20), x4 = 240 volumes +block, n_cycles = 20, 4 +n_volumes = 3 * block * n_cycles +stim1 = np.tile(np.r_[np.ones(block), np.zeros(2 * block)], n_cycles) +stim2 = np.tile(np.r_[np.zeros(block), np.ones(block), np.zeros(block)], n_cycles) + +# --- BOLD regressors: convolved with a canonical HRF (TR = 2 s) +tr = 2.0 +t_hrf = np.arange(0, 32, tr) +hrf = gamma.pdf(t_hrf, 6) - 0.35 * gamma.pdf(t_hrf, 12) +hrf /= hrf.sum() +reg1 = np.convolve(stim1, hrf)[:n_volumes] +reg2 = np.convolve(stim2, hrf)[:n_volumes] + +# --- six blobs centered on cortical voxels +mask = cortex.utils.get_cortical_mask(subject, xfmname, type="nearest") # (z, y, x) +gray = np.argwhere(mask) + +tuning = [(1.0, 0.2), # blob 1: stimulus 1 (high SNR) + (0.2, 1.0), # blob 2: stimulus 2 (high SNR) + (1.0, 1.0), # blob 3: both (high SNR) + (0.6, 0.1), # blob 4: stimulus 1 (low SNR) + (0.1, 0.6), # blob 5: stimulus 2 (low SNR) + (0.6, 0.6)] # blob 6: both (low SNR) + +n_blobs, radius = len(tuning), 6.0 + +centers = pick_centers(gray, n_blobs, min_dist=4*radius) +zz, yy, xx = np.mgrid[:shape[0], :shape[1], :shape[2]] + +# --- actual and predicted responses +predicted = np.zeros((n_volumes,) + shape) +for center, (a1, a2) in zip(centers, tuning): + predicted += blob(center, radius) * (a1 * reg1 + a2 * reg2)[:, None, None, None] +actual = predicted + 0.4 * np.random.randn(n_volumes, *shape) +predicted += 0.2 * np.random.randn(n_volumes, *shape) + +# --- prediction performance per voxel +r2 = 1 - ((actual - predicted) ** 2).sum(0) / ((actual - actual.mean(0)) ** 2).sum(0) +r2 = np.clip(r2, 0, 1) +r2[r2<=0.0] = np.nan # thresholding + +volumes = { + "R2": cortex.Volume(r2, subject, xfmname, + cmap="hot", vmin=0.0, vmax=1.0), + "actual": cortex.Volume(actual, subject, xfmname, + vmin=-1.5, vmax=1.5), + "predicted": cortex.Volume(predicted, subject, xfmname, + vmin=-1.5, vmax=1.5), + "stimulus 1": stim1, # block design, shown as reference traces + "stimulus 2": stim2, +} + +cortex.webshow(volumes) +print("blob centers (z, y, x):", centers.tolist()) +# Like the other WebGL examples, run this from ipython/jupyter or with +# `python -i` so the interpreter (and the viewer's server) stays alive. diff --git a/examples/timeseries/rgb_timeseries.py b/examples/timeseries/rgb_timeseries.py new file mode 100644 index 000000000..131884dfe --- /dev/null +++ b/examples/timeseries/rgb_timeseries.py @@ -0,0 +1,50 @@ +""" +============================= +Timeseries of RGB Vertex Data +============================= + +An RGB dataset carries three timecourses per vertex, one per color channel. +With the timeseries panel open (Open Controls -> movie -> timeseries), click +a vertex and the panel plots all three channels, each with its own checkbox +and color picker, above a strip showing the blended color over time - the +color actually painted on the brain at each volume. + +The data is synthetic, built for the example subject(S1). Three patches on +the cortical surface pulse in the red, green and blue channels at different +rates, so each patch blinks in its own color and clicking inside one shows +which channel drives it. +""" + +import numpy as np +import cortex + +np.random.seed(42) +subject = "S1" +n_volumes = 120 +t = np.arange(n_volumes) + +(lpts, _), (rpts, _) = cortex.db.get_surf(subject, "wm") +pts = np.vstack([lpts, rpts]) +n_vertices = len(pts) + +def patch(center_vertex, radius_mm=18.0): + """Gaussian surface patch (0-1) around one vertex, by 3D distance.""" + d2 = ((pts - pts[center_vertex]) ** 2).sum(1) + return np.exp(-d2 / (2 * radius_mm ** 2)).astype("float32") + +centers = np.random.choice(n_vertices, 3, replace=False) +rates = [8.0, 12.0, 4.0] # pulse periods +channels = [] +for center, period in zip(centers, rates): + pulse = 0.5 + 0.5 * np.sin(2 * np.pi * t / period) # (t,) + signal = pulse[:, None] * patch(center)[None, :] # (t, v) + noise = 0.08 * np.random.rand(n_volumes, n_vertices) + channels.append((np.clip(signal + noise, 0, 1) * 255).astype(np.uint8)) +red, green, blue = channels +alpha = np.ones((n_volumes, n_vertices), dtype="float32") + +rgb = cortex.VertexRGB(red, green, blue, subject, alpha=alpha) + +cortex.webshow({"rgb": rgb}) +# Like the other WebGL examples, run this from ipython/jupyter or with +# `python -i` so the interpreter (and the viewer's server) stays alive. From ee0a2ce3d69849a2f068c7186d4d38ff97b1cf01 Mon Sep 17 00:00:00 2001 From: Jiwoong Park Date: Thu, 1 Oct 2026 15:59:24 -0700 Subject: [PATCH 2/2] webgl: drag to scrub the timeseries plot, sync the frame slider when seeking --- cortex/webgl/resources/js/dataset.js | 21 +++++++++++-- cortex/webgl/resources/js/figure.js | 46 ++++++++++++++++++++++++---- cortex/webgl/resources/js/mriview.js | 7 +++++ 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/cortex/webgl/resources/js/dataset.js b/cortex/webgl/resources/js/dataset.js index 5d89608c8..dfed6bde8 100644 --- a/cortex/webgl/resources/js/dataset.js +++ b/cortex/webgl/resources/js/dataset.js @@ -294,6 +294,16 @@ var dataset = (function(module) { this.frame = time; var frame = ((time + this.delay) * this.rate).mod(this.frames); var fframe = Math.floor(frame); + // Vertex movies stream in as one array, frame by frame in order, so a + // seek can ask for a frame that has not arrived yet. Keep showing the + // current frame (data and nanmask together) until it lands; + // _frameArrived then re-applies it. this.frame already holds the + // requested time, so that check knows what to wait for. + if (this.vertex) { + for (var i = 0; i < this.data.length; i++) + if (this.data[i].verts && this.data[i].verts[fframe] === undefined) + return; + } this.uniforms.framemix.value = frame - fframe; for (var i = 0; i < this.data.length; i++) { this.data[i].set(this.uniforms, i, fframe, this._dispatch); @@ -623,7 +633,9 @@ var dataset = (function(module) { lattr.needsUpdate = true; rattr.needsUpdate = true; this.verts.push([lattr, rattr]); - this.loaded.notify(available); + // (count, index of the frame that just arrived), as for + // volumes, so a frame held by a seek refreshes on arrival + this.loaded.notify(available, available - 1); }.bind(this)); }.bind(this)).done(function(){ this.loaded.resolve(); @@ -638,8 +650,13 @@ var dataset = (function(module) { } module.VertexData.prototype.set = function(uniforms, dim, fframe, dispatch) { var name = dim == 0 ? "data0":"data2"; + // The next frame may not have streamed in yet: blend with the current + // one rather than wrapping (by the loaded count) round to frame 0. + var next = this.verts[(fframe+1).mod(this.frames)]; + if (next === undefined) + next = this.verts[fframe]; dispatch({type:"attribute", name:"data"+(2*dim), value:this.verts[fframe]}); - dispatch({type:"attribute", name:"data"+(2*dim+1), value:this.verts[(fframe+1).mod(this.verts.length)]}); + dispatch({type:"attribute", name:"data"+(2*dim+1), value:next}); // The combined nanmask is dispatched by DataView.setFrame after // every dim's data has been set, so we don't dispatch it here. } diff --git a/cortex/webgl/resources/js/figure.js b/cortex/webgl/resources/js/figure.js index 5ec673631..97c1f2e3e 100644 --- a/cortex/webgl/resources/js/figure.js +++ b/cortex/webgl/resources/js/figure.js @@ -375,16 +375,50 @@ var jsplot = (function (module) { this.frame = null; // playhead position, in data frames this._xmap = null; // plot x-geometry of the last draw, for seeking - // click a timepoint -> seek the brain to that volume - this.canvas.addEventListener("click", function(evt) { + // click or drag along the plot -> seek the brain to that volume, + // continuously while dragging, like the movie folder's frame slider. + // Pointer capture keeps the drag going if the mouse leaves the + // panel; past either end it clamps to the first or last volume. + this._dragging = false; + this._dragFrame = null; + var seekAt = function(evt, clamp) { if (!this._xmap || !this.viewer) - return; + return false; var rect = this.canvas.getBoundingClientRect(); var fx = (evt.clientX - rect.left - this._xmap.x0) / this._xmap.w; - if (fx < 0 || fx > 1) + if (!clamp && (fx < 0 || fx > 1)) + return false; + fx = Math.min(1, Math.max(0, fx)); + var frame = Math.round(fx * (this._xmap.n - 1)); + if (frame !== this._dragFrame) { // only seek when the frame changes + this._dragFrame = frame; + this.viewer.seekFrame(frame); + } + return true; + }.bind(this); + this.canvas.addEventListener("pointerdown", function(evt) { + if (evt.button !== 0) return; - this.viewer.seekFrame(Math.round(fx * (this._xmap.n - 1))); + this._dragFrame = null; + if (seekAt(evt, false)) { + this._dragging = true; + this.canvas.setPointerCapture(evt.pointerId); + evt.preventDefault(); // no text selection while dragging + } }.bind(this)); + this.canvas.addEventListener("pointermove", function(evt) { + if (this._dragging) + seekAt(evt, true); + }.bind(this)); + var endDrag = function(evt) { + if (!this._dragging) + return; + this._dragging = false; + if (this.canvas.hasPointerCapture(evt.pointerId)) + this.canvas.releasePointerCapture(evt.pointerId); + }.bind(this); + this.canvas.addEventListener("pointerup", endDrag); + this.canvas.addEventListener("pointercancel", endDrag); // one control group per movie dataset (3D views have no // timecourse); the active one starts checked, or the first movie @@ -406,7 +440,7 @@ var jsplot = (function (module) { module.TimeseriesAxes.prototype = Object.create(module.Axes.prototype); module.TimeseriesAxes.prototype.constructor = module.TimeseriesAxes; module.TimeseriesAxes.prototype.style = { - bg: "#0D1117", text: "#E8ECF5", muted: "#9AA3B5", + bg: "#000000", text: "#E8ECF5", muted: "#9AA3B5", spine: "#3A4250", play: "#FFB454", font: "11px sans-serif", dataColors: ["#6FA8FF", "#FF6B6B", "#5DD97C", "#FFB454", "#B48EAD", "#66D9E8"], diff --git a/cortex/webgl/resources/js/mriview.js b/cortex/webgl/resources/js/mriview.js index 41822a845..e0961b725 100644 --- a/cortex/webgl/resources/js/mriview.js +++ b/cortex/webgl/resources/js/mriview.js @@ -1270,6 +1270,13 @@ var mriview = (function(module) { this.active.setFrame(t); if (this.movie) this.movie.setFrame(t); + // The movie folder keeps its frame slider in sync by wrapping + // setFrame (menu.js), and this bypasses setFrame, so update the + // slider the same way the wrapper would. + if (movie_ui && movie_ui._controls && movie_ui._controls.frame) { + movie_ui.frame = t; + movie_ui._controls.frame.updateDisplay(); + } this.schedule(); }