diff --git a/cortex/export/headless.py b/cortex/export/headless.py index 12d2182bc..6ed589e04 100644 --- a/cortex/export/headless.py +++ b/cortex/export/headless.py @@ -103,6 +103,50 @@ def _wait_for_viewer_loaded(handle, timeout: float = 60.0) -> None: ) +def _wait_for_tracts_loaded(handle, timeout: float = 60.0) -> None: + """Block until every tractogram's buffers have arrived in the browser. + + ``viewer.loaded`` deliberately does not wait for tractograms -- a slow + streamline download must not hold up a viewer that is perfectly usable + without it (mriview.js: ``addTracts``). That leaves a race for anything + that screenshots straight after loading: ``getImage``/``save_3d_views`` + would catch a scene whose streamlines are still in flight. So poll + ``viewer.tractsState()`` too, which answers "resolved" immediately when + there are no tractograms at all. + + A failed download is reported rather than waited out: the screenshot + would silently be missing streamlines. + """ + deadline = time.monotonic() + timeout + poll_interval = 0.1 + last_err: Optional[str] = None + while time.monotonic() < deadline: + try: + result = handle.send( + method="run", params=["window.viewer.tractsState", []] + ) + except Exception as exc: + last_err = repr(exc) + result = None + val = result[0] if isinstance(result, list) and result else result + if val == "resolved": + return + if val == "rejected": + raise RuntimeError( + "A tractogram's streamline buffers failed to download in the " + "headless browser; the viewer would render without them." + ) + if val is not None: + last_err = ( + str(val.get("error", val)) if isinstance(val, dict) else str(val) + ) + time.sleep(poll_interval) + raise RuntimeError( + f"Tractogram buffers did not finish loading within {timeout:.0f}s " + f"(last response: {last_err!r})." + ) + + # --------------------------------------------------------------------------- # # Helper: run Playwright in a dedicated thread to avoid asyncio conflicts # # --------------------------------------------------------------------------- # @@ -464,6 +508,10 @@ def _await_client() -> None: # calls in tests and callers, and shortens the wait when the # browser is faster than the worst-case timeout. _wait_for_viewer_loaded(handle, timeout=timeout) + # ... and until any tractogram has finished downloading: those load + # outside viewer.loaded on purpose, so without this a screenshot + # taken right away can come back without its streamlines. + _wait_for_tracts_loaded(handle, timeout=timeout) yield handle diff --git a/cortex/tests/test_webgl_headless.py b/cortex/tests/test_webgl_headless.py index 37cde0f1c..d838b9bb4 100644 --- a/cortex/tests/test_webgl_headless.py +++ b/cortex/tests/test_webgl_headless.py @@ -742,11 +742,17 @@ def _served_metadata(handle): ``show()`` regenerates the page from the same ``metadata`` dict that ``addData`` merges into, so this is how we check that a reload of the viewer would show everything that has been added so far. + + The payload is the right-hand side of ``mixer.html``'s ``metadata = ...`` + assignment, not the argument of the ``dataset.fromJSON()`` below it: that + call takes the variable, so decoding from there lands on an identifier and + raises. Line 4 of the template declares ``metadata`` without an ``=``, so + the assignment is the first ``metadata = `` in the page. """ url = "http://localhost:%d/mixer.html" % handle.server.port with urllib.request.urlopen(url, timeout=30) as resp: page = resp.read().decode("utf-8") - marker = "dataset.fromJSON(" + marker = "metadata = " start = page.index(marker) + len(marker) return json.JSONDecoder().raw_decode(page, start)[0] diff --git a/cortex/tests/test_webgl_tractogram.py b/cortex/tests/test_webgl_tractogram.py new file mode 100644 index 000000000..0f670c368 --- /dev/null +++ b/cortex/tests/test_webgl_tractogram.py @@ -0,0 +1,424 @@ +"""Tests for shipping a `cortex.Tractogram` to the WebGL viewer. + +Covers the python transport (`cortex.webgl.data.Package`, `make_static`) and, +when playwright + Chromium are available, the javascript side +(`resources/js/tractogram.js`). +""" + +import os +import re + +import numpy as np +import pytest + +import cortex +from cortex.webgl.data import Package, _tract_id + +from .testing_utils import has_playwright +from .test_tractogram import _make_tractogram, _overlapping_groups + +subj = "S1" + + +def _vertex(): + """A small Vertex dataview for S1, to accompany the tractogram.""" + pts, _ = cortex.db.get_surf(subj, "fiducial", merge=True) + return cortex.Vertex(np.zeros(pts.shape[0], dtype=np.float32), subj) + + +def _dataset(): + # Overlapping groups + an ungrouped streamline, to exercise the viewer's + # per-group visibility toggles (a streamline is visible iff it is a + # member of >=1 visible group; ungrouped streamlines follow a pseudo + # "(ungrouped)" group). + tract = _make_tractogram( + n_streamlines=8, n_points=15, groups=_overlapping_groups(8) + ) + return cortex.Dataset(overlay=_vertex(), af=tract), tract + + +# --------------------------------------------------------------------------- +# Package: wire format +# --------------------------------------------------------------------------- + + +def test_package_tract_metadata_and_buffers(): + ds, tract = _dataset() + pkg = Package(ds) + + meta = pkg.metadata() + assert "af" in meta["tracts"] + tmeta = meta["tracts"]["af"] + + assert tmeta["subject"] == subj + assert tmeta["n_points"] == tract.n_points + assert tmeta["n_streamlines"] == tract.n_streamlines + assert tmeta["alpha"] == tract.alpha + assert tmeta["linewidth"] == tract.linewidth + assert tmeta["visible"] is True + assert set(tmeta["urls"]) == {"points", "offsets", "colors", "groups"} + assert tmeta["urls"]["points"] == "/tract/af/points/" + + bufs = pkg.tracts["af"] + n, m = tract.n_points, tract.n_streamlines + n_group_entries = sum(len(idx) for idx in tract.groups.values()) + assert len(bufs["points"]) == 12 * n + assert len(bufs["offsets"]) == 4 * (m + 1) + assert len(bufs["colors"]) == 3 * n + assert len(bufs["groups"]) == 4 * n_group_entries + + # The buffers must round-trip as little-endian arrays of the right dtype. + points = np.frombuffer(bufs["points"], dtype=" fewer segments; on the + # indexed path only the index shrinks, so count segments rather + # than vertices), and showAllGroups() must restore it. JSProxy calls + # return the per-client response list, hence the [0]. + full_n_segments = handle.tracts.af.n_segments + assert full_n_segments == n - m + # Alphabetical, with the synthetic "(ungrouped)" entry last -- not + # the order the groups arrived in, which the fixture deliberately + # makes the reverse of alphabetical. + assert handle.tracts.af.groupNames()[0] == [ + "first_half", + "second_half", + "(ungrouped)", + ] + handle.tracts.af.setGroupVisible("first_half", False) + shrunk_n_segments = handle.tracts.af.n_segments + assert 0 < shrunk_n_segments < full_n_segments + + handle.tracts.af.showAllGroups() + assert handle.tracts.af.n_segments == full_n_segments + + # Translucent streamlines must keep writing depth. Without it they + # have nothing to depth-test against each other and blend in buffer + # order, so whichever bundle sits last in the geometry paints over + # the ones in front of it and the picture reorders itself the moment + # opacity leaves 1. + assert handle.tracts.af.material.depthWrite is True + handle.tracts.af.setOpacity(0.5) + assert handle.tracts.af.material.transparent is True + assert handle.tracts.af.material.depthWrite is True + # Called with no argument setOpacity is the getter; as with + # groupNames() above, a JSProxy call answers with the per-client + # response list. + assert handle.tracts.af.setOpacity()[0] == 0.5 + # Out-of-range and unparsable input is clamped / ignored rather than + # reaching the material (the panel's number box accepts typing). + handle.tracts.af.setOpacity(5) + assert handle.tracts.af.setOpacity()[0] == 1 + handle.tracts.af.setOpacity("") + assert handle.tracts.af.setOpacity()[0] == 1 + + # A tractogram belongs to one subject's scanner space, so it steps + # aside (lines and panel entry both) while another subject's dataview + # is the active one -- there is only one subject in the test + # filestore, so move the tractogram to a fictitious one instead of + # building a two-subject dataset. + handle.send( + method="set", + params=["window.viewer.tracts.af.meta.subject", "not_" + subj], + ) + handle.send(method="run", params=["window.viewer._updateTractSubjects", []]) + assert handle.tracts.af.object.visible is False + assert handle.tracts.af.setSubjectShown()[0] is False + handle.send( + method="set", params=["window.viewer.tracts.af.meta.subject", subj] + ) + handle.send(method="run", params=["window.viewer._updateTractSubjects", []]) + assert handle.tracts.af.object.visible is True + + +@pytest.mark.skipif( + not has_playwright, reason="playwright + Chromium not available" +) +def test_tractogram_index_modes_in_headless_viewer(): + """The three element-index strategies, and the fallback's geometry. + + Which one a geometry gets depends on its size and on whether the GL + context has OES_element_index_uint, so a small tractogram in one browser + only ever exercises one of them. `mriview.indexModeFor` is the choice on + its own (checkable with plain arguments), and `forceIndexMode` pins it, + so the duplicated-vertex fallback can be built and compared against the + indexed geometry it replaces without uploading 65536+ points. + """ + ds, tract = _dataset() + n, m = tract.n_points, tract.n_streamlines + + def run(func, *args): + return handle.send(method="run", params=[func, list(args)])[0] + + with cortex.export.headless_viewer(ds, viewer_params={}) as handle: + # Small geometries stay on uint16 whatever the extension says; above + # 65535 vertices the extension decides between a 32-bit index and + # duplicating the endpoints of every segment. + assert run("window.mriview.indexModeFor", 100, True) == "uint16" + assert run("window.mriview.indexModeFor", 100, False) == "uint16" + assert run("window.mriview.indexModeFor", 65535, False) == "uint16" + assert run("window.mriview.indexModeFor", 65536, True) == "uint32" + assert run("window.mriview.indexModeFor", 65536, False) == "duplicate" + + af = handle.tracts.af + assert af.indexType == "uint16" + indexed_segments = af.n_segments + assert indexed_segments == n - m + assert af.n_vertices == n + # Endpoints and colors of a segment, as they reach the GPU. + indexed = run("window.viewer.tracts.af.segment", 3) + + # Same geometry without an element index: the endpoints of every + # segment are duplicated instead, so the vertex count doubles per + # segment while the drawn segments -- and their positions and + # colors -- must come out identical. + handle.send( + method="set", + params=["window.viewer.tracts.af.forceIndexMode", "duplicate"], + ) + handle.tracts.af._rebuildGeometry() + assert handle.tracts.af.indexType == "duplicate" + assert handle.tracts.af.n_segments == indexed_segments + assert handle.tracts.af.n_vertices == 2 * indexed_segments + assert run("window.viewer.tracts.af.segment", 3) == indexed + + # ... and back, so the fallback is not left pinned on the viewer. + handle.send( + method="set", + params=["window.viewer.tracts.af.forceIndexMode", None], + ) + handle.tracts.af._rebuildGeometry() + assert handle.tracts.af.indexType == "uint16" + assert handle.tracts.af.n_vertices == n + + +@pytest.mark.skipif( + not has_playwright, reason="playwright + Chromium not available" +) +def test_tractogram_named_like_an_object_prototype_member(): + """A dataset key is an arbitrary string, including "constructor". + + The viewer keys its tractograms by name in a plain object, so a name that + collides with an inherited Object.prototype member used to look like a + tractogram that was already loaded -- addTracts would then "replace" it + and rmTracts would trip over the inherited value. + """ + tract = _make_tractogram(n_streamlines=4, n_points=10) + ds = cortex.Dataset(overlay=_vertex(), constructor=tract) + + with cortex.export.headless_viewer(ds, viewer_params={}) as handle: + assert handle.send( + method="run", params=["window.viewer.hasTract", ["constructor"]] + ) == [True] + assert handle.send( + method="run", params=["window.viewer.hasTract", ["toString"]] + ) == [False] + assert handle.tracts.constructor.n_points == tract.n_points diff --git a/cortex/webgl/data.py b/cortex/webgl/data.py index 0dfed7d4c..21bee2321 100644 --- a/cortex/webgl/data.py +++ b/cortex/webgl/data.py @@ -5,11 +5,33 @@ views = [ dict(name="proper name", cmap=cmap, vmin=vmin, vmax=vmax, data=["__braindata_name"]) ], data = dict(__braindata_name=dict(subject=subject, min=min, max=max)), images=(__braindata_name=["img1.png", "img2.png"]), + tracts = dict(name=dict(subject=subject, n_points=N, n_streamlines=M, ..., + urls=dict(points=url, offsets=url, colors=url, + groups=url))), ) + +Tractograms (`cortex.Tractogram`) are handled separately from the BrainData +based dataviews: they are not colormapped in the browser and carry no +volume/vertex arrays, so they never appear in ``views``/``data``/``images``. +Instead they contribute four little-endian binary buffers each -- ``points`` +(float32, N x 3), ``offsets`` (uint32, M + 1), ``colors`` (uint8, N x 3) and +``groups`` (uint32, concatenation of every group's streamline indices, in +the order they appear in ``tract_meta[name]["groups"]``, whose values are +``[start, stop]`` slice bounds into this buffer rather than counts) -- which +the viewer fetches and turns into a THREE.js line geometry +(``resources/js/tractogram.js``). + +The urls of those buffers (and the file names `make_static` writes them to) +are built from a *transport id* rather than from the tractogram's name: a +name is a dataset key, which may hold anything at all -- including a ``/`` +or a ``..`` that would otherwise escape the output directory (see +`_tract_id`). """ +import hashlib import os import json +import re from io import BytesIO import numpy as np @@ -17,14 +39,112 @@ from .. import volume +#: Characters allowed verbatim in a tractogram transport id: safe both as a +#: single url path segment and as a file name on every platform. +_TRACT_ID_UNSAFE = re.compile(r"[^A-Za-z0-9_-]") + +#: Longest slug kept before falling back to the hashed form, so that a very +#: long dataset key cannot produce an unopenable file name. +_TRACT_ID_MAXLEN = 48 + + +def _tract_id(name: str) -> str: + """A url- and filesystem-safe transport id for a tractogram name. + + Tractogram names are dataset keys, so they are arbitrary strings: + ``Dataset(**{"../escape": tract})`` is perfectly legal, and both the + ``/tract/{name}/{buf}/`` urls and the ``tracts/{name}_{buf}.bin`` files + written by `make_static` would take it literally. Ordinary names + (letters, digits, ``_`` and ``-``) are returned unchanged so the urls + and file names stay readable; anything else is slugified and + disambiguated with a digest of the original name, which keeps the + mapping deterministic and collision-free across separate `Package` + instances (`cortex.webgl.show` serves tractograms pushed later by + `addData` from the same table). + """ + slug = _TRACT_ID_UNSAFE.sub("_", name).strip("_") + if slug == name and len(slug) <= _TRACT_ID_MAXLEN: + return slug + digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:8] + slug = slug[:_TRACT_ID_MAXLEN] + return "%s-%s" % (slug, digest) if slug else "tract-%s" % digest + + # TODO: How to package multiviews? class Package(object): - """Package the data into a form usable by javascript""" + """Package the data into a form usable by javascript + + Parameters + ---------- + data : Dataset or Dataview + The data to package. + require_brains : bool + If True (the default), packaging a dataset that holds only + tractograms raises `ValueError`: the viewer cannot boot without a + Volume/Vertex dataview to build its surfaces from. Set to False when + pushing data into an already-running viewer. + """ - def __init__(self, data): + def __init__(self, data, require_brains=True): self.dataset = dataset.normalize(data) - self.uniques = list(data.uniques(collapse=True)) - self.subjects = set() + self._require_brains = require_brains + + # `normalize` returns a bare Dataview untouched (it does not wrap it + # into a Dataset), so handle both shapes here. + if isinstance(self.dataset, dataset.Dataset): + items = list(self.dataset) + else: + items = [(None, self.dataset)] + + # Tractograms are not BrainData: they are pulled out first, both + # because Dataset.uniques() would choke on them (only BrainData + # implements .uniques()) and because every existing javascript path + # expects views/data/images to hold colormapped brain data only. + self.tracts = dict() + self.tract_meta = dict() + #: tractogram name -> the url/file-name-safe id its buffers are + #: served under (see `_tract_id`). + self.tract_ids = dict() + brain_views = [] + for name, view in items: + if isinstance(view, dataset.Tractogram): + tname = name or view.description or view.name + if view.n_points > np.iinfo(np.uint32).max: + raise ValueError( + "Tractogram %r has %d points, more than the uint32 " + "offsets sent to the viewer can address; use " + "Tractogram.subsample() first." % (tname, view.n_points) + ) + group_indices, _ = view.groups_wire() + self.tracts[tname] = dict( + points=view.points.astype(" (focusable, keyboard-operable), stripped back to the + plain glyph it used to be. */ +.tract-header .tract-toggle { + cursor:pointer; + width:1.2em; + display:inline-block; + text-align:center; + user-select:none; + color:#DDD; + background:none; + border:none; + padding:0; + margin:0; + font:inherit; + line-height:1; +} +.tract-header .tract-name { + font-weight:bold; + margin-left:4px; + white-space:nowrap; + overflow:hidden; + text-overflow:ellipsis; +} +.tract-header input[type=checkbox] { + margin:0 4px 0 0; +} +.tract-body { + margin:6px 0 0 1.2em; +} +.tract-opacity-row { + display:flex; + align-items:center; + margin-bottom:4px; +} +.tract-opacity-row .tract-opacity-label { + margin-right:6px; + color:#CCC; +} +.tract-opacity-row input[type=range] { + flex:1 1 auto; + max-width:110px; +} +.tract-opacity-row input.tract-opacity-value { + flex:0 0 auto; + width:46px; + margin-left:6px; + background:rgba(0,0,0,0.35); + border:1px solid #666; + color:#EEE; + font-size:11px; + padding:1px 2px; +} +.tract-groups-actions { + margin-bottom:2px; +} +.tract-groups-actions .tract-groups-title { + color:#CCC; + margin-right:8px; +} +/* Buttons rather than links: they act on the scene, not on a url, and they + have to be reachable by keyboard. */ +.tract-groups-actions button.tract-link { + color:#9CF; + cursor:pointer; + text-decoration:underline; + margin-right:8px; + background:none; + border:none; + padding:0; + font:inherit; +} +.tract-group-row { + display:block; + cursor:pointer; + white-space:nowrap; + overflow:hidden; + text-overflow:ellipsis; +} +.tract-group-row input[type=checkbox] { + margin:0 4px 0 0; +} +.tract-group-count { + color:#AAA; + font-size:8pt; +} #helpmenu { position:absolute; z-index:8; diff --git a/cortex/webgl/resources/js/mriview.js b/cortex/webgl/resources/js/mriview.js index 790bb4135..0efe989c1 100644 --- a/cortex/webgl/resources/js/mriview.js +++ b/cortex/webgl/resources/js/mriview.js @@ -62,6 +62,13 @@ var mriview = (function(module) { //mix function to attach to surface when it's added this._mix = function(evt){ this.controls.setMix(evt.flat); + //Streamlines live in fiducial space: hide them as soon as the + //surface starts inflating or flattening. Every path that changes + //the morph (unfold slider, python _set_view, Viewer.setMix) + //ends up dispatching this event. + for (var name in this.tracts) + if (this.hasTract(name)) + this.tracts[name].setMix(evt.mix); }.bind(this); //allowTilt function to attach to surface when it's added @@ -104,6 +111,8 @@ var mriview = (function(module) { this.surfs = []; this.dataviews = {}; + //mriview.Tractogram objects, keyed by name (see addTracts) + this.tracts = {}; this.active = null; this.loaded = $.Deferred().done(function() { @@ -274,10 +283,17 @@ var mriview = (function(module) { //(see JSMixer.addData in cortex/webgl/view.py). It is recognizable //by its "images" key, and has to be turned into DataView objects //(which also registers the new BrainData in dataset.brains). - if (!(data instanceof Array) && data.images !== undefined) + var tracts; + if (!(data instanceof Array) && data.images !== undefined) { + //Streamlines travel next to the dataviews in the same package, but + //they are not dataviews (see cortex/webgl/data.py). + tracts = data.tracts; data = dataset.fromJSON(data); + } if (!(data instanceof Array)) data = [data]; + if (tracts !== undefined) + this.addTracts(tracts); var name, view, ui; @@ -303,7 +319,10 @@ var mriview = (function(module) { // this.dataui.addFolder(name, true, view.ui); } - this.setData(data[0].name); + //A python-side addData() push can carry tractograms only, in which + //case the currently displayed dataview stays put. + if (data.length > 0) + this.setData(data[0].name); }; module.Viewer.prototype.fitDataname = function() { @@ -709,6 +728,8 @@ var mriview = (function(module) { $("#dataopts").show(); } this.fitDataname(); + this._updateTractSubjects(); + this._updateTractsPanel(); this.schedule(); this.loaded.resolve(); @@ -745,6 +766,132 @@ var mriview = (function(module) { $(this).remove(); }) }; + module.Viewer.prototype.addTracts = function(meta) { + //`meta` is the `tracts` dict of the metadata package built by + //cortex/webgl/data.py: {name: {..., urls:{points, offsets, colors}}}. + if (meta === undefined || meta === null) + return; + + for (var name in meta) { + if (!Object.prototype.hasOwnProperty.call(meta, name)) + continue; + //Tractogram names are dataset keys, so they may well be + //"constructor" or "toString": every lookup in this.tracts goes + //through hasOwnProperty, or an inherited Object.prototype member + //would pass for a tractogram already on screen. + if (this.hasTract(name)) + this.rmTracts(name); + + var tract = new module.Tractogram(name, meta[name], this.renderer); + this.tracts[name] = tract; + this.root.add(tract.object); + //Tracts load asynchronously and the viewer's own `loaded` Deferred + //deliberately does not wait for them, so redraw when they land. + tract.loaded.done(function(tract) { + $(this.object).find("#tracts").append(tract.element); + //The element is built only once the buffers land, so the + //subject gate has to be reapplied to it here. + this._updateTractSubjects(); + this._updateTractsPanel(); + this.schedule(); + }.bind(this)); + //Keep a freshly added tract in step with the current morph state. + if (tract.setMix !== undefined && this.surfs.length > 0) + tract.setMix(this.setMix()); + } + this._updateTractSubjects(); + this.schedule(); + }; + + //Whether `name` is a tractogram this viewer holds. Never `in` or a plain + //`!== undefined`: see the comment in addTracts. + module.Viewer.prototype.hasTract = function(name) { + return Object.prototype.hasOwnProperty.call(this.tracts, name); + }; + + //Have every tractogram's buffers arrived? "resolved" once they all have + //(and immediately when there are none), "rejected" if any download failed, + //"pending" while any is still in flight. Polled by + //cortex/export/headless.py, which must not screenshot a half-loaded + //scene; the viewer's own `loaded` Deferred deliberately does not wait + //for tracts, so that a slow tractogram cannot hold up the viewer. + module.Viewer.prototype.tractsState = function() { + var state = "resolved"; + for (var name in this.tracts) { + if (!this.hasTract(name)) + continue; + var s = this.tracts[name].loaded.state(); + if (s === "rejected") + return "rejected"; + if (s !== "resolved") + state = "pending"; + } + return state; + }; + + //Streamlines live in one subject's scanner space, so a tractogram only + //means something while that subject's surface is the one on screen. In a + //multi-subject dataset, switching the active dataview swaps the surface: + //the tractograms of every other subject have to step aside (see + //Tractogram.setSubjectShown). + module.Viewer.prototype._updateTractSubjects = function() { + var subject = null; + if (this.active !== null && this.active !== undefined && + this.active.data !== undefined && this.active.data.length > 0) + subject = this.active.data[0].subject; + for (var name in this.tracts) { + if (!this.hasTract(name)) + continue; + var tract = this.tracts[name]; + //With no active dataview yet (tracts can arrive first) there is + //nothing to contradict, so leave them shown. + tract.setSubjectShown(subject === null || + tract.meta.subject === subject); + } + }; + + module.Viewer.prototype.rmTracts = function(name) { + if (!this.hasTract(name)) + return; + var tract = this.tracts[name]; + this.root.remove(tract.object); + if (tract.element !== null) + tract.element.remove(); + tract.dispose(); + delete this.tracts[name]; + this._updateTractsPanel(); + this.schedule(); + }; + + //Show/hide the #tracts panel depending on whether there is anything to + //show, and keep it positioned directly under the dataset box (#dataopts + //can change height -- a long description, or being hidden entirely -- + //so this is recomputed rather than fixed in CSS). + module.Viewer.prototype._updateTractsPanel = function() { + var panel = $(this.object).find("#tracts"); + if (panel.length === 0) + return; + //Only the entries actually on screen count: a panel holding nothing + //but another subject's (hidden) tractograms would be an empty box. + var hasTracts = false; + for (var name in this.tracts) + if (this.hasTract(name) && this.tracts[name].setSubjectShown()) + hasTracts = true; + panel.toggle(hasTracts); + if (!hasTracts) + return; + + var dataopts = $(this.object).find("#dataopts"); + if (dataopts.length && dataopts.is(":visible")) { + panel.css({ + left: dataopts.position().left, + top: dataopts.position().top + dataopts.outerHeight() + 10, + }); + } else { + panel.css({left: "", top: ""}); + } + }; + module.Viewer.prototype.addSurf = function(surftype, opts) { //Sets the slicing surface used to visualize the data var surf = new surftype(this.active, opts); @@ -964,6 +1111,7 @@ var mriview = (function(module) { for (var i = 0; i < this.surfs.length; i++) if (this.surfs[i].setMix !== undefined) this.surfs[i].setMix(mix); + //Tracts follow through the surfaces' "mix" event (see this._mix). } module.Viewer.prototype.pick = function(evt) { @@ -1169,7 +1317,7 @@ var mriview = (function(module) { var _bound = false; module.Viewer.prototype._bindUI = function() { $(window).scrollTop(0); - $(window).resize(function() { this.resize(); this.fitDataname(); }.bind(this)); + $(window).resize(function() { this.resize(); this.fitDataname(); this._updateTractsPanel(); }.bind(this)); this.canvas.resize(function() { this.resize(); }.bind(this)); var cam_ui = this.ui.addFolder("camera", true); @@ -1431,8 +1579,11 @@ var mriview = (function(module) { var dataset_cat = $(dataopts).find('#dataset_category'); dataset_cat.hide(); $(dataopts).find('#dataname').click(function(e) { - dataset_cat.slideToggle(); - }); + dataset_cat.slideToggle({ + step: function() { this._updateTractsPanel(); }.bind(this), + complete: function() { this._updateTractsPanel(); }.bind(this), + }); + }.bind(this)); var setdat = function(event, ui) { var names = []; diff --git a/cortex/webgl/resources/js/svgoverlay.js b/cortex/webgl/resources/js/svgoverlay.js index c9ccaca68..a9f756ca7 100644 --- a/cortex/webgl/resources/js/svgoverlay.js +++ b/cortex/webgl/resources/js/svgoverlay.js @@ -157,9 +157,22 @@ var svgoverlay = (function(module) { var clearAlpha = renderer.getClearAlpha(); var clearColor = renderer.getClearColor().clone(); renderer.setClearColor(0xffffff, 1); + //The depth shader only knows the surface's vertex attributes, so + //objects that opt out (e.g. tractography lines, see tractogram.js) + //are hidden for this pass -- rendering them with the override + //material would crash on the missing attributes. + var hidden = []; + scene.traverse(function(obj) { + if (obj.visible && obj.userData && obj.userData.skipOverrideMaterial) { + obj.visible = false; + hidden.push(obj); + } + }); scene.overrideMaterial = this.depthshade; renderer.render(scene, camera, this.depth); scene.overrideMaterial = null; + for (var i = 0; i < hidden.length; i++) + hidden[i].visible = true; renderer.setClearColor(clearColor, clearAlpha); this.labels.left.visible = true; this.labels.right.visible = true; diff --git a/cortex/webgl/resources/js/tractogram.js b/cortex/webgl/resources/js/tractogram.js new file mode 100644 index 000000000..d7995dc7a --- /dev/null +++ b/cortex/webgl/resources/js/tractogram.js @@ -0,0 +1,737 @@ +var mriview = (function(module) { + + //Name of the synthetic group covering streamlines that belong to no real + //one. Not a bundle, so it is listed after them (see sortGroupNames). + var UNGROUPED = "(ungrouped)"; + + //Source of unique DOM ids, for the aria-controls wiring in _buildElement. + module._tractElementId = 0; + + //Fetch a binary payload (one of the tractogram's buffers) as an + //ArrayBuffer. Same XMLHttpRequest style as surfload.js / CTMLoader.js. + function loadBuffer(url, callback, errback) { + var xhr = new XMLHttpRequest(); + xhr.onreadystatechange = function() { + if (xhr.readyState == 4) { + if (xhr.status == 200 || xhr.status == 206 || xhr.status == 0) { + callback(xhr.response); + } else { + console.error("mriview.Tractogram: couldn't load " + url + + " (" + xhr.status + ")"); + if (errback !== undefined) + errback(xhr.status); + } + } + }; + xhr.open("GET", url, true); + xhr.responseType = "arraybuffer"; + xhr.send(null); + } + + //Three.js r69 only enables 32-bit element indices when the + //OES_element_index_uint extension is present (see the THREE.Line branch of + //renderBufferDirect, three.js:20451, which picks UNSIGNED_INT purely from + //the array type -- an unsupported extension means a GL error rather than a + //fallback). Without it we duplicate the vertices instead, which also avoids + //needing r69's `geometry.offsets` drawcall chunking for >65535 vertices. + module.supportsUint32Index = function(renderer) { + var gl = null; + if (renderer !== undefined && renderer !== null) + gl = renderer.context; + else if (window.viewer !== undefined && window.viewer.renderer !== undefined) + gl = window.viewer.renderer.context; + + if (gl === null || gl === undefined) + return false; + try { + return !!gl.getExtension("OES_element_index_uint"); + } catch (e) { + return false; + } + }; + + //Which element-index strategy a geometry of `npts` vertices needs: + //"uint16" (indexed, 16-bit elements), "uint32" (indexed, 32-bit elements) + //or "duplicate" (no index buffer at all, the endpoints of every segment + //duplicated). Split out from `indexMode` so it can be checked directly, + //without a GL context to fake. + module.indexModeFor = function(npts, uint32ok) { + if (npts <= 65535) + return "uint16"; + return uint32ok ? "uint32" : "duplicate"; + }; + + module.indexMode = function(npts, renderer) { + return module.indexModeFor(npts, module.supportsUint32Index(renderer)); + }; + + //A bundle of streamlines, rendered as GL_LINES. + // + //`meta` is one entry of the `tracts` dict of the metadata package built by + //cortex/webgl/data.py: {subject, n_points, n_streamlines, alpha, linewidth, + //visible, color, groups: {name: [start, stop]}, description, + //urls:{points, offsets, colors, groups}}. + module.Tractogram = function(name, meta, renderer) { + this.name = name; + this.meta = meta; + this.renderer = renderer; + + this._visible = (meta.visible === undefined) ? true : !!meta.visible; + this._opacity = (meta.alpha === undefined) ? 1 : meta.alpha; + this._mix = 0; + //Whether this tractogram's subject is the one currently on screen; + //maintained by Viewer._updateTractSubjects (see mriview.js). + this._subjectShown = true; + + this.n_points = 0; + this.n_streamlines = 0; + //Number of vertices actually uploaded to the GPU: equals n_points + //for the indexed geometry, 2 * (number of segments) for the + //duplicated-vertex fallback. Plain numbers are also what the python + //JSProxy can read back (typed-array lengths are not enumerable). + this.n_vertices = 0; + this.geometry = null; + this.material = null; + this.line = null; + //"uint16", "uint32" or "duplicate": how the last built geometry + //addresses its vertices (see module.indexMode). Set by _buildLine. + this.indexType = null; + //Set to one of those three strings to override that choice, e.g. to + //force the duplicated-vertex fallback on a driver that advertises + //OES_element_index_uint but mishandles it. null follows the renderer. + this.forceIndexMode = null; + + //Per-group visibility (name -> bool), including a synthetic + //"(ungrouped)" entry when at least one streamline belongs to no + //named group. Populated once the "groups" buffer has loaded (see + //_build); empty until then, and left empty forever when the + //tractogram has no groups at all (n.b. distinct from "has groups + //but is entirely covered by them", which does not create the + //pseudo-group). this._hasGroups records whether group filtering + //applies at all, so setVisible-only tractograms skip it entirely. + this._hasGroups = false; + //Group names come from the dataset, so these maps are prototype-free: + //a bundle called "__proto__" or "constructor" must be an ordinary key + //rather than a reference to (or an overwrite of) Object.prototype. + this._groupVisible = Object.create(null); + //Uint32Array: concatenation of every real group's streamline + //indices (server "groups" buffer) followed by the synthetic + //"(ungrouped)" group's indices, if any -- see _build. + this._groupIndices = null; + //name -> [start, stop] slice into _groupIndices. + this._groupSlices = Object.create(null); + //Uint8Array of length n_streamlines, recomputed by + //_updateStreamlineVisibility whenever _groupVisible changes: 1 if + //the streamline is a member of >=1 visible group (or, with no + //groups at all, always 1). + this._streamlineVisible = null; + //Raw (unfiltered) buffers, kept around so _rebuildGeometry can + //recompute the segment index / duplicated arrays without re-fetching. + this._rawPoints = null; + this._rawOffsets = null; + this._rawColors = null; + + //The viewer's own `loaded` Deferred must NOT wait on this one: tracts + //are an overlay on top of a viewer that is usable without them. + this.loaded = $.Deferred(); + + this.object = new THREE.Group(); + this.object.name = "Tractogram:" + name; + this.object.visible = this._visible; + //Passes that render the scene with a surface-specific override + //material (the SVG label depth pass in svgoverlay.js) must skip us: + //those shaders expect surface attributes this geometry doesn't have. + this.object.userData.skipOverrideMaterial = true; + + //DOM element for this tractogram's controls, built once _build has + //run (see _buildElement); appended under #tracts by + //Viewer.addTracts. References to the individual inputs are kept so + //_syncControls can update them after a programmatic state change + //(e.g. a Python call through the JSProxy). + this.element = null; + this._visibleCheckbox = null; + this._opacitySlider = null; + this._opacityBox = null; + this._groupCheckboxes = Object.create(null); + + var buffers = {}, names = ["points", "offsets", "colors", "groups"]; + var pending = names.length; + var failed = false; + var ondone = function(bufname) { + return function(data) { + buffers[bufname] = data; + if (--pending === 0 && !failed) + this._build(buffers); + }.bind(this); + }.bind(this); + var onfail = function(status) { + if (!failed) { + failed = true; + this.loaded.reject(status); + } + }.bind(this); + + for (var i = 0; i < names.length; i++) + loadBuffer(meta.urls[names[i]], ondone(names[i]), onfail); + }; + + //Turn the four raw buffers into a THREE.Line of segments, and set up + //per-group visibility (this._groupVisible / this._groupSlices / + //this._groupIndices) plus the "groups" dat.gui sub-folder, if any. + module.Tractogram.prototype._build = function(buffers) { + var points = new Float32Array(buffers.points); + var offsets = new Uint32Array(buffers.offsets); + var rawcolors = new Uint8Array(buffers.colors); + + var npts = points.length / 3; + this.n_points = npts; + //The offsets array carries a trailing sentinel equal to npts, so there + //is one streamline per pair of consecutive entries. + var nstream = Math.max(offsets.length - 1, 0); + this.n_streamlines = nstream; + + //r69 always uploads vertex attributes as gl.FLOAT (three.js:20276 + //hardcodes it in setupVertexAttributes), so the uint8 colors must be + //expanded to normalized floats -- normalized uint8 attributes are not + //an option in this version. + var colors = new Float32Array(rawcolors.length); + for (var i = 0; i < rawcolors.length; i++) + colors[i] = rawcolors[i] / 255; + + this._rawPoints = points; + this._rawOffsets = offsets; + this._rawColors = colors; + + this._setupGroups(buffers.groups, nstream); + this._updateStreamlineVisibility(); + + var alpha = this._opacity; + this.material = new THREE.LineBasicMaterial({ + vertexColors: THREE.VertexColors, + transparent: alpha < 1, + opacity: alpha, + //Always write depth, at every opacity -- see setOpacity for why. + depthWrite: true, + //...which on its own makes a fully transparent tractogram visible + //rather than invisible: fragments that contribute no colour still + //write depth, so they hide the translucent surface behind them and + //the streamlines show up as silhouettes cut out of the brain. + //r69's basic shader runs the alphatest discard while gl_FragColor.a + //is still just `opacity` (before vertex colours are folded in), so + //a small alphaTest drops exactly the fragments that would have been + //invisible anyway, depth write and all. It is a constant, so the + //ALPHATEST define is compiled once and opacity changes stay cheap. + alphaTest: 0.01, + linewidth: (this.meta.linewidth === undefined) ? 1 : this.meta.linewidth, + }); + + this._buildLine(this._computeGeometryArrays()); + + this._buildElement(); + + this.loaded.resolve(this); + }; + + //Parse the "groups" buffer (uint32 streamline indices, the + //concatenation of every group in metadata order -- see + //Tractogram.groups_wire in cortex/dataset/tractogram.py) plus + //`meta.groups` ({name: [start, stop]}, same order) into + //this._groupSlices / this._groupIndices, and append a synthetic + //"(ungrouped)" group covering any streamline that is not a member of a + //real group. Initializes every group (including the pseudo one) to + //visible. A tractogram with no groups at all leaves this._hasGroups + //false and group filtering out of the picture entirely. + module.Tractogram.prototype._setupGroups = function(groupsBuffer, nstream) { + var groupIndices = new Uint32Array(groupsBuffer); + //Prototype-free: group names are dataset-supplied, so "__proto__" has + //to land as an ordinary key rather than reset the prototype. + var groupSlices = Object.create(null); + //Object key order for non-integer-like string keys follows + //insertion order in all JS engines we target, and JSON.parse + //preserves the order the metadata was serialized in -- but a group + //named e.g. "2" would be reordered numerically by the JS engine; + //not handled here (group names are expected to be bundle names). + var metagroups = this.meta.groups || {}; + for (var gname in metagroups) + if (Object.prototype.hasOwnProperty.call(metagroups, gname)) + groupSlices[gname] = metagroups[gname]; + var hasGroups = Object.keys(groupSlices).length > 0; + + if (hasGroups) { + var inGroup = new Uint8Array(nstream); + for (var gi = 0; gi < groupIndices.length; gi++) + inGroup[groupIndices[gi]] = 1; + var ungrouped = []; + for (var s = 0; s < nstream; s++) + if (!inGroup[s]) + ungrouped.push(s); + if (ungrouped.length > 0) { + var combined = new Uint32Array(groupIndices.length + ungrouped.length); + combined.set(groupIndices, 0); + combined.set(ungrouped, groupIndices.length); + groupSlices[UNGROUPED] = [groupIndices.length, combined.length]; + groupIndices = combined; + } + } + + this._hasGroups = hasGroups; + this._groupIndices = groupIndices; + this._groupSlices = groupSlices; + this._groupVisible = Object.create(null); + for (var name in groupSlices) + this._groupVisible[name] = true; + }; + + //Recompute this._streamlineVisible (Uint8Array, one entry per + //streamline) from this._groupVisible. A streamline is visible iff it is + //a member of >=1 visible group (ungrouped streamlines follow the + //"(ungrouped)" pseudo-group); with no groups at all, every streamline + //is visible. + module.Tractogram.prototype._updateStreamlineVisibility = function() { + var nstream = this.n_streamlines; + var vis = new Uint8Array(nstream); + if (!this._hasGroups) { + vis.fill(1); + } else { + for (var name in this._groupSlices) { + if (!this._groupVisible[name]) + continue; + var se = this._groupSlices[name]; + for (var k = se[0]; k < se[1]; k++) + vis[this._groupIndices[k]] = 1; + } + } + this._streamlineVisible = vis; + }; + + //Build the position/color/index arrays for the currently-visible + //streamlines only, in the shape _buildLine expects. Shared by the first + //_build and every subsequent _rebuildGeometry. + module.Tractogram.prototype._computeGeometryArrays = function() { + var points = this._rawPoints, offsets = this._rawOffsets, colors = this._rawColors; + var nstream = this.n_streamlines; + var visible = this._streamlineVisible; + var npts = points.length / 3; + + //Number of segments among visible streamlines: every streamline of + //L points yields L-1 segments. + var nseg = 0; + for (var s = 0; s < nstream; s++) { + if (visible[s]) + nseg += Math.max(offsets[s+1] - offsets[s] - 1, 0); + } + + var mode = this.forceIndexMode || module.indexMode(npts, this.renderer); + if (mode !== "duplicate") { + //Uint16 is enough (and universally supported) for small tractograms. + var index = (mode === "uint16") ? new Uint16Array(2 * nseg) : new Uint32Array(2 * nseg); + var k = 0; + for (var s = 0; s < nstream; s++) { + if (!visible[s]) + continue; + for (var j = offsets[s]; j + 1 < offsets[s+1]; j++) { + index[k++] = j; + index[k++] = j + 1; + } + } + //Position/color stay the full (unfiltered) buffers -- only the + //index changes with visibility. + return {mode: mode, indexed: true, position: points, color: colors, index: index}; + } else { + //Fallback: duplicate the endpoints of every segment so no element + //index buffer is needed at all. + var pos = new Float32Array(6 * nseg); + var col = new Float32Array(6 * nseg); + var k = 0; + for (var s = 0; s < nstream; s++) { + if (!visible[s]) + continue; + for (var j = offsets[s]; j + 1 < offsets[s+1]; j++) { + for (var d = 0; d < 3; d++) { + pos[6*k + d] = points[3*j + d]; + pos[6*k + 3 + d] = points[3*(j+1) + d]; + col[6*k + d] = colors[3*j + d]; + col[6*k + 3 + d] = colors[3*(j+1) + d]; + } + k++; + } + } + return {mode: mode, indexed: false, position: pos, color: col}; + } + }; + + //Replace this.geometry/this.line with a fresh geometry built from + //`arrays` (as returned by _computeGeometryArrays), disposing the old + //ones. this.material is reused across rebuilds (only the geometry + //changes), so opacity/depthWrite state carries over unchanged. + module.Tractogram.prototype._buildLine = function(arrays) { + var geometry = new THREE.BufferGeometry(); + geometry.addAttribute("position", new THREE.BufferAttribute(arrays.position, 3)); + geometry.addAttribute("color", new THREE.BufferAttribute(arrays.color, 3)); + if (arrays.indexed) + geometry.addAttribute("index", new THREE.BufferAttribute(arrays.index, 1)); + geometry.computeBoundingSphere(); + + if (this.line !== null) + this.object.remove(this.line); + if (this.geometry !== null) + this.geometry.dispose(); + + this.geometry = geometry; + this.indexType = arrays.mode; + this.line = new THREE.Line(geometry, this.material, THREE.LinePieces); + this.line.name = "Tractogram:" + this.name + ":lines"; + this.n_vertices = geometry.attributes.position.array.length / 3; + //Segments actually drawn: shrinks when groups are hidden on both + //the indexed path (index halves) and the fallback path (vertices + //are duplicated per segment). n_vertices only shrinks on the latter. + this.n_segments = arrays.indexed + ? arrays.index.length / 2 + : this.n_vertices / 2; + this._updateRenderOrder(); + this.object.add(this.line); + }; + + //Positions and colors of drawn segment `i`, as they reach the GPU: + //{position:[x0,y0,z0,x1,y1,z1], color:[r0,g0,b0,r1,g1,b1]} with colors in + //0-1. Both geometry paths answer identically, which is what makes the + //duplicated-vertex fallback checkable against the indexed geometry (it is + //the only way to see the difference from outside, since the two draw the + //same picture). Returns null for an out-of-range segment. + module.Tractogram.prototype.segment = function(i) { + if (this.geometry === null || i < 0 || i >= this.n_segments) + return null; + var pos = this.geometry.attributes.position.array; + var col = this.geometry.attributes.color.array; + var index = this.geometry.attributes.index; + var a, b; + if (index !== undefined) { + a = index.array[2*i]; + b = index.array[2*i + 1]; + } else { + a = 2*i; + b = 2*i + 1; + } + var out = {position: [], color: []}, d; + for (d = 0; d < 3; d++) { + out.position.push(pos[3*a + d]); + out.color.push(col[3*a + d]); + } + for (d = 0; d < 3; d++) { + out.position.push(pos[3*b + d]); + out.color.push(col[3*b + d]); + } + return out; + }; + + //Rebuild the rendered geometry from the current this._streamlineVisible + //(called after any change to group visibility) and ask the viewer to + //redraw. A no-op before the buffers have loaded. + module.Tractogram.prototype._rebuildGeometry = function() { + if (this._rawPoints === null) + return; + this._buildLine(this._computeGeometryArrays()); + this._syncControls(); + if (window.viewer !== undefined && window.viewer.schedule !== undefined) + window.viewer.schedule(); + }; + + //Build this.element: the DOM controls for this tractogram, appended + //under #tracts by Viewer.addTracts once this.loaded resolves. Header + //row (visibility checkbox, name, collapse toggle) always present; body + //(opacity slider, and a per-group "bundles" section when the + //tractogram has groups) can be collapsed. Collapsed by default when + //there are more than 8 groups, to keep the panel from taking over the + //screen for tractograms with many bundles. + module.Tractogram.prototype._buildElement = function() { + var names = this.groupNames(); + var collapsed = this._hasGroups && names.length > 8; + //Every control carries its own accessible name: the panel is built + //from bare inputs, so nothing else would announce what they act on. + //The collapse control is a real ") + .text(collapsed ? "▸" : "▾") + .attr({"aria-expanded": collapsed ? "false" : "true", + "aria-controls": bodyId, + "aria-label": "settings for " + this.name, + title: "show/hide the settings for " + this.name}); + var visibleCheckbox = $("") + .attr({"aria-label": "show " + this.name, + title: "show/hide " + this.name}); + var nameLabel = $("").text(this.name); + header.append(visibleCheckbox, toggle, nameLabel); + + var body = $("
").attr("id", bodyId); + if (collapsed) + body.hide(); + + var opacityRow = $("
"); + var opacitySlider = $("") + .attr({"aria-label": "opacity of " + this.name, + title: "opacity of " + this.name}); + var opacityBox = $("") + .attr({"aria-label": "opacity of " + this.name + ", as a number", + title: "opacity of " + this.name}); + opacityRow.append($("opacity"), + opacitySlider, opacityBox); + body.append(opacityRow); + + this._groupCheckboxes = Object.create(null); + if (this._hasGroups) { + var groupsSection = $("
") + .attr({role: "group", "aria-label": "bundles of " + this.name}); + var actions = $("
"); + actions.append($("bundles")); + var allLink = $("") + .attr("aria-label", "show every bundle of " + this.name); + var noneLink = $("") + .attr("aria-label", "hide every bundle of " + this.name); + actions.append(allLink, noneLink); + groupsSection.append(actions); + + for (var i = 0; i < names.length; i++) { + (function(tract, gname) { + var slice = tract._groupSlices[gname]; + var count = slice[1] - slice[0]; + //A