[dev] Use datalad-fuse's remfile backend and all other dev branches

.github/workflows/run-tests.yml

@@ -76,6 +76,9 @@ jobs:
           - os: ubuntu-latest
             python: '3.12'
             mode: datalad
+          - os: ubuntu-latest
+            python: '3.12'
+            mode: datalad-dev
 
     steps:
     - name: Set up environment
@@ -130,6 +133,13 @@ jobs:
       if: matrix.mode == 'datalad'
       run: pip install git-annex ".[datalad]"
 
+    # TODO: drop once datalad/datalad-fuse#131 is released
+    - name: Install git-annex, remfile and development datalad-fuse
+      if: matrix.mode == 'datalad-dev'
+      run: |
+        pip install git-annex remfile ".[datalad]" \
+            "datalad-fuse @ git+https://github.com/datalad/datalad-fuse@refs/pull/131/head"
+
     - name: Create NFS filesystem
       if: matrix.mode == 'nfs'
       run: |

dandi/support/datalad_fuse.py

@@ -28,10 +28,12 @@
 from __future__ import annotations
 
 import atexit
+from collections.abc import Iterator
 from dataclasses import dataclass, field
 from datetime import datetime
 from functools import cache
 import gc
+from itertools import chain
 import logging
 import os
 from pathlib import Path
@@ -51,11 +53,17 @@ def get_adapter() -> Any:
     root so that it serves files of any dataset, and closed at exit
     """
     # Optional dependency:
-    from datalad_fuse.fsspec import FsspecAdapter
+    try:
+        # datalad-fuse with pluggable backends (datalad/datalad-fuse#131):
+        # remfile for HDF5-based files (such as NWB) if installed, fsspec
+        # otherwise, as configured with datalad.fusefs.backends
+        from datalad_fuse.adapter import RemoteFilesystemAdapter as Adapter
+    except ImportError:
+        from datalad_fuse.fsspec import FsspecAdapter as Adapter
 
     # caching=False: do not keep the streamed blocks on disk, in the dataset.
     # The root and all paths passed to the adapter must be absolute.
-    adapter = FsspecAdapter(Path(os.path.abspath(os.sep)), caching=False)
+    adapter = Adapter(Path(os.path.abspath(os.sep)), caching=False)
     atexit.register(adapter.__exit__, None, None, None)
     return adapter
 
@@ -296,9 +304,12 @@ def get_annexed_readable(path: str | Path) -> AnnexedReadableFile | None:
         # Not a broken symbolic link
         return None
     try:
-        from datalad_fuse.fsspec import FileState
+        from datalad_fuse.adapter import FileState
     except ImportError:
-        return None
+        try:
+            from datalad_fuse.fsspec import FileState
+        except ImportError:
+            return None
     if not annex_initialized(filepath.parent):
         return None
     try:
@@ -311,7 +322,11 @@ def get_annexed_readable(path: str | Path) -> AnnexedReadableFile | None:
         return None
     if state is not FileState.NO_CONTENT or key is None or key.size is None:
         return None
-    url = next(dsap.get_urls(str(key)), None)
+    urls: Iterator[str] = dsap.get_urls(str(key))
+    if hasattr(dsap, "get_exporttree_urls"):
+        # Fallback to S3 exports of datalad/datalad-fuse#131
+        urls = chain(urls, dsap.get_exporttree_urls(relpath, key))
+    url = next(urls, None)
     if url is None:
         lgr.debug("%s: git-annex knows no URL for key %s", filepath, key)
         return None

dandi/support/tests/test_datalad_fuse.py

@@ -115,8 +115,15 @@ def test_annexed_readable_reads_little(
     r = get_annexed_readable(tmp_path / "ds" / "sub-01" / "sub-01.nwb")
     assert r is not None
     assert get_neurodata_types.__wrapped__(r) == get_neurodata_types.__wrapped__(local)
-    # With fsspec's default 5 MiB read-ahead, most of the file was downloaded
-    assert 0 < server.bytes_served < size / 10
+    assert server.bytes_served > 0
+    with r.open() as fp:
+        remfile_used = type(fp._fp).__name__ == "RemfileWrapper"  # type: ignore[attr-defined]
+    if not remfile_used:
+        # With fsspec's default 5 MiB read-ahead, most of the file was
+        # downloaded.  (remfile, used for HDF5 files by
+        # datalad/datalad-fuse#131 when installed, reads ahead by itself, in
+        # chunks growing up to 100 MiB that datalad-fuse does not configure.)
+        assert server.bytes_served < size / 10
 
 
 @pytest.mark.ai_generated
@@ -214,3 +221,29 @@ def test_annexed_readable_cached_by_key(
     assert neurodata_types(get_annexed_readable(nwb)) == expected
     assert neurodata_types(nwb) == expected
     assert server.ranged_requests == requests
+
+
+@pytest.mark.ai_generated
+def test_annexed_readable_remfile(
+    streamed_nwb: tuple[Path, RangeHTTPServer], simple2_nwb: Path
+) -> None:
+    """With datalad/datalad-fuse#131 and remfile, NWB files are read with remfile"""
+    pytest.importorskip("datalad_fuse.adapter")
+    pytest.importorskip("remfile")
+    from datalad_fuse.remfile import RemfileWrapper
+
+    nwb, server = streamed_nwb
+    r = get_annexed_readable(nwb)
+    assert r is not None
+    fp = r.open()
+    try:
+        # (wrapped to pause garbage collection while it is open)
+        assert isinstance(fp._fp, RemfileWrapper)  # type: ignore[attr-defined]
+        with h5py.File(fp, "r") as h5:
+            assert h5.attrs["nwb_version"]
+    finally:
+        fp.close()
+    assert server.ranged_requests > 0
+    assert get_neurodata_types.__wrapped__(r) == get_neurodata_types.__wrapped__(
+        simple2_nwb
+    )

docs/source/cmdline/validate.rst

@@ -122,6 +122,11 @@ Notes:
 
 - git-annex must be initialized in the clone, which ``datalad clone`` does
   (after ``git clone``, run ``git annex init``).
+- With the development version of datalad-fuse from
+  https://github.com/datalad/datalad-fuse/pull/131 and remfile_ installed
+  (``pip install remfile``), NWB and other HDF5-based files are read with
+  remfile, which is optimized for HDF5's access pattern, and content exported
+  to S3 without a URL registered in git-annex can be found too.
 - datalad-fuse does not use the HTTP proxy environment variables
   (``HTTPS_PROXY`` etc.), so the URLs must be reachable directly.
 - Some nwbinspector checks read data arrays (e.g., timestamps), so the amount
@@ -145,6 +150,7 @@ Notes:
   the file and folder names of the annexed files themselves.
 
 .. _datalad-fuse: https://github.com/datalad/datalad-fuse
+.. _remfile: https://github.com/flatironinstitute/remfile
 
 
 Development Options