Stream missing annexed content with remfile, cached by annex key

.github/workflows/run-tests.yml

@@ -73,6 +73,9 @@ jobs:
           - os: ubuntu-latest
             python: '3.11'
             mode: lowest-deps
+          - os: ubuntu-latest
+            python: '3.12'
+            mode: datalad-fuse
 
     steps:
     - name: Set up environment
@@ -123,6 +126,10 @@ jobs:
             git+https://github.com/hdmf-dev/hdmf \
             git+https://github.com/hdmf-dev/hdmf-zarr
 
+    - name: Install git-annex and datalad-fuse (for streaming annexed content)
+      if: matrix.mode == 'datalad-fuse'
+      run: pip install git-annex ".[datalad]"
+
     - name: Create NFS filesystem
       if: matrix.mode == 'nfs'
       run: |

dandi/metadata/nwb.py

@@ -22,13 +22,14 @@
     metadata_cache,
     nwb_has_external_links,
 )
+from ..support.annex_stream import annex_fingerprint
 from ..utils import find_parent_directory_containing
 
 lgr = get_logger()
 
 
 # Disable this for clean hacking
-@metadata_cache.memoize_path
+@metadata_cache.memoize_path(custom_fingerprint=annex_fingerprint)
 def get_metadata(
     path: str | Path | Readable, digest: Digest | None = None
 ) -> dict[str, Any]:

dandi/misctypes.py

@@ -11,9 +11,11 @@
 from dataclasses import dataclass
 from datetime import datetime
 from fnmatch import fnmatchcase
+import gc
 import os.path
 from pathlib import Path
-from typing import IO, TypeVar, cast
+import threading
+from typing import IO, Any, TypeVar, cast
 
 from dandischema.models import DigestType
 
@@ -355,15 +357,22 @@ def open(self) -> IO[bytes]:
         # (or minio in tests) causes fsspec's background IO thread to block
         # forever, which in turn blocks the calling thread in
         # threading.Event.wait() — see https://github.com/fsspec/filesystem_spec/issues/1666
+        #
+        # Automatic garbage collection is disabled while the file is open, as
+        # it could otherwise deadlock fsspec's IO thread (see `_GCPausedFile`).
         return cast(
             IO[bytes],
-            fsspec.open(
-                self.url,
-                mode="rb",
-                client_kwargs={
-                    "timeout": ClientTimeout(total=120, sock_read=60, sock_connect=30)
-                },
-            ).open(),
+            _GCPausedFile(
+                fsspec.open(
+                    self.url,
+                    mode="rb",
+                    client_kwargs={
+                        "timeout": ClientTimeout(
+                            total=120, sock_read=60, sock_connect=30
+                        )
+                    },
+                ).open()
+            ),
         )
 
     def get_size(self) -> int:
@@ -377,3 +386,62 @@ def get_filename(self) -> str:
 
     def __str__(self) -> str:
         return self.url
+
+
+#: Protects `_gc_pauses` and `_gc_was_enabled`
+_gc_lock = threading.Lock()
+#: The number of open `_GCPausedFile` instances
+_gc_pauses = 0
+#: Whether automatic garbage collection was enabled before the first open
+#: `_GCPausedFile` disabled it
+_gc_was_enabled = False
+
+
+class _GCPausedFile:
+    """
+    A file object during whose lifetime automatic garbage collection is
+    disabled
+
+    h5py holds a global lock while it reads from a Python file object, and the
+    read of a `RemoteReadableAsset` waits for fsspec's IO thread to fetch the
+    data.  If a garbage collection ran in that thread meanwhile, the finalizer
+    of an unreferenced object closing an HDF5 file (e.g., hdmf's
+    ``HDMFIO.__del__``) would wait for the same lock, and neither thread could
+    proceed (<https://github.com/dandi/dandi-cli/issues/1450>).
+    """
+
+    def __init__(self, fp: IO[bytes]) -> None:
+        global _gc_pauses, _gc_was_enabled
+        self._fp = fp
+        self._paused = False
+        with _gc_lock:
+            if _gc_pauses == 0:
+                _gc_was_enabled = gc.isenabled()
+                gc.disable()
+            _gc_pauses += 1
+            self._paused = True
+
+    def __getattr__(self, name: str) -> Any:
+        return getattr(self._fp, name)
+
+    def __enter__(self) -> _GCPausedFile:
+        return self
+
+    def __exit__(self, *_exc: object) -> None:
+        self.close()
+
+    def __del__(self) -> None:
+        if self._paused:
+            self.close()
+
+    def close(self) -> None:
+        global _gc_pauses
+        try:
+            self._fp.close()
+        finally:
+            with _gc_lock:
+                if self._paused:
+                    self._paused = False
+                    _gc_pauses -= 1
+                    if _gc_pauses == 0 and _gc_was_enabled:
+                        gc.enable()

dandi/pynwb_utils.py

@@ -42,6 +42,7 @@
     metadata_nwb_subject_fields,
 )
 from .misctypes import Readable
+from .support.annex_stream import annex_fingerprint
 from .utils import get_module_version, is_url
 from .validate._types import (
     Origin,
@@ -194,7 +195,7 @@ def get_neurodata_types_to_modalities_map() -> dict[str, str]:
     return ndtypes
 
 
-@metadata_cache.memoize_path
+@metadata_cache.memoize_path(custom_fingerprint=annex_fingerprint)
 def get_neurodata_types(filepath: str | Path | Readable) -> list[str]:
     with open_readable(filepath) as fp, h5py.File(fp, "r") as h5file:
         all_pairs = _scan_neurodata_types(h5file)
@@ -808,7 +809,7 @@ def copy_nwb_file(src: str | Path, dest: str | Path) -> str:
     return str(dest)
 
 
-@metadata_cache.memoize_path
+@metadata_cache.memoize_path(custom_fingerprint=annex_fingerprint)
 def nwb_has_external_links(filepath: str | Path | Readable) -> bool:
     with open_readable(filepath) as f, h5py.File(f, "r") as fp:
         visited = set()