Add code duplication analysis to the dev workflow

.github/workflows/duplication.yml

@@ -0,0 +1,32 @@
+name: Duplication check
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  duplication:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.11'
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v5
+        with:
+          node-version: '24'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade tox
+
+      - name: Run duplication check
+        run: tox -e duplication

.gitignore

@@ -9,6 +9,7 @@
 .mypy_cache/
 .pytest_cache/
 .ruff_cache/
+.tmp/
 .tox/
 .venv*/
 __pycache__/

.jscpd.json

@@ -0,0 +1,24 @@
+{
+  "threshold": 2,
+  "reporters": ["console"],
+  "ignore": [
+    "**/.git/**",
+    "**/.tox/**",
+    "**/.venv*/**",
+    "**/venv*/**",
+    "**/node_modules/**",
+    "**/__pycache__/**",
+    "**/.hypothesis/**",
+    "**/*.egg-info/**",
+    "**/.eggs/**",
+    "**/.npm/**",
+    "**/.tmp/**",
+    "**/dist/**",
+    "**/build/**",
+    "**/tests/data/**",
+    "**/versioneer.py",
+    "**/dandi/_version.py"
+  ],
+  "minTokens": 50,
+  "minLines": 5
+}

dandi/bids_validator_deno/_validator.py

@@ -206,22 +206,8 @@ def _bids_validate(
     recursive: bool = False,
 ) -> BidsValidationResult:
     """
-    Validate a file directory as a BIDS dataset with the deno-compiled BIDS validator
-
-    Parameters
-    ----------
-    dir_ : DirectoryPath
-        The path to the directory to validate
-    config : Optional[dict]
-        The configuration to use in the validation. This specifies a JSON configuration
-        file to be provided through the `--config` option when invoking the underlying
-        deno-compiled BIDS validator. If `None`, the deno-compiled BIDS validator will
-        be invoked without the `--config` option.
-    ignore_nifti_headers : bool
-        If `True`, disregard NIfTI header content during validation
-    recursive : bool
-        If `True`, validate datasets found in derivatives directories in addition to
-        root dataset
+    Implementation of `bids_validate()`. See that function for a description of the
+    parameters.
 
     Returns
     -------

dandi/cli/cmd_ls.py

@@ -286,17 +286,9 @@ def flatten_v(v):
 
 
 def flatten_meta_to_pyout_v1(meta):
-    """Given a meta record, possibly flatten record since no nested records
-    supported yet
-
-    lists become joined using ', ', dicts get individual key: values.
-    lists of dict - doing nothing magical.
-
-    Empty values are not considered.
-
-    Parameters
-    ----------
-    meta: dict
+    """Like `flatten_meta_to_pyout`, but nested dicts get flattened
+    recursively into individual "key: value" entries instead of being
+    joined into a single string.
     """
     out = {}
 

dandi/cli/tests/test_cmd_validate.py

@@ -345,13 +345,7 @@ def test_validate_load_mutual_exclusivity(simple2_nwb: Path, tmp_path: Path) ->
     assert "mutually exclusive" in r.output
 
 
-@pytest.mark.ai_generated
-@pytest.mark.parametrize(
-    "grouping",
-    ["severity", "id", "validator", "standard", "dandiset"],
-)
-def test_render_text_grouping(grouping: str, capsys: pytest.CaptureFixture) -> None:
-    """Test extended grouping renders section headers with counts."""
+def _sample_validation_issues() -> tuple[Origin, list[ValidationResult]]:
     origin = Origin(
         type=OriginType.VALIDATION,
         validator=Validator.nwbinspector,
@@ -377,6 +371,17 @@ def test_render_text_grouping(grouping: str, capsys: pytest.CaptureFixture) -> N
             dandiset_path=Path("/data/ds001"),
         ),
     ]
+    return origin, issues
+
+
+@pytest.mark.ai_generated
+@pytest.mark.parametrize(
+    "grouping",
+    ["severity", "id", "validator", "standard", "dandiset"],
+)
+def test_render_text_grouping(grouping: str, capsys: pytest.CaptureFixture) -> None:
+    """Test extended grouping renders section headers with counts."""
+    _, issues = _sample_validation_issues()
     _render_text(issues, grouping=(grouping,))
     captured = capsys.readouterr().out
 
@@ -435,30 +440,8 @@ def test_validate_grouping_text_cli(
 @pytest.mark.ai_generated
 def test_render_text_multilevel_grouping(capsys: pytest.CaptureFixture) -> None:
     """Test multi-level grouping renders nested section headers."""
-    origin = Origin(
-        type=OriginType.VALIDATION,
-        validator=Validator.nwbinspector,
-        validator_version="",
-    )
-    issues = [
-        ValidationResult(
-            id="NWBI.check_data_orientation",
-            origin=origin,
-            scope=Scope.FILE,
-            message="Data may be in the wrong orientation.",
-            path=Path("sub-01/sub-01.nwb"),
-            severity=Severity.WARNING,
-            dandiset_path=Path("/data/ds001"),
-        ),
-        ValidationResult(
-            id="NWBI.check_missing_unit",
-            origin=origin,
-            scope=Scope.FILE,
-            message="Missing text for attribute 'unit'.",
-            path=Path("sub-02/sub-02.nwb"),
-            severity=Severity.WARNING,
-            dandiset_path=Path("/data/ds001"),
-        ),
+    origin, issues = _sample_validation_issues()
+    issues.append(
         ValidationResult(
             id="NWBI.check_data_orientation",
             origin=origin,
@@ -467,8 +450,8 @@ def test_render_text_multilevel_grouping(capsys: pytest.CaptureFixture) -> None:
             path=Path("sub-03/sub-03.nwb"),
             severity=Severity.ERROR,
             dandiset_path=Path("/data/ds001"),
-        ),
-    ]
+        )
+    )
     _render_text(issues, grouping=("severity", "id"))
     captured = capsys.readouterr().out
 

dandi/cli/tests/test_digest.py

@@ -57,9 +57,9 @@ def test_digest(
     assert r.output == f"file.txt: {filehash}\n"
 
 
-def test_digest_zarr(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    # Expected digest is selected by the Zarr serialisation format that
-    # ``zarr.save`` actually produced (V2 vs V3 layouts have different digests).
+def _make_sample_zarr(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> tuple[CliRunner, str]:
     runner = CliRunner()
     monkeypatch.chdir(tmp_path)
     dt = np.dtype("<i8")
@@ -69,6 +69,13 @@ def test_digest_zarr(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
     expected = _EXPECTED_SAMPLE_ZARR_DIGEST_BY_FORMAT[
         zarr_format_of(Path("sample.zarr"))
     ]
+    return runner, expected
+
+
+def test_digest_zarr(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    # Expected digest is selected by the Zarr serialisation format that
+    # ``zarr.save`` actually produced (V2 vs V3 layouts have different digests).
+    runner, expected = _make_sample_zarr(tmp_path, monkeypatch)
     r = runner.invoke(digest, ["--digest", "zarr-checksum", "sample.zarr"])
     assert r.exit_code == 0
     assert r.output == f"sample.zarr: {expected}\n"
@@ -87,15 +94,7 @@ def test_digest_zarr_with_excluded_dotfiles(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ) -> None:
     # See comment in `test_digest_zarr` regarding V2 vs V3 serialisation.
-    runner = CliRunner()
-    monkeypatch.chdir(tmp_path)
-    dt = np.dtype("<i8")
-    zarr.save(
-        "sample.zarr", np.arange(1000, dtype=dt), np.arange(1000, 0, -1, dtype=dt)
-    )
-    expected = _EXPECTED_SAMPLE_ZARR_DIGEST_BY_FORMAT[
-        zarr_format_of(Path("sample.zarr"))
-    ]
+    runner, expected = _make_sample_zarr(tmp_path, monkeypatch)
     subprocess.run(["git", "init"], cwd="sample.zarr", check=True)
     os.mkdir("sample.zarr/.dandi")
     Path("sample.zarr", ".dandi", "somefile.txt").touch()