diff --git a/docs/list.md b/docs/list.md index 68cfaf3..11af455 100644 --- a/docs/list.md +++ b/docs/list.md @@ -12,6 +12,12 @@ Options: -q, --quiet Only errors shown in logs. --write Write the events to file. --json Output as JSON. + --match TEXT Comma-separated, case-insensitive terms a larger dataset must + all contain. + --expand Open each matched larger dataset one level and list its + children. + --recursive Open each matched larger dataset through all descendant + levels. --help Show this message and exit. ``` @@ -102,6 +108,104 @@ Within Datatrail, there are two types of datasets: Please see the CLI reference page for more information on the `list` command: [datatrail list](../cli/#datatrail-list) +## Finding datasets with `--match` and `--expand` + +Navigating the hierarchy one name at a time gets slow when you do not know +where a dataset lives. `--match` filters the larger datasets of a scope, or +of **every** scope when no scope is given, by one or more comma-separated, +case-insensitive terms, which must all appear in the combined +`scope dataset` text: + +```shell +$> datatrail ls chime.acquisition.processed --match gains + Datatrail: Dataset Map ++-----------------------------+---------------+ +| Scope | Dataset | ++-----------------------------+---------------+ +| chime.acquisition.processed | complex_gains | ++-----------------------------+---------------+ +``` + +A hit may be a container whose children are the datasets you actually want. +`--expand` opens each matched larger dataset one level and lists the children +it finds, recording the parent; a match whose children cannot be listed keeps +its own row: + +```shell +$> datatrail ls --match gain --expand + Datatrail: Dataset Map ++-----------------------------+---------------+---------------+ +| Scope | Dataset | Parent | ++-----------------------------+---------------+---------------+ +| chime.acquisition.processed | complex_gains | | ++-----------------------------+---------------+---------------+ +| gbo.acquisition.processed | 20230716 | complex_gains | +| gbo.acquisition.processed | 20230715 | complex_gains | +| ... | ... | ... | ++-----------------------------+---------------+---------------+ +``` + +Rows reached through a parent resolve directly with +`datatrail ps `; a row kept for a matched dataset whose +children could not be listed (or that has none) may still be a container. + +!!! warning "Incomplete maps" + + If Datatrail does not answer for a scope or dataset during the walk, the + map is reported as **incomplete** and the unanswered queries are listed, + rather than silently showing them as empty. With `--json`, those queries + appear in the `failed` list. A partial map still exits 0; a map with **no** + rows and unanswered queries exits 1, since nothing was determined. + +`--recursive` follows every descendant of each matched larger dataset instead +of stopping after one level. It emits terminal datasets and records the full +path used to reach each one: + +```shell +$> datatrail ls gbo.acquisition.processed --match gains --recursive +``` + +The walk visits each dataset once, so shared descendants are not duplicated +and hierarchy cycles cannot loop forever. The first path found in sorted order +is retained. An answered empty child list is a terminal dataset. A branch that +does not answer is retained as a partial row and also listed under `failed`. +Like `--expand`, a recursive walk across all scopes requires `--match` to keep +the request bounded. + +With `--json`, the map is emitted as structured rows for scripting; `parent` +is `null` for rows that were not reached through expansion. Recursive rows +also include `path`, from the matched larger dataset through the terminal row: + +```bash +$ datatrail ls --match gain --expand --json +{ + "results": [ + { + "scope": "gbo.acquisition.processed", + "dataset": "20230525", + "parent": "complex_gains" + }, + ... + ], + "failed": [] +} +``` + +```bash +$ datatrail ls gbo.acquisition.processed --match gains --recursive --json +{ + "results": [ + { + "scope": "gbo.acquisition.processed", + "dataset": "20230525", + "parent": "complex_gains", + "path": ["complex_gains", "20230525"] + } + ], + "failed": [] +} +``` + ## 🤖 Machine-readable JSON output The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines: diff --git a/dtcli/ls.py b/dtcli/ls.py index d6335da..babd1e1 100644 --- a/dtcli/ls.py +++ b/dtcli/ls.py @@ -2,7 +2,7 @@ import json import logging -from typing import Optional +from typing import Any, Dict, List, Optional import click from requests.exceptions import ConnectionError @@ -35,6 +35,22 @@ @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.") @click.option("--write", is_flag=True, help="Write the events to file.") @click.option("--json", "output_json", is_flag=True, help="Output as JSON.") +@click.option( + "--match", + type=click.STRING, + default=None, + help="Comma-separated, case-insensitive terms a larger dataset must all contain.", +) +@click.option( + "--expand", + is_flag=True, + help="Open each matched larger dataset one level and list its children.", +) +@click.option( + "--recursive", + is_flag=True, + help="Open each matched larger dataset through all descendant levels.", +) @click.pass_context def list( # noqa: C901 ctx: click.Context, @@ -44,6 +60,9 @@ def list( # noqa: C901 quiet: bool = False, write: bool = False, output_json: bool = False, + match: Optional[str] = None, + expand: bool = False, + recursive: bool = False, ): """List Datatrail Scopes & Datasets. @@ -55,6 +74,9 @@ def list( # noqa: C901 quiet (bool): Only errors shown in logs. write (bool): Write the events to file. output_json (bool): Output as JSON. + match (str): Comma-separated terms a larger dataset must all contain. + expand (bool): Open each matched larger dataset one level. + recursive (bool): Open all descendants of each matched larger dataset. """ # Set logging level. set_log_level(logger, verbose, quiet) @@ -74,6 +96,33 @@ def list( # noqa: C901 error_console.print(e) ctx.exit(1) return None + if match is not None or expand or recursive: + if datasets: + error_console.print( + "--match, --expand, and --recursive map larger datasets; " + "omit the DATASETS argument." + ) + ctx.exit(1) + return None + if (expand or recursive) and match is None and not scope: + error_console.print( + "Expansion alone would open every dataset in the archive; " + "give a SCOPE or --match to narrow it." + ) + ctx.exit(1) + return None + discovery = functions.discover_datasets( + scope=scope, + match=match, + expand=expand, + verbose=verbose, + quiet=quiet, + recursive=recursive, + ) + _display_discovery( + discovery, ctx, expand or recursive, write, output_json, scope, recursive + ) + return None results = functions.list(scope, datasets, verbose, quiet) # Output JSON if requested. @@ -143,3 +192,84 @@ def list( # noqa: C901 if "error" in results.keys(): error_console.print(results["error"]) ctx.exit(1) + + +def _display_discovery( + results: Dict[str, Any], + ctx: click.Context, + expand: bool, + write: bool, + output_json: bool, + scope: Optional[str], + recursive: bool = False, +) -> None: + """Display the dataset map built by functions.discover_datasets. + + An empty map with unanswered queries exits non-zero: nothing was + determined. A partial map is shown, with the unanswered queries listed. + + Args: + results (Dict[str, Any]): Dictionary from functions.discover_datasets. + ctx (click.Context): Click context. + expand (bool): Whether children were listed, adding a parent column. + write (bool): Write the map to file. + output_json (bool): Output as JSON. + scope (Optional[str]): Scope walked, None when all were. + recursive (bool): Whether rows include their full hierarchy path. + """ + if output_json: + print(json.dumps(results, indent=2)) + if "error" in results: + ctx.exit(1) + if not results["results"] and results["failed"]: + ctx.exit(1) + return + if "error" in results: + error_console.print(results["error"]) + ctx.exit(1) + return + rows = results["results"] + failed = results["failed"] + if write: + with open(f"./dataset_map_{scope if scope else 'all_scopes'}.json", "w") as f: + json.dump(results, f) + if rows: + with console.pager(styles=False): + console.print(_discovery_table(rows, expand, recursive)) + elif not failed: + console.print("No datasets matched.") + if failed: + error_console.print("Map is incomplete -- Datatrail did not answer for:") + for item in failed: + error_console.print(f" {item}") + if not rows: + ctx.exit(1) + + +def _discovery_table( + rows: List[Dict[str, Any]], expand: bool, recursive: bool = False +) -> Table: + """Build the dataset map table.""" + table = Table( + title="Datatrail: Dataset Map", + header_style="magenta", + title_style="bold magenta", + ) + table.add_column("Scope") + table.add_column("Dataset") + if expand: + table.add_column("Parent") + if recursive: + table.add_column("Path") + previous = None + for row in rows: + if previous is not None and row["scope"] != previous: + table.add_section() + previous = row["scope"] + line = [row["scope"], row["dataset"]] + if expand: + line.append(row["parent"] if row["parent"] else "") + if recursive: + line.append(" / ".join(row["path"])) + table.add_row(*line) + return table diff --git a/dtcli/src/functions.py b/dtcli/src/functions.py index d342e82..815571e 100644 --- a/dtcli/src/functions.py +++ b/dtcli/src/functions.py @@ -9,7 +9,7 @@ from collections import Counter from collections.abc import Sequence from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple import requests @@ -116,6 +116,164 @@ def list( # noqa: C901 return {} +def discover_datasets( + scope: Optional[str] = None, + match: Optional[str] = None, + expand: bool = False, + verbose: int = 0, + quiet: bool = False, + recursive: bool = False, +) -> Dict[str, Any]: + """Map larger datasets across scopes, with filtering and expansion. + + Walks one scope, or every scope when none is given, and keeps the larger + datasets whose "scope dataset" text contains every comma-separated, + case-insensitive match term. With expand, each kept dataset is opened one + level and its children become the rows, recording the opened dataset as + their parent. With recursive, each kept dataset is opened until terminal + datasets are reached. A dataset whose children cannot be listed keeps its + own row. A scope or dataset Datatrail does not answer for is reported in + 'failed' rather than shown as empty. + + Args: + scope (Optional[str], optional): Scope to walk. Defaults to None, + which walks every scope. + match (Optional[str], optional): Comma-separated terms a dataset must + all contain. Defaults to None. + expand (bool, optional): Open each kept dataset one level. Defaults + to False. + verbose (int, optional): Verbosity. Defaults to 0. + quiet (bool, optional): Minimal logging. Defaults to False. + recursive (bool, optional): Open all descendants of each kept dataset. + Defaults to False. + + Returns: + Dict[str, Any]: Keys 'results', rows of scope, dataset and parent, + plus path for recursive rows, and 'failed', the branches Datatrail + did not answer. Key 'error' on a configuration or connection + failure. + """ + # Set logging level. + utilities.set_log_level(logger, verbose, quiet) + terms = [t.strip().lower() for t in (match or "").split(",") if t.strip()] + if scope: + scopes = [scope] + else: + found = list(verbose=verbose, quiet=quiet) + if "error" in found: + return found + answer = found.get("scopes") + # A non-200 response body is passed through as a string; never walk + # it, or any other non-list shape, as if it were the scopes list. + # NB: isinstance against the builtin list is unavailable here, since + # this module's list() shadows it. + if isinstance(answer, str) or not isinstance(answer, Sequence): + return {"error": "Datatrail did not answer the scopes query."} + if not answer: + return { + "error": "Datatrail reports zero scopes: an account or " + "configuration problem, not an empty archive." + } + scopes = sorted(answer) + results: List[Dict[str, Optional[str]]] = [] + failed: List[str] = [] + for s in scopes: + listed = list(s, verbose=verbose, quiet=quiet) + datasets = None if "error" in listed else listed.get("larger_datasets") + if datasets is None: + failed.append(f"datasets in {s}") + continue + kept = [ + d for d in sorted(datasets) if all(t in f"{s} {d}".lower() for t in terms) + ] + if recursive: + rows, branch_failures = _discover_descendants( + s, kept, verbose=verbose, quiet=quiet + ) + results.extend(rows) + failed.extend(branch_failures) + continue + for d in kept: + if not expand: + results.append({"scope": s, "dataset": d, "parent": None}) + continue + opened = list(s, d, verbose=verbose, quiet=quiet) + children = None if "error" in opened else opened.get("datasets") + if children is None: + failed.append(f"children of {s} {d}") + results.append({"scope": s, "dataset": d, "parent": None}) + elif children: + for c in sorted(children, reverse=True): + results.append({"scope": s, "dataset": c, "parent": d}) + else: + results.append({"scope": s, "dataset": d, "parent": None}) + return {"results": results, "failed": failed} + + +def _discover_descendants( + scope: str, + roots: Sequence[str], + verbose: int = 0, + quiet: bool = False, +) -> Tuple[List[Dict[str, Any]], List[str]]: + """Return unique terminal datasets below the given roots.""" + results: List[Dict[str, Any]] = [] + failed: List[str] = [] + visited: Set[str] = set() + emitted: Set[str] = set() + stack: List[Tuple[str, Optional[str], Tuple[str, ...]]] = [ + (root, None, (root,)) for root in reversed(sorted(set(roots))) + ] + + def add_row(dataset: str, parent: Optional[str], path: Tuple[str, ...]) -> None: + if dataset in emitted: + return + results.append( + { + "scope": scope, + "dataset": dataset, + "parent": parent, + "path": [*path], + } + ) + emitted.add(dataset) + + while stack: + dataset, parent, path = stack.pop() + if dataset in visited: + continue + visited.add(dataset) + opened = list(scope, dataset, verbose=verbose, quiet=quiet) + children = None if "error" in opened else opened.get("datasets") + if ( + children is None + or isinstance(children, str) + or not isinstance(children, Sequence) + or any(not isinstance(child, str) or not child.strip() for child in children) + ): + failed.append(f"children of {scope} {' / '.join(path)}") + add_row(dataset, parent, path) + continue + + child_names = sorted(set(children)) + if not child_names: + add_row(dataset, parent, path) + continue + + cycle_found = False + for child in reversed(child_names): + if child in path: + failed.append(f"cycle in {scope}: {' / '.join(path + (child,))}") + cycle_found = True + continue + if child not in visited: + stack.append((child, dataset, path + (child,))) + if cycle_found: + add_row(dataset, parent, path) + + return results, failed + + def ps( scope: str, dataset: str, diff --git a/tests/test_cli.py b/tests/test_cli.py index 20983d0..120bba5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -106,6 +106,9 @@ def test_cli_list_help(runner: CliRunner) -> None: assert "--write" in result.output assert "--json" in result.output assert "Output as JSON" in result.output + assert "--match" in result.output + assert "--expand" in result.output + assert "--recursive" in result.output def test_cli_ps_help(runner: CliRunner) -> None: @@ -366,6 +369,142 @@ def test_cli_list_children(runner: CliRunner) -> None: assert "289007650" in result.output +def test_cli_list_match(runner: CliRunner) -> None: + """Test for CLI list to filter larger datasets with --match. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, ["ls", "chime.event.baseband.raw", "--match", "classified"] + ) + assert result.exit_code == 0 + assert "classified.FRB" in result.output + + +def test_cli_list_match_expand(runner: CliRunner) -> None: + """Test for CLI list to expand matched larger datasets one level. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "--match", "classified.FRB", "--expand"], + ) + assert result.exit_code == 0 + assert "289007650" in result.output + assert "classified.FRB" in result.output + + +def test_cli_list_match_no_hits(runner: CliRunner) -> None: + """Test for CLI list with --match matching nothing. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "--match", "no.such.dataset.term"], + ) + assert result.exit_code == 0 + assert "No datasets matched." in result.output + + +def test_cli_list_match_with_dataset_argument(runner: CliRunner) -> None: + """Test for CLI list rejecting --match combined with a dataset argument. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "classified.FRB", "--match", "FRB"], + ) + assert result.exit_code == 1 + + +def test_cli_list_bare_expand(runner: CliRunner) -> None: + """Test for CLI list rejecting --expand without a scope or --match. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke(datatrail, ["ls", "--expand"]) + assert result.exit_code == 1 + + +def test_cli_list_bare_recursive(runner: CliRunner) -> None: + """Test for CLI list rejecting an unconstrained recursive walk. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke(datatrail, ["ls", "--recursive"]) + assert result.exit_code == 1 + + +def test_cli_list_recursive_plain(runner: CliRunner, monkeypatch) -> None: + """Test for CLI list showing recursive paths in the table. + + Args: + runner (CliRunner): Click runner. + monkeypatch: Pytest monkeypatch fixture. + """ + + def fake_discovery(**kwargs): + assert kwargs["recursive"] is True + return { + "results": [ + { + "scope": "test.scope", + "dataset": "leaf", + "parent": "branch", + "path": ["root", "branch", "leaf"], + } + ], + "failed": [], + } + + monkeypatch.setattr("dtcli.ls.functions.discover_datasets", fake_discovery) + result = runner.invoke(datatrail, ["ls", "--match", "root", "--recursive"]) + assert result.exit_code == 0 + assert "leaf" in result.output + assert "root / branch / leaf" in result.output + + +def test_cli_list_recursive_json(runner: CliRunner, monkeypatch) -> None: + """Test for CLI list retaining recursive paths in JSON. + + Args: + runner (CliRunner): Click runner. + monkeypatch: Pytest monkeypatch fixture. + """ + import json + + expected = { + "results": [ + { + "scope": "test.scope", + "dataset": "leaf", + "parent": "branch", + "path": ["root", "branch", "leaf"], + } + ], + "failed": [], + } + + def fake_discovery(**kwargs): + assert kwargs["recursive"] is True + return expected + + monkeypatch.setattr("dtcli.ls.functions.discover_datasets", fake_discovery) + result = runner.invoke(datatrail, ["ls", "--match", "root", "--recursive", "--json"]) + assert result.exit_code == 0 + json_start = result.output.find("{") + assert json.loads(result.output[json_start:]) == expected + + @pytest.mark.cadc def test_cli_ps(runner: CliRunner) -> None: """Test for CLI ps command. @@ -682,6 +821,33 @@ def test_cli_list_children_json(runner: CliRunner) -> None: assert "289007650" in output_data["datasets"] +def test_cli_list_match_json(runner: CliRunner) -> None: + """Test for CLI list to output the dataset map as JSON. + + Args: + runner (CliRunner): Click runner. + """ + import json + + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "--match", "classified", "--json"], + ) + assert result.exit_code == 0 + # Extract JSON from output (skip version check message if present) + json_start = result.output.find("{") + json_output = result.output[json_start:] + # Parse the output as JSON + output_data = json.loads(json_output) + # Should have 'results' rows and a 'failed' list + assert { + "scope": "chime.event.baseband.raw", + "dataset": "classified.FRB", + "parent": None, + } in output_data["results"] + assert output_data["failed"] == [] + + @pytest.mark.cadc def test_cli_ps_json(runner: CliRunner) -> None: """Test for CLI ps command with JSON output. diff --git a/tests/test_functions.py b/tests/test_functions.py index 59155b1..d362206 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -112,3 +112,205 @@ def json() -> Dict[str, Any]: ) results: Dict[str, Any] = functions.list() assert results == {"error": "Datatrail did not answer the scopes query."} + + +def _fake_list(scope=None, dataset=None, verbose=0, quiet=False): + """Stand-in for functions.list with one scope not answering.""" + if scope is None: + return {"scopes": ["b.scope", "a.scope", "c.scope"]} + if dataset is None: + if scope == "a.scope": + return {"larger_datasets": ["data.other", "data.good", "skip.me"]} + if scope == "c.scope": + return {"larger_datasets": []} + return {"error": "Datatrail Server at CHIME is not responding."} + if dataset == "data.good": + return {"datasets": ["child1", "child2"]} + return {"error": "Datatrail Server at CHIME is not responding."} + + +def test_discover_datasets(monkeypatch) -> None: + """Test discover_datasets filtering, sorting, and expansion.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(match="data", expand=True) + assert results["results"] == [ + {"scope": "a.scope", "dataset": "child2", "parent": "data.good"}, + {"scope": "a.scope", "dataset": "child1", "parent": "data.good"}, + {"scope": "a.scope", "dataset": "data.other", "parent": None}, + ] + # An unanswered query is reported, never shown as empty. + assert results["failed"] == [ + "children of a.scope data.other", + "datasets in b.scope", + ] + + +def test_discover_datasets_no_expand(monkeypatch) -> None: + """Test discover_datasets without expansion, terms ANDed against scope.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(match="a.scope,data") + assert results["results"] == [ + {"scope": "a.scope", "dataset": "data.good", "parent": None}, + {"scope": "a.scope", "dataset": "data.other", "parent": None}, + ] + assert results["failed"] == ["datasets in b.scope"] + + +def test_discover_datasets_single_scope(monkeypatch) -> None: + """Test discover_datasets walking one named scope only.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(scope="a.scope") + assert [r["dataset"] for r in results["results"]] == [ + "data.good", + "data.other", + "skip.me", + ] + assert results["failed"] == [] + + +def test_discover_datasets_empty_scope_is_not_failure(monkeypatch) -> None: + """Test a scope that answers with no datasets is empty, not failed.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(scope="c.scope") + assert results["results"] == [] + assert results["failed"] == [] + + +def test_discover_datasets_unanswered_scopes_query(monkeypatch) -> None: + """Test a non-list scopes answer is an error, never walked as text.""" + + def bad_list(scope=None, dataset=None, verbose=0, quiet=False): + return {"scopes": "Bad Gateway"} + + monkeypatch.setattr(functions, "list", bad_list) + results: Dict[str, Any] = functions.discover_datasets(match="gain") + assert "error" in results + assert "results" not in results + + +def test_discover_datasets_recursive_paths(monkeypatch) -> None: + """Test recursive discovery paths, ordering, filtering, and duplicates.""" + calls = [] + children = { + "wanted.root": ["branch.b", "branch.a", "branch.a"], + "branch.a": ["leaf.shared", "leaf.a"], + "branch.b": ["leaf.b", "leaf.shared"], + } + + def fake_list(scope=None, dataset=None, verbose=0, quiet=False): + if dataset is None: + return { + "larger_datasets": [ + "wanted.root", + "skip.root", + "wanted.empty", + "wanted.root", + ] + } + calls.append(dataset) + return {"datasets": children.get(dataset, [])} + + monkeypatch.setattr(functions, "list", fake_list) + results = functions.discover_datasets( + scope="test.scope", match="wanted", recursive=True + ) + assert results == { + "results": [ + { + "scope": "test.scope", + "dataset": "wanted.empty", + "parent": None, + "path": ["wanted.empty"], + }, + { + "scope": "test.scope", + "dataset": "leaf.a", + "parent": "branch.a", + "path": ["wanted.root", "branch.a", "leaf.a"], + }, + { + "scope": "test.scope", + "dataset": "leaf.shared", + "parent": "branch.a", + "path": ["wanted.root", "branch.a", "leaf.shared"], + }, + { + "scope": "test.scope", + "dataset": "leaf.b", + "parent": "branch.b", + "path": ["wanted.root", "branch.b", "leaf.b"], + }, + ], + "failed": [], + } + assert "skip.root" not in calls + assert calls.count("wanted.root") == 1 + assert calls.count("leaf.shared") == 1 + + +def test_discover_datasets_recursive_empty_and_failed(monkeypatch) -> None: + """Test recursive discovery keeps empty and failed branches distinct.""" + + def fake_list(scope=None, dataset=None, verbose=0, quiet=False): + if dataset is None: + return {"larger_datasets": ["root"]} + if dataset == "root": + return {"datasets": ["offline", "malformed", "empty"]} + if dataset == "offline": + return {"error": "service unavailable"} + if dataset == "malformed": + return {"datasets": [None]} + return {"datasets": []} + + monkeypatch.setattr(functions, "list", fake_list) + results = functions.discover_datasets(scope="test.scope", recursive=True) + assert results["results"] == [ + { + "scope": "test.scope", + "dataset": "empty", + "parent": "root", + "path": ["root", "empty"], + }, + { + "scope": "test.scope", + "dataset": "malformed", + "parent": "root", + "path": ["root", "malformed"], + }, + { + "scope": "test.scope", + "dataset": "offline", + "parent": "root", + "path": ["root", "offline"], + }, + ] + assert results["failed"] == [ + "children of test.scope root / malformed", + "children of test.scope root / offline", + ] + + +def test_discover_datasets_recursive_cycle(monkeypatch) -> None: + """Test recursive discovery stops and reports a hierarchy cycle.""" + calls = [] + + def fake_list(scope=None, dataset=None, verbose=0, quiet=False): + if dataset is None: + return {"larger_datasets": ["root"]} + calls.append(dataset) + return {"datasets": ["branch"] if dataset == "root" else ["root"]} + + monkeypatch.setattr(functions, "list", fake_list) + results = functions.discover_datasets(scope="test.scope", recursive=True) + assert results["results"] == [ + { + "scope": "test.scope", + "dataset": "branch", + "parent": "root", + "path": ["root", "branch"], + } + ] + assert results["failed"] == [ + "cycle in test.scope: root / branch / root", + ] + assert calls == ["root", "branch"]