From b4d4f7522b0e01876afcab480475fae18e7ebe14 Mon Sep 17 00:00:00 2001 From: Dylan Gormley Date: Mon, 24 Aug 2026 15:16:22 -0500 Subject: [PATCH 1/2] feat(ls): add --match and --expand for dataset discovery --- docs/list.md | 71 +++++++++++++++++++++++++++ dtcli/ls.py | 103 +++++++++++++++++++++++++++++++++++++++- dtcli/src/functions.py | 82 ++++++++++++++++++++++++++++++++ tests/test_cli.py | 94 ++++++++++++++++++++++++++++++++++++ tests/test_functions.py | 74 +++++++++++++++++++++++++++++ 5 files changed, 423 insertions(+), 1 deletion(-) diff --git a/docs/list.md b/docs/list.md index 68cfaf3..a1744ce 100644 --- a/docs/list.md +++ b/docs/list.md @@ -12,6 +12,10 @@ Options: -q, --quiet Only errors shown in logs. --write Write the events to file. --json Output as JSON. + --match TEXT Comma-separated, case-insensitive terms a larger dataset must + all contain. + --expand Open each matched larger dataset one level and list its + children. --help Show this message and exit. ``` @@ -102,6 +106,73 @@ Within Datatrail, there are two types of datasets: Please see the CLI reference page for more information on the `list` command: [datatrail list](../cli/#datatrail-list) +## Finding datasets with `--match` and `--expand` + +Navigating the hierarchy one name at a time gets slow when you do not know +where a dataset lives. `--match` filters the larger datasets of a scope, or +of **every** scope when no scope is given, by one or more comma-separated, +case-insensitive terms, which must all appear in the combined +`scope dataset` text: + +```shell +$> datatrail ls chime.acquisition.processed --match gains + Datatrail: Dataset Map ++-----------------------------+---------------+ +| Scope | Dataset | ++-----------------------------+---------------+ +| chime.acquisition.processed | complex_gains | ++-----------------------------+---------------+ +``` + +A hit may be a container whose children are the datasets you actually want. +`--expand` opens each matched larger dataset one level and lists the children +it finds, recording the parent; a match whose children cannot be listed keeps +its own row: + +```shell +$> datatrail ls --match gain --expand + Datatrail: Dataset Map ++-----------------------------+---------------+---------------+ +| Scope | Dataset | Parent | ++-----------------------------+---------------+---------------+ +| chime.acquisition.processed | complex_gains | | ++-----------------------------+---------------+---------------+ +| gbo.acquisition.processed | 20230716 | complex_gains | +| gbo.acquisition.processed | 20230715 | complex_gains | +| ... | ... | ... | ++-----------------------------+---------------+---------------+ +``` + +Rows reached through a parent resolve directly with +`datatrail ps `; a row kept for a matched dataset whose +children could not be listed (or that has none) may still be a container. + +!!! warning "Incomplete maps" + + If Datatrail does not answer for a scope or dataset during the walk, the + map is reported as **incomplete** and the unanswered queries are listed, + rather than silently showing them as empty. With `--json`, those queries + appear in the `failed` list. A partial map still exits 0; a map with **no** + rows and unanswered queries exits 1, since nothing was determined. + +With `--json`, the map is emitted as structured rows for scripting; `parent` +is `null` for rows that were not reached through `--expand`: + +```bash +$ datatrail ls --match gain --expand --json +{ + "results": [ + { + "scope": "gbo.acquisition.processed", + "dataset": "20230525", + "parent": "complex_gains" + }, + ... + ], + "failed": [] +} +``` + ## 🤖 Machine-readable JSON output The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines: diff --git a/dtcli/ls.py b/dtcli/ls.py index d6335da..e30cca7 100644 --- a/dtcli/ls.py +++ b/dtcli/ls.py @@ -2,7 +2,7 @@ import json import logging -from typing import Optional +from typing import Any, Dict, Optional import click from requests.exceptions import ConnectionError @@ -35,6 +35,17 @@ @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.") @click.option("--write", is_flag=True, help="Write the events to file.") @click.option("--json", "output_json", is_flag=True, help="Output as JSON.") +@click.option( + "--match", + type=click.STRING, + default=None, + help="Comma-separated, case-insensitive terms a larger dataset must all contain.", +) +@click.option( + "--expand", + is_flag=True, + help="Open each matched larger dataset one level and list its children.", +) @click.pass_context def list( # noqa: C901 ctx: click.Context, @@ -44,6 +55,8 @@ def list( # noqa: C901 quiet: bool = False, write: bool = False, output_json: bool = False, + match: Optional[str] = None, + expand: bool = False, ): """List Datatrail Scopes & Datasets. @@ -55,6 +68,8 @@ def list( # noqa: C901 quiet (bool): Only errors shown in logs. write (bool): Write the events to file. output_json (bool): Output as JSON. + match (str): Comma-separated terms a larger dataset must all contain. + expand (bool): Open each matched larger dataset one level. """ # Set logging level. set_log_level(logger, verbose, quiet) @@ -74,6 +89,24 @@ def list( # noqa: C901 error_console.print(e) ctx.exit(1) return None + if match is not None or expand: + if datasets: + error_console.print( + "--match and --expand map larger datasets; " + "omit the DATASETS argument." + ) + ctx.exit(1) + return None + if expand and match is None and not scope: + error_console.print( + "--expand alone would open every dataset in the archive; " + "give a SCOPE or --match to narrow it." + ) + ctx.exit(1) + return None + discovery = functions.discover_datasets(scope, match, expand, verbose, quiet) + _display_discovery(discovery, ctx, expand, write, output_json, scope) + return None results = functions.list(scope, datasets, verbose, quiet) # Output JSON if requested. @@ -143,3 +176,71 @@ def list( # noqa: C901 if "error" in results.keys(): error_console.print(results["error"]) ctx.exit(1) + + +def _display_discovery( + results: Dict[str, Any], + ctx: click.Context, + expand: bool, + write: bool, + output_json: bool, + scope: Optional[str], +) -> None: + """Display the dataset map built by functions.discover_datasets. + + An empty map with unanswered queries exits non-zero: nothing was + determined. A partial map is shown, with the unanswered queries listed. + + Args: + results (Dict[str, Any]): Dictionary from functions.discover_datasets. + ctx (click.Context): Click context. + expand (bool): Whether children were listed, adding a parent column. + write (bool): Write the map to file. + output_json (bool): Output as JSON. + scope (Optional[str]): Scope walked, None when all were. + """ + if output_json: + print(json.dumps(results, indent=2)) + if "error" in results: + ctx.exit(1) + if not results["results"] and results["failed"]: + ctx.exit(1) + return + if "error" in results: + error_console.print(results["error"]) + ctx.exit(1) + return + rows = results["results"] + failed = results["failed"] + if write: + with open(f"./dataset_map_{scope if scope else 'all_scopes'}.json", "w") as f: + json.dump(results, f) + if rows: + table = Table( + title="Datatrail: Dataset Map", + header_style="magenta", + title_style="bold magenta", + ) + table.add_column("Scope") + table.add_column("Dataset") + if expand: + table.add_column("Parent") + previous = None + for row in rows: + if previous is not None and row["scope"] != previous: + table.add_section() + previous = row["scope"] + line = [row["scope"], row["dataset"]] + if expand: + line.append(row["parent"] if row["parent"] else "") + table.add_row(*line) + with console.pager(styles=False): + console.print(table) + elif not failed: + console.print("No datasets matched.") + if failed: + error_console.print("Map is incomplete -- Datatrail did not answer for:") + for item in failed: + error_console.print(f" {item}") + if not rows: + ctx.exit(1) diff --git a/dtcli/src/functions.py b/dtcli/src/functions.py index 2804c68..abbee2f 100644 --- a/dtcli/src/functions.py +++ b/dtcli/src/functions.py @@ -116,6 +116,88 @@ def list( # noqa: C901 return {} +def discover_datasets( + scope: Optional[str] = None, + match: Optional[str] = None, + expand: bool = False, + verbose: int = 0, + quiet: bool = False, +) -> Dict[str, Any]: + """Map larger datasets across scopes, with filtering and expansion. + + Walks one scope, or every scope when none is given, and keeps the larger + datasets whose "scope dataset" text contains every comma-separated, + case-insensitive match term. With expand, each kept dataset is opened one + level and its children become the rows, recording the opened dataset as + their parent; a dataset whose children cannot be listed keeps its own row. + A scope or dataset Datatrail does not answer for is reported in 'failed' + rather than shown as empty. + + Args: + scope (Optional[str], optional): Scope to walk. Defaults to None, + which walks every scope. + match (Optional[str], optional): Comma-separated terms a dataset must + all contain. Defaults to None. + expand (bool, optional): Open each kept dataset one level. Defaults + to False. + verbose (int, optional): Verbosity. Defaults to 0. + quiet (bool, optional): Minimal logging. Defaults to False. + + Returns: + Dict[str, Any]: Keys 'results', rows of scope, dataset and parent, + and 'failed', the queries Datatrail did not answer. Key 'error' + on a configuration or connection failure. + """ + # Set logging level. + utilities.set_log_level(logger, verbose, quiet) + terms = [t.strip().lower() for t in (match or "").split(",") if t.strip()] + if scope: + scopes = [scope] + else: + found = list(verbose=verbose, quiet=quiet) + if "error" in found: + return found + answer = found.get("scopes") + # A non-200 response body is passed through as a string; never walk + # it, or any other non-list shape, as if it were the scopes list. + # NB: isinstance against the builtin list is unavailable here, since + # this module's list() shadows it. + if isinstance(answer, str) or not isinstance(answer, Sequence): + return {"error": "Datatrail did not answer the scopes query."} + if not answer: + return { + "error": "Datatrail reports zero scopes: an account or " + "configuration problem, not an empty archive." + } + scopes = sorted(answer) + results: List[Dict[str, Optional[str]]] = [] + failed: List[str] = [] + for s in scopes: + listed = list(s, verbose=verbose, quiet=quiet) + datasets = None if "error" in listed else listed.get("larger_datasets") + if datasets is None: + failed.append(f"datasets in {s}") + continue + kept = [ + d for d in sorted(datasets) if all(t in f"{s} {d}".lower() for t in terms) + ] + for d in kept: + if not expand: + results.append({"scope": s, "dataset": d, "parent": None}) + continue + opened = list(s, d, verbose=verbose, quiet=quiet) + children = None if "error" in opened else opened.get("datasets") + if children is None: + failed.append(f"children of {s} {d}") + results.append({"scope": s, "dataset": d, "parent": None}) + elif children: + for c in sorted(children, reverse=True): + results.append({"scope": s, "dataset": c, "parent": d}) + else: + results.append({"scope": s, "dataset": d, "parent": None}) + return {"results": results, "failed": failed} + + def ps( scope: str, dataset: str, diff --git a/tests/test_cli.py b/tests/test_cli.py index 77ddee8..4f02f06 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -106,6 +106,8 @@ def test_cli_list_help(runner: CliRunner) -> None: assert "--write" in result.output assert "--json" in result.output assert "Output as JSON" in result.output + assert "--match" in result.output + assert "--expand" in result.output def test_cli_ps_help(runner: CliRunner) -> None: @@ -366,6 +368,71 @@ def test_cli_list_children(runner: CliRunner) -> None: assert "289007650" in result.output +def test_cli_list_match(runner: CliRunner) -> None: + """Test for CLI list to filter larger datasets with --match. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, ["ls", "chime.event.baseband.raw", "--match", "classified"] + ) + assert result.exit_code == 0 + assert "classified.FRB" in result.output + + +def test_cli_list_match_expand(runner: CliRunner) -> None: + """Test for CLI list to expand matched larger datasets one level. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "--match", "classified.FRB", "--expand"], + ) + assert result.exit_code == 0 + assert "289007650" in result.output + assert "classified.FRB" in result.output + + +def test_cli_list_match_no_hits(runner: CliRunner) -> None: + """Test for CLI list with --match matching nothing. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "--match", "no.such.dataset.term"], + ) + assert result.exit_code == 0 + assert "No datasets matched." in result.output + + +def test_cli_list_match_with_dataset_argument(runner: CliRunner) -> None: + """Test for CLI list rejecting --match combined with a dataset argument. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "classified.FRB", "--match", "FRB"], + ) + assert result.exit_code == 1 + + +def test_cli_list_bare_expand(runner: CliRunner) -> None: + """Test for CLI list rejecting --expand without a scope or --match. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke(datatrail, ["ls", "--expand"]) + assert result.exit_code == 1 + + @pytest.mark.cadc def test_cli_ps(runner: CliRunner) -> None: """Test for CLI ps command. @@ -682,6 +749,33 @@ def test_cli_list_children_json(runner: CliRunner) -> None: assert "289007650" in output_data["datasets"] +def test_cli_list_match_json(runner: CliRunner) -> None: + """Test for CLI list to output the dataset map as JSON. + + Args: + runner (CliRunner): Click runner. + """ + import json + + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "--match", "classified", "--json"], + ) + assert result.exit_code == 0 + # Extract JSON from output (skip version check message if present) + json_start = result.output.find("{") + json_output = result.output[json_start:] + # Parse the output as JSON + output_data = json.loads(json_output) + # Should have 'results' rows and a 'failed' list + assert { + "scope": "chime.event.baseband.raw", + "dataset": "classified.FRB", + "parent": None, + } in output_data["results"] + assert output_data["failed"] == [] + + @pytest.mark.cadc def test_cli_ps_json(runner: CliRunner) -> None: """Test for CLI ps command with JSON output. diff --git a/tests/test_functions.py b/tests/test_functions.py index 59155b1..f3922c1 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -112,3 +112,77 @@ def json() -> Dict[str, Any]: ) results: Dict[str, Any] = functions.list() assert results == {"error": "Datatrail did not answer the scopes query."} + + +def _fake_list(scope=None, dataset=None, verbose=0, quiet=False): + """Stand-in for functions.list with one scope not answering.""" + if scope is None: + return {"scopes": ["b.scope", "a.scope", "c.scope"]} + if dataset is None: + if scope == "a.scope": + return {"larger_datasets": ["data.other", "data.good", "skip.me"]} + if scope == "c.scope": + return {"larger_datasets": []} + return {"error": "Datatrail Server at CHIME is not responding."} + if dataset == "data.good": + return {"datasets": ["child1", "child2"]} + return {"error": "Datatrail Server at CHIME is not responding."} + + +def test_discover_datasets(monkeypatch) -> None: + """Test discover_datasets filtering, sorting, and expansion.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(match="data", expand=True) + assert results["results"] == [ + {"scope": "a.scope", "dataset": "child2", "parent": "data.good"}, + {"scope": "a.scope", "dataset": "child1", "parent": "data.good"}, + {"scope": "a.scope", "dataset": "data.other", "parent": None}, + ] + # An unanswered query is reported, never shown as empty. + assert results["failed"] == [ + "children of a.scope data.other", + "datasets in b.scope", + ] + + +def test_discover_datasets_no_expand(monkeypatch) -> None: + """Test discover_datasets without expansion, terms ANDed against scope.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(match="a.scope,data") + assert results["results"] == [ + {"scope": "a.scope", "dataset": "data.good", "parent": None}, + {"scope": "a.scope", "dataset": "data.other", "parent": None}, + ] + assert results["failed"] == ["datasets in b.scope"] + + +def test_discover_datasets_single_scope(monkeypatch) -> None: + """Test discover_datasets walking one named scope only.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(scope="a.scope") + assert [r["dataset"] for r in results["results"]] == [ + "data.good", + "data.other", + "skip.me", + ] + assert results["failed"] == [] + + +def test_discover_datasets_empty_scope_is_not_failure(monkeypatch) -> None: + """Test a scope that answers with no datasets is empty, not failed.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(scope="c.scope") + assert results["results"] == [] + assert results["failed"] == [] + + +def test_discover_datasets_unanswered_scopes_query(monkeypatch) -> None: + """Test a non-list scopes answer is an error, never walked as text.""" + + def bad_list(scope=None, dataset=None, verbose=0, quiet=False): + return {"scopes": "Bad Gateway"} + + monkeypatch.setattr(functions, "list", bad_list) + results: Dict[str, Any] = functions.discover_datasets(match="gain") + assert "error" in results + assert "results" not in results From 1d51ae012e06d860e3fb18bf25828edb5e8b9585 Mon Sep 17 00:00:00 2001 From: Dylan Gormley Date: Tue, 25 Aug 2026 12:15:07 -0500 Subject: [PATCH 2/2] feat(ls): add recursive dataset discovery --- docs/list.md | 35 ++++++++++- dtcli/ls.py | 81 +++++++++++++++++-------- dtcli/src/functions.py | 88 +++++++++++++++++++++++++-- tests/test_cli.py | 72 ++++++++++++++++++++++ tests/test_functions.py | 128 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 371 insertions(+), 33 deletions(-) diff --git a/docs/list.md b/docs/list.md index a1744ce..11af455 100644 --- a/docs/list.md +++ b/docs/list.md @@ -16,6 +16,8 @@ Options: all contain. --expand Open each matched larger dataset one level and list its children. + --recursive Open each matched larger dataset through all descendant + levels. --help Show this message and exit. ``` @@ -155,8 +157,24 @@ children could not be listed (or that has none) may still be a container. appear in the `failed` list. A partial map still exits 0; a map with **no** rows and unanswered queries exits 1, since nothing was determined. +`--recursive` follows every descendant of each matched larger dataset instead +of stopping after one level. It emits terminal datasets and records the full +path used to reach each one: + +```shell +$> datatrail ls gbo.acquisition.processed --match gains --recursive +``` + +The walk visits each dataset once, so shared descendants are not duplicated +and hierarchy cycles cannot loop forever. The first path found in sorted order +is retained. An answered empty child list is a terminal dataset. A branch that +does not answer is retained as a partial row and also listed under `failed`. +Like `--expand`, a recursive walk across all scopes requires `--match` to keep +the request bounded. + With `--json`, the map is emitted as structured rows for scripting; `parent` -is `null` for rows that were not reached through `--expand`: +is `null` for rows that were not reached through expansion. Recursive rows +also include `path`, from the matched larger dataset through the terminal row: ```bash $ datatrail ls --match gain --expand --json @@ -173,6 +191,21 @@ $ datatrail ls --match gain --expand --json } ``` +```bash +$ datatrail ls gbo.acquisition.processed --match gains --recursive --json +{ + "results": [ + { + "scope": "gbo.acquisition.processed", + "dataset": "20230525", + "parent": "complex_gains", + "path": ["complex_gains", "20230525"] + } + ], + "failed": [] +} +``` + ## 🤖 Machine-readable JSON output The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines: diff --git a/dtcli/ls.py b/dtcli/ls.py index e30cca7..babd1e1 100644 --- a/dtcli/ls.py +++ b/dtcli/ls.py @@ -2,7 +2,7 @@ import json import logging -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import click from requests.exceptions import ConnectionError @@ -46,6 +46,11 @@ is_flag=True, help="Open each matched larger dataset one level and list its children.", ) +@click.option( + "--recursive", + is_flag=True, + help="Open each matched larger dataset through all descendant levels.", +) @click.pass_context def list( # noqa: C901 ctx: click.Context, @@ -57,6 +62,7 @@ def list( # noqa: C901 output_json: bool = False, match: Optional[str] = None, expand: bool = False, + recursive: bool = False, ): """List Datatrail Scopes & Datasets. @@ -70,6 +76,7 @@ def list( # noqa: C901 output_json (bool): Output as JSON. match (str): Comma-separated terms a larger dataset must all contain. expand (bool): Open each matched larger dataset one level. + recursive (bool): Open all descendants of each matched larger dataset. """ # Set logging level. set_log_level(logger, verbose, quiet) @@ -89,23 +96,32 @@ def list( # noqa: C901 error_console.print(e) ctx.exit(1) return None - if match is not None or expand: + if match is not None or expand or recursive: if datasets: error_console.print( - "--match and --expand map larger datasets; " + "--match, --expand, and --recursive map larger datasets; " "omit the DATASETS argument." ) ctx.exit(1) return None - if expand and match is None and not scope: + if (expand or recursive) and match is None and not scope: error_console.print( - "--expand alone would open every dataset in the archive; " + "Expansion alone would open every dataset in the archive; " "give a SCOPE or --match to narrow it." ) ctx.exit(1) return None - discovery = functions.discover_datasets(scope, match, expand, verbose, quiet) - _display_discovery(discovery, ctx, expand, write, output_json, scope) + discovery = functions.discover_datasets( + scope=scope, + match=match, + expand=expand, + verbose=verbose, + quiet=quiet, + recursive=recursive, + ) + _display_discovery( + discovery, ctx, expand or recursive, write, output_json, scope, recursive + ) return None results = functions.list(scope, datasets, verbose, quiet) @@ -185,6 +201,7 @@ def _display_discovery( write: bool, output_json: bool, scope: Optional[str], + recursive: bool = False, ) -> None: """Display the dataset map built by functions.discover_datasets. @@ -198,6 +215,7 @@ def _display_discovery( write (bool): Write the map to file. output_json (bool): Output as JSON. scope (Optional[str]): Scope walked, None when all were. + recursive (bool): Whether rows include their full hierarchy path. """ if output_json: print(json.dumps(results, indent=2)) @@ -216,26 +234,8 @@ def _display_discovery( with open(f"./dataset_map_{scope if scope else 'all_scopes'}.json", "w") as f: json.dump(results, f) if rows: - table = Table( - title="Datatrail: Dataset Map", - header_style="magenta", - title_style="bold magenta", - ) - table.add_column("Scope") - table.add_column("Dataset") - if expand: - table.add_column("Parent") - previous = None - for row in rows: - if previous is not None and row["scope"] != previous: - table.add_section() - previous = row["scope"] - line = [row["scope"], row["dataset"]] - if expand: - line.append(row["parent"] if row["parent"] else "") - table.add_row(*line) with console.pager(styles=False): - console.print(table) + console.print(_discovery_table(rows, expand, recursive)) elif not failed: console.print("No datasets matched.") if failed: @@ -244,3 +244,32 @@ def _display_discovery( error_console.print(f" {item}") if not rows: ctx.exit(1) + + +def _discovery_table( + rows: List[Dict[str, Any]], expand: bool, recursive: bool = False +) -> Table: + """Build the dataset map table.""" + table = Table( + title="Datatrail: Dataset Map", + header_style="magenta", + title_style="bold magenta", + ) + table.add_column("Scope") + table.add_column("Dataset") + if expand: + table.add_column("Parent") + if recursive: + table.add_column("Path") + previous = None + for row in rows: + if previous is not None and row["scope"] != previous: + table.add_section() + previous = row["scope"] + line = [row["scope"], row["dataset"]] + if expand: + line.append(row["parent"] if row["parent"] else "") + if recursive: + line.append(" / ".join(row["path"])) + table.add_row(*line) + return table diff --git a/dtcli/src/functions.py b/dtcli/src/functions.py index abbee2f..245457e 100644 --- a/dtcli/src/functions.py +++ b/dtcli/src/functions.py @@ -9,7 +9,7 @@ from collections import Counter from collections.abc import Sequence from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple import requests @@ -122,6 +122,7 @@ def discover_datasets( expand: bool = False, verbose: int = 0, quiet: bool = False, + recursive: bool = False, ) -> Dict[str, Any]: """Map larger datasets across scopes, with filtering and expansion. @@ -129,9 +130,10 @@ def discover_datasets( datasets whose "scope dataset" text contains every comma-separated, case-insensitive match term. With expand, each kept dataset is opened one level and its children become the rows, recording the opened dataset as - their parent; a dataset whose children cannot be listed keeps its own row. - A scope or dataset Datatrail does not answer for is reported in 'failed' - rather than shown as empty. + their parent. With recursive, each kept dataset is opened until terminal + datasets are reached. A dataset whose children cannot be listed keeps its + own row. A scope or dataset Datatrail does not answer for is reported in + 'failed' rather than shown as empty. Args: scope (Optional[str], optional): Scope to walk. Defaults to None, @@ -142,11 +144,14 @@ def discover_datasets( to False. verbose (int, optional): Verbosity. Defaults to 0. quiet (bool, optional): Minimal logging. Defaults to False. + recursive (bool, optional): Open all descendants of each kept dataset. + Defaults to False. Returns: Dict[str, Any]: Keys 'results', rows of scope, dataset and parent, - and 'failed', the queries Datatrail did not answer. Key 'error' - on a configuration or connection failure. + plus path for recursive rows, and 'failed', the branches Datatrail + did not answer. Key 'error' on a configuration or connection + failure. """ # Set logging level. utilities.set_log_level(logger, verbose, quiet) @@ -181,6 +186,13 @@ def discover_datasets( kept = [ d for d in sorted(datasets) if all(t in f"{s} {d}".lower() for t in terms) ] + if recursive: + rows, branch_failures = _discover_descendants( + s, kept, verbose=verbose, quiet=quiet + ) + results.extend(rows) + failed.extend(branch_failures) + continue for d in kept: if not expand: results.append({"scope": s, "dataset": d, "parent": None}) @@ -198,6 +210,70 @@ def discover_datasets( return {"results": results, "failed": failed} +def _discover_descendants( + scope: str, + roots: Sequence[str], + verbose: int = 0, + quiet: bool = False, +) -> Tuple[List[Dict[str, Any]], List[str]]: + """Return unique terminal datasets below the given roots.""" + results: List[Dict[str, Any]] = [] + failed: List[str] = [] + visited: Set[str] = set() + emitted: Set[str] = set() + stack: List[Tuple[str, Optional[str], Tuple[str, ...]]] = [ + (root, None, (root,)) for root in reversed(sorted(set(roots))) + ] + + def add_row(dataset: str, parent: Optional[str], path: Tuple[str, ...]) -> None: + if dataset in emitted: + return + results.append( + { + "scope": scope, + "dataset": dataset, + "parent": parent, + "path": [*path], + } + ) + emitted.add(dataset) + + while stack: + dataset, parent, path = stack.pop() + if dataset in visited: + continue + visited.add(dataset) + opened = list(scope, dataset, verbose=verbose, quiet=quiet) + children = None if "error" in opened else opened.get("datasets") + if ( + children is None + or isinstance(children, str) + or not isinstance(children, Sequence) + or any(not isinstance(child, str) or not child.strip() for child in children) + ): + failed.append(f"children of {scope} {' / '.join(path)}") + add_row(dataset, parent, path) + continue + + child_names = sorted(set(children)) + if not child_names: + add_row(dataset, parent, path) + continue + + cycle_found = False + for child in reversed(child_names): + if child in path: + failed.append(f"cycle in {scope}: {' / '.join(path + (child,))}") + cycle_found = True + continue + if child not in visited: + stack.append((child, dataset, path + (child,))) + if cycle_found: + add_row(dataset, parent, path) + + return results, failed + + def ps( scope: str, dataset: str, diff --git a/tests/test_cli.py b/tests/test_cli.py index 4f02f06..e91ea4b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -108,6 +108,7 @@ def test_cli_list_help(runner: CliRunner) -> None: assert "Output as JSON" in result.output assert "--match" in result.output assert "--expand" in result.output + assert "--recursive" in result.output def test_cli_ps_help(runner: CliRunner) -> None: @@ -433,6 +434,77 @@ def test_cli_list_bare_expand(runner: CliRunner) -> None: assert result.exit_code == 1 +def test_cli_list_bare_recursive(runner: CliRunner) -> None: + """Test for CLI list rejecting an unconstrained recursive walk. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke(datatrail, ["ls", "--recursive"]) + assert result.exit_code == 1 + + +def test_cli_list_recursive_plain(runner: CliRunner, monkeypatch) -> None: + """Test for CLI list showing recursive paths in the table. + + Args: + runner (CliRunner): Click runner. + monkeypatch: Pytest monkeypatch fixture. + """ + + def fake_discovery(**kwargs): + assert kwargs["recursive"] is True + return { + "results": [ + { + "scope": "test.scope", + "dataset": "leaf", + "parent": "branch", + "path": ["root", "branch", "leaf"], + } + ], + "failed": [], + } + + monkeypatch.setattr("dtcli.ls.functions.discover_datasets", fake_discovery) + result = runner.invoke(datatrail, ["ls", "--match", "root", "--recursive"]) + assert result.exit_code == 0 + assert "leaf" in result.output + assert "root / branch / leaf" in result.output + + +def test_cli_list_recursive_json(runner: CliRunner, monkeypatch) -> None: + """Test for CLI list retaining recursive paths in JSON. + + Args: + runner (CliRunner): Click runner. + monkeypatch: Pytest monkeypatch fixture. + """ + import json + + expected = { + "results": [ + { + "scope": "test.scope", + "dataset": "leaf", + "parent": "branch", + "path": ["root", "branch", "leaf"], + } + ], + "failed": [], + } + + def fake_discovery(**kwargs): + assert kwargs["recursive"] is True + return expected + + monkeypatch.setattr("dtcli.ls.functions.discover_datasets", fake_discovery) + result = runner.invoke(datatrail, ["ls", "--match", "root", "--recursive", "--json"]) + assert result.exit_code == 0 + json_start = result.output.find("{") + assert json.loads(result.output[json_start:]) == expected + + @pytest.mark.cadc def test_cli_ps(runner: CliRunner) -> None: """Test for CLI ps command. diff --git a/tests/test_functions.py b/tests/test_functions.py index f3922c1..d362206 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -186,3 +186,131 @@ def bad_list(scope=None, dataset=None, verbose=0, quiet=False): results: Dict[str, Any] = functions.discover_datasets(match="gain") assert "error" in results assert "results" not in results + + +def test_discover_datasets_recursive_paths(monkeypatch) -> None: + """Test recursive discovery paths, ordering, filtering, and duplicates.""" + calls = [] + children = { + "wanted.root": ["branch.b", "branch.a", "branch.a"], + "branch.a": ["leaf.shared", "leaf.a"], + "branch.b": ["leaf.b", "leaf.shared"], + } + + def fake_list(scope=None, dataset=None, verbose=0, quiet=False): + if dataset is None: + return { + "larger_datasets": [ + "wanted.root", + "skip.root", + "wanted.empty", + "wanted.root", + ] + } + calls.append(dataset) + return {"datasets": children.get(dataset, [])} + + monkeypatch.setattr(functions, "list", fake_list) + results = functions.discover_datasets( + scope="test.scope", match="wanted", recursive=True + ) + assert results == { + "results": [ + { + "scope": "test.scope", + "dataset": "wanted.empty", + "parent": None, + "path": ["wanted.empty"], + }, + { + "scope": "test.scope", + "dataset": "leaf.a", + "parent": "branch.a", + "path": ["wanted.root", "branch.a", "leaf.a"], + }, + { + "scope": "test.scope", + "dataset": "leaf.shared", + "parent": "branch.a", + "path": ["wanted.root", "branch.a", "leaf.shared"], + }, + { + "scope": "test.scope", + "dataset": "leaf.b", + "parent": "branch.b", + "path": ["wanted.root", "branch.b", "leaf.b"], + }, + ], + "failed": [], + } + assert "skip.root" not in calls + assert calls.count("wanted.root") == 1 + assert calls.count("leaf.shared") == 1 + + +def test_discover_datasets_recursive_empty_and_failed(monkeypatch) -> None: + """Test recursive discovery keeps empty and failed branches distinct.""" + + def fake_list(scope=None, dataset=None, verbose=0, quiet=False): + if dataset is None: + return {"larger_datasets": ["root"]} + if dataset == "root": + return {"datasets": ["offline", "malformed", "empty"]} + if dataset == "offline": + return {"error": "service unavailable"} + if dataset == "malformed": + return {"datasets": [None]} + return {"datasets": []} + + monkeypatch.setattr(functions, "list", fake_list) + results = functions.discover_datasets(scope="test.scope", recursive=True) + assert results["results"] == [ + { + "scope": "test.scope", + "dataset": "empty", + "parent": "root", + "path": ["root", "empty"], + }, + { + "scope": "test.scope", + "dataset": "malformed", + "parent": "root", + "path": ["root", "malformed"], + }, + { + "scope": "test.scope", + "dataset": "offline", + "parent": "root", + "path": ["root", "offline"], + }, + ] + assert results["failed"] == [ + "children of test.scope root / malformed", + "children of test.scope root / offline", + ] + + +def test_discover_datasets_recursive_cycle(monkeypatch) -> None: + """Test recursive discovery stops and reports a hierarchy cycle.""" + calls = [] + + def fake_list(scope=None, dataset=None, verbose=0, quiet=False): + if dataset is None: + return {"larger_datasets": ["root"]} + calls.append(dataset) + return {"datasets": ["branch"] if dataset == "root" else ["root"]} + + monkeypatch.setattr(functions, "list", fake_list) + results = functions.discover_datasets(scope="test.scope", recursive=True) + assert results["results"] == [ + { + "scope": "test.scope", + "dataset": "branch", + "parent": "root", + "path": ["root", "branch"], + } + ] + assert results["failed"] == [ + "cycle in test.scope: root / branch / root", + ] + assert calls == ["root", "branch"]