diff --git a/docs/list.md b/docs/list.md index 68cfaf3..a1744ce 100644 --- a/docs/list.md +++ b/docs/list.md @@ -12,6 +12,10 @@ Options: -q, --quiet Only errors shown in logs. --write Write the events to file. --json Output as JSON. + --match TEXT Comma-separated, case-insensitive terms a larger dataset must + all contain. + --expand Open each matched larger dataset one level and list its + children. --help Show this message and exit. ``` @@ -102,6 +106,73 @@ Within Datatrail, there are two types of datasets: Please see the CLI reference page for more information on the `list` command: [datatrail list](../cli/#datatrail-list) +## Finding datasets with `--match` and `--expand` + +Navigating the hierarchy one name at a time gets slow when you do not know +where a dataset lives. `--match` filters the larger datasets of a scope, or +of **every** scope when no scope is given, by one or more comma-separated, +case-insensitive terms, which must all appear in the combined +`scope dataset` text: + +```shell +$> datatrail ls chime.acquisition.processed --match gains + Datatrail: Dataset Map ++-----------------------------+---------------+ +| Scope | Dataset | ++-----------------------------+---------------+ +| chime.acquisition.processed | complex_gains | ++-----------------------------+---------------+ +``` + +A hit may be a container whose children are the datasets you actually want. +`--expand` opens each matched larger dataset one level and lists the children +it finds, recording the parent; a match whose children cannot be listed keeps +its own row: + +```shell +$> datatrail ls --match gain --expand + Datatrail: Dataset Map ++-----------------------------+---------------+---------------+ +| Scope | Dataset | Parent | ++-----------------------------+---------------+---------------+ +| chime.acquisition.processed | complex_gains | | ++-----------------------------+---------------+---------------+ +| gbo.acquisition.processed | 20230716 | complex_gains | +| gbo.acquisition.processed | 20230715 | complex_gains | +| ... | ... | ... | ++-----------------------------+---------------+---------------+ +``` + +Rows reached through a parent resolve directly with +`datatrail ps `; a row kept for a matched dataset whose +children could not be listed (or that has none) may still be a container. + +!!! warning "Incomplete maps" + + If Datatrail does not answer for a scope or dataset during the walk, the + map is reported as **incomplete** and the unanswered queries are listed, + rather than silently showing them as empty. With `--json`, those queries + appear in the `failed` list. A partial map still exits 0; a map with **no** + rows and unanswered queries exits 1, since nothing was determined. + +With `--json`, the map is emitted as structured rows for scripting; `parent` +is `null` for rows that were not reached through `--expand`: + +```bash +$ datatrail ls --match gain --expand --json +{ + "results": [ + { + "scope": "gbo.acquisition.processed", + "dataset": "20230525", + "parent": "complex_gains" + }, + ... + ], + "failed": [] +} +``` + ## 🤖 Machine-readable JSON output The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines: diff --git a/dtcli/ls.py b/dtcli/ls.py index d6335da..e30cca7 100644 --- a/dtcli/ls.py +++ b/dtcli/ls.py @@ -2,7 +2,7 @@ import json import logging -from typing import Optional +from typing import Any, Dict, Optional import click from requests.exceptions import ConnectionError @@ -35,6 +35,17 @@ @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.") @click.option("--write", is_flag=True, help="Write the events to file.") @click.option("--json", "output_json", is_flag=True, help="Output as JSON.") +@click.option( + "--match", + type=click.STRING, + default=None, + help="Comma-separated, case-insensitive terms a larger dataset must all contain.", +) +@click.option( + "--expand", + is_flag=True, + help="Open each matched larger dataset one level and list its children.", +) @click.pass_context def list( # noqa: C901 ctx: click.Context, @@ -44,6 +55,8 @@ def list( # noqa: C901 quiet: bool = False, write: bool = False, output_json: bool = False, + match: Optional[str] = None, + expand: bool = False, ): """List Datatrail Scopes & Datasets. @@ -55,6 +68,8 @@ def list( # noqa: C901 quiet (bool): Only errors shown in logs. write (bool): Write the events to file. output_json (bool): Output as JSON. + match (str): Comma-separated terms a larger dataset must all contain. + expand (bool): Open each matched larger dataset one level. """ # Set logging level. set_log_level(logger, verbose, quiet) @@ -74,6 +89,24 @@ def list( # noqa: C901 error_console.print(e) ctx.exit(1) return None + if match is not None or expand: + if datasets: + error_console.print( + "--match and --expand map larger datasets; " + "omit the DATASETS argument." + ) + ctx.exit(1) + return None + if expand and match is None and not scope: + error_console.print( + "--expand alone would open every dataset in the archive; " + "give a SCOPE or --match to narrow it." + ) + ctx.exit(1) + return None + discovery = functions.discover_datasets(scope, match, expand, verbose, quiet) + _display_discovery(discovery, ctx, expand, write, output_json, scope) + return None results = functions.list(scope, datasets, verbose, quiet) # Output JSON if requested. @@ -143,3 +176,71 @@ def list( # noqa: C901 if "error" in results.keys(): error_console.print(results["error"]) ctx.exit(1) + + +def _display_discovery( + results: Dict[str, Any], + ctx: click.Context, + expand: bool, + write: bool, + output_json: bool, + scope: Optional[str], +) -> None: + """Display the dataset map built by functions.discover_datasets. + + An empty map with unanswered queries exits non-zero: nothing was + determined. A partial map is shown, with the unanswered queries listed. + + Args: + results (Dict[str, Any]): Dictionary from functions.discover_datasets. + ctx (click.Context): Click context. + expand (bool): Whether children were listed, adding a parent column. + write (bool): Write the map to file. + output_json (bool): Output as JSON. + scope (Optional[str]): Scope walked, None when all were. + """ + if output_json: + print(json.dumps(results, indent=2)) + if "error" in results: + ctx.exit(1) + if not results["results"] and results["failed"]: + ctx.exit(1) + return + if "error" in results: + error_console.print(results["error"]) + ctx.exit(1) + return + rows = results["results"] + failed = results["failed"] + if write: + with open(f"./dataset_map_{scope if scope else 'all_scopes'}.json", "w") as f: + json.dump(results, f) + if rows: + table = Table( + title="Datatrail: Dataset Map", + header_style="magenta", + title_style="bold magenta", + ) + table.add_column("Scope") + table.add_column("Dataset") + if expand: + table.add_column("Parent") + previous = None + for row in rows: + if previous is not None and row["scope"] != previous: + table.add_section() + previous = row["scope"] + line = [row["scope"], row["dataset"]] + if expand: + line.append(row["parent"] if row["parent"] else "") + table.add_row(*line) + with console.pager(styles=False): + console.print(table) + elif not failed: + console.print("No datasets matched.") + if failed: + error_console.print("Map is incomplete -- Datatrail did not answer for:") + for item in failed: + error_console.print(f" {item}") + if not rows: + ctx.exit(1) diff --git a/dtcli/src/functions.py b/dtcli/src/functions.py index d342e82..d2a006d 100644 --- a/dtcli/src/functions.py +++ b/dtcli/src/functions.py @@ -116,6 +116,88 @@ def list( # noqa: C901 return {} +def discover_datasets( + scope: Optional[str] = None, + match: Optional[str] = None, + expand: bool = False, + verbose: int = 0, + quiet: bool = False, +) -> Dict[str, Any]: + """Map larger datasets across scopes, with filtering and expansion. + + Walks one scope, or every scope when none is given, and keeps the larger + datasets whose "scope dataset" text contains every comma-separated, + case-insensitive match term. With expand, each kept dataset is opened one + level and its children become the rows, recording the opened dataset as + their parent; a dataset whose children cannot be listed keeps its own row. + A scope or dataset Datatrail does not answer for is reported in 'failed' + rather than shown as empty. + + Args: + scope (Optional[str], optional): Scope to walk. Defaults to None, + which walks every scope. + match (Optional[str], optional): Comma-separated terms a dataset must + all contain. Defaults to None. + expand (bool, optional): Open each kept dataset one level. Defaults + to False. + verbose (int, optional): Verbosity. Defaults to 0. + quiet (bool, optional): Minimal logging. Defaults to False. + + Returns: + Dict[str, Any]: Keys 'results', rows of scope, dataset and parent, + and 'failed', the queries Datatrail did not answer. Key 'error' + on a configuration or connection failure. + """ + # Set logging level. + utilities.set_log_level(logger, verbose, quiet) + terms = [t.strip().lower() for t in (match or "").split(",") if t.strip()] + if scope: + scopes = [scope] + else: + found = list(verbose=verbose, quiet=quiet) + if "error" in found: + return found + answer = found.get("scopes") + # A non-200 response body is passed through as a string; never walk + # it, or any other non-list shape, as if it were the scopes list. + # NB: isinstance against the builtin list is unavailable here, since + # this module's list() shadows it. + if isinstance(answer, str) or not isinstance(answer, Sequence): + return {"error": "Datatrail did not answer the scopes query."} + if not answer: + return { + "error": "Datatrail reports zero scopes: an account or " + "configuration problem, not an empty archive." + } + scopes = sorted(answer) + results: List[Dict[str, Optional[str]]] = [] + failed: List[str] = [] + for s in scopes: + listed = list(s, verbose=verbose, quiet=quiet) + datasets = None if "error" in listed else listed.get("larger_datasets") + if datasets is None: + failed.append(f"datasets in {s}") + continue + kept = [ + d for d in sorted(datasets) if all(t in f"{s} {d}".lower() for t in terms) + ] + for d in kept: + if not expand: + results.append({"scope": s, "dataset": d, "parent": None}) + continue + opened = list(s, d, verbose=verbose, quiet=quiet) + children = None if "error" in opened else opened.get("datasets") + if children is None: + failed.append(f"children of {s} {d}") + results.append({"scope": s, "dataset": d, "parent": None}) + elif children: + for c in sorted(children, reverse=True): + results.append({"scope": s, "dataset": c, "parent": d}) + else: + results.append({"scope": s, "dataset": d, "parent": None}) + return {"results": results, "failed": failed} + + def ps( scope: str, dataset: str, diff --git a/tests/test_cli.py b/tests/test_cli.py index 20983d0..fa6d09f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -106,6 +106,8 @@ def test_cli_list_help(runner: CliRunner) -> None: assert "--write" in result.output assert "--json" in result.output assert "Output as JSON" in result.output + assert "--match" in result.output + assert "--expand" in result.output def test_cli_ps_help(runner: CliRunner) -> None: @@ -366,6 +368,71 @@ def test_cli_list_children(runner: CliRunner) -> None: assert "289007650" in result.output +def test_cli_list_match(runner: CliRunner) -> None: + """Test for CLI list to filter larger datasets with --match. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, ["ls", "chime.event.baseband.raw", "--match", "classified"] + ) + assert result.exit_code == 0 + assert "classified.FRB" in result.output + + +def test_cli_list_match_expand(runner: CliRunner) -> None: + """Test for CLI list to expand matched larger datasets one level. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "--match", "classified.FRB", "--expand"], + ) + assert result.exit_code == 0 + assert "289007650" in result.output + assert "classified.FRB" in result.output + + +def test_cli_list_match_no_hits(runner: CliRunner) -> None: + """Test for CLI list with --match matching nothing. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "--match", "no.such.dataset.term"], + ) + assert result.exit_code == 0 + assert "No datasets matched." in result.output + + +def test_cli_list_match_with_dataset_argument(runner: CliRunner) -> None: + """Test for CLI list rejecting --match combined with a dataset argument. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "classified.FRB", "--match", "FRB"], + ) + assert result.exit_code == 1 + + +def test_cli_list_bare_expand(runner: CliRunner) -> None: + """Test for CLI list rejecting --expand without a scope or --match. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke(datatrail, ["ls", "--expand"]) + assert result.exit_code == 1 + + @pytest.mark.cadc def test_cli_ps(runner: CliRunner) -> None: """Test for CLI ps command. @@ -682,6 +749,33 @@ def test_cli_list_children_json(runner: CliRunner) -> None: assert "289007650" in output_data["datasets"] +def test_cli_list_match_json(runner: CliRunner) -> None: + """Test for CLI list to output the dataset map as JSON. + + Args: + runner (CliRunner): Click runner. + """ + import json + + result = runner.invoke( + datatrail, + ["ls", "chime.event.baseband.raw", "--match", "classified", "--json"], + ) + assert result.exit_code == 0 + # Extract JSON from output (skip version check message if present) + json_start = result.output.find("{") + json_output = result.output[json_start:] + # Parse the output as JSON + output_data = json.loads(json_output) + # Should have 'results' rows and a 'failed' list + assert { + "scope": "chime.event.baseband.raw", + "dataset": "classified.FRB", + "parent": None, + } in output_data["results"] + assert output_data["failed"] == [] + + @pytest.mark.cadc def test_cli_ps_json(runner: CliRunner) -> None: """Test for CLI ps command with JSON output. diff --git a/tests/test_functions.py b/tests/test_functions.py index 59155b1..f3922c1 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -112,3 +112,77 @@ def json() -> Dict[str, Any]: ) results: Dict[str, Any] = functions.list() assert results == {"error": "Datatrail did not answer the scopes query."} + + +def _fake_list(scope=None, dataset=None, verbose=0, quiet=False): + """Stand-in for functions.list with one scope not answering.""" + if scope is None: + return {"scopes": ["b.scope", "a.scope", "c.scope"]} + if dataset is None: + if scope == "a.scope": + return {"larger_datasets": ["data.other", "data.good", "skip.me"]} + if scope == "c.scope": + return {"larger_datasets": []} + return {"error": "Datatrail Server at CHIME is not responding."} + if dataset == "data.good": + return {"datasets": ["child1", "child2"]} + return {"error": "Datatrail Server at CHIME is not responding."} + + +def test_discover_datasets(monkeypatch) -> None: + """Test discover_datasets filtering, sorting, and expansion.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(match="data", expand=True) + assert results["results"] == [ + {"scope": "a.scope", "dataset": "child2", "parent": "data.good"}, + {"scope": "a.scope", "dataset": "child1", "parent": "data.good"}, + {"scope": "a.scope", "dataset": "data.other", "parent": None}, + ] + # An unanswered query is reported, never shown as empty. + assert results["failed"] == [ + "children of a.scope data.other", + "datasets in b.scope", + ] + + +def test_discover_datasets_no_expand(monkeypatch) -> None: + """Test discover_datasets without expansion, terms ANDed against scope.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(match="a.scope,data") + assert results["results"] == [ + {"scope": "a.scope", "dataset": "data.good", "parent": None}, + {"scope": "a.scope", "dataset": "data.other", "parent": None}, + ] + assert results["failed"] == ["datasets in b.scope"] + + +def test_discover_datasets_single_scope(monkeypatch) -> None: + """Test discover_datasets walking one named scope only.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(scope="a.scope") + assert [r["dataset"] for r in results["results"]] == [ + "data.good", + "data.other", + "skip.me", + ] + assert results["failed"] == [] + + +def test_discover_datasets_empty_scope_is_not_failure(monkeypatch) -> None: + """Test a scope that answers with no datasets is empty, not failed.""" + monkeypatch.setattr(functions, "list", _fake_list) + results: Dict[str, Any] = functions.discover_datasets(scope="c.scope") + assert results["results"] == [] + assert results["failed"] == [] + + +def test_discover_datasets_unanswered_scopes_query(monkeypatch) -> None: + """Test a non-list scopes answer is an error, never walked as text.""" + + def bad_list(scope=None, dataset=None, verbose=0, quiet=False): + return {"scopes": "Bad Gateway"} + + monkeypatch.setattr(functions, "list", bad_list) + results: Dict[str, Any] = functions.discover_datasets(match="gain") + assert "error" in results + assert "results" not in results