feat(ls): add --match and --expand for dataset discovery

docs/list.md

@@ -12,6 +12,10 @@ Options:
   -q, --quiet    Only errors shown in logs.
   --write        Write the events to file.
   --json         Output as JSON.
+  --match TEXT   Comma-separated, case-insensitive terms a larger dataset must
+                 all contain.
+  --expand       Open each matched larger dataset one level and list its
+                 children.
   --help         Show this message and exit.
 
 ```
@@ -102,6 +106,73 @@ Within Datatrail, there are two types of datasets:
     Please see the CLI reference page for more information on the `list` command:
     [datatrail list](../cli/#datatrail-list)
 
+## Finding datasets with `--match` and `--expand`
+
+Navigating the hierarchy one name at a time gets slow when you do not know
+where a dataset lives. `--match` filters the larger datasets of a scope, or
+of **every** scope when no scope is given, by one or more comma-separated,
+case-insensitive terms, which must all appear in the combined
+`scope dataset` text:
+
+```shell
+$> datatrail ls chime.acquisition.processed --match gains
+      Datatrail: Dataset Map
++-----------------------------+---------------+
+| Scope                       | Dataset       |
++-----------------------------+---------------+
+| chime.acquisition.processed | complex_gains |
++-----------------------------+---------------+
+```
+
+A hit may be a container whose children are the datasets you actually want.
+`--expand` opens each matched larger dataset one level and lists the children
+it finds, recording the parent; a match whose children cannot be listed keeps
+its own row:
+
+```shell
+$> datatrail ls --match gain --expand
+                    Datatrail: Dataset Map
++-----------------------------+---------------+---------------+
+| Scope                       | Dataset       | Parent        |
++-----------------------------+---------------+---------------+
+| chime.acquisition.processed | complex_gains |               |
++-----------------------------+---------------+---------------+
+| gbo.acquisition.processed   | 20230716      | complex_gains |
+| gbo.acquisition.processed   | 20230715      | complex_gains |
+| ...                         | ...           | ...           |
++-----------------------------+---------------+---------------+
+```
+
+Rows reached through a parent resolve directly with
+`datatrail ps <scope> <dataset>`; a row kept for a matched dataset whose
+children could not be listed (or that has none) may still be a container.
+
+!!! warning "Incomplete maps"
+
+    If Datatrail does not answer for a scope or dataset during the walk, the
+    map is reported as **incomplete** and the unanswered queries are listed,
+    rather than silently showing them as empty. With `--json`, those queries
+    appear in the `failed` list. A partial map still exits 0; a map with **no**
+    rows and unanswered queries exits 1, since nothing was determined.
+
+With `--json`, the map is emitted as structured rows for scripting; `parent`
+is `null` for rows that were not reached through `--expand`:
+
+```bash
+$ datatrail ls --match gain --expand --json
+{
+  "results": [
+    {
+      "scope": "gbo.acquisition.processed",
+      "dataset": "20230525",
+      "parent": "complex_gains"
+    },
+    ...
+  ],
+  "failed": []
+}
+```
+
 ## 🤖 Machine-readable JSON output
 
 The `--json` flag outputs structured JSON instead of formatted tables, making it easy to parse the output in scripts and pipelines:

dtcli/ls.py

@@ -2,7 +2,7 @@
 
 import json
 import logging
-from typing import Optional
+from typing import Any, Dict, Optional
 
 import click
 from requests.exceptions import ConnectionError
@@ -35,6 +35,17 @@
 @click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.")
 @click.option("--write", is_flag=True, help="Write the events to file.")
 @click.option("--json", "output_json", is_flag=True, help="Output as JSON.")
+@click.option(
+    "--match",
+    type=click.STRING,
+    default=None,
+    help="Comma-separated, case-insensitive terms a larger dataset must all contain.",
+)
+@click.option(
+    "--expand",
+    is_flag=True,
+    help="Open each matched larger dataset one level and list its children.",
+)
 @click.pass_context
 def list(  # noqa: C901
     ctx: click.Context,
@@ -44,6 +55,8 @@ def list(  # noqa: C901
     quiet: bool = False,
     write: bool = False,
     output_json: bool = False,
+    match: Optional[str] = None,
+    expand: bool = False,
 ):
     """List Datatrail Scopes & Datasets.
 
@@ -55,6 +68,8 @@ def list(  # noqa: C901
         quiet (bool): Only errors shown in logs.
         write (bool): Write the events to file.
         output_json (bool): Output as JSON.
+        match (str): Comma-separated terms a larger dataset must all contain.
+        expand (bool): Open each matched larger dataset one level.
     """
     # Set logging level.
     set_log_level(logger, verbose, quiet)
@@ -74,6 +89,24 @@ def list(  # noqa: C901
             error_console.print(e)
             ctx.exit(1)
             return None
+    if match is not None or expand:
+        if datasets:
+            error_console.print(
+                "--match and --expand map larger datasets; "
+                "omit the DATASETS argument."
+            )
+            ctx.exit(1)
+            return None
+        if expand and match is None and not scope:
+            error_console.print(
+                "--expand alone would open every dataset in the archive; "
+                "give a SCOPE or --match to narrow it."
+            )
+            ctx.exit(1)
+            return None
+        discovery = functions.discover_datasets(scope, match, expand, verbose, quiet)
+        _display_discovery(discovery, ctx, expand, write, output_json, scope)
+        return None
     results = functions.list(scope, datasets, verbose, quiet)
 
     # Output JSON if requested.
@@ -143,3 +176,71 @@ def list(  # noqa: C901
     if "error" in results.keys():
         error_console.print(results["error"])
         ctx.exit(1)
+
+
+def _display_discovery(
+    results: Dict[str, Any],
+    ctx: click.Context,
+    expand: bool,
+    write: bool,
+    output_json: bool,
+    scope: Optional[str],
+) -> None:
+    """Display the dataset map built by functions.discover_datasets.
+
+    An empty map with unanswered queries exits non-zero: nothing was
+    determined. A partial map is shown, with the unanswered queries listed.
+
+    Args:
+        results (Dict[str, Any]): Dictionary from functions.discover_datasets.
+        ctx (click.Context): Click context.
+        expand (bool): Whether children were listed, adding a parent column.
+        write (bool): Write the map to file.
+        output_json (bool): Output as JSON.
+        scope (Optional[str]): Scope walked, None when all were.
+    """
+    if output_json:
+        print(json.dumps(results, indent=2))
+        if "error" in results:
+            ctx.exit(1)
+        if not results["results"] and results["failed"]:
+            ctx.exit(1)
+        return
+    if "error" in results:
+        error_console.print(results["error"])
+        ctx.exit(1)
+        return
+    rows = results["results"]
+    failed = results["failed"]
+    if write:
+        with open(f"./dataset_map_{scope if scope else 'all_scopes'}.json", "w") as f:
+            json.dump(results, f)
+    if rows:
+        table = Table(
+            title="Datatrail: Dataset Map",
+            header_style="magenta",
+            title_style="bold magenta",
+        )
+        table.add_column("Scope")
+        table.add_column("Dataset")
+        if expand:
+            table.add_column("Parent")
+        previous = None
+        for row in rows:
+            if previous is not None and row["scope"] != previous:
+                table.add_section()
+            previous = row["scope"]
+            line = [row["scope"], row["dataset"]]
+            if expand:
+                line.append(row["parent"] if row["parent"] else "")
+            table.add_row(*line)
+        with console.pager(styles=False):
+            console.print(table)
+    elif not failed:
+        console.print("No datasets matched.")
+    if failed:
+        error_console.print("Map is incomplete -- Datatrail did not answer for:")
+        for item in failed:
+            error_console.print(f"  {item}")
+        if not rows:
+            ctx.exit(1)

dtcli/src/functions.py

@@ -116,6 +116,88 @@ def list(  # noqa: C901
         return {}
 
 
+def discover_datasets(
+    scope: Optional[str] = None,
+    match: Optional[str] = None,
+    expand: bool = False,
+    verbose: int = 0,
+    quiet: bool = False,
+) -> Dict[str, Any]:
+    """Map larger datasets across scopes, with filtering and expansion.
+
+    Walks one scope, or every scope when none is given, and keeps the larger
+    datasets whose "scope dataset" text contains every comma-separated,
+    case-insensitive match term. With expand, each kept dataset is opened one
+    level and its children become the rows, recording the opened dataset as
+    their parent; a dataset whose children cannot be listed keeps its own row.
+    A scope or dataset Datatrail does not answer for is reported in 'failed'
+    rather than shown as empty.
+
+    Args:
+        scope (Optional[str], optional): Scope to walk. Defaults to None,
+            which walks every scope.
+        match (Optional[str], optional): Comma-separated terms a dataset must
+            all contain. Defaults to None.
+        expand (bool, optional): Open each kept dataset one level. Defaults
+            to False.
+        verbose (int, optional): Verbosity. Defaults to 0.
+        quiet (bool, optional): Minimal logging. Defaults to False.
+
+    Returns:
+        Dict[str, Any]: Keys 'results', rows of scope, dataset and parent,
+            and 'failed', the queries Datatrail did not answer. Key 'error'
+            on a configuration or connection failure.
+    """
+    # Set logging level.
+    utilities.set_log_level(logger, verbose, quiet)
+    terms = [t.strip().lower() for t in (match or "").split(",") if t.strip()]
+    if scope:
+        scopes = [scope]
+    else:
+        found = list(verbose=verbose, quiet=quiet)
+        if "error" in found:
+            return found
+        answer = found.get("scopes")
+        # A non-200 response body is passed through as a string; never walk
+        # it, or any other non-list shape, as if it were the scopes list.
+        # NB: isinstance against the builtin list is unavailable here, since
+        # this module's list() shadows it.
+        if isinstance(answer, str) or not isinstance(answer, Sequence):
+            return {"error": "Datatrail did not answer the scopes query."}
+        if not answer:
+            return {
+                "error": "Datatrail reports zero scopes: an account or "
+                "configuration problem, not an empty archive."
+            }
+        scopes = sorted(answer)
+    results: List[Dict[str, Optional[str]]] = []
+    failed: List[str] = []
+    for s in scopes:
+        listed = list(s, verbose=verbose, quiet=quiet)
+        datasets = None if "error" in listed else listed.get("larger_datasets")
+        if datasets is None:
+            failed.append(f"datasets in {s}")
+            continue
+        kept = [
+            d for d in sorted(datasets) if all(t in f"{s} {d}".lower() for t in terms)
+        ]
+        for d in kept:
+            if not expand:
+                results.append({"scope": s, "dataset": d, "parent": None})
+                continue
+            opened = list(s, d, verbose=verbose, quiet=quiet)
+            children = None if "error" in opened else opened.get("datasets")
+            if children is None:
+                failed.append(f"children of {s} {d}")
+                results.append({"scope": s, "dataset": d, "parent": None})
+            elif children:
+                for c in sorted(children, reverse=True):
+                    results.append({"scope": s, "dataset": c, "parent": d})
+            else:
+                results.append({"scope": s, "dataset": d, "parent": None})
+    return {"results": results, "failed": failed}
+
+
 def ps(
     scope: str,
     dataset: str,