diff --git a/commands/cat/dr_egeria_folder.py b/commands/cat/dr_egeria_folder.py index 23138863..bd411d0b 100644 --- a/commands/cat/dr_egeria_folder.py +++ b/commands/cat/dr_egeria_folder.py @@ -6,13 +6,26 @@ documented there in PORTAL_STARTUP.md) so "run all commands in a folder" means the same thing across both repos: -- Ordering: an optional `_batch.json` manifest at the folder root, shaped - {"files": ["a.md", "b.md", ...]}, gives the explicit order for those files. - Any other *.md file in the folder not mentioned there is appended after, - alphabetically. No manifest at all -> pure alphabetical. A manifest entry - naming a file that no longer exists on disk is silently dropped (stale - reference, not an error) -- this mirrors bootstrap_batches.py exactly. -- Only *.md files are processed; `_batch.json` itself is always excluded. +- Ordering: an optional `_batch.json` manifest in a folder gives the explicit + order in its "files" list. An entry is a file ("a.md"), a file in a + subfolder ("sub/x.md"), or a whole subfolder ("sub/"), which runs in place + in its own order (its own `_batch.json`, recursively). Anything not listed + is appended after: *.md files alphabetically, then subfolders + alphabetically, each flagged with a warning when the folder has a "files" + list. A stale entry is skipped with a warning. No manifest -> alphabetical + files, then alphabetical subfolders. +- Skipped: README.md, and the subfolders dr-egeria-outbox, egeria-outbox, + logs, data, templates, __pycache__, .ipynb_checkpoints and any dot-folder + (the outbox folders hold processed copies with live commands), unless named + explicitly in "files". "exclude" in `_batch.json` skips more. +- Run-as user: a "userid" in `_batch.json` (folder default, inherited by + subfolders) or per entry as {"file": "a.md", "userid": "juleskeeper"}; the + nearest declaration wins. Passwords never come from the manifest: + EGERIA_BOOTSTRAP_PASSWORD_ (upper-cased), else --user_pass. Files + with no declared userid run as --userid. --no-manifest-userids runs + everything as --userid. +- A file reachable twice (listed directly and via its folder, or through a + symlink) runs once. - Every file is expected to be upsert-safe (Create -> Update transitions are handled by the processors themselves), so this command is safe to re-run against the same folder repeatedly -- there is no staleness tracking here, @@ -73,39 +86,126 @@ MANIFEST_NAME = "_batch.json" -def resolve_batch_order(folder: Path) -> list[str]: +_SKIP_SUBDIRS = {"logs", ".ipynb_checkpoints", "__pycache__", "dr-egeria-outbox", + "egeria-outbox", "data", "templates"} +_SKIP_FILES = {"readme.md"} + + +def _read_manifest(folder: Path, warnings: list[str]) -> dict: + manifest_path = folder / MANIFEST_NAME + if not manifest_path.is_file(): + return {} + try: + data = json.loads(manifest_path.read_text()) + return data if isinstance(data, dict) else {} + except (json.JSONDecodeError, OSError) as e: + warnings.append(f"could not read {manifest_path}: {e} -- that folder falls back to alphabetical order") + return {} + + +def _entry_parts(entry) -> tuple: + """A "files" entry is a plain string, or {"file": ..., "userid": ...}.""" + if isinstance(entry, str): + return entry, None + if isinstance(entry, dict) and isinstance(entry.get("file"), str): + return entry["file"], (str(entry["userid"]) if entry.get("userid") else None) + return None, None + + +def _is_md(p: Path) -> bool: + return p.is_file() and p.suffix.lower() == ".md" + + +def resolve_batch(folder: Path) -> tuple[list[tuple[str, str | None]], list[str]]: """ - Resolve the ordered list of *.md filenames to process in `folder`, - following the same manifest semantics as egeria-workspaces-fs's - bootstrap_batches.py: explicit order from _batch.json's "files" list - (dropping any entry that no longer exists on disk), then every other - *.md file in the folder appended alphabetically. + Resolve the ordered (relative path, userid) pairs to process under + `folder`, plus any warnings about the ordering, following the same + manifest semantics as egeria-workspaces-fs's bootstrap_batches.py (see the + module docstring). userid is None where no manifest declares one. """ - all_md = sorted(p.name for p in folder.glob("*.md")) + out: list[tuple[str, str | None]] = [] + warnings: list[str] = [] + seen_files: set[Path] = set() + visited_dirs: set[Path] = set() + + def expand(d: Path, prefix: str, inherited_user: str | None) -> None: + real = d.resolve() + if real in visited_dirs: + warnings.append(f"{prefix or './'} is reachable twice (symlink loop?) -- expanded once") + return + visited_dirs.add(real) + + manifest = _read_manifest(d, warnings) + folder_user = str(manifest["userid"]) if manifest.get("userid") else inherited_user + listed = manifest.get("files") or [] + excluded = {str(x).rstrip("/") for x in (manifest.get("exclude") or [])} + placed: set[str] = set() + + def add_file(p: Path, rel: str, user: str | None) -> None: + rp = p.resolve() + if rp in seen_files: + return + seen_files.add(rp) + out.append((rel, user)) + + for entry in listed: + raw, entry_user = _entry_parts(entry) + if raw is None: + warnings.append(f"{MANIFEST_NAME} in {prefix or './'} has an unreadable files entry {entry!r} -- skipped") + continue + name = raw.rstrip("/") + if name in excluded: + continue + p = d / name + user = entry_user or folder_user + if "/" not in name: + placed.add(name) # a nested entry ("sub/x.md") leaves the rest of sub/ to the remainder pass + if p.is_dir(): + expand(p, f"{prefix}{name}/", user) + elif _is_md(p): + add_file(p, prefix + name, user) + else: + warnings.append(f"{MANIFEST_NAME} in {prefix or './'} lists {raw!r}, which isn't there -- skipped") + + has_order = bool(listed) + children = sorted(d.iterdir(), key=lambda c: c.name) + for c in children: + if (_is_md(c) and c.name not in placed and c.name not in excluded + and c.name.lower() not in _SKIP_FILES): + if has_order: + warnings.append(f"{prefix}{c.name} isn't in {MANIFEST_NAME}'s files list -- " + "runs after the listed files, alphabetically") + add_file(c, prefix + c.name, folder_user) + for c in children: + if (c.is_dir() and c.name not in placed and c.name not in excluded + and c.name not in _SKIP_SUBDIRS and not c.name.startswith(".")): + before = len(out) + expand(c, f"{prefix}{c.name}/", folder_user) + if has_order and len(out) > before: + warnings.append(f"{prefix}{c.name}/ isn't in {MANIFEST_NAME}'s files list -- " + "runs after the listed files, alphabetically") + + expand(folder, "", None) + return out, warnings - manifest_path = folder / MANIFEST_NAME - ordered: list[str] = [] - if manifest_path.is_file(): - try: - manifest = json.loads(manifest_path.read_text()) - except (json.JSONDecodeError, OSError) as e: - console.print(f"[yellow]Warning:[/yellow] could not read {MANIFEST_NAME}: {e} -- falling back to alphabetical order") - manifest = {} - for name in manifest.get("files", []): - if name in all_md: - ordered.append(name) - # else: stale manifest entry, silently dropped (matches peer semantics) - remainder = sorted(name for name in all_md if name not in ordered) - return ordered + remainder +def resolve_batch_order(folder: Path) -> list[str]: + """Ordered relative paths only -- see resolve_batch.""" + return [rel for rel, _user in resolve_batch(folder)[0]] -async def run_one_file(input_file: Path, directive: str, client, parse_summary: str, +def password_for(userid: str, default: str) -> str: + """A manifest names a user, never a password: EGERIA_BOOTSTRAP_PASSWORD_ + (the same variable the Portal reads), else the CLI's --user_pass.""" + return os.environ.get(f"EGERIA_BOOTSTRAP_PASSWORD_{userid.upper()}") or default + + +async def run_one_file(input_file: Path, display_name: str, directive: str, client, parse_summary: str, attribute_logs: str, usage_level: str, debug: bool) -> tuple[str, int, int, int, str]: """ Run a single file through process_md_file_v2, capturing its console output the same way tests/dr-egeria-command-tests/run_dr_tests.py does, - and return (filename, success_count, failure_count, warning_count, tail_of_output). + and return (relative name, success_count, failure_count, warning_count, tail_of_output). """ from md_processing.dr_egeria import process_md_file_v2 import md_processing.dr_egeria as dre_module @@ -136,7 +236,7 @@ async def run_one_file(input_file: Path, directive: str, client, parse_summary: failures = output.count("FAILURE") warnings = output.count("WARNING") tail = output[-800:] if failures else "" - return input_file.name, successes, failures, warnings, tail + return display_name, successes, failures, warnings, tail @click.command("dr_egeria_folder", help="Run every Dr.Egeria markdown command file in a folder.") @@ -161,11 +261,13 @@ async def run_one_file(input_file: Path, directive: str, client, parse_summary: help="Use Advanced usage level -- shows additional attributes (default: Basic)") @click.option("--debug", is_flag=True, default=False, help="Print each Egeria API request URL and body to the console") @click.option("--results-file", default="", help="Optional path to also write the full per-file report to.") +@click.option("--no-manifest-userids", is_flag=True, default=False, + help="Ignore userid settings in _batch.json and run every file as --userid.") @logger.catch def dr_egeria_folder(folder: str, directive: str, do_validate: bool, do_process: bool, server: str, url: str, userid: str, user_pass: str, parse_summary: str, attribute_logs: str, advanced: bool, - debug: bool, results_file: str) -> None: + debug: bool, results_file: str, no_manifest_userids: bool) -> None: """ Discover and run every *.md file in FOLDER through Dr.Egeria, in order (see module docstring for the _batch.json manifest / ordering rules). @@ -180,29 +282,47 @@ def dr_egeria_folder(folder: str, directive: str, do_validate: bool, do_process: usage_level = "Advanced" if advanced else "Basic" folder_path = Path(folder) - md_files = resolve_batch_order(folder_path) - if not md_files: + plan, order_warnings = resolve_batch(folder_path) + if no_manifest_userids: + plan = [(rel, None) for rel, _user in plan] + if not plan: console.print(f"[yellow]No *.md files found in {folder_path}[/yellow]") return - console.print(f"[bold]Dr.Egeria folder batch[/bold]: {folder_path} | directive={directive} | {len(md_files)} file(s)") - for name in md_files: - console.print(f" - {name}") + console.print(f"[bold]Dr.Egeria folder batch[/bold]: {folder_path} | directive={directive} | {len(plan)} file(s)") + for rel, file_user in plan: + console.print(f" - {rel} [dim](as {file_user or userid})[/dim]") + for w in order_warnings: + console.print(f" [yellow]![/yellow] {w}") from pyegeria import EgeriaTech - client = EgeriaTech(server, url, userid, user_pass) - client.create_egeria_bearer_token() + clients: dict[str, EgeriaTech] = {} + + def client_for(user: str) -> EgeriaTech: + """One client (one bearer token) per user for the whole run.""" + if user not in clients: + pwd = user_pass if user == userid else password_for(user, user_pass) + c = EgeriaTech(server, url, user, pwd) + c.create_egeria_bearer_token() + clients[user] = c + return clients[user] results = [] - for name in md_files: - input_path = folder_path / name - result = asyncio.run(run_one_file( - input_path, directive, client, parse_summary, attribute_logs, usage_level, debug - )) + for rel, file_user in plan: + input_path = folder_path / rel + run_as = file_user or userid + try: + client = client_for(run_as) + except Exception as e: # noqa: BLE001 -- report per file, keep going + result = (rel, 0, 1, 0, f"could not sign in as {run_as}: {e}") + else: + result = asyncio.run(run_one_file( + input_path, rel, directive, client, parse_summary, attribute_logs, usage_level, debug + )) results.append(result) fname, s, f, w, _tail = result status = "[red]FAILED[/red]" if f else "[green]ok[/green]" - console.print(f" {status} {fname} ({s} success, {f} failure, {w} warning)") + console.print(f" {status} {fname} as {run_as} ({s} success, {f} failure, {w} warning)") table = Table(title="Dr.Egeria Folder Batch Summary") table.add_column("File") diff --git a/commands/cat/execute_dr_egeria_folder.py b/commands/cat/execute_dr_egeria_folder.py deleted file mode 100644 index b21dc2ee..00000000 --- a/commands/cat/execute_dr_egeria_folder.py +++ /dev/null @@ -1,151 +0,0 @@ -""" -Execute all Dr. Egeria markdown files in a folder, following the load order in README.md if present. -""" -import os -import sys -import click -import re -import asyncio -from loguru import logger -from rich.console import Console - -from pyegeria.core._exceptions import PyegeriaException, print_basic_exception -from pyegeria.core.config import settings -from md_processing.dr_egeria import process_md_file_v2 -from pyegeria import EgeriaTech - -# Configure logging -log_format = "{time} | {level} | {function} | {line} | {message} | {extra}" -logger.remove() -logger.add(sys.stderr, level="WARNING", format=log_format, colorize=True) -logger.add("debug_log.log", rotation="1 day", retention="1 week", compression="zip", level="WARNING", format=log_format, - colorize=True) - -# Load configuration -app_config = settings.Environment - -EGERIA_VIEW_SERVER = os.environ.get("EGERIA_VIEW_SERVER", app_config.egeria_view_server) -EGERIA_VIEW_SERVER_URL = os.environ.get("EGERIA_VIEW_SERVER_URL", app_config.egeria_view_server_url) -EGERIA_USER = os.environ.get("EGERIA_USER", "erinoverview") -EGERIA_USER_PASSWORD = os.environ.get("EGERIA_USER_PASSWORD", "secret") -EGERIA_WIDTH = int(os.environ.get("EGERIA_WIDTH", settings.Environment.egeria_width or 190)) - -console = Console(width=EGERIA_WIDTH) - -def get_load_order(folder_path: str) -> list[str]: - """Extract load order from README.md if it exists.""" - readme_path = os.path.join(folder_path, "README.md") - if not os.path.exists(readme_path): - return [] - - with open(readme_path, 'r') as f: - content = f.read() - - # Look for "## Load order" section and then a code block - match = re.search(r'## Load order\s+.*?```\s+(.*?)\s+```', content, re.DOTALL) - if not match: - return [] - - lines = match.group(1).splitlines() - files = [] - for line in lines: - line = line.strip() - if not line: - continue - # Extract filename - assuming it's the last word on the line - parts = line.split() - if parts: - files.append(parts[-1]) - return files - -async def run_folder(folder_path: str, directive: str, client: EgeriaTech, - parse_summary: str, attribute_logs: str, usage_level: str, - summary_only: bool, debug: bool) -> None: - """Execute all files in the folder.""" - files = get_load_order(folder_path) - - if not files: - # Fallback to all .md files in alphabetical order, excluding README.md - files = sorted([f for f in os.listdir(folder_path) - if f.endswith(".md") and f.lower() != "readme.md"]) - console.print(f"[yellow]No load order found in README.md, using alphabetical order.[/yellow]") - else: - console.print(f"[green]Following load order from README.md[/green]") - - for filename in files: - file_path = os.path.join(folder_path, filename) - if not os.path.exists(file_path): - console.print(f"[red]File not found: {file_path}[/red]") - continue - - console.print(f"\n[bold blue]Processing {filename}...[/bold blue]") - try: - await process_md_file_v2( - input_file=file_path, - output_folder="", - directive=directive, - client=client, - parse_summary=parse_summary, - attribute_logs=attribute_logs, - usage_level=usage_level, - summary_only=summary_only, - debug=debug, - ) - except Exception as e: - console.print(f"[red]Error processing {filename}: {e}[/red]") - if debug: - console.print_exception() - -@click.command("execute_dr_egeria_folder", help="Execute all Dr. Egeria markdown files in a folder.") -@click.argument("folder_path", type=click.Path(exists=True, file_okay=False, dir_okay=True)) -@click.option("--directive", default="validate", help="How to process the files (display/validate/process).", - type=click.Choice(["display", "validate", "process"], case_sensitive=False)) -@click.option("--validate", "do_validate", is_flag=True, default=False, help="Shortcut for --directive validate") -@click.option("--process", "do_process", is_flag=True, default=False, help="Shortcut for --directive process") -@click.option("--server", default=EGERIA_VIEW_SERVER, help="Egeria view server to use.") -@click.option("--url", default=EGERIA_VIEW_SERVER_URL, help="URL of Egeria platform to connect to") -@click.option("--userid", default=EGERIA_USER, help="Egeria user") -@click.option("--user_pass", default=EGERIA_USER_PASSWORD, help="Egeria user password") -@click.option("--parse-summary", default="none", help="When to show parse summaries", - type=click.Choice(["all", "errors", "none"], case_sensitive=False)) -@click.option("--attribute-logs", default="info", help="Per-attribute log verbosity", - type=click.Choice(["debug", "info", "none"], case_sensitive=False)) -@click.option("--advanced", is_flag=True, default=False, help="Use Advanced usage level") -@click.option("--summary-only", is_flag=True, default=False, help="Only display the summary table") -@click.option("--debug", is_flag=True, default=False, help="Print each Egeria API request URL and body") -def main(folder_path: str, directive: str, do_validate: bool, do_process: bool, - server: str, url: str, userid: str, user_pass: str, - parse_summary: str, attribute_logs: str, advanced: bool, - summary_only: bool, debug: bool) -> None: - - if do_process: - directive = "process" - elif do_validate: - directive = "validate" - - usage_level = "Advanced" if advanced else "Basic" - - try: - client = EgeriaTech(server, url, userid, user_pass) - client.create_egeria_bearer_token() - - asyncio.run(run_folder( - folder_path=folder_path, - directive=directive, - client=client, - parse_summary=parse_summary, - attribute_logs=attribute_logs, - usage_level=usage_level, - summary_only=summary_only, - debug=debug - )) - - except PyegeriaException as e: - console.print_exception() - print_basic_exception(e) - except Exception as e: - console.print_exception() - logger.error(f"Unexpected error: {e}") - -if __name__ == "__main__": - main() diff --git a/docs/dr_egeria_manual.md b/docs/dr_egeria_manual.md index caa98016..4dc77a14 100644 --- a/docs/dr_egeria_manual.md +++ b/docs/dr_egeria_manual.md @@ -15,6 +15,9 @@ Dr.Egeria is a Markdown-based processing engine for Egeria. It allows users to i - [Reference Resolution](#reference-resolution) 4. [Supported Command Families](#supported-command-families) 5. [CLI Utility: dr_egeria](#cli-utility-dr_egeria) + - [Running a folder: dr_egeria_folder](#running-a-folder-dr_egeria_folder) + - [The `_batch.json` manifest](#the-_batchjson-manifest) + - [Automatic loading in the Egeria Portal](#automatic-loading-in-the-egeria-portal) 6. [Python API](#python-api) - [process_md_file (synchronous)](#process_md_file-synchronous) - [process_md_file_v2 (async)](#process_md_file_v2-async) @@ -400,7 +403,77 @@ dr_egeria [OPTIONS] [INPUT_FILE] - `--advanced`: Enables advanced usage level, exposing additional attributes in templates and validation. - `--summary-only`: Suppresses per-command diagnostic output, showing only the final summary table. - `--debug`: Prints every Egeria API request URL and body to the console for troubleshooting. -- `--server`, `--url`, `--userid`: Connection details for the Egeria platform. +- `--server`, `--url`: Connection details for the Egeria platform. +- `--userid`, `--user_pass`: The Egeria user to run as, and its password. They override `EGERIA_USER` and `EGERIA_USER_PASSWORD`. Elements the file creates are attributed to this user. + +### Running a folder: dr_egeria_folder + +`dr_egeria_folder` runs every Dr.Egeria file in a folder and its subfolders, in order, and prints a summary table at the end. It keeps going after a failure and exits with status 1 if any file failed. + +```bash +dr_egeria_folder "coco-workbooks/0. data-governance-program" --validate # check only (the default) +dr_egeria_folder "coco-workbooks/0. data-governance-program" --process # make the changes +``` + +It takes the same options as `dr_egeria`, plus: + +- `--results-file PATH`: also write the per-file report to `PATH`. +- `--no-manifest-userids`: ignore `userid` settings in `_batch.json` and run every file as `--userid`. + +Before running, it lists each file with the user it will run as, and prints any ordering warnings (see below). The order and the run-as users come from the folder's `_batch.json` files. The Egeria Portal's automatic loading reads the same files with the same rules, so a folder runs the same way from either. + +### The `_batch.json` manifest + +An optional `_batch.json` in a folder controls how that folder's files run. Every field is optional. + +| Field | Meaning | +| --- | --- | +| `files` | Run order. An entry is a file (`"glossary.md"`), a file in a subfolder (`"sub/x.md"`), or a whole subfolder (`"sub/"`), which runs at that position in its own order. An entry can also be an object, `{"file": "risk-register.md", "userid": "juleskeeper"}`, to set its run-as user. | +| `exclude` | Files or subfolders never to run, such as prose documents beside the command files | +| `userid` | The Egeria user this folder's files run as; subfolders inherit it | +| `displayName`, `description`, `canary`, `defaultEnabled`, `idempotent` | Used only by the Egeria Portal (see below) | + +Discovery rules: + +- **Unlisted content still runs.** Anything not in `files` runs after the listed items: `.md` files alphabetically, then subfolders alphabetically. Each is reported as a warning, because its position was defaulted rather than chosen. With no `_batch.json`, everything runs in that alphabetical order. +- **Always skipped,** unless named in `files`: `README.md`, and the subfolders `dr-egeria-outbox`, `egeria-outbox`, `logs`, `data`, `templates`, `__pycache__`, `.ipynb_checkpoints` and any folder starting with `.`. The outbox folders hold Dr.Egeria's processed copies of earlier runs; those still contain live commands. +- **Stale entries are skipped.** An entry that no longer exists is skipped with a warning. +- **Each file runs once.** A file reachable twice, for example listed directly and also through its folder, runs once. + +**Run-as user.** The nearest declaration wins: the file's own entry, then the `userid` of the `_batch.json` in its folder, then the setting passed down from parent folders. A file with no declared user runs as `--userid`. Passwords never go in `_batch.json`. For a user named in a manifest, the password comes from `EGERIA_BOOTSTRAP_PASSWORD_` (userid in upper case, e.g. `EGERIA_BOOTSTRAP_PASSWORD_JULESKEEPER`), else `--user_pass`. + +Example, following the step order in a README that mixes files and subfolders: + +```json +{ + "userid": "erinoverview", + "files": [ + "strategic-supply-chain-analysis.md", + "extending-the-systems-inventory/", + {"file": "mapping-the-systems/", "userid": "peterprofile"}, + "data-field-naming/", + "strategic-digital-products/", + "solution-design.md", + {"file": "software-development-governance-program.md", "userid": "pollytasker"} + ], + "exclude": ["founders-briefing-script.md", "strategic-supply-chain-system-matches.md"] +} +``` + +If a folder's README describes a load order, put that order in `files`: neither `dr_egeria_folder` nor the Portal reads READMEs. + +### Automatic loading in the Egeria Portal + +The Egeria Portal in [egeria-workspaces](https://github.com/odpi/egeria-workspaces) loads folders of Dr.Egeria files automatically. Each folder under its `dr-egeria-inbox` is a batch, run with the `_batch.json` rules above. The Portal adds: + +- **Auto-heal.** A `canary` names one element the batch creates, e.g. `{"type": "Glossary", "name": "Employee Glossary"}`. If that element goes missing, for example after Egeria is reset, the Portal re-runs the batch. +- **Cross-folder order.** A `_folder_order.json` in `dr-egeria-inbox` lists batch names to run first, in order. +- **Batch ownership.** A subfolder that has its own symlink in `dr-egeria-inbox` runs as its own batch, and its parent skips it. +- **An admin panel** to enable batches, run them on demand, and see each file's run-as user, run status and ordering warnings. + +Adding a file to a batch that is already loaded does not run it automatically. Run the batch from the admin panel, or run the folder with `dr_egeria_folder --process`. + +The user guide is `DR_EGERIA_AUTO_LOADING_GUIDE.md` at the root of egeria-workspaces. ---