From bba88bb34f43de7642866fe7285d129262a6972b Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Thu, 9 Jul 2026 01:12:25 +0200 Subject: [PATCH 01/59] feat(workflows): align workflow CLI with extension command surface Adds the missing workflow commands and flags so the workflow CLI matches the extension/preset pattern: add --dev and --from, search --author, update, enable and disable. Disabled workflows are blocked from running and marked in list output. Fixes #2342 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 274 ++++++++++++++++-- src/specify_cli/workflows/catalog.py | 5 + tests/test_workflows.py | 377 +++++++++++++++++++++++++ 3 files changed, 631 insertions(+), 25 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index e1d29b47d5..e206be07f8 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -348,6 +348,18 @@ def workflow_run( engine.on_step_start = lambda sid, label: console.print(f" \u25b8 [{sid}] {label} \u2026") err = _error_console(json_output) + + if not is_file_source: + from .catalog import WorkflowRegistry + + installed_meta = WorkflowRegistry(project_root).get(source) + if installed_meta is not None and installed_meta.get("enabled", True) is False: + err.print( + f"[red]Error:[/red] Workflow '{_escape_markup(source)}' is disabled. " + f"Enable with: specify workflow enable {_escape_markup(source)}" + ) + raise typer.Exit(1) + try: definition = engine.load_workflow(source_path if is_file_source else source) except FileNotFoundError: @@ -570,7 +582,8 @@ def workflow_list(): console.print("\n[bold cyan]Installed Workflows:[/bold cyan]\n") for wf_id, wf_data in installed.items(): - console.print(f" [bold]{wf_data.get('name', wf_id)}[/bold] ({wf_id}) v{wf_data.get('version', '?')}") + marker = "" if wf_data.get("enabled", True) else " [red]\\[disabled][/red]" + console.print(f" [bold]{wf_data.get('name', wf_id)}[/bold] ({wf_id}) v{wf_data.get('version', '?')}{marker}") desc = wf_data.get("description", "") if desc: console.print(f" {desc}") @@ -580,9 +593,11 @@ def workflow_list(): @workflow_app.command("add") def workflow_add( source: str = typer.Argument(..., help="Workflow ID, URL, or local path"), + dev: bool = typer.Option(False, "--dev", help="Install from a local workflow YAML file or directory"), + from_url: str | None = typer.Option(None, "--from", help="Install from a custom URL"), ): """Install a workflow from catalog, URL, or local path.""" - from .catalog import WorkflowCatalog, WorkflowRegistry, WorkflowCatalogError + from .catalog import WorkflowRegistry from .engine import WorkflowDefinition project_root = _require_specify_project() @@ -594,7 +609,9 @@ def workflow_add( _reject_unsafe_dir(project_root / ".specify", ".specify") _reject_unsafe_dir(workflows_dir, ".specify/workflows") - def _validate_and_install_local(yaml_path: Path, source_label: str) -> None: + def _validate_and_install_local( + yaml_path: Path, source_label: str, expected_id: str | None = None + ) -> None: """Validate and install a workflow from a local YAML file.""" try: definition = WorkflowDefinition.from_yaml(yaml_path) @@ -621,6 +638,13 @@ def _validate_and_install_local(yaml_path: Path, source_label: str) -> None: console.print(f" \u2022 {err}") raise typer.Exit(1) + if expected_id is not None and definition.id != expected_id: + console.print( + f"[red]Error:[/red] Workflow ID in YAML ({definition.id!r}) " + f"does not match the requested workflow ID ({expected_id!r})." + ) + raise typer.Exit(1) + dest_dir = _safe_workflow_id_dir(workflows_dir, definition.id) dest_dir.mkdir(parents=True, exist_ok=True) import shutil @@ -633,16 +657,40 @@ def _validate_and_install_local(yaml_path: Path, source_label: str) -> None: }) console.print(f"[green]✓[/green] Workflow '{definition.name}' ({definition.id}) installed") - # Try as URL (http/https) - if source.startswith("http://") or source.startswith("https://"): + # Explicit local install (mirrors `extension add --dev`). --dev takes + # precedence over --from so a URL that would be ignored is never fetched. + if dev: + dev_path = Path(source).expanduser() + if dev_path.is_file() and dev_path.suffix in (".yml", ".yaml"): + _validate_and_install_local(dev_path, str(dev_path)) + return + if dev_path.is_dir(): + dev_wf_file = dev_path / "workflow.yml" + if not dev_wf_file.exists(): + console.print(f"[red]Error:[/red] No workflow.yml found in {_escape_markup(source)}") + raise typer.Exit(1) + _validate_and_install_local(dev_wf_file, str(dev_path)) + return + console.print( + "[red]Error:[/red] --dev source must be a workflow YAML file or a " + f"directory containing workflow.yml: {_escape_markup(source)}" + ) + raise typer.Exit(1) + + # Try as URL (http/https) — either the positional source is a URL, or an + # explicit --from URL names where to fetch it (mirrors `extension add --from`). + download_url = from_url or ( + source if source.startswith(("http://", "https://")) else None + ) + if download_url is not None: from ipaddress import ip_address from urllib.parse import urlparse from specify_cli.authentication.http import open_url as _open_url try: - parsed_src = urlparse(source) + parsed_src = urlparse(download_url) except ValueError: - console.print(f"[red]Error:[/red] Invalid URL: {_escape_markup(source)}") + console.print(f"[red]Error:[/red] Invalid URL: {_escape_markup(download_url)}") raise typer.Exit(1) src_host = parsed_src.hostname or "" src_loopback = src_host == "localhost" @@ -661,15 +709,15 @@ def _validate_and_install_local(yaml_path: Path, source_label: str) -> None: _wf_url_extra_headers = None _resolved_wf_url = _resolve_gh_asset( - source, _open_url, timeout=30, github_hosts=_github_provider_hosts() + download_url, _open_url, timeout=30, github_hosts=_github_provider_hosts() ) if _resolved_wf_url: - source = _resolved_wf_url + download_url = _resolved_wf_url _wf_url_extra_headers = {"Accept": "application/octet-stream"} import tempfile try: - with _open_url(source, timeout=30, extra_headers=_wf_url_extra_headers) as resp: + with _open_url(download_url, timeout=30, extra_headers=_wf_url_extra_headers) as resp: final_url = resp.geturl() final_parsed = urlparse(final_url) final_host = final_parsed.hostname or "" @@ -692,7 +740,13 @@ def _validate_and_install_local(yaml_path: Path, source_label: str) -> None: console.print(f"[red]Error:[/red] Failed to download workflow: {exc}") raise typer.Exit(1) try: - _validate_and_install_local(tmp_path, source) + # When installed via --from, the positional argument names the + # workflow the user expects — enforce it like the catalog branch. + _validate_and_install_local( + tmp_path, + download_url, + expected_id=source if from_url else None, + ) finally: tmp_path.unlink(missing_ok=True) return @@ -712,25 +766,42 @@ def _validate_and_install_local(yaml_path: Path, source_label: str) -> None: return # Try from catalog + _install_workflow_from_catalog(project_root, registry, workflows_dir, source) + + +def _install_workflow_from_catalog( + project_root: Path, + registry: Any, + workflows_dir: Path, + workflow_id: str, +) -> None: + """Download, validate, and register a catalog workflow. + + Shared by ``workflow add`` and ``workflow update``. Raises ``typer.Exit`` + on any failure; the registry entry is only written on full success. + """ + from .catalog import WorkflowCatalog, WorkflowCatalogError + from .engine import WorkflowDefinition + catalog = WorkflowCatalog(project_root) try: - info = catalog.get_workflow_info(source) + info = catalog.get_workflow_info(workflow_id) except WorkflowCatalogError as exc: console.print(f"[red]Error:[/red] {exc}") raise typer.Exit(1) if not info: - console.print(f"[red]Error:[/red] Workflow '{source}' not found in catalog") + console.print(f"[red]Error:[/red] Workflow '{workflow_id}' not found in catalog") raise typer.Exit(1) if not info.get("_install_allowed", True): - console.print(f"[yellow]Warning:[/yellow] Workflow '{source}' is from a discovery-only catalog") + console.print(f"[yellow]Warning:[/yellow] Workflow '{workflow_id}' is from a discovery-only catalog") console.print("Direct installation is not enabled for this catalog source.") raise typer.Exit(1) workflow_url = info.get("url") if not workflow_url: - console.print(f"[red]Error:[/red] Workflow '{source}' does not have an install URL in the catalog") + console.print(f"[red]Error:[/red] Workflow '{workflow_id}' does not have an install URL in the catalog") raise typer.Exit(1) # Validate URL scheme (HTTPS required, HTTP allowed for localhost only) @@ -750,14 +821,14 @@ def _validate_and_install_local(yaml_path: Path, source_label: str) -> None: pass if parsed_url.scheme != "https" and not (parsed_url.scheme == "http" and is_loopback): console.print( - f"[red]Error:[/red] Workflow '{source}' has an invalid install URL. " + f"[red]Error:[/red] Workflow '{workflow_id}' has an invalid install URL. " "Only HTTPS URLs are allowed, except HTTP for localhost/loopback." ) raise typer.Exit(1) # Reject path traversal, symlinked , and a symlinked workflow.yml leaf # before any mkdir/download writes beneath the install directory. - workflow_dir = _safe_workflow_id_dir(workflows_dir, source) + workflow_dir = _safe_workflow_id_dir(workflows_dir, workflow_id) workflow_file = workflow_dir / "workflow.yml" try: @@ -791,7 +862,7 @@ def _validate_and_install_local(yaml_path: Path, source_label: str) -> None: import shutil shutil.rmtree(workflow_dir, ignore_errors=True) console.print( - f"[red]Error:[/red] Workflow '{source}' redirected to non-HTTPS URL: {final_url}" + f"[red]Error:[/red] Workflow '{workflow_id}' redirected to non-HTTPS URL: {final_url}" ) raise typer.Exit(1) workflow_file.write_bytes(response.read()) @@ -799,7 +870,7 @@ def _validate_and_install_local(yaml_path: Path, source_label: str) -> None: if workflow_dir.exists(): import shutil shutil.rmtree(workflow_dir, ignore_errors=True) - console.print(f"[red]Error:[/red] Failed to install workflow '{source}' from catalog: {exc}") + console.print(f"[red]Error:[/red] Failed to install workflow '{workflow_id}' from catalog: {exc}") raise typer.Exit(1) # Validate the downloaded workflow before registering @@ -822,25 +893,25 @@ def _validate_and_install_local(yaml_path: Path, source_label: str) -> None: raise typer.Exit(1) # Enforce that the workflow's internal ID matches the catalog key - if definition.id and definition.id != source: + if definition.id and definition.id != workflow_id: import shutil shutil.rmtree(workflow_dir, ignore_errors=True) console.print( f"[red]Error:[/red] Workflow ID in YAML ({definition.id!r}) " - f"does not match catalog key ({source!r}). " + f"does not match catalog key ({workflow_id!r}). " f"The catalog entry may be misconfigured." ) raise typer.Exit(1) - registry.add(source, { - "name": definition.name or info.get("name", source), + registry.add(workflow_id, { + "name": definition.name or info.get("name", workflow_id), "version": definition.version or info.get("version", "0.0.0"), "description": definition.description or info.get("description", ""), "source": "catalog", "catalog_name": info.get("_catalog_name", ""), "url": workflow_url, }) - console.print(f"[green]✓[/green] Workflow '{info.get('name', source)}' installed from catalog") + console.print(f"[green]✓[/green] Workflow '{info.get('name', workflow_id)}' installed from catalog") @workflow_app.command("remove") @@ -904,10 +975,163 @@ def workflow_remove( console.print(f"[green]✓[/green] Workflow '{workflow_id}' removed") +@workflow_app.command("update") +def workflow_update( + workflow_id: str | None = typer.Argument(None, help="Workflow ID to update (default: all)"), +): + """Update installed workflow(s) to the latest catalog version.""" + from packaging import version as pkg_version + + from .catalog import WorkflowCatalog, WorkflowCatalogError, WorkflowRegistry + + project_root = _require_specify_project() + registry = WorkflowRegistry(project_root) + workflows_dir = project_root / ".specify" / "workflows" + _reject_unsafe_dir(project_root / ".specify", ".specify") + _reject_unsafe_dir(workflows_dir, ".specify/workflows") + + installed = registry.list() + if workflow_id: + if not registry.is_installed(workflow_id): + console.print(f"[red]Error:[/red] Workflow '{_escape_markup(workflow_id)}' is not installed") + raise typer.Exit(1) + targets = [workflow_id] + else: + targets = list(installed) + + if not targets: + console.print("[yellow]No workflows installed[/yellow]") + raise typer.Exit(0) + + catalog = WorkflowCatalog(project_root) + console.print("🔄 Checking for updates...\n") + + updates_available: list[dict[str, str]] = [] + for wf_id in targets: + safe_id = _escape_markup(str(wf_id)) + metadata = installed.get(wf_id) or {} + if metadata.get("source") != "catalog": + console.print(f"⚠ {safe_id}: Installed from a local path or URL — re-add to update (skipping)") + continue + try: + installed_version = pkg_version.Version(str(metadata.get("version"))) + except pkg_version.InvalidVersion: + console.print( + f"⚠ {safe_id}: Invalid installed version '{_escape_markup(str(metadata.get('version')))}' in registry (skipping)" + ) + continue + try: + info = catalog.get_workflow_info(wf_id) + except WorkflowCatalogError as exc: + console.print(f"[red]Error:[/red] {exc}") + raise typer.Exit(1) + if not info: + console.print(f"⚠ {safe_id}: Not found in catalog (skipping)") + continue + if not info.get("_install_allowed", True): + console.print( + f"⚠ {safe_id}: Updates not allowed from '{_escape_markup(str(info.get('_catalog_name', 'catalog')))}' (skipping)" + ) + continue + try: + catalog_version = pkg_version.Version(str(info.get("version"))) + except pkg_version.InvalidVersion: + console.print( + f"⚠ {safe_id}: Invalid catalog version '{_escape_markup(str(info.get('version')))}' (skipping)" + ) + continue + if catalog_version > installed_version: + updates_available.append( + {"id": wf_id, "installed": str(installed_version), "available": str(catalog_version)} + ) + else: + console.print(f"✓ {safe_id}: Up to date (v{installed_version})") + + if not updates_available: + console.print("\n[green]All workflows are up to date![/green]") + raise typer.Exit(0) + + console.print("\n[bold]Updates available:[/bold]\n") + for update in updates_available: + console.print( + f" • {_escape_markup(update['id'])}: {update['installed']} → {update['available']}" + ) + console.print() + if not typer.confirm("Update these workflows?"): + console.print("Cancelled") + raise typer.Exit(0) + + console.print() + failed: list[str] = [] + for update in updates_available: + # Installed workflows are a single workflow.yml — back it up so a + # failed download/validation doesn't destroy the working copy. + wf_dir = _safe_workflow_id_dir(workflows_dir, update["id"]) + wf_file = wf_dir / "workflow.yml" + backup = wf_file.read_bytes() if wf_file.is_file() else None + try: + _install_workflow_from_catalog(project_root, registry, workflows_dir, update["id"]) + except typer.Exit: + if backup is not None: + wf_dir.mkdir(parents=True, exist_ok=True) + wf_file.write_bytes(backup) + failed.append(update["id"]) + + if failed: + console.print( + f"\n[red]Failed to update:[/red] {', '.join(_escape_markup(f) for f in failed)}" + ) + raise typer.Exit(1) + + +@workflow_app.command("enable") +def workflow_enable( + workflow_id: str = typer.Argument(..., help="Workflow ID to enable"), +): + """Enable a disabled workflow.""" + from .catalog import WorkflowRegistry + + project_root = _require_specify_project() + registry = WorkflowRegistry(project_root) + metadata = registry.get(workflow_id) + if metadata is None: + console.print(f"[red]Error:[/red] Workflow '{_escape_markup(workflow_id)}' is not installed") + raise typer.Exit(1) + if metadata.get("enabled", True): + console.print(f"[yellow]Workflow '{_escape_markup(workflow_id)}' is already enabled[/yellow]") + raise typer.Exit(0) + metadata["enabled"] = True + registry.add(workflow_id, metadata) + console.print(f"[green]✓[/green] Workflow '{_escape_markup(workflow_id)}' enabled") + + +@workflow_app.command("disable") +def workflow_disable( + workflow_id: str = typer.Argument(..., help="Workflow ID to disable"), +): + """Disable a workflow without removing it.""" + from .catalog import WorkflowRegistry + + project_root = _require_specify_project() + registry = WorkflowRegistry(project_root) + metadata = registry.get(workflow_id) + if metadata is None: + console.print(f"[red]Error:[/red] Workflow '{_escape_markup(workflow_id)}' is not installed") + raise typer.Exit(1) + if not metadata.get("enabled", True): + console.print(f"[yellow]Workflow '{_escape_markup(workflow_id)}' is already disabled[/yellow]") + raise typer.Exit(0) + metadata["enabled"] = False + registry.add(workflow_id, metadata) + console.print(f"[green]✓[/green] Workflow '{_escape_markup(workflow_id)}' disabled") + console.print(f"To re-enable: specify workflow enable {_escape_markup(workflow_id)}") + + @workflow_app.command("search") def workflow_search( query: str | None = typer.Argument(None, help="Search query"), tag: str | None = typer.Option(None, "--tag", help="Filter by tag"), + author: str | None = typer.Option(None, "--author", help="Filter by author"), ): """Search workflow catalogs.""" from .catalog import WorkflowCatalog, WorkflowCatalogError @@ -916,7 +1140,7 @@ def workflow_search( catalog = WorkflowCatalog(project_root) try: - results = catalog.search(query=query, tag=tag) + results = catalog.search(query=query, tag=tag, author=author) except WorkflowCatalogError as exc: console.print(f"[red]Error:[/red] {exc}") raise typer.Exit(1) diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 97bf58a04e..d8855a06aa 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -427,6 +427,7 @@ def search( self, query: str | None = None, tag: str | None = None, + author: str | None = None, ) -> list[dict[str, Any]]: """Search workflows across all configured catalogs.""" merged = self._get_merged_workflows() @@ -451,6 +452,10 @@ def search( normalized_tags = [t.lower() for t in tags if isinstance(t, str)] if tag.lower() not in normalized_tags: continue + if author: + wf_author = wf_data.get("author", "") + if not isinstance(wf_author, str) or wf_author.lower() != author.lower(): + continue results.append(wf_data) return results diff --git a/tests/test_workflows.py b/tests/test_workflows.py index d7cff20f6d..ab995113c9 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7194,3 +7194,380 @@ def test_add_non_string_step_id_reports_validation_error( assert result.exit_code == 1 assert result.exception is None or isinstance(result.exception, SystemExit) assert "Step ID" in result.output + + +class TestWorkflowCliAlignment: + """CLI alignment with extension/preset commands (#2342).""" + + WORKFLOW_YAML = """ +schema_version: "1.0" +workflow: + id: "align-wf" + name: "Align Workflow" + version: "{version}" + description: "CLI alignment test workflow" +steps: + - id: step-one + type: shell + run: "echo hello" +""" + + def _write_workflow_dir(self, base, version="1.0.0"): + d = base / "wf-src" + d.mkdir(parents=True, exist_ok=True) + (d / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version=version), encoding="utf-8" + ) + return d + + def _install_dev(self, runner, app, project_dir): + src = self._write_workflow_dir(project_dir) + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + assert result.exit_code == 0, result.output + return src + + # -- add --dev ----------------------------------------------------- + + def test_add_dev_directory_installs(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + assert WorkflowRegistry(project_dir).is_installed("align-wf") + + def test_add_dev_yaml_file_installs(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + src = self._write_workflow_dir(project_dir) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "add", str(src / "workflow.yml"), "--dev"]) + assert result.exit_code == 0, result.output + assert WorkflowRegistry(project_dir).is_installed("align-wf") + + def test_add_dev_missing_path_errors(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "add", str(project_dir / "missing"), "--dev"]) + assert result.exit_code != 0 + assert "--dev" in result.output + + def test_add_dev_dir_without_workflow_yml_errors(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + empty = project_dir / "empty-src" + empty.mkdir() + runner = CliRunner() + result = runner.invoke(app, ["workflow", "add", str(empty), "--dev"]) + assert result.exit_code != 0 + assert "No workflow.yml found" in result.output + + # -- add --from ---------------------------------------------------- + + class _FakeResponse: + def __init__(self, data, url="https://example.com/workflow.yml"): + self._data = data + self._url = url + + def read(self): + return self._data + + def geturl(self): + return self._url + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def test_add_from_url_installs(self, project_dir, monkeypatch): + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(data, url), + ): + result = runner.invoke( + app, + ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + ) + assert result.exit_code == 0, result.output + assert WorkflowRegistry(project_dir).is_installed("align-wf") + + def test_add_from_url_id_mismatch_errors(self, project_dir, monkeypatch): + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(data, url), + ): + result = runner.invoke( + app, + ["workflow", "add", "other-id", "--from", "https://example.com/workflow.yml"], + ) + assert result.exit_code != 0 + assert "does not match" in result.output + assert not WorkflowRegistry(project_dir).is_installed("align-wf") + + # -- search --author ----------------------------------------------- + + def test_search_author_filters(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog + + monkeypatch.chdir(project_dir) + workflows = { + "wf-a": {"name": "Workflow A", "version": "1.0.0", "description": "", "author": "alice"}, + "wf-b": {"name": "Workflow B", "version": "1.0.0", "description": "", "author": "bob"}, + } + monkeypatch.setattr( + WorkflowCatalog, + "_get_merged_workflows", + lambda self, force_refresh=False: {k: dict(v) for k, v in workflows.items()}, + ) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "search", "--author", "Alice"]) + assert result.exit_code == 0, result.output + assert "wf-a" in result.output + assert "wf-b" not in result.output + + # -- update ---------------------------------------------------------- + + def test_update_no_workflows_installed(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "update"]) + assert result.exit_code == 0, result.output + assert "No workflows installed" in result.output + + def test_update_not_installed_errors(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "update", "ghost"]) + assert result.exit_code != 0 + assert "not installed" in result.output + + def test_update_skips_non_catalog_sources(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + result = runner.invoke(app, ["workflow", "update"]) + assert result.exit_code == 0, result.output + assert "re-add to update" in result.output + + def test_update_installs_newer_catalog_version(self, project_dir, monkeypatch): + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + registry = WorkflowRegistry(project_dir) + registry.add("align-wf", { + "name": "Align Workflow", + "version": "1.0.0", + "description": "CLI alignment test workflow", + "source": "catalog", + "catalog_name": "test-catalog", + "url": "https://example.com/workflow.yml", + }) + wf_dir = project_dir / ".specify" / "workflows" / "align-wf" + wf_dir.mkdir(parents=True) + (wf_dir / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(data, url), + ): + result = runner.invoke(app, ["workflow", "update"], input="y\n") + assert result.exit_code == 0, result.output + assert "1.0.0" in result.output and "2.0.0" in result.output + meta = WorkflowRegistry(project_dir).get("align-wf") + assert meta["version"] == "2.0.0" + assert "2.0.0" in (wf_dir / "workflow.yml").read_text(encoding="utf-8") + + def test_update_up_to_date_reports_and_exits_zero(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + WorkflowRegistry(project_dir).add("align-wf", { + "name": "Align Workflow", + "version": "1.0.0", + "description": "", + "source": "catalog", + "url": "https://example.com/workflow.yml", + }) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + }, + ) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "update"]) + assert result.exit_code == 0, result.output + assert "Up to date" in result.output + assert "All workflows are up to date!" in result.output + + def test_update_restores_backup_on_failed_download(self, project_dir, monkeypatch): + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + WorkflowRegistry(project_dir).add("align-wf", { + "name": "Align Workflow", + "version": "1.0.0", + "description": "", + "source": "catalog", + "url": "https://example.com/workflow.yml", + }) + wf_dir = project_dir / ".specify" / "workflows" / "align-wf" + wf_dir.mkdir(parents=True) + original = self.WORKFLOW_YAML.format(version="1.0.0") + (wf_dir / "workflow.yml").write_text(original, encoding="utf-8") + + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + }, + ) + + def boom(url, timeout=None, extra_headers=None): + raise OSError("network down") + + runner = CliRunner() + with patch("specify_cli.authentication.http.open_url", side_effect=boom): + result = runner.invoke(app, ["workflow", "update"], input="y\n") + assert result.exit_code != 0 + assert "Failed to update" in result.output + # Working copy and registry version are untouched + assert (wf_dir / "workflow.yml").read_text(encoding="utf-8") == original + assert WorkflowRegistry(project_dir).get("align-wf")["version"] == "1.0.0" + + # -- enable / disable ------------------------------------------------ + + def test_disable_blocks_run_enable_restores(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + + result = runner.invoke(app, ["workflow", "disable", "align-wf"]) + assert result.exit_code == 0, result.output + assert WorkflowRegistry(project_dir).get("align-wf")["enabled"] is False + + result = runner.invoke(app, ["workflow", "run", "align-wf"]) + assert result.exit_code != 0 + assert "disabled" in result.output + + result = runner.invoke(app, ["workflow", "enable", "align-wf"]) + assert result.exit_code == 0, result.output + assert WorkflowRegistry(project_dir).get("align-wf")["enabled"] is True + + result = runner.invoke(app, ["workflow", "run", "align-wf"]) + assert result.exit_code == 0, result.output + + def test_disable_shows_marker_in_list(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + runner.invoke(app, ["workflow", "disable", "align-wf"]) + result = runner.invoke(app, ["workflow", "list"]) + assert result.exit_code == 0, result.output + assert "[disabled]" in result.output + + def test_enable_disable_not_installed_errors(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + for cmd in ("enable", "disable"): + result = runner.invoke(app, ["workflow", cmd, "ghost"]) + assert result.exit_code != 0 + assert "not installed" in result.output + + def test_enable_disable_idempotent_warnings(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + + result = runner.invoke(app, ["workflow", "enable", "align-wf"]) + assert result.exit_code == 0 + assert "already enabled" in result.output + + runner.invoke(app, ["workflow", "disable", "align-wf"]) + result = runner.invoke(app, ["workflow", "disable", "align-wf"]) + assert result.exit_code == 0 + assert "already disabled" in result.output From cc8143ea73db35d28f228ec69fd20c2e796c3e89 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Thu, 9 Jul 2026 01:34:19 +0200 Subject: [PATCH 02/59] fix(workflows): preserve disabled state on update, guard corrupted registry entries Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 26 +++++++-- tests/test_workflows.py | 74 ++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index e206be07f8..95dff473b3 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -353,7 +353,7 @@ def workflow_run( from .catalog import WorkflowRegistry installed_meta = WorkflowRegistry(project_root).get(source) - if installed_meta is not None and installed_meta.get("enabled", True) is False: + if isinstance(installed_meta, dict) and installed_meta.get("enabled", True) is False: err.print( f"[red]Error:[/red] Workflow '{_escape_markup(source)}' is disabled. " f"Enable with: specify workflow enable {_escape_markup(source)}" @@ -903,14 +903,19 @@ def _install_workflow_from_catalog( ) raise typer.Exit(1) - registry.add(workflow_id, { + entry = { "name": definition.name or info.get("name", workflow_id), "version": definition.version or info.get("version", "0.0.0"), "description": definition.description or info.get("description", ""), "source": "catalog", "catalog_name": info.get("_catalog_name", ""), "url": workflow_url, - }) + } + # Preserve a prior disabled state across updates/reinstalls. + existing = registry.get(workflow_id) + if isinstance(existing, dict) and existing.get("enabled", True) is False: + entry["enabled"] = False + registry.add(workflow_id, entry) console.print(f"[green]✓[/green] Workflow '{info.get('name', workflow_id)}' installed from catalog") @@ -1009,7 +1014,10 @@ def workflow_update( updates_available: list[dict[str, str]] = [] for wf_id in targets: safe_id = _escape_markup(str(wf_id)) - metadata = installed.get(wf_id) or {} + metadata = installed.get(wf_id) + if not isinstance(metadata, dict): + console.print(f"⚠ {safe_id}: Registry entry is corrupted (skipping)") + continue if metadata.get("source") != "catalog": console.print(f"⚠ {safe_id}: Installed from a local path or URL — re-add to update (skipping)") continue @@ -1097,6 +1105,11 @@ def workflow_enable( if metadata is None: console.print(f"[red]Error:[/red] Workflow '{_escape_markup(workflow_id)}' is not installed") raise typer.Exit(1) + if not isinstance(metadata, dict): + console.print( + f"[red]Error:[/red] Registry entry for '{_escape_markup(workflow_id)}' is corrupted" + ) + raise typer.Exit(1) if metadata.get("enabled", True): console.print(f"[yellow]Workflow '{_escape_markup(workflow_id)}' is already enabled[/yellow]") raise typer.Exit(0) @@ -1118,6 +1131,11 @@ def workflow_disable( if metadata is None: console.print(f"[red]Error:[/red] Workflow '{_escape_markup(workflow_id)}' is not installed") raise typer.Exit(1) + if not isinstance(metadata, dict): + console.print( + f"[red]Error:[/red] Registry entry for '{_escape_markup(workflow_id)}' is corrupted" + ) + raise typer.Exit(1) if not metadata.get("enabled", True): console.print(f"[yellow]Workflow '{_escape_markup(workflow_id)}' is already disabled[/yellow]") raise typer.Exit(0) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index ab995113c9..85b53810e7 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7435,6 +7435,80 @@ def test_update_installs_newer_catalog_version(self, project_dir, monkeypatch): assert meta["version"] == "2.0.0" assert "2.0.0" in (wf_dir / "workflow.yml").read_text(encoding="utf-8") + def test_update_preserves_disabled_state(self, project_dir, monkeypatch): + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + WorkflowRegistry(project_dir).add("align-wf", { + "name": "Align Workflow", + "version": "1.0.0", + "description": "", + "source": "catalog", + "url": "https://example.com/workflow.yml", + "enabled": False, + }) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + }, + ) + data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(data, url), + ): + result = runner.invoke(app, ["workflow", "update"], input="y\n") + assert result.exit_code == 0, result.output + meta = WorkflowRegistry(project_dir).get("align-wf") + assert meta["version"] == "2.0.0" + assert meta["enabled"] is False + + def test_update_skips_corrupted_registry_entry(self, project_dir, monkeypatch): + import json + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + registry_path = WorkflowRegistry(project_dir).registry_path + registry_path.parent.mkdir(parents=True, exist_ok=True) + registry_path.write_text( + json.dumps({"schema_version": "1.0", "workflows": {"broken": "not-a-dict"}}), + encoding="utf-8", + ) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "update"]) + assert result.exit_code == 0, result.output + assert "corrupted" in result.output + + def test_enable_disable_corrupted_registry_entry_errors(self, project_dir, monkeypatch): + import json + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + registry_path = WorkflowRegistry(project_dir).registry_path + registry_path.parent.mkdir(parents=True, exist_ok=True) + registry_path.write_text( + json.dumps({"schema_version": "1.0", "workflows": {"broken": "not-a-dict"}}), + encoding="utf-8", + ) + runner = CliRunner() + for cmd in ("enable", "disable"): + result = runner.invoke(app, ["workflow", cmd, "broken"]) + assert result.exit_code != 0 + assert "corrupted" in result.output + def test_update_up_to_date_reports_and_exits_zero(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app From 59a791b940ca41c68c7a57d04c282feb1970ba52 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Thu, 9 Jul 2026 01:58:39 +0200 Subject: [PATCH 03/59] fix(workflows): guard list against corrupted registry entries, re-raise typer.Exit in catalog install workflow list now skips non-dict registry entries with a warning instead of crashing, matching update/enable/disable. The broad except in _install_workflow_from_catalog no longer swallows typer.Exit, so precise errors like the non-HTTPS redirect message are not duplicated. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 5 +++++ tests/test_workflows.py | 27 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 95dff473b3..975488bfbf 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -582,6 +582,9 @@ def workflow_list(): console.print("\n[bold cyan]Installed Workflows:[/bold cyan]\n") for wf_id, wf_data in installed.items(): + if not isinstance(wf_data, dict): + console.print(f" [yellow]Warning:[/yellow] Skipping corrupted registry entry '{wf_id}'.\n") + continue marker = "" if wf_data.get("enabled", True) else " [red]\\[disabled][/red]" console.print(f" [bold]{wf_data.get('name', wf_id)}[/bold] ({wf_id}) v{wf_data.get('version', '?')}{marker}") desc = wf_data.get("description", "") @@ -866,6 +869,8 @@ def _install_workflow_from_catalog( ) raise typer.Exit(1) workflow_file.write_bytes(response.read()) + except typer.Exit: + raise except Exception as exc: if workflow_dir.exists(): import shutil diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 85b53810e7..e1613563c3 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7490,6 +7490,33 @@ def test_update_skips_corrupted_registry_entry(self, project_dir, monkeypatch): assert result.exit_code == 0, result.output assert "corrupted" in result.output + def test_list_skips_corrupted_registry_entry(self, project_dir, monkeypatch): + import json + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + registry_path = WorkflowRegistry(project_dir).registry_path + registry_path.parent.mkdir(parents=True, exist_ok=True) + registry_path.write_text( + json.dumps( + { + "schema_version": "1.0", + "workflows": { + "broken": "not-a-dict", + "ok": {"name": "OK Workflow", "version": "1.0.0"}, + }, + } + ), + encoding="utf-8", + ) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "list"]) + assert result.exit_code == 0, result.output + assert "corrupted" in result.output + assert "OK Workflow" in result.output + def test_enable_disable_corrupted_registry_entry_errors(self, project_dir, monkeypatch): import json from typer.testing import CliRunner From 52ab7ffdf05060460c4fda538ece4c02a55403d4 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Thu, 9 Jul 2026 10:32:39 +0200 Subject: [PATCH 04/59] fix(workflows): escape rich markup in id-mismatch errors and validate --from source early The two id-mismatch error paths interpolated repr() into Rich markup, so a stray bracket in a user typo could be parsed as markup. Route both through rich.markup.escape. `workflow add --from ` also validated the source only after downloading. Validate it up front so a URL/path/typo fails without a network fetch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 12 ++++++++---- tests/test_workflows.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 975488bfbf..8e245efbec 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -606,6 +606,10 @@ def workflow_add( project_root = _require_specify_project() registry = WorkflowRegistry(project_root) workflows_dir = project_root / ".specify" / "workflows" + # With --from, source names the expected workflow ID: validate it up + # front so a URL/path/typo fails without a network fetch. + if from_url is not None and not dev: + _validate_workflow_id_or_exit(source) # Reject a symlinked .specify / .specify/workflows before any write so an # install can't escape the project root (covers the local, URL, and # catalog branches below — all write beneath workflows_dir). @@ -643,8 +647,8 @@ def _validate_and_install_local( if expected_id is not None and definition.id != expected_id: console.print( - f"[red]Error:[/red] Workflow ID in YAML ({definition.id!r}) " - f"does not match the requested workflow ID ({expected_id!r})." + f"[red]Error:[/red] Workflow ID in YAML ({_escape_markup(repr(definition.id))}) " + f"does not match the requested workflow ID ({_escape_markup(repr(expected_id))})." ) raise typer.Exit(1) @@ -902,8 +906,8 @@ def _install_workflow_from_catalog( import shutil shutil.rmtree(workflow_dir, ignore_errors=True) console.print( - f"[red]Error:[/red] Workflow ID in YAML ({definition.id!r}) " - f"does not match catalog key ({workflow_id!r}). " + f"[red]Error:[/red] Workflow ID in YAML ({_escape_markup(repr(definition.id))}) " + f"does not match catalog key ({_escape_markup(repr(workflow_id))}). " f"The catalog entry may be misconfigured." ) raise typer.Exit(1) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index e1613563c3..7304b7c24c 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7332,6 +7332,30 @@ def test_add_from_url_id_mismatch_errors(self, project_dir, monkeypatch): assert "does not match" in result.output assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_from_rejects_invalid_source_id_without_fetch(self, project_dir, monkeypatch): + """--from with a non-workflow-id source (URL, path, uppercase) fails before any network fetch.""" + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + calls: list[str] = [] + + def _fake_open(url, timeout=None, extra_headers=None): + calls.append(url) + raise AssertionError(f"network fetch attempted: {url}") + + runner = CliRunner() + with patch("specify_cli.authentication.http.open_url", side_effect=_fake_open): + for bad_source in ("https://x/y.yml", "./local.yml", "BadCase"): + result = runner.invoke( + app, + ["workflow", "add", bad_source, "--from", "https://example.com/workflow.yml"], + ) + assert result.exit_code != 0 + assert "Invalid workflow ID" in result.output + assert calls == [] + # -- search --author ----------------------------------------------- def test_search_author_filters(self, project_dir, monkeypatch): From 4c356c413edbf7c3de3fa031ea2d728cec767115 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:57:06 +0200 Subject: [PATCH 05/59] fix(workflows): escape rich markup in list output and catalog install errors, isolate update failures workflow list now escapes id/name/version/description before printing, matching how extensions render user-editable fields. The catalog install helper computes safe_wf_id once and uses it for every early error path plus the final failure message. workflow update wraps _safe_workflow_id_dir and the backup read inside the try/except typer.Exit block so an unsafe id in a corrupted registry fails that one workflow and the rest continue. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 34 ++++++++----- tests/test_workflows.py | 67 ++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 8e245efbec..cc124d80d9 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -582,14 +582,17 @@ def workflow_list(): console.print("\n[bold cyan]Installed Workflows:[/bold cyan]\n") for wf_id, wf_data in installed.items(): + safe_id = _escape_markup(wf_id) if not isinstance(wf_data, dict): - console.print(f" [yellow]Warning:[/yellow] Skipping corrupted registry entry '{wf_id}'.\n") + console.print(f" [yellow]Warning:[/yellow] Skipping corrupted registry entry '{safe_id}'.\n") continue marker = "" if wf_data.get("enabled", True) else " [red]\\[disabled][/red]" - console.print(f" [bold]{wf_data.get('name', wf_id)}[/bold] ({wf_id}) v{wf_data.get('version', '?')}{marker}") + name = _escape_markup(str(wf_data.get("name", wf_id))) + version = _escape_markup(str(wf_data.get("version", "?"))) + console.print(f" [bold]{name}[/bold] ({safe_id}) v{version}{marker}") desc = wf_data.get("description", "") if desc: - console.print(f" {desc}") + console.print(f" {_escape_markup(str(desc))}") console.print() @@ -790,6 +793,8 @@ def _install_workflow_from_catalog( from .catalog import WorkflowCatalog, WorkflowCatalogError from .engine import WorkflowDefinition + safe_wf_id = _escape_markup(workflow_id) + catalog = WorkflowCatalog(project_root) try: info = catalog.get_workflow_info(workflow_id) @@ -798,17 +803,17 @@ def _install_workflow_from_catalog( raise typer.Exit(1) if not info: - console.print(f"[red]Error:[/red] Workflow '{workflow_id}' not found in catalog") + console.print(f"[red]Error:[/red] Workflow '{safe_wf_id}' not found in catalog") raise typer.Exit(1) if not info.get("_install_allowed", True): - console.print(f"[yellow]Warning:[/yellow] Workflow '{workflow_id}' is from a discovery-only catalog") + console.print(f"[yellow]Warning:[/yellow] Workflow '{safe_wf_id}' is from a discovery-only catalog") console.print("Direct installation is not enabled for this catalog source.") raise typer.Exit(1) workflow_url = info.get("url") if not workflow_url: - console.print(f"[red]Error:[/red] Workflow '{workflow_id}' does not have an install URL in the catalog") + console.print(f"[red]Error:[/red] Workflow '{safe_wf_id}' does not have an install URL in the catalog") raise typer.Exit(1) # Validate URL scheme (HTTPS required, HTTP allowed for localhost only) @@ -828,7 +833,7 @@ def _install_workflow_from_catalog( pass if parsed_url.scheme != "https" and not (parsed_url.scheme == "http" and is_loopback): console.print( - f"[red]Error:[/red] Workflow '{workflow_id}' has an invalid install URL. " + f"[red]Error:[/red] Workflow '{safe_wf_id}' has an invalid install URL. " "Only HTTPS URLs are allowed, except HTTP for localhost/loopback." ) raise typer.Exit(1) @@ -869,7 +874,7 @@ def _install_workflow_from_catalog( import shutil shutil.rmtree(workflow_dir, ignore_errors=True) console.print( - f"[red]Error:[/red] Workflow '{workflow_id}' redirected to non-HTTPS URL: {final_url}" + f"[red]Error:[/red] Workflow '{safe_wf_id}' redirected to non-HTTPS URL: {_escape_markup(final_url)}" ) raise typer.Exit(1) workflow_file.write_bytes(response.read()) @@ -879,7 +884,7 @@ def _install_workflow_from_catalog( if workflow_dir.exists(): import shutil shutil.rmtree(workflow_dir, ignore_errors=True) - console.print(f"[red]Error:[/red] Failed to install workflow '{workflow_id}' from catalog: {exc}") + console.print(f"[red]Error:[/red] Failed to install workflow '{safe_wf_id}' from catalog: {_escape_markup(str(exc))}") raise typer.Exit(1) # Validate the downloaded workflow before registering @@ -1083,13 +1088,16 @@ def workflow_update( for update in updates_available: # Installed workflows are a single workflow.yml — back it up so a # failed download/validation doesn't destroy the working copy. - wf_dir = _safe_workflow_id_dir(workflows_dir, update["id"]) - wf_file = wf_dir / "workflow.yml" - backup = wf_file.read_bytes() if wf_file.is_file() else None + wf_dir: Path | None = None + wf_file: Path | None = None + backup: bytes | None = None try: + wf_dir = _safe_workflow_id_dir(workflows_dir, update["id"]) + wf_file = wf_dir / "workflow.yml" + backup = wf_file.read_bytes() if wf_file.is_file() else None _install_workflow_from_catalog(project_root, registry, workflows_dir, update["id"]) except typer.Exit: - if backup is not None: + if backup is not None and wf_dir is not None and wf_file is not None: wf_dir.mkdir(parents=True, exist_ok=True) wf_file.write_bytes(backup) failed.append(update["id"]) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 7304b7c24c..97e5350842 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7541,6 +7541,73 @@ def test_list_skips_corrupted_registry_entry(self, project_dir, monkeypatch): assert "corrupted" in result.output assert "OK Workflow" in result.output + def test_list_escapes_rich_markup_in_registry_fields(self, project_dir, monkeypatch): + """User-editable name/description/id fields must not be parsed as Rich markup.""" + import json + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + registry_path = WorkflowRegistry(project_dir).registry_path + registry_path.parent.mkdir(parents=True, exist_ok=True) + registry_path.write_text( + json.dumps( + { + "schema_version": "1.0", + "workflows": { + "ok": { + "name": "Bracket [Test]", + "version": "1.0.0", + "description": "desc [with] brackets", + }, + }, + } + ), + encoding="utf-8", + ) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "list"]) + assert result.exit_code == 0, result.output + assert "Bracket [Test]" in result.output + assert "desc [with] brackets" in result.output + + def test_update_reports_unsafe_registry_id_per_workflow(self, project_dir, monkeypatch): + """An unsafe workflow id in the registry must fail that one entry, not abort the whole update.""" + import json + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry, WorkflowCatalog + + monkeypatch.chdir(project_dir) + registry_path = WorkflowRegistry(project_dir).registry_path + registry_path.parent.mkdir(parents=True, exist_ok=True) + registry_path.write_text( + json.dumps( + { + "schema_version": "1.0", + "workflows": { + "../evil": { + "name": "Bad", + "version": "0.0.1", + "source": "catalog", + "url": "https://example.com/evil.yml", + }, + }, + } + ), + encoding="utf-8", + ) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: {"version": "9.9.9", "url": "https://example.com/evil.yml", "_install_allowed": True}, + ) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "update"], input="y\n") + assert result.exit_code != 0 + assert "Failed to update" in result.output + def test_enable_disable_corrupted_registry_entry_errors(self, project_dir, monkeypatch): import json from typer.testing import CliRunner From 8409bca244400afba6d4ecb3583da982b3124763 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Thu, 9 Jul 2026 19:45:51 +0200 Subject: [PATCH 06/59] fix(workflows): escape rich markup in --from download exception message Matches how the catalog install path escapes exception strings. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index cc124d80d9..dbd07ae8ee 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -747,7 +747,7 @@ def _validate_and_install_local( except typer.Exit: raise except Exception as exc: - console.print(f"[red]Error:[/red] Failed to download workflow: {exc}") + console.print(f"[red]Error:[/red] Failed to download workflow: {_escape_markup(str(exc))}") raise typer.Exit(1) try: # When installed via --from, the positional argument names the From 129362f29b2313ddef156442770fb9b7034c21e4 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 08:26:37 +0200 Subject: [PATCH 07/59] fix(workflows): catch OSError in per-workflow update loop and make restore best-effort Transient FS errors (perms, disk full) from backup read or write no longer abort the whole update run. The restore is wrapped in its own try/except so a failed write only warns, and the offending workflow is reported via 'Failed to update' like other per-workflow failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 17 +++++++-- tests/test_workflows.py | 50 ++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index dbd07ae8ee..73f8d7b2d3 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -1096,10 +1096,21 @@ def workflow_update( wf_file = wf_dir / "workflow.yml" backup = wf_file.read_bytes() if wf_file.is_file() else None _install_workflow_from_catalog(project_root, registry, workflows_dir, update["id"]) - except typer.Exit: + except (typer.Exit, OSError) as exc: if backup is not None and wf_dir is not None and wf_file is not None: - wf_dir.mkdir(parents=True, exist_ok=True) - wf_file.write_bytes(backup) + try: + wf_dir.mkdir(parents=True, exist_ok=True) + wf_file.write_bytes(backup) + except OSError as restore_exc: + console.print( + f"[yellow]Warning:[/yellow] Could not restore backup for " + f"'{_escape_markup(update['id'])}': {_escape_markup(str(restore_exc))}" + ) + if isinstance(exc, OSError): + console.print( + f"[red]Error:[/red] Filesystem error updating " + f"'{_escape_markup(update['id'])}': {_escape_markup(str(exc))}" + ) failed.append(update["id"]) if failed: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 97e5350842..1ea8a1d0ee 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7608,6 +7608,56 @@ def test_update_reports_unsafe_registry_id_per_workflow(self, project_dir, monke assert result.exit_code != 0 assert "Failed to update" in result.output + def test_update_survives_oserror_from_backup_read(self, project_dir, monkeypatch): + """OSError while reading the backup for one workflow must not abort the whole update.""" + import json + from pathlib import Path + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry, WorkflowCatalog + + monkeypatch.chdir(project_dir) + registry_path = WorkflowRegistry(project_dir).registry_path + registry_path.parent.mkdir(parents=True, exist_ok=True) + registry_path.write_text( + json.dumps( + { + "schema_version": "1.0", + "workflows": { + "wobbly": { + "name": "Wobbly", + "version": "0.0.1", + "source": "catalog", + "url": "https://example.com/wobbly.yml", + }, + }, + } + ), + encoding="utf-8", + ) + # An existing installed file whose read_bytes will raise OSError. + wf_file = project_dir / ".specify" / "workflows" / "wobbly" / "workflow.yml" + wf_file.parent.mkdir(parents=True, exist_ok=True) + wf_file.write_bytes(b"schema_version: '1.0'\nworkflow:\n id: wobbly\n name: Wobbly\n version: 0.0.1\nsteps: []\n") + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: {"version": "9.9.9", "url": "https://example.com/wobbly.yml", "_install_allowed": True}, + ) + real_read = Path.read_bytes + + def _boom(self, *args, **kwargs): + if self.name == "workflow.yml" and "wobbly" in str(self): + raise OSError("simulated permission denied") + return real_read(self, *args, **kwargs) + + monkeypatch.setattr(Path, "read_bytes", _boom) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "update"], input="y\n") + assert result.exit_code != 0, result.output + assert "Filesystem error" in result.output + assert "Failed to update" in result.output + def test_enable_disable_corrupted_registry_entry_errors(self, project_dir, monkeypatch): import json from typer.testing import CliRunner From 4ca50f588b1d87ee994ca469b2a51031dc144696 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 08:38:12 +0200 Subject: [PATCH 08/59] fix(workflows): escape rich markup in search output workflow search now escapes catalog-derived name/id/version/description/ tags before printing, matching extension search and workflow list. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 10 +++++++--- tests/test_workflows.py | 27 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 73f8d7b2d3..e73fdf1a0e 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -1197,13 +1197,17 @@ def workflow_search( console.print(f"\n[bold cyan]Workflows ({len(results)}):[/bold cyan]\n") for wf in results: - console.print(f" [bold]{wf.get('name', wf.get('id', '?'))}[/bold] ({wf.get('id', '?')}) v{wf.get('version', '?')}") + name = _escape_markup(str(wf.get("name", wf.get("id", "?")))) + wf_id = _escape_markup(str(wf.get("id", "?"))) + version = _escape_markup(str(wf.get("version", "?"))) + console.print(f" [bold]{name}[/bold] ({wf_id}) v{version}") desc = wf.get("description", "") if desc: - console.print(f" {desc}") + console.print(f" {_escape_markup(str(desc))}") tags = wf.get("tags", []) if tags: - console.print(f" [dim]Tags: {', '.join(tags)}[/dim]") + safe_tags = _escape_markup(", ".join(str(t) for t in tags)) + console.print(f" [dim]Tags: {safe_tags}[/dim]") console.print() diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 1ea8a1d0ee..dc3cd0ec25 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7379,6 +7379,33 @@ def test_search_author_filters(self, project_dir, monkeypatch): assert "wf-a" in result.output assert "wf-b" not in result.output + def test_search_escapes_rich_markup_in_catalog_fields(self, project_dir, monkeypatch): + """Catalog-derived name/description/tags must not be parsed as Rich markup.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog + + monkeypatch.chdir(project_dir) + workflows = { + "wf-a": { + "name": "Bracket [Search]", + "version": "1.0.0", + "description": "desc [with] brackets", + "tags": ["tag[1]", "tag2"], + }, + } + monkeypatch.setattr( + WorkflowCatalog, + "_get_merged_workflows", + lambda self, force_refresh=False: {k: dict(v) for k, v in workflows.items()}, + ) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "search"]) + assert result.exit_code == 0, result.output + assert "Bracket [Search]" in result.output + assert "desc [with] brackets" in result.output + assert "tag[1]" in result.output + # -- update ---------------------------------------------------------- def test_update_no_workflows_installed(self, project_dir, monkeypatch): From 10f9770f0cf97f4a14589adbd8e60f93af3a82a0 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 19:58:14 +0200 Subject: [PATCH 09/59] fix: escape workflow validation errors before Rich output Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 6 +++--- tests/test_workflows.py | 28 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index e73fdf1a0e..fd83bebdc1 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -374,7 +374,7 @@ def workflow_run( if errors: err.print("[red]Workflow validation failed:[/red]") for verr in errors: - err.print(f" • {verr}") + err.print(f" • {_escape_markup(str(verr))}") raise typer.Exit(1) # Parse inputs @@ -645,7 +645,7 @@ def _validate_and_install_local( if errors: console.print("[red]Error:[/red] Workflow validation failed:") for err in errors: - console.print(f" \u2022 {err}") + console.print(f" \u2022 {_escape_markup(str(err))}") raise typer.Exit(1) if expected_id is not None and definition.id != expected_id: @@ -903,7 +903,7 @@ def _install_workflow_from_catalog( shutil.rmtree(workflow_dir, ignore_errors=True) console.print("[red]Error:[/red] Downloaded workflow validation failed:") for err in errors: - console.print(f" \u2022 {err}") + console.print(f" \u2022 {_escape_markup(str(err))}") raise typer.Exit(1) # Enforce that the workflow's internal ID matches the catalog key diff --git a/tests/test_workflows.py b/tests/test_workflows.py index dc3cd0ec25..4e18557ed7 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -5905,6 +5905,34 @@ def test_add_refuses_symlinked_workflows_dir(self, temp_dir, monkeypatch): assert result.exit_code != 0 assert "symlinked .specify/workflows" in result.output + def test_add_escapes_rich_markup_in_validation_errors(self, temp_dir, monkeypatch): + """User-controlled YAML values in validation errors must not be parsed as Rich markup.""" + from typer.testing import CliRunner + from specify_cli import app + + (temp_dir / ".specify" / "workflows").mkdir(parents=True) + src = temp_dir / "incoming.yml" + src.write_text( + """ +schema_version: "1.0" +workflow: + id: "markup-wf" + name: "Markup" + version: "[bold]bad[/bold]" + +steps: + - id: step-one + command: speckit.specify +""", + encoding="utf-8", + ) + + monkeypatch.chdir(temp_dir) + result = CliRunner().invoke(app, ["workflow", "add", str(src)]) + + assert result.exit_code != 0 + assert "[bold]bad[/bold]" in result.output + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") def test_add_refuses_symlinked_id_dir(self, temp_dir, monkeypatch, sample_workflow_yaml): """A symlinked install dir must not let a copy escape the project root.""" From 91c71655c7fe718e04b96dc36124bb19dea3a7d2 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:25:33 +0200 Subject: [PATCH 10/59] fix(workflows): escape remaining unescaped Rich markup paths Covers the last few review threads not yet addressed: - Escape yaml.YAMLError text in the local workflow add install path (matches the already-escaped download/catalog paths). - Escape the non---dev local directory fallback's "No workflow.yml found in " message (the --dev branch already escaped it). - Escape the redirected final_url in the --from non-HTTPS redirect error (IPv6 literals like http://[::1]/... are legal and contain brackets). - Escape the "Downloaded workflow is invalid" exception message in _install_workflow_from_catalog, matching the sibling catalog-install exception handler a few lines above it. Adds regression tests for each in TestWorkflowCliAlignment, following the existing escaping-test pattern in this class. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 10 ++- tests/test_workflows.py | 105 +++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 4 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index fd83bebdc1..502e1e02f2 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -626,7 +626,7 @@ def _validate_and_install_local( try: definition = WorkflowDefinition.from_yaml(yaml_path) except (ValueError, yaml.YAMLError) as exc: - console.print(f"[red]Error:[/red] Invalid workflow YAML: {exc}") + console.print(f"[red]Error:[/red] Invalid workflow YAML: {_escape_markup(str(exc))}") raise typer.Exit(1) # Non-string ids (e.g. unquoted ``id: 123`` or ``id: 0``) fall through # to validate_workflow below, which reports a typed error instead of @@ -739,7 +739,9 @@ def _validate_and_install_local( # Redirect host is not an IP literal; keep loopback as determined above. pass if final_parsed.scheme != "https" and not (final_parsed.scheme == "http" and final_lb): - console.print(f"[red]Error:[/red] URL redirected to non-HTTPS: {final_url}") + console.print( + f"[red]Error:[/red] URL redirected to non-HTTPS: {_escape_markup(final_url)}" + ) raise typer.Exit(1) with tempfile.NamedTemporaryFile(suffix=".yml", delete=False) as tmp: tmp.write(resp.read()) @@ -770,7 +772,7 @@ def _validate_and_install_local( elif source_path.is_dir(): wf_file = source_path / "workflow.yml" if not wf_file.exists(): - console.print(f"[red]Error:[/red] No workflow.yml found in {source}") + console.print(f"[red]Error:[/red] No workflow.yml found in {_escape_markup(source)}") raise typer.Exit(1) _validate_and_install_local(wf_file, str(source_path)) return @@ -893,7 +895,7 @@ def _install_workflow_from_catalog( except (ValueError, yaml.YAMLError) as exc: import shutil shutil.rmtree(workflow_dir, ignore_errors=True) - console.print(f"[red]Error:[/red] Downloaded workflow is invalid: {exc}") + console.print(f"[red]Error:[/red] Downloaded workflow is invalid: {_escape_markup(str(exc))}") raise typer.Exit(1) from .engine import validate_workflow diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 4e18557ed7..96578d1e0f 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7300,6 +7300,40 @@ def test_add_dev_dir_without_workflow_yml_errors(self, project_dir, monkeypatch) assert result.exit_code != 0 assert "No workflow.yml found" in result.output + def test_add_local_dir_without_workflow_yml_errors(self, project_dir, monkeypatch): + """Same as the --dev case, but for the plain local-path fallback (no --dev).""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + empty = project_dir / "empty-src-[bracket]" + empty.mkdir() + runner = CliRunner() + result = runner.invoke(app, ["workflow", "add", str(empty)]) + assert result.exit_code != 0 + assert "No workflow.yml found" in result.output + assert "[bracket]" in result.output + + def test_add_yaml_parse_error_escapes_rich_markup(self, project_dir, monkeypatch): + """A YAML syntax error can quote the offending line verbatim; brackets in it must not be Rich markup.""" + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.engine import WorkflowDefinition + + monkeypatch.chdir(project_dir) + bad = project_dir / "bad.yml" + bad.write_text("workflow:\n id: wf\n", encoding="utf-8") + runner = CliRunner() + with patch.object( + WorkflowDefinition, + "from_yaml", + side_effect=ValueError('bad snippet: "New [Feature]"'), + ): + result = runner.invoke(app, ["workflow", "add", str(bad)]) + assert result.exit_code != 0 + assert 'bad snippet: "New [Feature]"' in result.output + # -- add --from ---------------------------------------------------- class _FakeResponse: @@ -7360,6 +7394,26 @@ def test_add_from_url_id_mismatch_errors(self, project_dir, monkeypatch): assert "does not match" in result.output assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_from_url_non_https_redirect_escapes_rich_markup(self, project_dir, monkeypatch): + """A redirect to a non-HTTPS IPv6 literal (legally bracketed) must not be parsed as Rich markup.""" + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + redirected_url = "http://[2001:db8::1]/workflow.yml" + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(b"", redirected_url), + ): + result = runner.invoke( + app, + ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + ) + assert result.exit_code != 0 + assert redirected_url in result.output + def test_add_from_rejects_invalid_source_id_without_fetch(self, project_dir, monkeypatch): """--from with a non-workflow-id source (URL, path, uppercase) fails before any network fetch.""" from unittest.mock import patch @@ -7514,6 +7568,57 @@ def test_update_installs_newer_catalog_version(self, project_dir, monkeypatch): assert meta["version"] == "2.0.0" assert "2.0.0" in (wf_dir / "workflow.yml").read_text(encoding="utf-8") + def test_update_downloaded_invalid_yaml_escapes_rich_markup(self, project_dir, monkeypatch): + """A malformed downloaded workflow can quote the offending line verbatim; escape it before printing.""" + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + from specify_cli.workflows.engine import WorkflowDefinition + + monkeypatch.chdir(project_dir) + registry = WorkflowRegistry(project_dir) + registry.add("align-wf", { + "name": "Align Workflow", + "version": "1.0.0", + "description": "CLI alignment test workflow", + "source": "catalog", + "catalog_name": "test-catalog", + "url": "https://example.com/workflow.yml", + }) + wf_dir = project_dir / ".specify" / "workflows" / "align-wf" + wf_dir.mkdir(parents=True) + (wf_dir / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(b"", url), + ), patch.object( + WorkflowDefinition, + "from_yaml", + side_effect=ValueError('bad snippet: "New [Feature]"'), + ): + result = runner.invoke(app, ["workflow", "update"], input="y\n") + assert 'bad snippet: "New [Feature]"' in result.output + assert "Failed to update" in result.output + # The previously installed workflow must survive a failed update. + assert "1.0.0" in (wf_dir / "workflow.yml").read_text(encoding="utf-8") + def test_update_preserves_disabled_state(self, project_dir, monkeypatch): from unittest.mock import patch from typer.testing import CliRunner From 7100211ffcd71cd16b24f9c927f2df9858a8fece Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:46:45 +0200 Subject: [PATCH 11/59] fix(workflows): escape workflow name/id in install success messages Workflow names and ids come from user-controlled YAML or external catalog data; printing them unescaped lets bracket characters be interpreted as Rich tags. Escape them in the add/catalog-install success messages and the remaining catalog error paths, matching the rest of the output hardening. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 502e1e02f2..36fb1b135a 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -665,7 +665,10 @@ def _validate_and_install_local( "description": definition.description, "source": source_label, }) - console.print(f"[green]✓[/green] Workflow '{definition.name}' ({definition.id}) installed") + console.print( + f"[green]✓[/green] Workflow '{_escape_markup(definition.name)}' " + f"({_escape_markup(definition.id)}) installed" + ) # Explicit local install (mirrors `extension add --dev`). --dev takes # precedence over --from so a URL that would be ignored is never fetched. @@ -801,7 +804,7 @@ def _install_workflow_from_catalog( try: info = catalog.get_workflow_info(workflow_id) except WorkflowCatalogError as exc: - console.print(f"[red]Error:[/red] {exc}") + console.print(f"[red]Error:[/red] {_escape_markup(str(exc))}") raise typer.Exit(1) if not info: @@ -932,7 +935,10 @@ def _install_workflow_from_catalog( if isinstance(existing, dict) and existing.get("enabled", True) is False: entry["enabled"] = False registry.add(workflow_id, entry) - console.print(f"[green]✓[/green] Workflow '{info.get('name', workflow_id)}' installed from catalog") + console.print( + f"[green]✓[/green] Workflow '{_escape_markup(str(info.get('name', workflow_id)))}' " + "installed from catalog" + ) @workflow_app.command("remove") @@ -1047,7 +1053,7 @@ def workflow_update( try: info = catalog.get_workflow_info(wf_id) except WorkflowCatalogError as exc: - console.print(f"[red]Error:[/red] {exc}") + console.print(f"[red]Error:[/red] {_escape_markup(str(exc))}") raise typer.Exit(1) if not info: console.print(f"⚠ {safe_id}: Not found in catalog (skipping)") @@ -1190,7 +1196,7 @@ def workflow_search( try: results = catalog.search(query=query, tag=tag, author=author) except WorkflowCatalogError as exc: - console.print(f"[red]Error:[/red] {exc}") + console.print(f"[red]Error:[/red] {_escape_markup(str(exc))}") raise typer.Exit(1) if not results: From a5307256ca9c1406f787e99eb7f277cb9f6cfd91 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 23:04:29 +0200 Subject: [PATCH 12/59] fix(workflows): fail cleanly on unparseable catalog install URLs urlparse raises ValueError on e.g. an unbalanced IPv6 literal before the invalid-URL branch is reached; on workflow update that also bypassed the per-workflow handler and aborted the whole command. Convert the parse failure into a clean error so add fails cleanly and update skips just the affected workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 10 +++++-- tests/test_workflows.py | 41 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 36fb1b135a..062b85116a 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -825,8 +825,14 @@ def _install_workflow_from_catalog( from ipaddress import ip_address from urllib.parse import urlparse - parsed_url = urlparse(workflow_url) - url_host = parsed_url.hostname or "" + try: + parsed_url = urlparse(workflow_url) + url_host = parsed_url.hostname or "" + except ValueError: + console.print( + f"[red]Error:[/red] Workflow '{safe_wf_id}' has a malformed install URL." + ) + raise typer.Exit(1) is_loopback = False if url_host == "localhost": is_loopback = True diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 96578d1e0f..96c2449d58 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7619,6 +7619,47 @@ def test_update_downloaded_invalid_yaml_escapes_rich_markup(self, project_dir, m # The previously installed workflow must survive a failed update. assert "1.0.0" in (wf_dir / "workflow.yml").read_text(encoding="utf-8") + def test_update_malformed_catalog_url_fails_cleanly(self, project_dir, monkeypatch): + """An unparseable catalog URL (unbalanced IPv6 literal) must not abort the whole update.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + registry = WorkflowRegistry(project_dir) + registry.add("align-wf", { + "name": "Align Workflow", + "version": "1.0.0", + "description": "CLI alignment test workflow", + "source": "catalog", + "catalog_name": "test-catalog", + "url": "https://[::1/workflow.yml", + }) + wf_dir = project_dir / ".specify" / "workflows" / "align-wf" + wf_dir.mkdir(parents=True) + (wf_dir / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "2.0.0", + "url": "https://[::1/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "update"], input="y\n") + assert "malformed install URL" in result.output + assert "Failed to update" in result.output + # The previously installed workflow must survive. + assert "1.0.0" in (wf_dir / "workflow.yml").read_text(encoding="utf-8") + def test_update_preserves_disabled_state(self, project_dir, monkeypatch): from unittest.mock import patch from typer.testing import CliRunner From 560c7cdd5fbd340c63303bcc4ebd4809534f4517 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 23:13:55 +0200 Subject: [PATCH 13/59] fix(workflows): reject catalog updates whose downloaded version mismatches The update path never verified the downloaded workflow carries the catalog version that triggered the update, so a stale or misconfigured URL could report success while leaving the old version installed or downgrading it. Pass the expected version into the install helper and fail the update when the downloaded definition does not match. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 27 ++++++++++++++- tests/test_workflows.py | 48 ++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 062b85116a..5b48aed0aa 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -789,11 +789,14 @@ def _install_workflow_from_catalog( registry: Any, workflows_dir: Path, workflow_id: str, + expected_version: str | None = None, ) -> None: """Download, validate, and register a catalog workflow. Shared by ``workflow add`` and ``workflow update``. Raises ``typer.Exit`` on any failure; the registry entry is only written on full success. + ``expected_version``, when given, rejects a downloaded workflow whose + version does not match the catalog version that triggered the install. """ from .catalog import WorkflowCatalog, WorkflowCatalogError from .engine import WorkflowDefinition @@ -928,6 +931,25 @@ def _install_workflow_from_catalog( ) raise typer.Exit(1) + # A stale or misconfigured URL can serve a different version than the + # catalog advertised; without this check `update` would report success + # while leaving the old version installed (or even downgrading). + if expected_version is not None: + from packaging import version as pkg_version + try: + version_matches = pkg_version.Version(str(definition.version)) == pkg_version.Version(expected_version) + except pkg_version.InvalidVersion: + version_matches = str(definition.version) == expected_version + if not version_matches: + import shutil + shutil.rmtree(workflow_dir, ignore_errors=True) + console.print( + f"[red]Error:[/red] Downloaded workflow version ({_escape_markup(str(definition.version))}) " + f"does not match the catalog version ({_escape_markup(expected_version)}). " + f"The catalog entry may be stale or misconfigured." + ) + raise typer.Exit(1) + entry = { "name": definition.name or info.get("name", workflow_id), "version": definition.version or info.get("version", "0.0.0"), @@ -1109,7 +1131,10 @@ def workflow_update( wf_dir = _safe_workflow_id_dir(workflows_dir, update["id"]) wf_file = wf_dir / "workflow.yml" backup = wf_file.read_bytes() if wf_file.is_file() else None - _install_workflow_from_catalog(project_root, registry, workflows_dir, update["id"]) + _install_workflow_from_catalog( + project_root, registry, workflows_dir, update["id"], + expected_version=update["available"], + ) except (typer.Exit, OSError) as exc: if backup is not None and wf_dir is not None and wf_file is not None: try: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 96c2449d58..0725db8d9f 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7660,6 +7660,54 @@ def test_update_malformed_catalog_url_fails_cleanly(self, project_dir, monkeypat # The previously installed workflow must survive. assert "1.0.0" in (wf_dir / "workflow.yml").read_text(encoding="utf-8") + def test_update_rejects_version_mismatch_from_stale_url(self, project_dir, monkeypatch): + """A URL serving a different version than the catalog advertised must fail the update.""" + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + WorkflowRegistry(project_dir).add("align-wf", { + "name": "Align Workflow", + "version": "1.0.0", + "description": "CLI alignment test workflow", + "source": "catalog", + "catalog_name": "test-catalog", + "url": "https://example.com/workflow.yml", + }) + wf_dir = project_dir / ".specify" / "workflows" / "align-wf" + wf_dir.mkdir(parents=True) + (wf_dir / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + # The URL still serves the old 1.0.0 payload. + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(data, url), + ): + result = runner.invoke(app, ["workflow", "update"], input="y\n") + assert "does not match the catalog version" in result.output + assert "Failed to update" in result.output + meta = WorkflowRegistry(project_dir).get("align-wf") + assert meta["version"] == "1.0.0" + assert "1.0.0" in (wf_dir / "workflow.yml").read_text(encoding="utf-8") + def test_update_preserves_disabled_state(self, project_dir, monkeypatch): from unittest.mock import patch from typer.testing import CliRunner From e44d21436f8a41a668b703031d0956440afa14e6 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 23:24:12 +0200 Subject: [PATCH 14/59] fix(workflows): validate workflow ID in run command and document new CLI flags Path-equivalent spellings like "align-wf/" previously bypassed the registry disabled check because the engine normalizes the path while the registry matches the raw string. workflow run now validates non-file sources against the workflow ID pattern before lookup. Also updates docs/reference/workflows.md with --dev/--from install options, update/enable/disable commands, and the search --author flag. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/reference/workflows.md | 29 +++++++++++++++++++++++--- src/specify_cli/workflows/_commands.py | 9 ++++++++ tests/test_workflows.py | 17 +++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/docs/reference/workflows.md b/docs/reference/workflows.md index 16bbe0893e..c80ea7cd3c 100644 --- a/docs/reference/workflows.md +++ b/docs/reference/workflows.md @@ -86,8 +86,30 @@ Lists workflows installed in the current project. specify workflow add ``` +| Option | Description | +| -------- | ------------------------------------------------------ | +| `--dev` | Install from a local workflow YAML file or directory | +| `--from` | Install from a custom URL (`` names the expected workflow ID) | + Installs a workflow from the catalog, a URL (HTTPS required), or a local file path. +## Update Workflows + +```bash +specify workflow update [workflow_id] +``` + +Updates one installed catalog workflow — or all of them when no ID is given — to the latest catalog version. Prompts for confirmation and keeps the installed copy if a download or validation fails. + +## Enable or Disable a Workflow + +```bash +specify workflow enable +specify workflow disable +``` + +Disabled workflows stay installed and listed (marked `[disabled]`) but refuse to run until re-enabled. + ## Remove a Workflow ```bash @@ -102,9 +124,10 @@ Removes an installed workflow from the project. specify workflow search [query] ``` -| Option | Description | -| ------- | --------------- | -| `--tag` | Filter by tag | +| Option | Description | +| ---------- | ----------------- | +| `--tag` | Filter by tag | +| `--author` | Filter by author | Searches all active catalogs for workflows matching the query. diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 5b48aed0aa..be6e766c5f 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -352,6 +352,15 @@ def workflow_run( if not is_file_source: from .catalog import WorkflowRegistry + # Reject path-equivalent spellings ("align-wf/", "align-wf/.") that + # would miss the registry lookup yet still load the installed file, + # bypassing the disabled check below. + if source in _RESERVED_WORKFLOW_IDS or not _WORKFLOW_ID_PATTERN.match(source): + err.print( + f"[red]Error:[/red] Invalid workflow ID: {_escape_markup(repr(source))}" + ) + raise typer.Exit(1) + installed_meta = WorkflowRegistry(project_root).get(source) if isinstance(installed_meta, dict) and installed_meta.get("enabled", True) is False: err.print( diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 0725db8d9f..c6ef0191ce 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8023,6 +8023,23 @@ def test_disable_blocks_run_enable_restores(self, project_dir, monkeypatch): result = runner.invoke(app, ["workflow", "run", "align-wf"]) assert result.exit_code == 0, result.output + def test_disable_blocks_run_via_path_equivalent_id(self, project_dir, monkeypatch): + """"align-wf/" must not run a disabled workflow by dodging the registry lookup.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + + result = runner.invoke(app, ["workflow", "disable", "align-wf"]) + assert result.exit_code == 0, result.output + + for spelling in ("align-wf/", "align-wf/."): + result = runner.invoke(app, ["workflow", "run", spelling]) + assert result.exit_code != 0, spelling + assert "Invalid workflow ID" in result.output, spelling + def test_disable_shows_marker_in_list(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app From 8efa3c02dca5eb1bb4498d62fe81c4dcd1e26518 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 23:33:50 +0200 Subject: [PATCH 15/59] fix(workflows): enforce disabled state for direct paths to installed workflows Running the installed copy's YAML directly (specify workflow run .specify/workflows/align-wf/workflow.yml) skipped the registry check. File sources resolving inside .specify/workflows// now map back to the workflow ID and refuse to run while disabled. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 24 ++++++++++++++++++------ tests/test_workflows.py | 7 +++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index be6e766c5f..2ef5146436 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -349,9 +349,10 @@ def workflow_run( err = _error_console(json_output) - if not is_file_source: - from .catalog import WorkflowRegistry + from .catalog import WorkflowRegistry + registered_id: str | None = None + if not is_file_source: # Reject path-equivalent spellings ("align-wf/", "align-wf/.") that # would miss the registry lookup yet still load the installed file, # bypassing the disabled check below. @@ -360,12 +361,23 @@ def workflow_run( f"[red]Error:[/red] Invalid workflow ID: {_escape_markup(repr(source))}" ) raise typer.Exit(1) - - installed_meta = WorkflowRegistry(project_root).get(source) + registered_id = source + else: + # A direct YAML path may still point at an installed workflow's own + # file; map it back to its ID so disabled state is enforced there too. + workflows_root = (project_root / ".specify" / "workflows").resolve() + resolved = source_path.resolve() + if resolved.is_relative_to(workflows_root): + rel_parts = resolved.relative_to(workflows_root).parts + if rel_parts: + registered_id = rel_parts[0] + + if registered_id is not None: + installed_meta = WorkflowRegistry(project_root).get(registered_id) if isinstance(installed_meta, dict) and installed_meta.get("enabled", True) is False: err.print( - f"[red]Error:[/red] Workflow '{_escape_markup(source)}' is disabled. " - f"Enable with: specify workflow enable {_escape_markup(source)}" + f"[red]Error:[/red] Workflow '{_escape_markup(registered_id)}' is disabled. " + f"Enable with: specify workflow enable {_escape_markup(registered_id)}" ) raise typer.Exit(1) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index c6ef0191ce..2277db359b 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8040,6 +8040,13 @@ def test_disable_blocks_run_via_path_equivalent_id(self, project_dir, monkeypatc assert result.exit_code != 0, spelling assert "Invalid workflow ID" in result.output, spelling + # Direct path to the installed workflow's own YAML must also refuse. + installed_yaml = ".specify/workflows/align-wf/workflow.yml" + assert (project_dir / installed_yaml).is_file() + result = runner.invoke(app, ["workflow", "run", installed_yaml]) + assert result.exit_code != 0 + assert "disabled" in result.output + def test_disable_shows_marker_in_list(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app From 96d3b381e9890b466e52d20bec71001d045104e9 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 23:43:08 +0200 Subject: [PATCH 16/59] fix(workflows): reject explicit empty --from URL instead of catalog fallback 'workflow add foo --from ""' fell through 'from_url or ...' to a catalog install. Distinguish None from empty string so explicit values stay on the URL-validation path and fail closed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 6 ++++-- tests/test_workflows.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 2ef5146436..c517bc6208 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -713,8 +713,10 @@ def _validate_and_install_local( # Try as URL (http/https) — either the positional source is a URL, or an # explicit --from URL names where to fetch it (mirrors `extension add --from`). - download_url = from_url or ( - source if source.startswith(("http://", "https://")) else None + download_url = ( + from_url + if from_url is not None + else (source if source.startswith(("http://", "https://")) else None) ) if download_url is not None: from ipaddress import ip_address diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 2277db359b..7554deb0d4 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7394,6 +7394,19 @@ def test_add_from_url_id_mismatch_errors(self, project_dir, monkeypatch): assert "does not match" in result.output assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_from_empty_url_rejected_not_catalog_fallback(self, project_dir, monkeypatch): + """--from "" must fail URL validation, not silently install from the catalog.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "add", "align-wf", "--from", ""]) + assert result.exit_code != 0 + assert "HTTPS" in result.output + assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_from_url_non_https_redirect_escapes_rich_markup(self, project_dir, monkeypatch): """A redirect to a non-HTTPS IPv6 literal (legally bracketed) must not be parsed as Rich markup.""" from unittest.mock import patch From f28e125338352d0bbdb72f3bd3e5966fce973e6f Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Fri, 10 Jul 2026 23:54:37 +0200 Subject: [PATCH 17/59] fix(workflows): registry rollback on save failure, consistent disabled check, honest update summary - WorkflowRegistry.add now rolls back its in-memory mutation when save() raises, so a later successful save cannot persist metadata for a failed update alongside the restored YAML backup. - workflow run uses the same truthiness check for 'enabled' as list and disable, so malformed values like 0 or null refuse to run. - workflow update reports 'No workflows were eligible for update' when every target was skipped instead of claiming all are up to date. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 12 ++++++-- src/specify_cli/workflows/catalog.py | 12 +++++++- tests/test_workflows.py | 41 ++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index c517bc6208..691fa8945c 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -374,7 +374,7 @@ def workflow_run( if registered_id is not None: installed_meta = WorkflowRegistry(project_root).get(registered_id) - if isinstance(installed_meta, dict) and installed_meta.get("enabled", True) is False: + if isinstance(installed_meta, dict) and not installed_meta.get("enabled", True): err.print( f"[red]Error:[/red] Workflow '{_escape_markup(registered_id)}' is disabled. " f"Enable with: specify workflow enable {_escape_markup(registered_id)}" @@ -983,7 +983,7 @@ def _install_workflow_from_catalog( } # Preserve a prior disabled state across updates/reinstalls. existing = registry.get(workflow_id) - if isinstance(existing, dict) and existing.get("enabled", True) is False: + if isinstance(existing, dict) and not existing.get("enabled", True): entry["enabled"] = False registry.add(workflow_id, entry) console.print( @@ -1085,6 +1085,7 @@ def workflow_update( console.print("🔄 Checking for updates...\n") updates_available: list[dict[str, str]] = [] + checked = 0 for wf_id in targets: safe_id = _escape_markup(str(wf_id)) metadata = installed.get(wf_id) @@ -1122,14 +1123,19 @@ def workflow_update( ) continue if catalog_version > installed_version: + checked += 1 updates_available.append( {"id": wf_id, "installed": str(installed_version), "available": str(catalog_version)} ) else: + checked += 1 console.print(f"✓ {safe_id}: Up to date (v{installed_version})") if not updates_available: - console.print("\n[green]All workflows are up to date![/green]") + if checked: + console.print("\n[green]All workflows are up to date![/green]") + else: + console.print("\n[yellow]No workflows were eligible for update[/yellow]") raise typer.Exit(0) console.print("\n[bold]Updates available:[/bold]\n") diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index d8855a06aa..203fb6753b 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -93,12 +93,22 @@ def add(self, workflow_id: str, metadata: dict[str, Any]) -> None: from datetime import datetime, timezone existing = self.data["workflows"].get(workflow_id, {}) + had_entry = workflow_id in self.data["workflows"] metadata["installed_at"] = existing.get( "installed_at", datetime.now(timezone.utc).isoformat() ) metadata["updated_at"] = datetime.now(timezone.utc).isoformat() self.data["workflows"][workflow_id] = metadata - self.save() + try: + self.save() + except OSError: + # Roll back the in-memory mutation so a later successful save + # cannot persist metadata for a write that failed. + if had_entry: + self.data["workflows"][workflow_id] = existing + else: + del self.data["workflows"][workflow_id] + raise def remove(self, workflow_id: str) -> bool: """Remove an installed workflow entry. Returns True if found.""" diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 7554deb0d4..083e2638e3 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7533,6 +7533,47 @@ def test_update_skips_non_catalog_sources(self, project_dir, monkeypatch): result = runner.invoke(app, ["workflow", "update"]) assert result.exit_code == 0, result.output assert "re-add to update" in result.output + # Every target was skipped — must not claim everything is up to date. + assert "No workflows were eligible for update" in result.output + assert "up to date!" not in result.output + + def test_registry_add_rolls_back_memory_on_save_failure(self, project_dir, monkeypatch): + from specify_cli.workflows.catalog import WorkflowRegistry + + registry = WorkflowRegistry(project_dir) + registry.add("align-wf", {"version": "1.0.0", "source": "catalog"}) + + def boom(): + raise OSError("disk full") + + monkeypatch.setattr(registry, "save", boom) + with pytest.raises(OSError): + registry.add("align-wf", {"version": "2.0.0", "source": "catalog"}) + assert registry.get("align-wf")["version"] == "1.0.0" + + with pytest.raises(OSError): + registry.add("other-wf", {"version": "1.0.0", "source": "catalog"}) + assert registry.get("other-wf") is None + + def test_run_refuses_falsy_non_bool_enabled(self, project_dir, monkeypatch): + """"enabled": 0 shows as disabled in list — run must agree.""" + import json as json_mod + + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + + registry = WorkflowRegistry(project_dir) + registry.data["workflows"]["align-wf"]["enabled"] = 0 + registry.registry_path.write_text(json_mod.dumps(registry.data), encoding="utf-8") + + result = runner.invoke(app, ["workflow", "run", "align-wf"]) + assert result.exit_code != 0 + assert "disabled" in result.output def test_update_installs_newer_catalog_version(self, project_dir, monkeypatch): from unittest.mock import patch From 338f7eecb0edaae22becedccd3273425e1bc9d13 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:05:26 +0200 Subject: [PATCH 18/59] fix(workflows): guard non-string catalog URL and keep enable/disable rollback intact - A truthy non-string catalog url (e.g. 123) reached urlparse and raised AttributeError, escaping the clean error path; validate it is a string. - enable/disable mutated the live registry entry before add(), so add's rollback snapshot captured the already-toggled object; pass a fresh mapping instead. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 15 +++++--- tests/test_workflows.py | 49 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 691fa8945c..6ce5ceb274 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -846,6 +846,12 @@ def _install_workflow_from_catalog( if not workflow_url: console.print(f"[red]Error:[/red] Workflow '{safe_wf_id}' does not have an install URL in the catalog") raise typer.Exit(1) + if not isinstance(workflow_url, str): + # Untrusted catalog payload; a non-string would crash urlparse below. + console.print( + f"[red]Error:[/red] Workflow '{safe_wf_id}' has a malformed install URL." + ) + raise typer.Exit(1) # Validate URL scheme (HTTPS required, HTTP allowed for localhost only) from ipaddress import ip_address @@ -1209,8 +1215,9 @@ def workflow_enable( if metadata.get("enabled", True): console.print(f"[yellow]Workflow '{_escape_markup(workflow_id)}' is already enabled[/yellow]") raise typer.Exit(0) - metadata["enabled"] = True - registry.add(workflow_id, metadata) + # Fresh mapping: registry.get() returns the live entry, and mutating it + # in place would defeat WorkflowRegistry.add's rollback-on-save-failure. + registry.add(workflow_id, {**metadata, "enabled": True}) console.print(f"[green]✓[/green] Workflow '{_escape_markup(workflow_id)}' enabled") @@ -1235,8 +1242,8 @@ def workflow_disable( if not metadata.get("enabled", True): console.print(f"[yellow]Workflow '{_escape_markup(workflow_id)}' is already disabled[/yellow]") raise typer.Exit(0) - metadata["enabled"] = False - registry.add(workflow_id, metadata) + # Fresh mapping for the same rollback reason as workflow_enable. + registry.add(workflow_id, {**metadata, "enabled": False}) console.print(f"[green]✓[/green] Workflow '{_escape_markup(workflow_id)}' disabled") console.print(f"To re-enable: specify workflow enable {_escape_markup(workflow_id)}") diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 083e2638e3..8514017c62 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7714,6 +7714,55 @@ def test_update_malformed_catalog_url_fails_cleanly(self, project_dir, monkeypat # The previously installed workflow must survive. assert "1.0.0" in (wf_dir / "workflow.yml").read_text(encoding="utf-8") + def test_add_non_string_catalog_url_fails_cleanly(self, project_dir, monkeypatch): + """A truthy non-string catalog URL must hit the clean error path, not AttributeError.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog + + monkeypatch.chdir(project_dir) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": 123, + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "malformed install URL" in result.output + + def test_enable_failed_save_leaves_workflow_disabled(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + result = runner.invoke(app, ["workflow", "disable", "align-wf"]) + assert result.exit_code == 0, result.output + + def boom(self): + raise OSError("disk full") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(WorkflowRegistry, "save", boom) + result = runner.invoke(app, ["workflow", "enable", "align-wf"]) + assert result.exit_code != 0 + + assert WorkflowRegistry(project_dir).get("align-wf")["enabled"] is False + result = runner.invoke(app, ["workflow", "enable", "align-wf"]) + assert result.exit_code == 0, result.output + assert WorkflowRegistry(project_dir).get("align-wf")["enabled"] is True + def test_update_rejects_version_mismatch_from_stale_url(self, project_dir, monkeypatch): """A URL serving a different version than the catalog advertised must fail the update.""" from unittest.mock import patch From 82ecd053315bc236da9c667fa0b15508b8700b61 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:14:40 +0200 Subject: [PATCH 19/59] fix(workflows): tolerate non-dict registry entries in add and clarify test docstrings A corrupted-but-parseable registry entry (e.g. a string value) crashed WorkflowRegistry.add with AttributeError on existing.get. Guard the non-dict case while still restoring the original raw value on rollback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/catalog.py | 6 ++++-- tests/test_workflows.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 203fb6753b..0ab5d81c55 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -92,8 +92,10 @@ def add(self, workflow_id: str, metadata: dict[str, Any]) -> None: """Add or update an installed workflow entry.""" from datetime import datetime, timezone - existing = self.data["workflows"].get(workflow_id, {}) + raw_existing = self.data["workflows"].get(workflow_id) had_entry = workflow_id in self.data["workflows"] + # Corrupted-but-parseable registries may hold non-dict entries. + existing = raw_existing if isinstance(raw_existing, dict) else {} metadata["installed_at"] = existing.get( "installed_at", datetime.now(timezone.utc).isoformat() ) @@ -105,7 +107,7 @@ def add(self, workflow_id: str, metadata: dict[str, Any]) -> None: # Roll back the in-memory mutation so a later successful save # cannot persist metadata for a write that failed. if had_entry: - self.data["workflows"][workflow_id] = existing + self.data["workflows"][workflow_id] = raw_existing else: del self.data["workflows"][workflow_id] raise diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 8514017c62..e773f3b607 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7555,8 +7555,16 @@ def boom(): registry.add("other-wf", {"version": "1.0.0", "source": "catalog"}) assert registry.get("other-wf") is None + def test_registry_add_survives_non_dict_existing_entry(self, project_dir): + from specify_cli.workflows.catalog import WorkflowRegistry + + registry = WorkflowRegistry(project_dir) + registry.data["workflows"]["align-wf"] = "corrupted" + registry.add("align-wf", {"version": "1.0.0", "source": "catalog"}) + assert registry.get("align-wf")["version"] == "1.0.0" + def test_run_refuses_falsy_non_bool_enabled(self, project_dir, monkeypatch): - """"enabled": 0 shows as disabled in list — run must agree.""" + """A falsy non-bool "enabled" (0) shows as disabled in list — run must agree.""" import json as json_mod from typer.testing import CliRunner @@ -8127,7 +8135,7 @@ def test_disable_blocks_run_enable_restores(self, project_dir, monkeypatch): assert result.exit_code == 0, result.output def test_disable_blocks_run_via_path_equivalent_id(self, project_dir, monkeypatch): - """"align-wf/" must not run a disabled workflow by dodging the registry lookup.""" + """Path spelling "align-wf/" must not run a disabled workflow by dodging the registry lookup.""" from typer.testing import CliRunner from specify_cli import app From 6b25c75a05ed1c717fc171a6b0ea42bcc21b46c3 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:23:49 +0200 Subject: [PATCH 20/59] fix(workflows): atomic registry save and accurate mixed-target update summary - save() wrote the registry with open('w'), so a failed dump truncated the file and the next load reset every entry. Write to a sibling temp file and os.replace into place. - workflow update no longer claims all workflows are up to date when some targets were skipped; it reports checked-only status with a skipped count. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 9 ++++- src/specify_cli/workflows/catalog.py | 16 ++++++-- tests/test_workflows.py | 53 ++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 6ce5ceb274..973e69fe06 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -1138,10 +1138,15 @@ def workflow_update( console.print(f"✓ {safe_id}: Up to date (v{installed_version})") if not updates_available: - if checked: + if not checked: + console.print("\n[yellow]No workflows were eligible for update[/yellow]") + elif checked == len(targets): console.print("\n[green]All workflows are up to date![/green]") else: - console.print("\n[yellow]No workflows were eligible for update[/yellow]") + console.print( + f"\n[green]All checked workflows are up to date[/green] " + f"[yellow]({len(targets) - checked} skipped)[/yellow]" + ) raise typer.Exit(0) console.print("\n[bold]Updates available:[/bold]\n") diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 0ab5d81c55..942a6cfd0c 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -83,10 +83,20 @@ def _load(self) -> dict[str, Any]: return {"schema_version": self.SCHEMA_VERSION, "workflows": {}} def save(self) -> None: - """Persist registry to disk.""" + """Persist registry to disk atomically.""" self.workflows_dir.mkdir(parents=True, exist_ok=True) - with open(self.registry_path, "w", encoding="utf-8") as f: - json.dump(self.data, f, indent=2) + # Write-then-replace so a failed dump cannot truncate the registry. + tmp_path = self.registry_path.with_name(self.registry_path.name + ".tmp") + try: + with open(tmp_path, "w", encoding="utf-8") as f: + json.dump(self.data, f, indent=2) + os.replace(tmp_path, self.registry_path) + except OSError: + try: + tmp_path.unlink() + except OSError: + pass + raise def add(self, workflow_id: str, metadata: dict[str, Any]) -> None: """Add or update an installed workflow entry.""" diff --git a/tests/test_workflows.py b/tests/test_workflows.py index e773f3b607..77589fb611 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7563,6 +7563,59 @@ def test_registry_add_survives_non_dict_existing_entry(self, project_dir): registry.add("align-wf", {"version": "1.0.0", "source": "catalog"}) assert registry.get("align-wf")["version"] == "1.0.0" + def test_registry_save_failure_preserves_file_on_disk(self, project_dir, monkeypatch): + """A failed dump must not truncate the persisted registry.""" + from specify_cli.workflows.catalog import WorkflowRegistry + + registry = WorkflowRegistry(project_dir) + registry.add("align-wf", {"version": "1.0.0", "source": "catalog"}) + + import specify_cli.workflows.catalog as catalog_mod + + def boom(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr(catalog_mod.json, "dump", boom) + with pytest.raises(OSError): + registry.add("align-wf", {"version": "2.0.0", "source": "catalog"}) + monkeypatch.undo() + + fresh = WorkflowRegistry(project_dir) + assert fresh.get("align-wf")["version"] == "1.0.0" + assert not list(registry.workflows_dir.glob("*.tmp")) + + def test_update_mixed_targets_does_not_claim_all_up_to_date(self, project_dir, monkeypatch): + """Skipped targets must not be presented as verified up to date.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) # local source → skipped + WorkflowRegistry(project_dir).add("catalog-wf", { + "name": "Catalog Workflow", + "version": "1.0.0", + "description": "", + "source": "catalog", + "url": "https://example.com/workflow.yml", + }) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + }, + ) + result = runner.invoke(app, ["workflow", "update"]) + assert result.exit_code == 0, result.output + assert "All workflows are up to date!" not in result.output + assert "All checked workflows are up to date" in result.output + assert "skipped" in result.output + def test_run_refuses_falsy_non_bool_enabled(self, project_dir, monkeypatch): """A falsy non-bool "enabled" (0) shows as disabled in list — run must agree.""" import json as json_mod From eabce4efe1d2ac2b6a1b85a1bd50c943f77d3312 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:34:05 +0200 Subject: [PATCH 21/59] fix(workflows): exclusive temp file for registry save and cwd-independent disabled guard - save() now uses tempfile.mkstemp in the workflows dir (matching the engine's atomic writer), so a pre-created symlink at a predictable .tmp path cannot redirect the write and concurrent processes cannot collide. - The direct-path disabled guard derives the owning project from the resolved file path instead of the caller's cwd, so running an installed workflow's YAML from outside the project still refuses when disabled. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 17 ++++++++++------- src/specify_cli/workflows/catalog.py | 19 +++++++++++++------ tests/test_workflows.py | 10 ++++++++++ 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 973e69fe06..334c7e2541 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -352,6 +352,7 @@ def workflow_run( from .catalog import WorkflowRegistry registered_id: str | None = None + registry_root = project_root if not is_file_source: # Reject path-equivalent spellings ("align-wf/", "align-wf/.") that # would miss the registry lookup yet still load the installed file, @@ -364,16 +365,18 @@ def workflow_run( registered_id = source else: # A direct YAML path may still point at an installed workflow's own - # file; map it back to its ID so disabled state is enforced there too. - workflows_root = (project_root / ".specify" / "workflows").resolve() + # file; map it back to its owning project and ID from the canonical + # path itself so the guard is independent of the caller's cwd. resolved = source_path.resolve() - if resolved.is_relative_to(workflows_root): - rel_parts = resolved.relative_to(workflows_root).parts - if rel_parts: - registered_id = rel_parts[0] + parts = resolved.parts + for i in range(len(parts) - 2): + if parts[i] == ".specify" and parts[i + 1] == "workflows": + registry_root = Path(*parts[:i]) if i else Path(resolved.anchor or ".") + registered_id = parts[i + 2] + break if registered_id is not None: - installed_meta = WorkflowRegistry(project_root).get(registered_id) + installed_meta = WorkflowRegistry(registry_root).get(registered_id) if isinstance(installed_meta, dict) and not installed_meta.get("enabled", True): err.print( f"[red]Error:[/red] Workflow '{_escape_markup(registered_id)}' is disabled. " diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 942a6cfd0c..87fe1ca0b4 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -13,6 +13,7 @@ import hashlib import json import os +import tempfile import time from dataclasses import dataclass from pathlib import Path @@ -85,15 +86,21 @@ def _load(self) -> dict[str, Any]: def save(self) -> None: """Persist registry to disk atomically.""" self.workflows_dir.mkdir(parents=True, exist_ok=True) - # Write-then-replace so a failed dump cannot truncate the registry. - tmp_path = self.registry_path.with_name(self.registry_path.name + ".tmp") + # Unique, exclusive temp then replace: a failed dump cannot truncate + # the registry, a pre-created symlink cannot redirect the write, and + # concurrent CLI processes cannot collide on the same temp path. + fd, tmp = tempfile.mkstemp( + dir=str(self.registry_path.parent), + prefix=f".{self.registry_path.name}.", + suffix=".tmp", + ) try: - with open(tmp_path, "w", encoding="utf-8") as f: + with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(self.data, f, indent=2) - os.replace(tmp_path, self.registry_path) - except OSError: + os.replace(tmp, self.registry_path) + except BaseException: try: - tmp_path.unlink() + os.unlink(tmp) except OSError: pass raise diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 77589fb611..810a94dc31 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8211,6 +8211,16 @@ def test_disable_blocks_run_via_path_equivalent_id(self, project_dir, monkeypatc assert result.exit_code != 0 assert "disabled" in result.output + # Same guard must hold when invoked from outside the project. + outside = project_dir.parent / "outside-cwd" + outside.mkdir(exist_ok=True) + monkeypatch.chdir(outside) + result = runner.invoke( + app, ["workflow", "run", str(project_dir / installed_yaml)] + ) + assert result.exit_code != 0 + assert "disabled" in result.output + def test_disable_shows_marker_in_list(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app From cac5300f89c8bed41198591edd88a0a50cc75c2f Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:43:11 +0200 Subject: [PATCH 22/59] fix(workflows): symlink guards and shape validation in workflow registry, dev-dir file check - WorkflowRegistry now mirrors StepRegistry: _load refuses symlinked parents/registry file and normalizes a non-dict workflows field; save() rejects symlinked paths before writing. - workflow add --dev requires workflow.yml to be a regular file so a directory named workflow.yml gets the documented CLI error instead of an uncaught IsADirectoryError. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 2 +- src/specify_cli/workflows/catalog.py | 37 ++++++++++++++++++++++--- tests/test_workflows.py | 38 ++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 334c7e2541..03d5a4b8a7 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -703,7 +703,7 @@ def _validate_and_install_local( return if dev_path.is_dir(): dev_wf_file = dev_path / "workflow.yml" - if not dev_wf_file.exists(): + if not dev_wf_file.is_file(): console.print(f"[red]Error:[/red] No workflow.yml found in {_escape_markup(source)}") raise typer.Exit(1) _validate_and_install_local(dev_wf_file, str(dev_path)) diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 87fe1ca0b4..092b4b6bf1 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -72,19 +72,48 @@ def __init__(self, project_root: Path) -> None: self.registry_path = self.workflows_dir / self.REGISTRY_FILE self.data = self._load() + def _has_symlinked_parent(self) -> bool: + """Return True if any directory under .specify/workflows is a symlink.""" + current = self.project_root + for part in (".specify", "workflows"): + current = current / part + if current.is_symlink(): + return True + return False + def _load(self) -> dict[str, Any]: """Load registry from disk or create default.""" + default_registry: dict[str, Any] = { + "schema_version": self.SCHEMA_VERSION, + "workflows": {}, + } + # Defense-in-depth: refuse to read through symlinked parents or a + # symlinked registry file (mirrors StepRegistry._load). + if self._has_symlinked_parent() or self.registry_path.is_symlink(): + return default_registry if self.registry_path.exists(): try: with open(self.registry_path, encoding="utf-8") as f: - return json.load(f) - except (json.JSONDecodeError, ValueError): + data = json.load(f) + # Validate shape: must be a dict with a dict "workflows" field. + if not isinstance(data, dict): + return default_registry + if not isinstance(data.get("workflows"), dict): + data["workflows"] = {} + return data + except (json.JSONDecodeError, ValueError, OSError, UnicodeError): # Corrupted registry file — reset to default - return {"schema_version": self.SCHEMA_VERSION, "workflows": {}} - return {"schema_version": self.SCHEMA_VERSION, "workflows": {}} + return default_registry + return default_registry def save(self) -> None: """Persist registry to disk atomically.""" + # Refuse to write through symlinked parents (mirrors StepRegistry.save + # and the CLI-level _reject_unsafe_dir guard). + if self._has_symlinked_parent() or self.registry_path.is_symlink(): + raise OSError( + "Refusing to write workflow registry through a symlinked path." + ) self.workflows_dir.mkdir(parents=True, exist_ok=True) # Unique, exclusive temp then replace: a failed dump cannot truncate # the registry, a pre-created symlink cannot redirect the write, and diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 810a94dc31..d8c702b18d 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7563,6 +7563,44 @@ def test_registry_add_survives_non_dict_existing_entry(self, project_dir): registry.add("align-wf", {"version": "1.0.0", "source": "catalog"}) assert registry.get("align-wf")["version"] == "1.0.0" + def test_registry_load_normalizes_malformed_workflows_field(self, project_dir): + from specify_cli.workflows.catalog import WorkflowRegistry + + registry = WorkflowRegistry(project_dir) + registry.workflows_dir.mkdir(parents=True, exist_ok=True) + registry.registry_path.write_text('{"workflows": "broken"}', encoding="utf-8") + fresh = WorkflowRegistry(project_dir) + assert fresh.get("anything") is None + fresh.add("align-wf", {"version": "1.0.0", "source": "catalog"}) + assert fresh.get("align-wf")["version"] == "1.0.0" + + def test_registry_save_refuses_symlinked_parent(self, project_dir, tmp_path): + from specify_cli.workflows.catalog import WorkflowRegistry + + outside = tmp_path / "outside-specify" + outside.mkdir() + specify_dir = project_dir / ".specify" + if specify_dir.exists(): + shutil.rmtree(specify_dir) + specify_dir.symlink_to(outside) + registry = WorkflowRegistry(project_dir) + with pytest.raises(OSError, match="symlink"): + registry.add("align-wf", {"version": "1.0.0", "source": "catalog"}) + assert not (outside / "workflows").exists() + + def test_add_dev_dir_with_workflow_yml_directory_errors_cleanly(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + dev_dir = project_dir / "dev-wf" + (dev_dir / "workflow.yml").mkdir(parents=True) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "add", "--dev", str(dev_dir)]) + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "No workflow.yml found" in result.output + def test_registry_save_failure_preserves_file_on_disk(self, project_dir, monkeypatch): """A failed dump must not truncate the persisted registry.""" from specify_cli.workflows.catalog import WorkflowRegistry From a69bb180a118c7bb33a7182e89f8b970a8191139 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:54:27 +0200 Subject: [PATCH 23/59] fix(workflows): validate download redirects before following them All three workflow download sites (add --from, catalog install, step install) passed no redirect_validator to open_url, so an HTTPS URL redirecting to cleartext HTTP issued the insecure request before the post-hoc geturl() check reported it. Shared validator now rejects non-HTTPS redirects (loopback HTTP allowed) pre-follow, matching the preset download path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 38 ++++++++++++-- tests/test_workflows.py | 73 +++++++++++++++++++++----- 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 03d5a4b8a7..71422b623d 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -108,6 +108,26 @@ def _reject_unsafe_workflow_storage(project_root: Path) -> None: _RESERVED_WORKFLOW_IDS: frozenset[str] = frozenset({"runs", "steps"}) +def _reject_insecure_download_redirect(old_url: str, new_url: str) -> None: + """Reject a redirect before it is followed unless HTTPS (or loopback HTTP).""" + import urllib.error + from ipaddress import ip_address + from urllib.parse import urlparse + + parsed = urlparse(new_url) + host = parsed.hostname or "" + loopback = host == "localhost" + if not loopback: + try: + loopback = ip_address(host).is_loopback + except ValueError: + pass + if parsed.scheme != "https" and not (parsed.scheme == "http" and loopback): + raise urllib.error.URLError( + "redirect target must use HTTPS, or HTTP for localhost/loopback" + ) + + def _validate_workflow_id_or_exit(workflow_id: str) -> None: """Validate that ``workflow_id`` is a safe installed-workflow directory name.""" if ( @@ -756,7 +776,12 @@ def _validate_and_install_local( import tempfile try: - with _open_url(download_url, timeout=30, extra_headers=_wf_url_extra_headers) as resp: + with _open_url( + download_url, + timeout=30, + extra_headers=_wf_url_extra_headers, + redirect_validator=_reject_insecure_download_redirect, + ) as resp: final_url = resp.geturl() final_parsed = urlparse(final_url) final_host = final_parsed.hostname or "" @@ -903,7 +928,12 @@ def _install_workflow_from_catalog( _wf_cat_extra_headers = {"Accept": "application/octet-stream"} workflow_dir.mkdir(parents=True, exist_ok=True) - with _open_url(workflow_url, timeout=30, extra_headers=_wf_cat_extra_headers) as response: + with _open_url( + workflow_url, + timeout=30, + extra_headers=_wf_cat_extra_headers, + redirect_validator=_reject_insecure_download_redirect, + ) as response: # Validate final URL after redirects final_url = response.geturl() final_parsed = urlparse(final_url) @@ -1621,7 +1651,9 @@ def _safe_fetch(url: str) -> bytes: raise ValueError(f"Refusing to fetch from non-HTTPS URL: {url}") if not parsed.hostname: raise ValueError(f"Refusing to fetch from URL with no hostname: {url}") - with _open_url(url, timeout=30) as resp: + with _open_url( + url, timeout=30, redirect_validator=_reject_insecure_download_redirect + ) as resp: final_url = resp.geturl() final_parsed = urlparse(final_url) final_is_localhost = final_parsed.hostname in ("localhost", "127.0.0.1", "::1") diff --git a/tests/test_workflows.py b/tests/test_workflows.py index d8c702b18d..cad52b1486 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -6579,7 +6579,7 @@ def __enter__(self): def __exit__(self, *a): return False - def fake_open_url(url, timeout=None, extra_headers=None): + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): captured_urls.append((url, extra_headers, timeout)) if "releases/tags/" in url: return FakeResponse(json.dumps({ @@ -6631,7 +6631,7 @@ def __enter__(self): def __exit__(self, *a): return False - def fake_open_url(url, timeout=None, extra_headers=None): + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): captured_urls.append((url, extra_headers)) return FakeResponse(self.VALID_WORKFLOW_YAML.encode()) @@ -6674,7 +6674,7 @@ def __enter__(self): def __exit__(self, *a): return False - def fake_open_url(url, timeout=None, extra_headers=None): + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): captured_urls.append((url, extra_headers)) if "releases/tags/" in url: return FakeResponse(json.dumps({ @@ -6750,7 +6750,7 @@ def __enter__(self): def __exit__(self, *a): return False - def fake_open_url(url, timeout=None, extra_headers=None): + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): captured_urls.append((url, extra_headers)) if "releases/tags/" in url: return FakeResponse(json.dumps({ @@ -6818,7 +6818,7 @@ def __exit__(self, *a): run: "echo hello" """ - def fake_open_url(url, timeout=None, extra_headers=None): + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): captured_urls.append((url, extra_headers)) if "releases/tags/" in url: return FakeResponse(json.dumps({ @@ -7364,7 +7364,7 @@ def test_add_from_url_installs(self, project_dir, monkeypatch): runner = CliRunner() with patch( "specify_cli.authentication.http.open_url", - side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(data, url), + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), ): result = runner.invoke( app, @@ -7384,7 +7384,7 @@ def test_add_from_url_id_mismatch_errors(self, project_dir, monkeypatch): runner = CliRunner() with patch( "specify_cli.authentication.http.open_url", - side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(data, url), + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), ): result = runner.invoke( app, @@ -7418,7 +7418,7 @@ def test_add_from_url_non_https_redirect_escapes_rich_markup(self, project_dir, runner = CliRunner() with patch( "specify_cli.authentication.http.open_url", - side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(b"", redirected_url), + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(b"", redirected_url), ): result = runner.invoke( app, @@ -7436,7 +7436,7 @@ def test_add_from_rejects_invalid_source_id_without_fetch(self, project_dir, mon monkeypatch.chdir(project_dir) calls: list[str] = [] - def _fake_open(url, timeout=None, extra_headers=None): + def _fake_open(url, timeout=None, extra_headers=None, redirect_validator=None): calls.append(url) raise AssertionError(f"network fetch attempted: {url}") @@ -7601,6 +7601,51 @@ def test_add_dev_dir_with_workflow_yml_directory_errors_cleanly(self, project_di assert result.exception is None or isinstance(result.exception, SystemExit) assert "No workflow.yml found" in result.output + def test_download_redirect_validator_rejects_http_before_follow(self): + import urllib.error + + from specify_cli.workflows._commands import _reject_insecure_download_redirect + + with pytest.raises(urllib.error.URLError): + _reject_insecure_download_redirect( + "https://example.com/wf.yml", "http://evil.example.com/wf.yml" + ) + # Allowed: HTTPS anywhere, HTTP on loopback. + _reject_insecure_download_redirect( + "https://example.com/wf.yml", "https://cdn.example.com/wf.yml" + ) + _reject_insecure_download_redirect( + "https://example.com/wf.yml", "http://localhost:8000/wf.yml" + ) + _reject_insecure_download_redirect( + "https://example.com/wf.yml", "http://127.0.0.1/wf.yml" + ) + + def test_add_from_url_passes_redirect_validator(self, project_dir, monkeypatch): + from unittest.mock import patch + + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + seen: dict[str, object] = {} + + def fake_open(url, timeout=None, extra_headers=None, redirect_validator=None): + seen["validator"] = redirect_validator + return self._FakeResponse(data, url) + + runner = CliRunner() + with patch("specify_cli.authentication.http.open_url", side_effect=fake_open): + result = runner.invoke( + app, + ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + ) + assert result.exit_code == 0, result.output + from specify_cli.workflows._commands import _reject_insecure_download_redirect + + assert seen["validator"] is _reject_insecure_download_redirect + def test_registry_save_failure_preserves_file_on_disk(self, project_dir, monkeypatch): """A failed dump must not truncate the persisted registry.""" from specify_cli.workflows.catalog import WorkflowRegistry @@ -7712,7 +7757,7 @@ def test_update_installs_newer_catalog_version(self, project_dir, monkeypatch): runner = CliRunner() with patch( "specify_cli.authentication.http.open_url", - side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(data, url), + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), ): result = runner.invoke(app, ["workflow", "update"], input="y\n") assert result.exit_code == 0, result.output @@ -7760,7 +7805,7 @@ def test_update_downloaded_invalid_yaml_escapes_rich_markup(self, project_dir, m runner = CliRunner() with patch( "specify_cli.authentication.http.open_url", - side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(b"", url), + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(b"", url), ), patch.object( WorkflowDefinition, "from_yaml", @@ -7901,7 +7946,7 @@ def test_update_rejects_version_mismatch_from_stale_url(self, project_dir, monke runner = CliRunner() with patch( "specify_cli.authentication.http.open_url", - side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(data, url), + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), ): result = runner.invoke(app, ["workflow", "update"], input="y\n") assert "does not match the catalog version" in result.output @@ -7939,7 +7984,7 @@ def test_update_preserves_disabled_state(self, project_dir, monkeypatch): runner = CliRunner() with patch( "specify_cli.authentication.http.open_url", - side_effect=lambda url, timeout=None, extra_headers=None: self._FakeResponse(data, url), + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), ): result = runner.invoke(app, ["workflow", "update"], input="y\n") assert result.exit_code == 0, result.output @@ -8187,7 +8232,7 @@ def test_update_restores_backup_on_failed_download(self, project_dir, monkeypatc }, ) - def boom(url, timeout=None, extra_headers=None): + def boom(url, timeout=None, extra_headers=None, redirect_validator=None): raise OSError("network down") runner = CliRunner() From 6a0c462f6996f661aacba1cdadd1433a50ed3584 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:59:11 +0200 Subject: [PATCH 24/59] test(workflows): accept redirect_validator kwarg in step-add open_url fakes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/test_workflows.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index cad52b1486..495cdacff1 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -6151,7 +6151,7 @@ def read(self): def geturl(self): return self.url - def _fake_open_url(url, timeout=30): + def _fake_open_url(url, timeout=30, redirect_validator=None): return _FakeResponse(url) monkeypatch.setattr(StepCatalog, "get_step_info", _fake_get_step_info) @@ -6210,7 +6210,7 @@ def read(self): def geturl(self): return self.url - def _fake_open_url(url, timeout=30): + def _fake_open_url(url, timeout=30, redirect_validator=None): return _FakeResponse(url) monkeypatch.setattr(StepCatalog, "get_step_info", _fake_get_step_info) @@ -6258,7 +6258,7 @@ def read(self): def geturl(self): return self.url - def _fake_open_url(url, timeout=30): + def _fake_open_url(url, timeout=30, redirect_validator=None): return _FakeResponse(url) monkeypatch.setattr(StepCatalog, "get_step_info", _fake_get_step_info) From 3763e6fa35e06cb631b31365923c69930e72eac7 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 01:38:47 +0200 Subject: [PATCH 25/59] fix(workflows): guard directory-shaped workflow.yml and unreadable registry - workflow add's plain local-path fallback (no --dev) checked wf_file.exists() before installing, so a directory literally named workflow.yml passed the guard and _validate_and_install_local() leaked an uncaught IsADirectoryError instead of the documented CLI error. Use is_file(), matching the --dev branch's existing guard. - WorkflowRegistry._load() treated any OSError while reading an existing registry the same as corrupted JSON, resetting to an empty in-memory registry. A later save() would then silently persist that empty state via os.replace, discarding every previously installed workflow entry. Track a _load_error flag on OSError-during-read and have save() refuse to write when it is set, so a transient I/O failure can no longer overwrite intact data on disk. - docs/reference/workflows.md: document `--from ` with its value placeholder, matching extensions.md and presets.md. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/reference/workflows.md | 8 ++--- src/specify_cli/workflows/_commands.py | 2 +- src/specify_cli/workflows/catalog.py | 32 +++++++++++++++----- tests/test_workflows.py | 42 ++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/docs/reference/workflows.md b/docs/reference/workflows.md index c80ea7cd3c..8bd4a8778d 100644 --- a/docs/reference/workflows.md +++ b/docs/reference/workflows.md @@ -86,10 +86,10 @@ Lists workflows installed in the current project. specify workflow add ``` -| Option | Description | -| -------- | ------------------------------------------------------ | -| `--dev` | Install from a local workflow YAML file or directory | -| `--from` | Install from a custom URL (`` names the expected workflow ID) | +| Option | Description | +| --------------- | ------------------------------------------------------ | +| `--dev` | Install from a local workflow YAML file or directory | +| `--from ` | Install from a custom URL (`` names the expected workflow ID) | Installs a workflow from the catalog, a URL (HTTPS required), or a local file path. diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 71422b623d..ab95767f95 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -825,7 +825,7 @@ def _validate_and_install_local( return elif source_path.is_dir(): wf_file = source_path / "workflow.yml" - if not wf_file.exists(): + if not wf_file.is_file(): console.print(f"[red]Error:[/red] No workflow.yml found in {_escape_markup(source)}") raise typer.Exit(1) _validate_and_install_local(wf_file, str(source_path)) diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 092b4b6bf1..4bf341198d 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -70,6 +70,10 @@ def __init__(self, project_root: Path) -> None: self.project_root = project_root self.workflows_dir = project_root / ".specify" / "workflows" self.registry_path = self.workflows_dir / self.REGISTRY_FILE + # Set before _load() so a read failure (distinct from a corrupted or + # missing file) can flip it to block a later save() from silently + # persisting an empty registry over unreadable-but-intact data. + self._load_error = False self.data = self._load() def _has_symlinked_parent(self) -> bool: @@ -95,15 +99,23 @@ def _load(self) -> dict[str, Any]: try: with open(self.registry_path, encoding="utf-8") as f: data = json.load(f) - # Validate shape: must be a dict with a dict "workflows" field. - if not isinstance(data, dict): - return default_registry - if not isinstance(data.get("workflows"), dict): - data["workflows"] = {} - return data - except (json.JSONDecodeError, ValueError, OSError, UnicodeError): + except OSError: + # An I/O failure (e.g. permissions, transient FS issue) is not + # the same as a corrupted file: the real data may still be + # intact on disk. Flag it so save() refuses to overwrite that + # data with this in-memory default instead of silently + # discarding every prior entry. + self._load_error = True + return default_registry + except (json.JSONDecodeError, ValueError, UnicodeError): # Corrupted registry file — reset to default return default_registry + # Validate shape: must be a dict with a dict "workflows" field. + if not isinstance(data, dict): + return default_registry + if not isinstance(data.get("workflows"), dict): + data["workflows"] = {} + return data return default_registry def save(self) -> None: @@ -114,6 +126,12 @@ def save(self) -> None: raise OSError( "Refusing to write workflow registry through a symlinked path." ) + if self._load_error: + raise OSError( + f"Refusing to save workflow registry at {self.registry_path}: " + "the existing file could not be read, so saving now would " + "discard its contents." + ) self.workflows_dir.mkdir(parents=True, exist_ok=True) # Unique, exclusive temp then replace: a failed dump cannot truncate # the registry, a pre-created symlink cannot redirect the write, and diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 495cdacff1..0625b4e521 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -4746,6 +4746,33 @@ def test_persistence(self, project_dir): registry2 = WorkflowRegistry(project_dir) assert registry2.is_installed("test-wf") + def test_load_read_oserror_refuses_to_save_over_existing_data(self, project_dir, monkeypatch): + """A transient read failure (e.g. temporarily unreadable file) must not be + treated the same as a corrupted/missing registry: saving afterwards would + silently discard every previously persisted workflow entry.""" + from specify_cli.workflows.catalog import WorkflowRegistry + import builtins + + registry1 = WorkflowRegistry(project_dir) + registry1.add("test-wf", {"name": "Test", "version": "1.0.0"}) + registry_path = registry1.registry_path + real_open = builtins.open + + def _raising_open(file, mode="r", *args, **kwargs): + if Path(file) == registry_path and "r" in mode: + raise OSError("simulated read failure") + return real_open(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", _raising_open) + registry2 = WorkflowRegistry(project_dir) + # The in-memory view may fall back to empty, but a save must not be + # allowed to persist that empty state over the real file on disk. + with pytest.raises(OSError): + registry2.save() + # The original entry must survive on disk untouched. + data = json.loads(registry_path.read_text(encoding="utf-8")) + assert "test-wf" in data["workflows"] + # ===== Workflow Catalog Tests ===== @@ -7314,6 +7341,21 @@ def test_add_local_dir_without_workflow_yml_errors(self, project_dir, monkeypatc assert "No workflow.yml found" in result.output assert "[bracket]" in result.output + def test_add_local_dir_with_workflow_yml_directory_errors_cleanly(self, project_dir, monkeypatch): + """Same as the --dev case, but for the plain local-path fallback (no --dev): + a directory named workflow.yml must not reach open() and leak IsADirectoryError.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + src_dir = project_dir / "local-wf" + (src_dir / "workflow.yml").mkdir(parents=True) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "add", str(src_dir)]) + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "No workflow.yml found" in result.output + def test_add_yaml_parse_error_escapes_rich_markup(self, project_dir, monkeypatch): """A YAML syntax error can quote the offending line verbatim; brackets in it must not be Rich markup.""" from unittest.mock import patch From 86de599d3e43b4764d5063be2b43d72b6bf404e8 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:11:31 +0200 Subject: [PATCH 26/59] fix(workflow): rollback registry.remove() and guard OSError at CLI boundaries Critical: WorkflowRegistry.remove() deleted the in-memory entry then called save() with no rollback, unlike add(). Combined with workflow_remove deleting the workflow directory before calling registry.remove(), a save failure permanently destroyed the workflow's files, left the on-disk registry still claiming it installed, and surfaced a raw unhandled OSError with no CLI message. - WorkflowRegistry.remove() now rolls back the in-memory entry on a save() OSError, mirroring add()'s existing rollback pattern. - workflow_remove persists the registry removal (registry.remove(), wrapped in try/except OSError -> clean escaped message) before deleting any files, so a save failure never touches the workflow directory. Important sibling paths: workflow add (local/--dev/--from and catalog), enable, and disable all called registry.add() without catching its deliberate OSError, so a save failure surfaced either an orphaned install directory (fresh local/catalog installs) or a raw/unhandled exception with no clean CLI output. - _validate_and_install_local (backs local/--dev/--from) now removes the freshly created directory on a fresh install, or restores the prior workflow.yml bytes on a reinstall-over-existing-local install, before raising a clean escaped error. - _install_workflow_from_catalog wraps the final registry.add() using the function's own established convention (rmtree the just-downloaded workflow_dir, then a clean escaped error) -- workflow_update's existing backup/restore around this function is unaffected. - workflow_enable/workflow_disable catch registry.add()'s OSError and print a clean escaped message instead of leaking the exception. Added failing-first tests proving each behavior (registry-unit rollback test, CLI-level remove/add/enable/disable save-failure tests parametrized where they share one root cause), all confirmed red before the fix and green after. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 77 +++++++++-- src/specify_cli/workflows/catalog.py | 10 +- tests/test_workflows.py | 175 +++++++++++++++++++++++++ 3 files changed, 250 insertions(+), 12 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index ab95767f95..cd24b22bcf 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -700,15 +700,35 @@ def _validate_and_install_local( raise typer.Exit(1) dest_dir = _safe_workflow_id_dir(workflows_dir, definition.id) + dest_file = dest_dir / "workflow.yml" + existed_before = dest_dir.is_dir() + backup_bytes = ( + dest_file.read_bytes() if existed_before and dest_file.is_file() else None + ) dest_dir.mkdir(parents=True, exist_ok=True) import shutil - shutil.copy2(yaml_path, dest_dir / "workflow.yml") - registry.add(definition.id, { - "name": definition.name, - "version": definition.version, - "description": definition.description, - "source": source_label, - }) + shutil.copy2(yaml_path, dest_file) + try: + registry.add(definition.id, { + "name": definition.name, + "version": definition.version, + "description": definition.description, + "source": source_label, + }) + except OSError as exc: + # Don't leave an orphan directory behind for a fresh install; for + # a reinstall over an existing local workflow, restore the prior + # workflow.yml instead of clobbering it with the failed update. + if existed_before: + if backup_bytes is not None: + dest_file.write_bytes(backup_bytes) + else: + shutil.rmtree(dest_dir, ignore_errors=True) + console.print( + f"[red]Error:[/red] Failed to update workflow registry for " + f"'{_escape_markup(definition.id)}': {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) console.print( f"[green]✓[/green] Workflow '{_escape_markup(definition.name)}' " f"({_escape_markup(definition.id)}) installed" @@ -1024,7 +1044,16 @@ def _install_workflow_from_catalog( existing = registry.get(workflow_id) if isinstance(existing, dict) and not existing.get("enabled", True): entry["enabled"] = False - registry.add(workflow_id, entry) + try: + registry.add(workflow_id, entry) + except OSError as exc: + import shutil + shutil.rmtree(workflow_dir, ignore_errors=True) + console.print( + f"[red]Error:[/red] Failed to update workflow registry for " + f"'{_escape_markup(workflow_id)}': {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) console.print( f"[green]✓[/green] Workflow '{_escape_markup(str(info.get('name', workflow_id)))}' " "installed from catalog" @@ -1078,6 +1107,19 @@ def workflow_remove( ) raise typer.Exit(1) + # Persist the registry removal before touching any files: if save() + # fails, WorkflowRegistry.remove() rolls back its in-memory state and + # raises, so the workflow stays fully installed (files + registry) rather + # than being deleted while the registry still (or no longer) claims it. + try: + registry.remove(workflow_id) + except OSError as exc: + console.print( + f"[red]Error:[/red] Failed to update workflow registry for '{safe_id}': " + f"{_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + if workflow_dir.exists(): import shutil try: @@ -1088,7 +1130,6 @@ def workflow_remove( ) raise typer.Exit(1) - registry.remove(workflow_id) console.print(f"[green]✓[/green] Workflow '{workflow_id}' removed") @@ -1255,7 +1296,14 @@ def workflow_enable( raise typer.Exit(0) # Fresh mapping: registry.get() returns the live entry, and mutating it # in place would defeat WorkflowRegistry.add's rollback-on-save-failure. - registry.add(workflow_id, {**metadata, "enabled": True}) + try: + registry.add(workflow_id, {**metadata, "enabled": True}) + except OSError as exc: + console.print( + f"[red]Error:[/red] Failed to update workflow registry for " + f"'{_escape_markup(workflow_id)}': {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) console.print(f"[green]✓[/green] Workflow '{_escape_markup(workflow_id)}' enabled") @@ -1281,7 +1329,14 @@ def workflow_disable( console.print(f"[yellow]Workflow '{_escape_markup(workflow_id)}' is already disabled[/yellow]") raise typer.Exit(0) # Fresh mapping for the same rollback reason as workflow_enable. - registry.add(workflow_id, {**metadata, "enabled": False}) + try: + registry.add(workflow_id, {**metadata, "enabled": False}) + except OSError as exc: + console.print( + f"[red]Error:[/red] Failed to update workflow registry for " + f"'{_escape_markup(workflow_id)}': {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) console.print(f"[green]✓[/green] Workflow '{_escape_markup(workflow_id)}' disabled") console.print(f"To re-enable: specify workflow enable {_escape_markup(workflow_id)}") diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 4bf341198d..26ffe6051c 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -179,8 +179,16 @@ def add(self, workflow_id: str, metadata: dict[str, Any]) -> None: def remove(self, workflow_id: str) -> bool: """Remove an installed workflow entry. Returns True if found.""" if workflow_id in self.data["workflows"]: + removed_entry = self.data["workflows"][workflow_id] del self.data["workflows"][workflow_id] - self.save() + try: + self.save() + except OSError: + # Roll back the in-memory deletion so a save failure can't + # desync this instance from the untouched file on disk, + # mirroring add()'s rollback-on-save-failure. + self.data["workflows"][workflow_id] = removed_entry + raise return True return False diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 0625b4e521..eb59c50d4e 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -4716,6 +4716,29 @@ def test_remove(self, project_dir): registry.remove("test-wf") assert not registry.is_installed("test-wf") + def test_remove_rolls_back_in_memory_on_save_failure(self, project_dir, monkeypatch): + """A save() failure during remove() must not leave the in-memory registry + out of sync with the (unchanged) file on disk, mirroring add()'s rollback.""" + from specify_cli.workflows.catalog import WorkflowRegistry + import specify_cli.workflows.catalog as catalog_mod + + registry = WorkflowRegistry(project_dir) + registry.add("test-wf", {"name": "Test", "version": "1.0.0"}) + + def boom(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr(catalog_mod.json, "dump", boom) + with pytest.raises(OSError): + registry.remove("test-wf") + monkeypatch.undo() + + # In-memory state must still show the entry (rolled back), matching + # the untouched file on disk. + assert registry.is_installed("test-wf") + fresh = WorkflowRegistry(project_dir) + assert fresh.is_installed("test-wf") + def test_list(self, project_dir): from specify_cli.workflows.catalog import WorkflowRegistry @@ -5880,6 +5903,38 @@ def test_remove_refuses_non_directory_workflow_path(self, project_dir, monkeypat assert workflow_path.read_text(encoding="utf-8") == "not a directory" assert WorkflowRegistry(project_dir).is_installed("test-wf") + def test_remove_registry_save_failure_preserves_files_and_registry( + self, project_dir, monkeypatch + ): + """If persisting the registry removal fails, the workflow's files must + not have already been deleted: the CLI must not delete files before the + registry successfully records the removal, and it must fail cleanly.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + registry = WorkflowRegistry(project_dir) + registry.add("test-wf", {"name": "Test", "version": "1.0.0"}) + workflow_dir = project_dir / ".specify" / "workflows" / "test-wf" + workflow_dir.mkdir(parents=True, exist_ok=True) + (workflow_dir / "workflow.yml").write_text("keep-me", encoding="utf-8") + + def boom(self): + raise OSError("disk full") + + monkeypatch.chdir(project_dir) + with pytest.MonkeyPatch.context() as mp: + mp.setattr(WorkflowRegistry, "save", boom) + result = CliRunner().invoke(app, ["workflow", "remove", "test-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + # Files must survive a registry-save failure. + assert (workflow_dir / "workflow.yml").read_text(encoding="utf-8") == "keep-me" + # The on-disk registry must still claim the workflow installed. + assert WorkflowRegistry(project_dir).is_installed("test-wf") + class TestWorkflowAddSymlinkGuard: def test_add_malformed_ipv6_url_exits_cleanly(self, temp_dir, monkeypatch): @@ -7643,6 +7698,90 @@ def test_add_dev_dir_with_workflow_yml_directory_errors_cleanly(self, project_di assert result.exception is None or isinstance(result.exception, SystemExit) assert "No workflow.yml found" in result.output + @pytest.mark.parametrize("mode", ["dev", "local", "from_url"]) + def test_add_save_failure_leaves_no_orphan_directory(self, project_dir, monkeypatch, mode): + """A registry.add() save failure during a fresh install must not leave + an orphaned workflow directory on disk, and must fail with a clean + escaped message instead of a raw OSError traceback. Shared by --dev, + the plain local-path fallback, and --from since all three funnel + through _validate_and_install_local's single install choke point.""" + import contextlib + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + + def boom(self): + raise OSError("disk full") + + if mode == "from_url": + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + args = ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"] + url_patch = patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), + ) + else: + src = self._write_workflow_dir(project_dir) + args = ["workflow", "add", str(src)] + (["--dev"] if mode == "dev" else []) + url_patch = contextlib.nullcontext() + + with url_patch, pytest.MonkeyPatch.context() as mp: + mp.setattr(WorkflowRegistry, "save", boom) + result = runner.invoke(app, args) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + dest_dir = project_dir / ".specify" / "workflows" / "align-wf" + assert not dest_dir.exists() + assert not WorkflowRegistry(project_dir).is_installed("align-wf") + + def test_add_catalog_save_failure_leaves_no_orphan_directory(self, project_dir, monkeypatch): + """Same guarantee as the local-install paths, but for a fresh catalog + install: a registry.add() failure must clean up the freshly-downloaded + directory and fail with a clean escaped message.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + + def boom(self): + raise OSError("disk full") + + runner = CliRunner() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), + ) + mp.setattr(WorkflowRegistry, "save", boom) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + dest_dir = project_dir / ".specify" / "workflows" / "align-wf" + assert not dest_dir.exists() + assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_download_redirect_validator_rejects_http_before_follow(self): import urllib.error @@ -7949,6 +8088,42 @@ def boom(self): assert result.exit_code == 0, result.output assert WorkflowRegistry(project_dir).get("align-wf")["enabled"] is True + @pytest.mark.parametrize("command", ["enable", "disable"]) + def test_enable_disable_save_failure_gives_clean_output( + self, project_dir, monkeypatch, command + ): + """A save() failure in enable/disable must produce a clean escaped CLI + error, not surface the raw OSError as an unhandled exception. Shared + root behavior: both call registry.add() with a fresh mapping and must + catch its deliberate OSError the same way.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + # disable starts from the enabled default; enable needs a prior disable. + starting_enabled = command == "disable" + if command == "enable": + pre = runner.invoke(app, ["workflow", "disable", "align-wf"]) + assert pre.exit_code == 0, pre.output + + def boom(self): + raise OSError("disk full") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(WorkflowRegistry, "save", boom) + result = runner.invoke(app, ["workflow", command, "align-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + assert ( + WorkflowRegistry(project_dir).get("align-wf").get("enabled", True) + is starting_enabled + ) + def test_update_rejects_version_mismatch_from_stale_url(self, project_dir, monkeypatch): """A URL serving a different version than the catalog advertised must fail the update.""" from unittest.mock import patch From fb64ddd48c33b57327ec916406ad9ed464ba62bb Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 02:28:48 +0200 Subject: [PATCH 27/59] fix(workflow): preserve prior catalog install on reinstall registry-save failure _install_workflow_from_catalog's final registry.add() failure handler unconditionally rmtree'd workflow_dir. That's safe for a brand-new install, but plain `workflow add ` also allows re-adding an already-installed workflow, downloading the new version over the existing directory first. If registry.add() then failed to save, the unconditional rmtree deleted the prior working install while the registry (after its own rollback) still reported it installed -- data loss with no way back. workflow_update already avoids this via an outer backup/restore around this function, but plain add has no such caller. Fix mirrors _validate_and_install_local's existed-before/backup-aware handling: capture whether workflow_dir existed and back up its workflow.yml bytes before any download write, then on a registry.add() OSError, restore those bytes for a reinstall or rmtree only a brand-new directory. Only one file (workflow.yml) is ever written by this path, so no further per-file bookkeeping is needed. Added a failing-first regression: install a catalog workflow, re-add it with a simulated registry save OSError, and assert a clean error, the original workflow.yml restored byte-for-byte, and the registry still reporting the original version installed. Confirmed red (prior file deleted) before the fix, green after. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 20 +++++++- tests/test_workflows.py | 65 ++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index cd24b22bcf..fdbc9c504f 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -934,6 +934,15 @@ def _install_workflow_from_catalog( workflow_dir = _safe_workflow_id_dir(workflows_dir, workflow_id) workflow_file = workflow_dir / "workflow.yml" + # Captured before any mkdir/download writes so a registry.add() failure + # at the end of this function can tell a fresh install from a + # reinstall-over-an-existing-one, mirroring _validate_and_install_local's + # existed-before/backup-aware rollback. + existed_before = workflow_dir.is_dir() + prior_workflow_bytes = ( + workflow_file.read_bytes() if existed_before and workflow_file.is_file() else None + ) + try: from specify_cli.authentication.http import open_url as _open_url from specify_cli.authentication.http import github_provider_hosts as _github_provider_hosts @@ -1047,8 +1056,15 @@ def _install_workflow_from_catalog( try: registry.add(workflow_id, entry) except OSError as exc: - import shutil - shutil.rmtree(workflow_dir, ignore_errors=True) + # Don't destroy a prior working install on a reinstall: only a + # brand-new directory is safe to remove wholesale; an existing one + # gets its previous workflow.yml restored instead. + if existed_before: + if prior_workflow_bytes is not None: + workflow_file.write_bytes(prior_workflow_bytes) + else: + import shutil + shutil.rmtree(workflow_dir, ignore_errors=True) console.print( f"[red]Error:[/red] Failed to update workflow registry for " f"'{_escape_markup(workflow_id)}': {_escape_markup(str(exc))}" diff --git a/tests/test_workflows.py b/tests/test_workflows.py index eb59c50d4e..cf76d49c0c 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7782,6 +7782,71 @@ def boom(self): assert not dest_dir.exists() assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_catalog_reinstall_save_failure_restores_prior_file(self, project_dir, monkeypatch): + """Re-adding an already-installed catalog workflow downloads the new + version over the existing install directory. If registry.add() then + fails to save, the prior working workflow.yml must be restored + byte-for-byte (not left overwritten with the new download, and not + deleted like a fresh install) and the registry must remain valid and + still point at the original version -- the update path's caller has + an outer backup/restore for this, but plain `workflow add` does not, + so _install_workflow_from_catalog must handle it itself.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + original_data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + runner = CliRunner() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + original_data, url + ), + ) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + assert result.exit_code == 0, result.output + + dest_file = project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" + assert dest_file.read_bytes() == original_data + + new_data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + + def boom(self): + raise OSError("disk full") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + new_data, url + ), + ) + mp.setattr(WorkflowRegistry, "save", boom) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + # The prior working install must survive untouched, byte-for-byte. + assert dest_file.read_bytes() == original_data + registry = WorkflowRegistry(project_dir) + assert registry.is_installed("align-wf") + assert registry.get("align-wf")["version"] == "1.0.0" + def test_download_redirect_validator_rejects_http_before_follow(self): import urllib.error From 49f6fb6279435f8993741d7afda1d2d2d515836b Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 08:27:06 +0200 Subject: [PATCH 28/59] fix(workflow): centralize catalog-install cleanup across all failure branches _install_workflow_from_catalog is new in this PR and has seven failure branches after the mkdir/download step, each independently rmtree'ing workflow_dir: redirect-to-non-HTTPS rejection, a generic download exception, invalid downloaded YAML, a validate_workflow failure, a workflow-id/catalog-key mismatch, a version mismatch, and (fixed in the prior commit) a registry.add() OSError. Only the last one had been special-cased to spare a prior working install on reinstall; the other six still unconditionally deleted the whole directory, so re-adding an already-installed catalog workflow and hitting any of those six earlier failures destroyed the working install even though nothing about it had actually changed. Replaced all seven ad hoc rmtree call sites with a single local _cleanup_failed_install() helper that closes over the existed_before / prior_workflow_bytes captured once at the top of the function: restore the prior workflow.yml for a reinstall, or rmtree only a directory that this attempt itself created. Every failure branch now calls this one helper, so the fix is structural rather than duplicated, and every existing error message/exit code is unchanged -- only the cleanup performed before each message is different. Added a parametrized regression test covering the four early-failure trigger points reachable from plain workflow add (redirect rejection, download exception, invalid YAML, ID mismatch): each installs a catalog workflow, re-adds it while forcing that specific failure, and asserts a clean error plus the original workflow.yml surviving byte-for-byte. Confirmed red against the unfixed code (all four raised FileNotFoundError reading the deleted file) before applying the helper, green after. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 53 +++++++++--------- tests/test_workflows.py | 74 ++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 27 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index fdbc9c504f..f425878306 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -934,15 +934,30 @@ def _install_workflow_from_catalog( workflow_dir = _safe_workflow_id_dir(workflows_dir, workflow_id) workflow_file = workflow_dir / "workflow.yml" - # Captured before any mkdir/download writes so a registry.add() failure - # at the end of this function can tell a fresh install from a - # reinstall-over-an-existing-one, mirroring _validate_and_install_local's - # existed-before/backup-aware rollback. + # Captured before any mkdir/download writes so every failure branch below + # can tell a fresh install from a reinstall-over-an-existing-one, + # mirroring _validate_and_install_local's existed-before/backup-aware + # rollback. existed_before = workflow_dir.is_dir() prior_workflow_bytes = ( workflow_file.read_bytes() if existed_before and workflow_file.is_file() else None ) + def _cleanup_failed_install() -> None: + """Restore the prior workflow.yml on a reinstall, or remove the + directory entirely for a fresh install. Every failure branch that + runs after the mkdir/download step -- redirect rejection, download + exception, invalid YAML, ID mismatch, version mismatch, and + registry.add() failure -- must call this instead of rmtree'ing + directly, so none of them can destroy a working install that + predates this attempt.""" + if existed_before: + if prior_workflow_bytes is not None: + workflow_file.write_bytes(prior_workflow_bytes) + else: + import shutil + shutil.rmtree(workflow_dir, ignore_errors=True) + try: from specify_cli.authentication.http import open_url as _open_url from specify_cli.authentication.http import github_provider_hosts as _github_provider_hosts @@ -975,9 +990,7 @@ def _install_workflow_from_catalog( # Host is not an IP literal (e.g., a regular hostname); treat as non-loopback. pass if final_parsed.scheme != "https" and not (final_parsed.scheme == "http" and final_loopback): - if workflow_dir.exists(): - import shutil - shutil.rmtree(workflow_dir, ignore_errors=True) + _cleanup_failed_install() console.print( f"[red]Error:[/red] Workflow '{safe_wf_id}' redirected to non-HTTPS URL: {_escape_markup(final_url)}" ) @@ -986,9 +999,7 @@ def _install_workflow_from_catalog( except typer.Exit: raise except Exception as exc: - if workflow_dir.exists(): - import shutil - shutil.rmtree(workflow_dir, ignore_errors=True) + _cleanup_failed_install() console.print(f"[red]Error:[/red] Failed to install workflow '{safe_wf_id}' from catalog: {_escape_markup(str(exc))}") raise typer.Exit(1) @@ -996,16 +1007,14 @@ def _install_workflow_from_catalog( try: definition = WorkflowDefinition.from_yaml(workflow_file) except (ValueError, yaml.YAMLError) as exc: - import shutil - shutil.rmtree(workflow_dir, ignore_errors=True) + _cleanup_failed_install() console.print(f"[red]Error:[/red] Downloaded workflow is invalid: {_escape_markup(str(exc))}") raise typer.Exit(1) from .engine import validate_workflow errors = validate_workflow(definition) if errors: - import shutil - shutil.rmtree(workflow_dir, ignore_errors=True) + _cleanup_failed_install() console.print("[red]Error:[/red] Downloaded workflow validation failed:") for err in errors: console.print(f" \u2022 {_escape_markup(str(err))}") @@ -1013,8 +1022,7 @@ def _install_workflow_from_catalog( # Enforce that the workflow's internal ID matches the catalog key if definition.id and definition.id != workflow_id: - import shutil - shutil.rmtree(workflow_dir, ignore_errors=True) + _cleanup_failed_install() console.print( f"[red]Error:[/red] Workflow ID in YAML ({_escape_markup(repr(definition.id))}) " f"does not match catalog key ({_escape_markup(repr(workflow_id))}). " @@ -1032,8 +1040,7 @@ def _install_workflow_from_catalog( except pkg_version.InvalidVersion: version_matches = str(definition.version) == expected_version if not version_matches: - import shutil - shutil.rmtree(workflow_dir, ignore_errors=True) + _cleanup_failed_install() console.print( f"[red]Error:[/red] Downloaded workflow version ({_escape_markup(str(definition.version))}) " f"does not match the catalog version ({_escape_markup(expected_version)}). " @@ -1056,15 +1063,7 @@ def _install_workflow_from_catalog( try: registry.add(workflow_id, entry) except OSError as exc: - # Don't destroy a prior working install on a reinstall: only a - # brand-new directory is safe to remove wholesale; an existing one - # gets its previous workflow.yml restored instead. - if existed_before: - if prior_workflow_bytes is not None: - workflow_file.write_bytes(prior_workflow_bytes) - else: - import shutil - shutil.rmtree(workflow_dir, ignore_errors=True) + _cleanup_failed_install() console.print( f"[red]Error:[/red] Failed to update workflow registry for " f"'{_escape_markup(workflow_id)}': {_escape_markup(str(exc))}" diff --git a/tests/test_workflows.py b/tests/test_workflows.py index cf76d49c0c..67a0554f95 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7847,6 +7847,80 @@ def boom(self): assert registry.is_installed("align-wf") assert registry.get("align-wf")["version"] == "1.0.0" + @pytest.mark.parametrize( + "mode", ["redirect_rejected", "download_exception", "invalid_yaml", "id_mismatch"] + ) + def test_add_catalog_reinstall_early_failure_restores_prior_file( + self, project_dir, monkeypatch, mode + ): + """Every _install_workflow_from_catalog failure branch that runs after + the mkdir/download step -- not just the registry.add() OSError case + -- must route through the same existed-before/backup-aware cleanup: + on a reinstall, a redirect rejection, a download exception, invalid + YAML, or a workflow-id mismatch must restore the prior working + workflow.yml rather than deleting the whole directory. One shared + root cause (the cleanup helper), so parametrized over trigger point.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + original_data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + runner = CliRunner() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + original_data, url + ), + ) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + assert result.exit_code == 0, result.output + + dest_file = project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" + assert dest_file.read_bytes() == original_data + + if mode == "redirect_rejected": + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): + return self._FakeResponse(b"irrelevant", "http://evil.example.com/workflow.yml") + elif mode == "download_exception": + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): + raise OSError("network down") + elif mode == "invalid_yaml": + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): + return self._FakeResponse(b": : not valid yaml: [", url) + else: # id_mismatch + mismatched_yaml = self.WORKFLOW_YAML.format(version="2.0.0").replace( + 'id: "align-wf"', 'id: "different-workflow"' + ) + + def fake_open_url(url, timeout=None, extra_headers=None, redirect_validator=None): + return self._FakeResponse(mismatched_yaml.encode(), url) + + with pytest.MonkeyPatch.context() as mp: + mp.setattr("specify_cli.authentication.http.open_url", fake_open_url) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + assert dest_file.read_bytes() == original_data + registry = WorkflowRegistry(project_dir) + assert registry.is_installed("align-wf") + assert registry.get("align-wf")["version"] == "1.0.0" + def test_download_redirect_validator_rejects_http_before_follow(self): import urllib.error From 812050a4f9491ba6bcab3a2c438af7fb6713a5d8 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 08:55:31 +0200 Subject: [PATCH 29/59] fix(workflow): restore registry entry verbatim on post-removal rmtree failure workflow_remove now persists registry.remove() before deleting any files (fixed previously), but if the registry write succeeds and the subsequent shutil.rmtree(workflow_dir) then fails, the registry was left claiming the workflow uninstalled while its directory remained on disk -- an orphaned install with no path back to a clean state. workflow_step_remove already handles this exact sequencing by capturing the registry entry before removal and restoring it directly into registry.data plus save() (bypassing add(), which would stamp a new updated_at) if the directory removal fails afterwards. Applied the same pattern to workflow_remove: capture registry_metadata via registry.get() before registry.remove(), and on an rmtree OSError, write it straight back into registry.data["workflows"][workflow_id] and save(), matching workflow_step_remove's restore-failure handling (a yellow warning, not a hard failure, since the primary error is already about to be reported). Existing error message and exit behavior for the rmtree failure are unchanged. Added a failing-first regression: install a workflow, monkeypatch shutil.rmtree to raise OSError, and assert a clean existing error message, the directory remaining (rmtree never actually deleted anything), and the registry entry restored byte-for-byte identical (including installed_at/updated_at) -- proving the fix bypasses add() and doesn't re-stamp timestamps. Confirmed red (registry entry stayed None) before the fix, green after. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 17 +++++++++++ tests/test_workflows.py | 42 ++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index f425878306..dd42d2f5c0 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -1122,6 +1122,11 @@ def workflow_remove( ) raise typer.Exit(1) + # Captured before the registry write so a subsequent directory-removal + # failure can restore it verbatim (bypassing add(), which would stamp a + # new updated_at), mirroring workflow_step_remove's same restore pattern. + registry_metadata = registry.get(workflow_id) + # Persist the registry removal before touching any files: if save() # fails, WorkflowRegistry.remove() rolls back its in-memory state and # raises, so the workflow stays fully installed (files + registry) rather @@ -1140,6 +1145,18 @@ def workflow_remove( try: shutil.rmtree(workflow_dir) except OSError as exc: + # The registry removal already succeeded; restore the original + # entry verbatim so the registry doesn't claim this workflow is + # uninstalled while its directory is still sitting on disk. + try: + if registry_metadata is not None: + registry.data["workflows"][workflow_id] = registry_metadata + registry.save() + except Exception as restore_exc: # noqa: BLE001 + console.print( + f"[yellow]Warning:[/yellow] Failed to restore registry entry " + f"for '{safe_id}' after directory removal failure: {restore_exc}" + ) console.print( f"[red]Error:[/red] Failed to remove workflow directory {workflow_dir}: {exc}" ) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 67a0554f95..2908f3f7d6 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -5935,6 +5935,48 @@ def boom(self): # The on-disk registry must still claim the workflow installed. assert WorkflowRegistry(project_dir).is_installed("test-wf") + def test_remove_directory_failure_restores_registry_entry_verbatim( + self, project_dir, monkeypatch + ): + """If the registry removal already persisted successfully but the + subsequent shutil.rmtree fails, the directory was never actually + deleted (rmtree raised before removing anything usable), so the + registry must not be left claiming the workflow uninstalled. The + restored entry must be byte-for-byte the original (same + installed_at/updated_at) -- calling add() again would stamp a new + updated_at, which is why workflow_step_remove restores directly via + registry.data and save() instead of add().""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + registry = WorkflowRegistry(project_dir) + registry.add("test-wf", {"name": "Test", "version": "1.0.0"}) + workflow_dir = project_dir / ".specify" / "workflows" / "test-wf" + workflow_dir.mkdir(parents=True, exist_ok=True) + (workflow_dir / "workflow.yml").write_text("keep-me", encoding="utf-8") + + original_entry = WorkflowRegistry(project_dir).get("test-wf") + + def boom(*args, **kwargs): + raise OSError("permission denied") + + monkeypatch.chdir(project_dir) + with pytest.MonkeyPatch.context() as mp: + mp.setattr("shutil.rmtree", boom) + result = CliRunner().invoke(app, ["workflow", "remove", "test-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "Failed to remove workflow directory" in result.output + # rmtree raised, so nothing was actually deleted. + assert workflow_dir.exists() + assert (workflow_dir / "workflow.yml").read_text(encoding="utf-8") == "keep-me" + # The registry entry must come back exactly as it was, not re-added. + restored = WorkflowRegistry(project_dir).get("test-wf") + assert restored == original_entry + assert WorkflowRegistry(project_dir).is_installed("test-wf") + class TestWorkflowAddSymlinkGuard: def test_add_malformed_ipv6_url_exits_cleanly(self, temp_dir, monkeypatch): From 6a127b32469fd43ac4f94823871a56099ae83bc8 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 09:32:43 +0200 Subject: [PATCH 30/59] Fix 4 current Copilot review findings on workflow run/registry/install 1. workflow run ownership check followed symlinks via Path.resolve() before mapping a direct YAML path back to its installed workflow ID. A symlinked .specify/workflows//workflow.yml resolved outside the tree, missed the ownership match entirely, and let the disabled-workflow guard be silently skipped while engine.load_workflow still followed the symlink. Now maps ownership from a lexically-normalized path (os.path. normpath, no symlink following) and explicitly refuses to run if the installed directory or workflow.yml leaf is itself a symlink. Direct external workflow paths that don't match .specify/workflows/... are unaffected. 2. WorkflowRegistry._load() caught a read OSError and silently fell back to an empty in-memory registry, only blocking a later save(). Callers that only query is_installed()/get()/list() before writing a file (e.g. commands/init.py's bundled speckit install, which overwrites workflow.yml once is_installed() reports false) could act on that false-empty state and destroy real data before ever reaching save(). _load() now raises OSError immediately so an unreadable registry fails closed at construction, before any query or side effect is possible. Added _open_workflow_registry() to give every CLI command a consistent clean-error boundary around registry construction. 3. _validate_and_install_local's mkdir/copy2 ran before the try/except that protected registry.add(); a copy2 failure (e.g. a truncating partial write on a reinstall) was not caught at all, so the existing backup-restore cleanup never ran and the prior working workflow.yml was corrupted with a raw traceback surfaced to the user. mkdir/copy2 now run inside the same rollback-protected section as registry.add(), sharing one _cleanup_failed_install() helper. 4. workflow update's skip message claimed any non-catalog source was installed "from a local path or URL", which is wrong for the bundled speckit workflow (source: "bundled"). Message is now source-neutral. Verified all 4 threads are current (not outdated) via GraphQL review thread query on PR #3419, HEAD 812050a. Tests: strict TDD per fix (red test proving each bug, minimal production change, green). tests/test_workflows.py: 474 passed. Full suite: 3976 passed, 110 skipped. ruff check: all checks passed on touched files and full src tree. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 111 +++++++++++------ src/specify_cli/workflows/catalog.py | 25 ++-- tests/test_workflows.py | 161 ++++++++++++++++++++++++- 3 files changed, 238 insertions(+), 59 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index dd42d2f5c0..267aea757c 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -58,6 +58,25 @@ def _error_console(json_output: bool): return err_console if json_output else console +def _open_workflow_registry(project_root: Path, out=None): + """Construct a WorkflowRegistry, exiting cleanly on an unreadable file. + + WorkflowRegistry fails closed (raises OSError) at construction when its + file can't be read, rather than falling back to an empty registry a + caller could mistake for "nothing installed". Every CLI command that + opens a registry needs this same clean-error boundary. + """ + from .catalog import WorkflowRegistry + + try: + return WorkflowRegistry(project_root) + except OSError as exc: + (out or console).print( + f"[red]Error:[/red] Failed to read workflow registry: {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + + def _parse_input_values( input_values: list[str] | None, *, json_output: bool = False ) -> dict[str, Any]: @@ -369,8 +388,6 @@ def workflow_run( err = _error_console(json_output) - from .catalog import WorkflowRegistry - registered_id: str | None = None registry_root = project_root if not is_file_source: @@ -385,18 +402,35 @@ def workflow_run( registered_id = source else: # A direct YAML path may still point at an installed workflow's own - # file; map it back to its owning project and ID from the canonical - # path itself so the guard is independent of the caller's cwd. - resolved = source_path.resolve() - parts = resolved.parts + # file; map it back to its owning project and ID from the *lexical* + # path (collapsing .. / . without resolving symlinks) rather than + # resolve(): resolving first would follow a symlinked workflow.yml + # out of .specify/workflows, fail to find an owner, and let + # engine.load_workflow below run the symlink target unchecked -- + # silently bypassing a disabled workflow's guard. + lexical = Path(os.path.normpath(str(source_path.absolute()))) + parts = lexical.parts for i in range(len(parts) - 2): if parts[i] == ".specify" and parts[i + 1] == "workflows": - registry_root = Path(*parts[:i]) if i else Path(resolved.anchor or ".") + registry_root = Path(*parts[:i]) if i else Path(lexical.anchor or ".") registered_id = parts[i + 2] + # A legitimately installed workflow's own directory tree + # never contains a symlink (workflow add/remove both refuse + # one at install time); one appearing here means the file + # actually loaded below would not be the file this ownership + # match is based on, so refuse rather than silently mismatch. + for k in range(i + 2, len(parts) + 1): + if Path(*parts[:k]).is_symlink(): + err.print( + "[red]Error:[/red] Refusing to run: " + f".specify/workflows/{_escape_markup(registered_id)} " + "contains a symlinked path component" + ) + raise typer.Exit(1) break if registered_id is not None: - installed_meta = WorkflowRegistry(registry_root).get(registered_id) + installed_meta = _open_workflow_registry(registry_root, err).get(registered_id) if isinstance(installed_meta, dict) and not installed_meta.get("enabled", True): err.print( f"[red]Error:[/red] Workflow '{_escape_markup(registered_id)}' is disabled. " @@ -612,10 +646,8 @@ def workflow_status( @workflow_app.command("list") def workflow_list(): """List installed workflows.""" - from .catalog import WorkflowRegistry - project_root = _require_specify_project() - registry = WorkflowRegistry(project_root) + registry = _open_workflow_registry(project_root) installed = registry.list() if not installed: @@ -647,11 +679,10 @@ def workflow_add( from_url: str | None = typer.Option(None, "--from", help="Install from a custom URL"), ): """Install a workflow from catalog, URL, or local path.""" - from .catalog import WorkflowRegistry from .engine import WorkflowDefinition project_root = _require_specify_project() - registry = WorkflowRegistry(project_root) + registry = _open_workflow_registry(project_root) workflows_dir = project_root / ".specify" / "workflows" # With --from, source names the expected workflow ID: validate it up # front so a URL/path/typo fails without a network fetch. @@ -705,17 +736,9 @@ def _validate_and_install_local( backup_bytes = ( dest_file.read_bytes() if existed_before and dest_file.is_file() else None ) - dest_dir.mkdir(parents=True, exist_ok=True) import shutil - shutil.copy2(yaml_path, dest_file) - try: - registry.add(definition.id, { - "name": definition.name, - "version": definition.version, - "description": definition.description, - "source": source_label, - }) - except OSError as exc: + + def _cleanup_failed_install() -> None: # Don't leave an orphan directory behind for a fresh install; for # a reinstall over an existing local workflow, restore the prior # workflow.yml instead of clobbering it with the failed update. @@ -724,6 +747,26 @@ def _validate_and_install_local( dest_file.write_bytes(backup_bytes) else: shutil.rmtree(dest_dir, ignore_errors=True) + + try: + dest_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(yaml_path, dest_file) + except OSError as exc: + _cleanup_failed_install() + console.print( + f"[red]Error:[/red] Failed to install workflow " + f"'{_escape_markup(definition.id)}': {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + try: + registry.add(definition.id, { + "name": definition.name, + "version": definition.version, + "description": definition.description, + "source": source_label, + }) + except OSError as exc: + _cleanup_failed_install() console.print( f"[red]Error:[/red] Failed to update workflow registry for " f"'{_escape_markup(definition.id)}': {_escape_markup(str(exc))}" @@ -1080,13 +1123,11 @@ def workflow_remove( workflow_id: str = typer.Argument(..., help="Workflow ID to uninstall"), ): """Uninstall a workflow.""" - from .catalog import WorkflowRegistry - project_root = _require_specify_project() workflows_dir = project_root / ".specify" / "workflows" _validate_workflow_id_or_exit(workflow_id) - registry = WorkflowRegistry(project_root) + registry = _open_workflow_registry(project_root) if not registry.is_installed(workflow_id): console.print(f"[red]Error:[/red] Workflow '{workflow_id}' is not installed") @@ -1172,10 +1213,10 @@ def workflow_update( """Update installed workflow(s) to the latest catalog version.""" from packaging import version as pkg_version - from .catalog import WorkflowCatalog, WorkflowCatalogError, WorkflowRegistry + from .catalog import WorkflowCatalog, WorkflowCatalogError project_root = _require_specify_project() - registry = WorkflowRegistry(project_root) + registry = _open_workflow_registry(project_root) workflows_dir = project_root / ".specify" / "workflows" _reject_unsafe_dir(project_root / ".specify", ".specify") _reject_unsafe_dir(workflows_dir, ".specify/workflows") @@ -1205,7 +1246,7 @@ def workflow_update( console.print(f"⚠ {safe_id}: Registry entry is corrupted (skipping)") continue if metadata.get("source") != "catalog": - console.print(f"⚠ {safe_id}: Installed from a local path or URL — re-add to update (skipping)") + console.print(f"⚠ {safe_id}: Not installed from a catalog — re-add to update (skipping)") continue try: installed_version = pkg_version.Version(str(metadata.get("version"))) @@ -1310,10 +1351,8 @@ def workflow_enable( workflow_id: str = typer.Argument(..., help="Workflow ID to enable"), ): """Enable a disabled workflow.""" - from .catalog import WorkflowRegistry - project_root = _require_specify_project() - registry = WorkflowRegistry(project_root) + registry = _open_workflow_registry(project_root) metadata = registry.get(workflow_id) if metadata is None: console.print(f"[red]Error:[/red] Workflow '{_escape_markup(workflow_id)}' is not installed") @@ -1344,10 +1383,8 @@ def workflow_disable( workflow_id: str = typer.Argument(..., help="Workflow ID to disable"), ): """Disable a workflow without removing it.""" - from .catalog import WorkflowRegistry - project_root = _require_specify_project() - registry = WorkflowRegistry(project_root) + registry = _open_workflow_registry(project_root) metadata = registry.get(workflow_id) if metadata is None: console.print(f"[red]Error:[/red] Workflow '{_escape_markup(workflow_id)}' is not installed") @@ -1416,13 +1453,13 @@ def workflow_info( workflow_id: str = typer.Argument(..., help="Workflow ID"), ): """Show workflow details and step graph.""" - from .catalog import WorkflowCatalog, WorkflowRegistry, WorkflowCatalogError + from .catalog import WorkflowCatalog, WorkflowCatalogError from .engine import WorkflowEngine project_root = _require_specify_project() # Check installed first - registry = WorkflowRegistry(project_root) + registry = _open_workflow_registry(project_root) installed = registry.get(workflow_id) engine = WorkflowEngine(project_root) diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 26ffe6051c..8a10db66f9 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -70,10 +70,6 @@ def __init__(self, project_root: Path) -> None: self.project_root = project_root self.workflows_dir = project_root / ".specify" / "workflows" self.registry_path = self.workflows_dir / self.REGISTRY_FILE - # Set before _load() so a read failure (distinct from a corrupted or - # missing file) can flip it to block a later save() from silently - # persisting an empty registry over unreadable-but-intact data. - self._load_error = False self.data = self._load() def _has_symlinked_parent(self) -> bool: @@ -99,14 +95,17 @@ def _load(self) -> dict[str, Any]: try: with open(self.registry_path, encoding="utf-8") as f: data = json.load(f) - except OSError: + except OSError as exc: # An I/O failure (e.g. permissions, transient FS issue) is not # the same as a corrupted file: the real data may still be - # intact on disk. Flag it so save() refuses to overwrite that - # data with this in-memory default instead of silently - # discarding every prior entry. - self._load_error = True - return default_registry + # intact on disk. Fail closed here, at construction, rather + # than falling back to an empty registry -- a caller that + # only queries is_installed()/get()/list() before writing a + # file (never reaching save()) would otherwise mistake this + # for "nothing installed" and overwrite real data. + raise OSError( + f"Failed to read workflow registry at {self.registry_path}: {exc}" + ) from exc except (json.JSONDecodeError, ValueError, UnicodeError): # Corrupted registry file — reset to default return default_registry @@ -126,12 +125,6 @@ def save(self) -> None: raise OSError( "Refusing to write workflow registry through a symlinked path." ) - if self._load_error: - raise OSError( - f"Refusing to save workflow registry at {self.registry_path}: " - "the existing file could not be read, so saving now would " - "discard its contents." - ) self.workflows_dir.mkdir(parents=True, exist_ok=True) # Unique, exclusive temp then replace: a failed dump cannot truncate # the registry, a pre-created symlink cannot redirect the write, and diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 2908f3f7d6..0fce0d9688 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -4771,8 +4771,10 @@ def test_persistence(self, project_dir): def test_load_read_oserror_refuses_to_save_over_existing_data(self, project_dir, monkeypatch): """A transient read failure (e.g. temporarily unreadable file) must not be - treated the same as a corrupted/missing registry: saving afterwards would - silently discard every previously persisted workflow entry.""" + treated the same as a corrupted/missing registry: constructing a registry + on top of it -- and thus any query a caller makes before ever calling + save() -- must fail closed instead of silently reporting an empty + registry that a caller could then act on and overwrite.""" from specify_cli.workflows.catalog import WorkflowRegistry import builtins @@ -4787,15 +4789,37 @@ def _raising_open(file, mode="r", *args, **kwargs): return real_open(file, mode, *args, **kwargs) monkeypatch.setattr(builtins, "open", _raising_open) - registry2 = WorkflowRegistry(project_dir) - # The in-memory view may fall back to empty, but a save must not be - # allowed to persist that empty state over the real file on disk. with pytest.raises(OSError): - registry2.save() + WorkflowRegistry(project_dir) # The original entry must survive on disk untouched. data = json.loads(registry_path.read_text(encoding="utf-8")) assert "test-wf" in data["workflows"] + def test_load_read_oserror_fails_closed_not_silently_empty(self, project_dir, monkeypatch): + """Root cause: a registry that failed to read must never let a query + method (is_installed/get/list) report as if nothing were installed -- + a caller (e.g. bundled workflow install) that only checks + is_installed() before writing a file would otherwise overwrite real + data on a transient read failure, long before any save() call could + catch it. The failure must surface at construction, before any query + or side effect is possible.""" + from specify_cli.workflows.catalog import WorkflowRegistry + import builtins + + registry1 = WorkflowRegistry(project_dir) + registry1.add("test-wf", {"name": "Test", "version": "1.0.0"}) + registry_path = registry1.registry_path + real_open = builtins.open + + def _raising_open(file, mode="r", *args, **kwargs): + if Path(file) == registry_path and "r" in mode: + raise OSError("simulated read failure") + return real_open(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", _raising_open) + with pytest.raises(OSError): + WorkflowRegistry(project_dir) + # ===== Workflow Catalog Tests ===== @@ -7676,6 +7700,26 @@ def test_update_skips_non_catalog_sources(self, project_dir, monkeypatch): assert "No workflows were eligible for update" in result.output assert "up to date!" not in result.output + def test_update_skip_message_accurate_for_bundled_source(self, project_dir, monkeypatch): + """A workflow registered with source "bundled" (e.g. the speckit + workflow installed by `specify init`) was never installed from a + local path or URL; the skip message must not claim otherwise.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + registry = WorkflowRegistry(project_dir) + registry.add( + "speckit", + {"name": "Speckit", "version": "1.0.0", "source": "bundled"}, + ) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "update"]) + assert result.exit_code == 0, result.output + assert "local path or URL" not in result.output + assert "re-add to update" in result.output + def test_registry_add_rolls_back_memory_on_save_failure(self, project_dir, monkeypatch): from specify_cli.workflows.catalog import WorkflowRegistry @@ -7782,6 +7826,46 @@ def boom(self): assert not dest_dir.exists() assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_dev_reinstall_copy_failure_restores_prior_file(self, project_dir, monkeypatch): + """_validate_and_install_local's copy2 call currently runs *before* the + try/except block that protects registry.add(): a copy2 failure (e.g. a + truncating partial write on a reinstall) is not caught at all, so the + existing backup-restore cleanup never runs and the prior working + workflow.yml is left corrupted. copy2 must be covered by the same + rollback-protected section as registry.add().""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + src = self._install_dev(runner, app, project_dir) + installed_yaml = project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" + original_bytes = installed_yaml.read_bytes() + original_registry_entry = WorkflowRegistry(project_dir).get("align-wf") + + # Point --dev at a new version of the same workflow to trigger a + # reinstall (overwrite) rather than a fresh install. + (src / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="2.0.0"), encoding="utf-8" + ) + + def boom(*args, **kwargs): + # Simulate a truncating partial write followed by an OSError, + # mirroring a real disk-full/interrupted-copy failure. + installed_yaml.write_bytes(b"") + raise OSError("disk full") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr("shutil.copy2", boom) + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + assert installed_yaml.read_bytes() == original_bytes + assert WorkflowRegistry(project_dir).get("align-wf") == original_registry_entry + def test_add_catalog_save_failure_leaves_no_orphan_directory(self, project_dir, monkeypatch): """Same guarantee as the local-install paths, but for a fresh catalog install: a registry.add() failure must clean up the freshly-downloaded @@ -8435,6 +8519,37 @@ def test_list_skips_corrupted_registry_entry(self, project_dir, monkeypatch): assert "corrupted" in result.output assert "OK Workflow" in result.output + def test_list_unreadable_registry_fails_closed_with_clean_error( + self, project_dir, monkeypatch + ): + """An unreadable registry file must produce a clean CLI error, not a + raw traceback and not a silent "nothing installed" list -- the latter + is exactly the fail-open state a caller could otherwise mistake for + "safe to (re)install", overwriting real files. Covers the read/query + boundary fix required at every WorkflowRegistry call site.""" + import builtins + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + + registry_path = WorkflowRegistry(project_dir).registry_path.resolve() + real_open = builtins.open + + def _raising_open(file, mode="r", *args, **kwargs): + if Path(file).resolve() == registry_path and "r" in mode: + raise OSError("simulated read failure") + return real_open(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", _raising_open) + result = runner.invoke(app, ["workflow", "list"]) + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "Error" in result.output + def test_list_escapes_rich_markup_in_registry_fields(self, project_dir, monkeypatch): """User-editable name/description/id fields must not be parsed as Rich markup.""" import json @@ -8702,6 +8817,40 @@ def test_disable_blocks_run_via_path_equivalent_id(self, project_dir, monkeypatc assert result.exit_code != 0 assert "disabled" in result.output + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") + def test_disable_blocks_run_when_installed_yaml_is_symlinked( + self, project_dir, monkeypatch + ): + """A disabled workflow's own workflow.yml being replaced with a symlink + must not bypass the disabled check. Resolving the path before mapping + it back to its registry owner would follow the symlink out of + .specify/workflows, fail to find an owner, and let engine.load_workflow + run the original symlink target anyway -- ownership must be + determined from the normalized *lexical* path (not resolve()), and a + symlinked path component in the installed tree must be refused.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + + result = runner.invoke(app, ["workflow", "disable", "align-wf"]) + assert result.exit_code == 0, result.output + + installed_yaml = project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" + external_target = project_dir / "external-workflow.yml" + external_target.write_text( + self.WORKFLOW_YAML.format(version="9.9.9"), encoding="utf-8" + ) + installed_yaml.unlink() + installed_yaml.symlink_to(external_target) + + result = runner.invoke(app, ["workflow", "run", str(installed_yaml)]) + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "disabled" in result.output or "symlink" in result.output.lower() + def test_disable_shows_marker_in_list(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app From 12d985b76b4418cf8d3d11a314043d788a66c01a Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 09:51:49 +0200 Subject: [PATCH 31/59] Fix disabled-workflow bypass via symlinked .specify project root workflow run's ownership check derived registry_root/registered_id from the lexical path, then checked the id directory and workflow.yml leaf for symlinks -- but never checked .specify or .specify/workflows themselves for that derived root. _reject_unsafe_workflow_storage only guards the cwd's project_root, which can differ from the path-derived registry_root (a direct path into an unrelated project, or that project's own .specify being a symlink to an attacker-controlled tree). WorkflowRegistry's own symlinked-parent handling silently substitutes an empty registry instead of raising, so a query against it (is_installed/get returning "not found") is not a safety signal a caller can rely on: with a symlinked .specify, the disabled check saw no registry entry and let a disabled workflow run anyway. Fix: reject an unsafe .specify/.specify-workflows for the actual derived registry_root before ever consulting the registry, reusing the existing _reject_unsafe_dir helper already used by _reject_unsafe_workflow_storage. Red-first end-to-end repro: victim project's .specify symlinked to an attacker-controlled tree containing a disabled workflow entry, run invoked with a direct path from an unrelated cwd -- confirmed the disabled workflow executed (exit 0) before the fix, now refused cleanly. Tests: tests/test_workflows.py 475 passed. Full suite: 3977 passed, 110 skipped. ruff check: all checks passed. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 14 +++++++ tests/test_workflows.py | 54 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 267aea757c..d356895252 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -414,6 +414,20 @@ def workflow_run( if parts[i] == ".specify" and parts[i + 1] == "workflows": registry_root = Path(*parts[:i]) if i else Path(lexical.anchor or ".") registered_id = parts[i + 2] + # The path-derived registry_root here may differ from the + # cwd's project_root already checked by + # _reject_unsafe_workflow_storage above (e.g. this path + # points into another project entirely, or this project's + # own .specify is itself a symlink to an attacker-controlled + # tree) -- check it explicitly rather than trusting that + # cwd-scoped guard, and don't rely on WorkflowRegistry's own + # symlinked-parent handling below (it silently substitutes + # an empty registry instead of raising, so a query against + # it can't be trusted as a safety signal here). + _reject_unsafe_dir(registry_root / ".specify", ".specify") + _reject_unsafe_dir( + registry_root / ".specify" / "workflows", ".specify/workflows" + ) # A legitimately installed workflow's own directory tree # never contains a symlink (workflow add/remove both refuse # one at install time); one appearing here means the file diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 0fce0d9688..e0cd927f26 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8851,6 +8851,60 @@ def test_disable_blocks_run_when_installed_yaml_is_symlinked( assert result.exception is None or isinstance(result.exception, SystemExit) assert "disabled" in result.output or "symlink" in result.output.lower() + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") + def test_run_refuses_symlinked_specify_dir_hiding_disabled_workflow( + self, temp_dir, monkeypatch + ): + """A victim project's own .specify directory being a symlink to an + attacker-controlled tree must not bypass the disabled-workflow guard. + _reject_unsafe_workflow_storage only checks the *cwd's* project root + (unrelated here); the id/leaf symlink-component loop only checks + components from the id directory onward, missing .specify/ + .specify/workflows themselves. The ownership check must reject an + unsafe .specify/.specify-workflows for the actual path-derived + registry root before ever consulting the registry -- it must not + rely on WorkflowRegistry's own symlinked-parent handling, which + silently returns an empty registry instead of raising and so is not + a safety signal a caller can depend on.""" + from typer.testing import CliRunner + from specify_cli import app + + victim = temp_dir / "victim" + victim.mkdir() + attacker_real = temp_dir / "attacker-real" + (attacker_real / "workflows" / "evil").mkdir(parents=True) + (attacker_real / "workflows" / "evil" / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + (attacker_real / "workflows" / "workflow-registry.json").write_text( + json.dumps( + { + "schema_version": "1.0", + "workflows": { + "evil": { + "name": "Evil", + "version": "1.0.0", + "source": "dev", + "enabled": False, + } + }, + } + ), + encoding="utf-8", + ) + (victim / ".specify").symlink_to(attacker_real) + + unrelated_cwd = temp_dir / "unrelated-cwd" + unrelated_cwd.mkdir() + monkeypatch.chdir(unrelated_cwd) + + runner = CliRunner() + target = victim / ".specify" / "workflows" / "evil" / "workflow.yml" + result = runner.invoke(app, ["workflow", "run", str(target)]) + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "symlink" in result.output.lower() + def test_disable_shows_marker_in_list(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app From ce3e0abacb3e9dd9fe830210438a93c3fd86cee2 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 09:58:17 +0200 Subject: [PATCH 32/59] Fix raw exception leak in bundle remove primitive boundary remove_bundle() had no exception handling around its component removal loop, unlike install_bundle() which converts any raw exception into a clean BundlerError. Since WorkflowRegistry now fails closed (raises OSError) on an unreadable registry file, and _WorkflowKindManager.__init__ constructs WorkflowRegistry with no try/except, an unreadable workflow registry surfaced as a raw OSError through remove_bundle(). The bundle_remove CLI command only catches BundlerError, so the raw OSError propagated uncaught, producing exit_code=1 with empty output instead of a clean, actionable message. Wrap remove_bundle()'s component loop in the same try/except BundlerError: raise / except Exception: raise BundlerError(...) from exc pattern already used by install_bundle(), converting any raw exception at this shared boundary. save_records() remains outside the try block, so a failure still leaves the bundle's record untouched (no removal side effects recorded). Tests: - tests/integration/test_bundler_install_flow.py::test_remove_converts_raw_installer_exception_to_bundler_error (function-level regression: a raw OSError from installer.is_installed must become a clean BundlerError, and the bundle record must survive) - tests/contract/test_bundle_cli.py::test_remove_reports_clean_error_when_primitive_raises_raw_exception (CLI-level regression: `specify bundle remove` must print a clean actionable message and exit non-zero instead of raw/empty output) Both tests were confirmed red beforehand: the raw OSError propagated uncaught out of remove_bundle(), and the CLI-level CliRunner result showed exit_code=1 with empty output. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/bundler/services/installer.py | 28 +++++++++------ tests/contract/test_bundle_cli.py | 36 +++++++++++++++++++ .../integration/test_bundler_install_flow.py | 26 ++++++++++++++ 3 files changed, 80 insertions(+), 10 deletions(-) diff --git a/src/specify_cli/bundler/services/installer.py b/src/specify_cli/bundler/services/installer.py index 3e61ded040..486b146bc5 100644 --- a/src/specify_cli/bundler/services/installer.py +++ b/src/specify_cli/bundler/services/installer.py @@ -188,16 +188,24 @@ def remove_bundle( still_needed = components_still_needed(records, exclude_bundle_id=bundle_id) result = InstallResult(bundle_id=bundle_id) - for component in target.contributed_components: - key = (component.kind, component.id) - if key in still_needed: - result.skipped.append(component) - continue - if installer.is_installed(project_root, component): - installer.remove(project_root, component) - result.uninstalled.append(component) - else: - result.skipped.append(component) + try: + for component in target.contributed_components: + key = (component.kind, component.id) + if key in still_needed: + result.skipped.append(component) + continue + if installer.is_installed(project_root, component): + installer.remove(project_root, component) + result.uninstalled.append(component) + else: + result.skipped.append(component) + except BundlerError: + raise + except Exception as exc: # noqa: BLE001 + raise BundlerError( + f"Failed to remove bundle '{bundle_id}': {exc}. " + "No changes were recorded." + ) from exc save_records(project_root, remove_record(records, bundle_id)) return result diff --git a/tests/contract/test_bundle_cli.py b/tests/contract/test_bundle_cli.py index 1705c5945d..f76fafe32c 100644 --- a/tests/contract/test_bundle_cli.py +++ b/tests/contract/test_bundle_cli.py @@ -63,6 +63,42 @@ def test_commands_outside_project_fail_with_guidance(tmp_path: Path, monkeypatch assert "Spec Kit project" in result.output +def test_remove_reports_clean_error_when_primitive_raises_raw_exception( + project: Path, +): + """A raw exception from a primitive installer (e.g. an OSError from an + unreadable workflow registry surfacing through _WorkflowKindManager's + fail-closed construction) must not propagate uncaught through + `specify bundle remove` -- the command only catches BundlerError, so + without a conversion at the remove_bundle boundary this would exit + with an unhandled exception and empty/raw output instead of a clean, + actionable message, and no removal side effects should occur either.""" + from specify_cli.bundler.models.manifest import BundleManifest + from specify_cli.bundler.models.records import load_records + from specify_cli.bundler.services.adapters import DefaultPrimitiveInstaller + from specify_cli.bundler.services.installer import install_bundle + from specify_cli.bundler.services.resolver import resolve_install_plan + from tests.bundler_helpers import FakeInstaller + + manifest = BundleManifest.from_dict(valid_manifest_dict()) + plan = resolve_install_plan( + manifest, speckit_version="0.11.2", active_integration="copilot" + ) + install_bundle(project, plan, FakeInstaller(), manifest=manifest) + + def boom(self, project_root, component): + raise OSError("workflow registry unreadable") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(DefaultPrimitiveInstaller, "is_installed", boom) + result = runner.invoke(app, ["bundle", "remove", "demo-bundle"]) + + assert result.exit_code != 0 + assert result.output.strip() != "" + assert result.exception is None or isinstance(result.exception, SystemExit) + assert {r.bundle_id for r in load_records(project)} == {"demo-bundle"} + + def test_fail_writes_error_to_stderr_not_stdout(capsys): """_fail must write to stderr, not stdout: every bundle command routes errors through it, and under --json the error would otherwise corrupt the JSON payload diff --git a/tests/integration/test_bundler_install_flow.py b/tests/integration/test_bundler_install_flow.py index bf066b660e..3ef8ff45b9 100644 --- a/tests/integration/test_bundler_install_flow.py +++ b/tests/integration/test_bundler_install_flow.py @@ -97,6 +97,32 @@ def test_remove_unknown_bundle_errors(tmp_path: Path): remove_bundle(tmp_path, "ghost", FakeInstaller()) +def test_remove_converts_raw_installer_exception_to_bundler_error(tmp_path: Path): + """A raw exception from a primitive installer (e.g. an OSError from an + unreadable workflow registry surfacing through _WorkflowKindManager's + fail-closed construction) must not propagate uncaught out of + remove_bundle: install_bundle already converts any non-BundlerError + exception into a clean BundlerError, but remove_bundle had no such + conversion, so the CLI's `bundle remove` (which only catches + BundlerError) would let a raw exception through with no clean message + and no removal side effects should occur either.""" + make_project(tmp_path) + manifest = BundleManifest.from_dict(valid_manifest_dict()) + installer = FakeInstaller() + install_bundle(tmp_path, _plan(manifest), installer, manifest=manifest) + + def boom(project_root, component): + raise OSError("workflow registry unreadable") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(installer, "is_installed", boom) + with pytest.raises(BundlerError): + remove_bundle(tmp_path, "demo-bundle", installer) + + # No removal side effects: the bundle record must still be present. + assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} + + def test_remove_reports_uninstalled_not_installed(tmp_path: Path): make_project(tmp_path) manifest = BundleManifest.from_dict(valid_manifest_dict()) From 6b593e3dd0092443394bbd1eda9cd0ec2ee82549 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 10:41:53 +0200 Subject: [PATCH 33/59] Fix 8 current Copilot review findings on registry fail-closed, rollback orphans, backup-read boundaries, and Rich escaping 1. WorkflowRegistry._load(): a symlinked .specify/.specify/workflows parent (or a symlinked registry file) silently returned an empty registry instead of raising, unlike an unreadable-file read failure. A read-only caller (notably the bundler's remove path) querying is_installed() before ever writing could conclude an installed workflow is absent, skip removing it, then delete the bundle record -- leaving the workflow untracked but still on disk. Now raises OSError immediately, matching the existing unreadable-file fail-closed behavior. 2/8. _validate_and_install_local and _install_workflow_from_catalog: when the destination directory already existed but had no prior workflow.yml (e.g. a leftover empty dir), existed_before was True but there were no backup bytes to restore, so the rollback closure did nothing on a later failure -- leaving the newly copied/ downloaded file behind. Both now unlink the newly created file in this case, restoring the pre-existing directory to its prior (empty) state. 3/4. Both install paths read the prior workflow.yml bytes (to seed the reinstall rollback) *before* any try/except boundary: a read failure on the existing file (e.g. a transient permission/FS issue) leaked a raw, unescaped OSError instead of the same clean CLI error used by every other failure branch in these functions. Both reads are now guarded by their own try/except OSError, with no writes attempted before the read succeeds (so there is nothing to roll back on this specific failure). 5. remove_bundle's exception-conversion message unconditionally claimed "No changes were recorded," even though a failure can occur after earlier components in the same bundle have already been removed from disk (save_records never runs on this path, so the record is left claiming the bundle fully installed). The message now reports how many components were already removed when that happened, instead of asserting no changes occurred. 6/7. workflow_remove's new post-registry-removal directory-failure error and its restore-failure warning interpolated workflow_dir and the exception values into Rich markup unescaped. A project path or OS error message containing Rich-markup-like brackets could be parsed as markup and hide/corrupt the displayed text. Both now use the existing _escape_markup helper, consistent with every other error path in this file. Tests (tests/test_workflows.py unless noted): - TestWorkflowRegistry::test_load_symlinked_workflows_dir_fails_closed_not_silently_empty (1) - TestWorkflowCliAlignment::test_add_dev_fresh_install_into_preexisting_empty_dir_cleans_new_file (2) - TestWorkflowCliAlignment::test_add_catalog_fresh_install_into_preexisting_empty_dir_cleans_new_file (8) - TestWorkflowCliAlignment::test_add_dev_reinstall_backup_read_failure_gives_clean_error (3) - TestWorkflowCliAlignment::test_add_catalog_reinstall_backup_read_failure_gives_clean_error (4) - tests/integration/test_bundler_install_flow.py::test_remove_partial_failure_message_reflects_partial_state (5) - TestWorkflowRemoveGuard::test_remove_directory_and_restore_failure_escapes_rich_markup (6/7) All seven were confirmed red beforehand, matching each thread's described failure mode exactly (silent empty registry instead of a raise; orphaned new file left behind; raw unescaped OSError leaking; a misleading "no changes were recorded" claim; Rich markup consuming bracketed path/exception text). Also updated test_registry_save_refuses_symlinked_parent, a pre-existing test that asserted the symlinked-parent raise at add()/save() time -- it now raises at construction instead, per fix #1, so the test was adjusted to match without weakening its guarantee (still asserts no writes occur under the symlinked target). Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/bundler/services/installer.py | 11 +- src/specify_cli/workflows/_commands.py | 44 ++- src/specify_cli/workflows/catalog.py | 13 +- .../integration/test_bundler_install_flow.py | 30 ++ tests/test_workflows.py | 259 +++++++++++++++++- 5 files changed, 342 insertions(+), 15 deletions(-) diff --git a/src/specify_cli/bundler/services/installer.py b/src/specify_cli/bundler/services/installer.py index 486b146bc5..9fc7d2a96b 100644 --- a/src/specify_cli/bundler/services/installer.py +++ b/src/specify_cli/bundler/services/installer.py @@ -202,9 +202,16 @@ def remove_bundle( except BundlerError: raise except Exception as exc: # noqa: BLE001 + if result.uninstalled: + detail = ( + f"{len(result.uninstalled)} component(s) were already removed " + "before this failure; the bundle record was left unchanged, " + "so the project may be partially uninstalled." + ) + else: + detail = "No components were removed." raise BundlerError( - f"Failed to remove bundle '{bundle_id}': {exc}. " - "No changes were recorded." + f"Failed to remove bundle '{bundle_id}': {exc}. {detail}" ) from exc save_records(project_root, remove_record(records, bundle_id)) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index d356895252..98fd7284f6 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -747,18 +747,30 @@ def _validate_and_install_local( dest_dir = _safe_workflow_id_dir(workflows_dir, definition.id) dest_file = dest_dir / "workflow.yml" existed_before = dest_dir.is_dir() - backup_bytes = ( - dest_file.read_bytes() if existed_before and dest_file.is_file() else None - ) + try: + backup_bytes = ( + dest_file.read_bytes() if existed_before and dest_file.is_file() else None + ) + except OSError as exc: + console.print( + f"[red]Error:[/red] Failed to read existing workflow " + f"'{_escape_markup(definition.id)}' before install: {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) import shutil def _cleanup_failed_install() -> None: # Don't leave an orphan directory behind for a fresh install; for # a reinstall over an existing local workflow, restore the prior # workflow.yml instead of clobbering it with the failed update. + # A pre-existing directory with no prior workflow.yml (no backup + # bytes) must have the newly written file removed instead of + # doing nothing, so it doesn't linger as an orphan. if existed_before: if backup_bytes is not None: dest_file.write_bytes(backup_bytes) + else: + dest_file.unlink(missing_ok=True) else: shutil.rmtree(dest_dir, ignore_errors=True) @@ -996,9 +1008,16 @@ def _install_workflow_from_catalog( # mirroring _validate_and_install_local's existed-before/backup-aware # rollback. existed_before = workflow_dir.is_dir() - prior_workflow_bytes = ( - workflow_file.read_bytes() if existed_before and workflow_file.is_file() else None - ) + try: + prior_workflow_bytes = ( + workflow_file.read_bytes() if existed_before and workflow_file.is_file() else None + ) + except OSError as exc: + console.print( + f"[red]Error:[/red] Failed to read existing workflow " + f"'{safe_wf_id}' before install: {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) def _cleanup_failed_install() -> None: """Restore the prior workflow.yml on a reinstall, or remove the @@ -1007,10 +1026,15 @@ def _cleanup_failed_install() -> None: exception, invalid YAML, ID mismatch, version mismatch, and registry.add() failure -- must call this instead of rmtree'ing directly, so none of them can destroy a working install that - predates this attempt.""" + predates this attempt. A pre-existing directory with no prior + workflow.yml (no backup bytes) must have the newly downloaded file + removed instead of doing nothing, so it doesn't linger as an + orphan.""" if existed_before: if prior_workflow_bytes is not None: workflow_file.write_bytes(prior_workflow_bytes) + else: + workflow_file.unlink(missing_ok=True) else: import shutil shutil.rmtree(workflow_dir, ignore_errors=True) @@ -1210,10 +1234,12 @@ def workflow_remove( except Exception as restore_exc: # noqa: BLE001 console.print( f"[yellow]Warning:[/yellow] Failed to restore registry entry " - f"for '{safe_id}' after directory removal failure: {restore_exc}" + f"for '{safe_id}' after directory removal failure: " + f"{_escape_markup(str(restore_exc))}" ) console.print( - f"[red]Error:[/red] Failed to remove workflow directory {workflow_dir}: {exc}" + f"[red]Error:[/red] Failed to remove workflow directory " + f"{_escape_markup(str(workflow_dir))}: {_escape_markup(str(exc))}" ) raise typer.Exit(1) diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 8a10db66f9..65b24e7265 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -88,9 +88,18 @@ def _load(self) -> dict[str, Any]: "workflows": {}, } # Defense-in-depth: refuse to read through symlinked parents or a - # symlinked registry file (mirrors StepRegistry._load). + # symlinked registry file. Unlike StepRegistry (read-only best-effort + # elsewhere), a fabricated empty registry here is not safe: read-only + # callers (notably the bundler's remove path) query is_installed() + # before ever writing, and would otherwise conclude an installed + # workflow is absent, skip removing it, then delete the bundle + # record -- leaving the workflow untracked but still on disk. Fail + # closed here just like the unreadable-file case below. if self._has_symlinked_parent() or self.registry_path.is_symlink(): - return default_registry + raise OSError( + f"Refusing to read workflow registry at {self.registry_path}: " + "a parent directory or the registry file itself is a symlink" + ) if self.registry_path.exists(): try: with open(self.registry_path, encoding="utf-8") as f: diff --git a/tests/integration/test_bundler_install_flow.py b/tests/integration/test_bundler_install_flow.py index 3ef8ff45b9..8b52d90fe3 100644 --- a/tests/integration/test_bundler_install_flow.py +++ b/tests/integration/test_bundler_install_flow.py @@ -123,6 +123,36 @@ def boom(project_root, component): assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} +def test_remove_partial_failure_message_reflects_partial_state(tmp_path: Path): + """A failure can occur after earlier components in the same bundle have + already been removed from disk. The bundle record is left unchanged + (save_records never runs on this path), so it still claims the bundle + fully installed -- but the message must not claim "No changes were + recorded" when components were, in fact, already removed.""" + make_project(tmp_path) + manifest = BundleManifest.from_dict(valid_manifest_dict()) + installer = FakeInstaller() + install_bundle(tmp_path, _plan(manifest), installer, manifest=manifest) + + real_remove = installer.remove + calls = {"n": 0} + + def remove_then_fail(project_root, component): + calls["n"] += 1 + if calls["n"] == 1: + return real_remove(project_root, component) + raise OSError("disk full") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(installer, "remove", remove_then_fail) + with pytest.raises(BundlerError) as exc_info: + remove_bundle(tmp_path, "demo-bundle", installer) + + message = str(exc_info.value) + assert "no changes were recorded" not in message.lower() + assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} + + def test_remove_reports_uninstalled_not_installed(tmp_path: Path): make_project(tmp_path) manifest = BundleManifest.from_dict(valid_manifest_dict()) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index e0cd927f26..ba1c355a03 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -4820,6 +4820,33 @@ def _raising_open(file, mode="r", *args, **kwargs): with pytest.raises(OSError): WorkflowRegistry(project_dir) + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") + def test_load_symlinked_workflows_dir_fails_closed_not_silently_empty( + self, project_dir + ): + """A symlinked .specify/workflows is the same fail-open hazard as a + read OSError: silently reporting an empty registry lets a read-only + caller (e.g. the bundler's remove path) conclude a workflow isn't + installed, skip removing it, and then delete the bundle record -- + leaving the workflow untracked but still on disk. Raise here too, + exactly like the unreadable-file case, so callers cannot act on + fabricated empty state.""" + from specify_cli.workflows.catalog import WorkflowRegistry + import json as _json + + outside = project_dir.parent / "outside-workflows" + outside.mkdir(parents=True, exist_ok=True) + (outside / "workflow-registry.json").write_text( + _json.dumps({"schema_version": "1.0", "workflows": {"evil": {}}}), + encoding="utf-8", + ) + workflows_link = project_dir / ".specify" / "workflows" + workflows_link.rmdir() + workflows_link.symlink_to(outside, target_is_directory=True) + + with pytest.raises(OSError): + WorkflowRegistry(project_dir) + # ===== Workflow Catalog Tests ===== @@ -6001,6 +6028,61 @@ def boom(*args, **kwargs): assert restored == original_entry assert WorkflowRegistry(project_dir).is_installed("test-wf") + def test_remove_directory_and_restore_failure_escapes_rich_markup( + self, temp_dir, monkeypatch + ): + """The project path (workflow_dir) and the rmtree/restore-save + exceptions interpolated into these new Rich error/warning messages + must be escaped like every other error path here -- unescaped Rich + markup characters (e.g. brackets) in a project directory name or an + OS/registry error message could otherwise be parsed as markup and + hide or corrupt the displayed text instead of showing it verbatim.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + project_dir = temp_dir / "weird[project]" + project_dir.mkdir() + (project_dir / ".specify").mkdir() + (project_dir / ".specify" / "workflows").mkdir() + + registry = WorkflowRegistry(project_dir) + registry.add("test-wf", {"name": "Test", "version": "1.0.0"}) + workflow_dir = project_dir / ".specify" / "workflows" / "test-wf" + workflow_dir.mkdir(parents=True, exist_ok=True) + (workflow_dir / "workflow.yml").write_text("keep-me", encoding="utf-8") + + def rmtree_boom(*args, **kwargs): + raise OSError("[disk] permission denied") + + real_save = WorkflowRegistry.save + call_count = {"n": 0} + + def save_boom(self): + # The first save() call is registry.remove()'s own persist, + # which must succeed so we reach the rmtree failure below; only + # the second call (the post-rmtree-failure restore attempt) + # should fail, to exercise the restore-failure warning path. + call_count["n"] += 1 + if call_count["n"] == 1: + return real_save(self) + raise Exception("[warn] save exploded") + + monkeypatch.chdir(project_dir) + with pytest.MonkeyPatch.context() as mp: + mp.setattr("shutil.rmtree", rmtree_boom) + mp.setattr(WorkflowRegistry, "save", save_boom) + result = CliRunner().invoke(app, ["workflow", "remove", "test-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + # Rich may soft-wrap the long path across lines; compare with + # whitespace collapsed so the wrap position doesn't affect the check. + output_compact = "".join(result.output.split()) + assert "".join(str(workflow_dir).split()) in output_compact + assert "[disk]permissiondenied" in output_compact + assert "[warn]saveexploded" in output_compact + class TestWorkflowAddSymlinkGuard: def test_add_malformed_ipv6_url_exits_cleanly(self, temp_dir, monkeypatch): @@ -7758,6 +7840,9 @@ def test_registry_load_normalizes_malformed_workflows_field(self, project_dir): assert fresh.get("align-wf")["version"] == "1.0.0" def test_registry_save_refuses_symlinked_parent(self, project_dir, tmp_path): + """Construction now fails closed on a symlinked .specify just like + an unreadable registry file: a symlinked parent must never be + silently tolerated up to save() -- it must raise immediately.""" from specify_cli.workflows.catalog import WorkflowRegistry outside = tmp_path / "outside-specify" @@ -7766,9 +7851,8 @@ def test_registry_save_refuses_symlinked_parent(self, project_dir, tmp_path): if specify_dir.exists(): shutil.rmtree(specify_dir) specify_dir.symlink_to(outside) - registry = WorkflowRegistry(project_dir) with pytest.raises(OSError, match="symlink"): - registry.add("align-wf", {"version": "1.0.0", "source": "catalog"}) + WorkflowRegistry(project_dir) assert not (outside / "workflows").exists() def test_add_dev_dir_with_workflow_yml_directory_errors_cleanly(self, project_dir, monkeypatch): @@ -7866,6 +7950,75 @@ def boom(*args, **kwargs): assert installed_yaml.read_bytes() == original_bytes assert WorkflowRegistry(project_dir).get("align-wf") == original_registry_entry + def test_add_dev_reinstall_backup_read_failure_gives_clean_error( + self, project_dir, monkeypatch + ): + """The prior-file backup read (used to restore on a later install + failure) ran before any guarded section: a read failure on the + existing workflow.yml (e.g. a transient permission/FS issue) leaked + a raw OSError instead of the clean escaped CLI error used by every + other failure branch here, and left the destination untouched since + it happens before any write.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + src = self._install_dev(runner, app, project_dir) + installed_yaml = project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" + original_bytes = installed_yaml.read_bytes() + + (src / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="2.0.0"), encoding="utf-8" + ) + + real_read_bytes = Path.read_bytes + + def boom(self_path, *args, **kwargs): + if self_path.resolve() == installed_yaml.resolve(): + raise OSError("permission denied") + return real_read_bytes(self_path, *args, **kwargs) + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(Path, "read_bytes", boom) + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + assert installed_yaml.read_bytes() == original_bytes + + def test_add_dev_fresh_install_into_preexisting_empty_dir_cleans_new_file( + self, project_dir, monkeypatch + ): + """When the destination directory already exists but has no + workflow.yml (e.g. an empty dir left over from elsewhere), a later + registry.add() failure must remove the newly copied file -- the + rollback previously did nothing in this case (existed_before=True + with no backup bytes), leaving the new file behind -- while leaving + the pre-existing directory itself intact.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + src = self._write_workflow_dir(project_dir) + dest_dir = project_dir / ".specify" / "workflows" / "align-wf" + dest_dir.mkdir(parents=True) # pre-existing, but empty: no workflow.yml + + def boom(self, *args, **kwargs): + raise OSError("disk full") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(WorkflowRegistry, "add", boom) + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + + assert result.exit_code != 0 + assert result.output.strip() != "" + assert dest_dir.is_dir() + assert not (dest_dir / "workflow.yml").exists() + def test_add_catalog_save_failure_leaves_no_orphan_directory(self, project_dir, monkeypatch): """Same guarantee as the local-install paths, but for a fresh catalog install: a registry.add() failure must clean up the freshly-downloaded @@ -7973,6 +8126,108 @@ def boom(self): assert registry.is_installed("align-wf") assert registry.get("align-wf")["version"] == "1.0.0" + def test_add_catalog_reinstall_backup_read_failure_gives_clean_error( + self, project_dir, monkeypatch + ): + """Same backup-read boundary gap as the local-install path: the + prior-file read used to seed the reinstall's rollback ran before + the download/validation error boundary, so a read failure on the + existing workflow.yml (e.g. a transient permission/FS issue) leaked + a raw OSError instead of a clean escaped CLI error, and must be + caught before any download/write is attempted.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog + + monkeypatch.chdir(project_dir) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + original_data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + runner = CliRunner() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + original_data, url + ), + ) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + assert result.exit_code == 0, result.output + + dest_file = project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" + real_read_bytes = Path.read_bytes + + def boom(self_path, *args, **kwargs): + if self_path.resolve() == dest_file.resolve(): + raise OSError("permission denied") + return real_read_bytes(self_path, *args, **kwargs) + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(Path, "read_bytes", boom) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + assert dest_file.read_bytes() == original_data + + def test_add_catalog_fresh_install_into_preexisting_empty_dir_cleans_new_file( + self, project_dir, monkeypatch + ): + """Same rollback orphan gap as the local-install path, but for a + fresh catalog install: a pre-existing empty destination directory + (no workflow.yml) sets existed_before=True with no backup bytes, so + the rollback previously did nothing on a later failure -- leaving + the freshly downloaded workflow.yml behind. It must be removed, + leaving the pre-existing directory itself intact.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + dest_dir = project_dir / ".specify" / "workflows" / "align-wf" + dest_dir.mkdir(parents=True) # pre-existing, but empty: no workflow.yml + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + + def boom(self): + raise OSError("disk full") + + runner = CliRunner() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), + ) + mp.setattr(WorkflowRegistry, "save", boom) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + + assert result.exit_code != 0 + assert result.output.strip() != "" + assert dest_dir.is_dir() + assert not (dest_dir / "workflow.yml").exists() + @pytest.mark.parametrize( "mode", ["redirect_rejected", "download_exception", "invalid_yaml", "id_mismatch"] ) From db45f6c2d1a2096015de5c505851db5a6312f869 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:05:15 +0200 Subject: [PATCH 34/59] Fix 3 current Copilot review findings: bookkeeping-aware BundlerError removal, bounded workflow downloads 1. bundle remove: BundlerError raised by the primitive installer itself (e.g. from a kind manager) bypassed the partial-removal bookkeeping message added previously via a bare `except BundlerError: raise`. Now routes through the same detail-construction logic as generic exceptions, so a mid-loop BundlerError after an earlier successful removal still reports that the project may be partially uninstalled, while a zero-removal BundlerError still reports "No components were removed." Both preserve the original exception message and chain `from exc`. 2/3. workflow add --from and catalog install/update downloads used unbounded `response.read()`, buffering the entire server-controlled body into memory before any size check, and trusted Content-Length alone where checked at all. Added a single shared `_read_response_within_limit()` helper reused by both call sites: it fails fast on an oversized declared Content-Length, and separately enforces the same cap while streaming in 64KiB chunks so a chunked or Content-Length-less response cannot bypass the limit by lying about or omitting its size. Chose 5 MiB as the cap: workflow YAML definitions are small step/metadata text, not binaries, so this is generous headroom against a malicious/misbehaving server without affecting any legitimate workflow definition. Both call sites already route any raised exception through their existing clean-error and rollback (`_cleanup_failed_install`) paths, so no additional error-handling plumbing was needed. Tests: extended the shared `_FakeResponse` test helper (and 5 duplicate per-test FakeResponse classes) to support `.read(amt)` chunked reads with an internal cursor (backward compatible with existing bare `.read()` callers) plus header simulation. Added red-first tests for: BundlerError after partial removal reporting partial state, BundlerError with zero removals reporting no changes, --from oversized-Content-Length rejection, --from oversized-streamed-body-without-Content-Length rejection, and the same two cases for the catalog install path (asserting no orphan directory/registry mutation on rejection). tests/integration/test_bundler_install_flow.py: 17 passed tests/test_workflows.py: 485 passed tests -q: 3992 passed, 110 skipped ruff check: clean on all touched files Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/bundler/services/installer.py | 2 - src/specify_cli/workflows/_commands.py | 50 +++- .../integration/test_bundler_install_flow.py | 60 +++++ tests/test_workflows.py | 220 ++++++++++++++++-- 4 files changed, 315 insertions(+), 17 deletions(-) diff --git a/src/specify_cli/bundler/services/installer.py b/src/specify_cli/bundler/services/installer.py index 9fc7d2a96b..61c882d1f4 100644 --- a/src/specify_cli/bundler/services/installer.py +++ b/src/specify_cli/bundler/services/installer.py @@ -199,8 +199,6 @@ def remove_bundle( result.uninstalled.append(component) else: result.skipped.append(component) - except BundlerError: - raise except Exception as exc: # noqa: BLE001 if result.uninstalled: detail = ( diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 98fd7284f6..c3ba89157f 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -147,6 +147,52 @@ def _reject_insecure_download_redirect(old_url: str, new_url: str) -> None: ) +# Workflow YAML definitions are small step/metadata text, not binaries, so +# this is generous headroom against a malicious or misbehaving server -- not +# a ceiling any legitimate workflow definition should ever approach. +_MAX_WORKFLOW_YAML_BYTES = 5 * 1024 * 1024 # 5 MiB +_DOWNLOAD_CHUNK_SIZE = 65536 + + +def _read_response_within_limit(response, max_bytes: int = _MAX_WORKFLOW_YAML_BYTES) -> bytes: + """Read *response* fully, enforcing *max_bytes* via bounded streaming. + + A ``Content-Length`` header is checked up front to fail fast, but it is + never trusted alone: the actual bytes read are also counted as they + stream in, so a chunked or ``Content-Length``-less response that lies + about (or omits) its size still cannot exceed the limit. + """ + content_length = None + getheader = getattr(response, "getheader", None) + if callable(getheader): + try: + raw_length = getheader("Content-Length") + except Exception: + raw_length = None + if raw_length is not None: + try: + content_length = int(raw_length) + except (TypeError, ValueError): + content_length = None + if content_length is not None and content_length > max_bytes: + raise ValueError( + f"response declared {content_length} bytes, exceeding the " + f"{max_bytes}-byte workflow size limit" + ) + + chunks: list[bytes] = [] + total = 0 + while True: + chunk = response.read(_DOWNLOAD_CHUNK_SIZE) + if not chunk: + break + total += len(chunk) + if total > max_bytes: + raise ValueError(f"response exceeds the {max_bytes}-byte workflow size limit") + chunks.append(chunk) + return b"".join(chunks) + + def _validate_workflow_id_or_exit(workflow_id: str) -> None: """Validate that ``workflow_id`` is a safe installed-workflow directory name.""" if ( @@ -887,7 +933,7 @@ def _cleanup_failed_install() -> None: ) raise typer.Exit(1) with tempfile.NamedTemporaryFile(suffix=".yml", delete=False) as tmp: - tmp.write(resp.read()) + tmp.write(_read_response_within_limit(resp)) tmp_path = Path(tmp.name) except typer.Exit: raise @@ -1076,7 +1122,7 @@ def _cleanup_failed_install() -> None: f"[red]Error:[/red] Workflow '{safe_wf_id}' redirected to non-HTTPS URL: {_escape_markup(final_url)}" ) raise typer.Exit(1) - workflow_file.write_bytes(response.read()) + workflow_file.write_bytes(_read_response_within_limit(response)) except typer.Exit: raise except Exception as exc: diff --git a/tests/integration/test_bundler_install_flow.py b/tests/integration/test_bundler_install_flow.py index 8b52d90fe3..a2e34797fd 100644 --- a/tests/integration/test_bundler_install_flow.py +++ b/tests/integration/test_bundler_install_flow.py @@ -153,6 +153,66 @@ def remove_then_fail(project_root, component): assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} +def test_remove_bundlerror_from_installer_after_partial_removal_reports_partial_state( + tmp_path: Path, +): + """If the primitive installer itself raises BundlerError (not a raw/ + unexpected exception) after an earlier component in the same bundle was + already removed, the surfaced message must still carry the same + partial-removal detail as the generic-exception path -- a bare + ``except BundlerError: raise`` would re-raise the installer's original + message verbatim with no mention that the project may now be partially + uninstalled.""" + make_project(tmp_path) + manifest = BundleManifest.from_dict(valid_manifest_dict()) + installer = FakeInstaller() + install_bundle(tmp_path, _plan(manifest), installer, manifest=manifest) + + real_remove = installer.remove + calls = {"n": 0} + + def remove_then_raise_bundler_error(project_root, component): + calls["n"] += 1 + if calls["n"] == 1: + return real_remove(project_root, component) + raise BundlerError("kind manager refused removal") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(installer, "remove", remove_then_raise_bundler_error) + with pytest.raises(BundlerError) as exc_info: + remove_bundle(tmp_path, "demo-bundle", installer) + + message = str(exc_info.value) + assert "no changes were recorded" not in message.lower() + assert "kind manager refused removal" in message + assert "partially uninstalled" in message.lower() + assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} + + +def test_remove_bundlerror_from_installer_with_zero_removed_reports_no_changes( + tmp_path: Path, +): + """When the installer raises BundlerError before anything was actually + removed, the message should not misleadingly claim partial state.""" + make_project(tmp_path) + manifest = BundleManifest.from_dict(valid_manifest_dict()) + installer = FakeInstaller() + install_bundle(tmp_path, _plan(manifest), installer, manifest=manifest) + + def boom(project_root, component): + raise BundlerError("kind manager unavailable") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(installer, "is_installed", boom) + with pytest.raises(BundlerError) as exc_info: + remove_bundle(tmp_path, "demo-bundle", installer) + + message = str(exc_info.value) + assert "no components were removed" in message.lower() + assert "kind manager unavailable" in message + assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} + + def test_remove_reports_uninstalled_not_installed(tmp_path: Path): make_project(tmp_path) manifest = BundleManifest.from_dict(valid_manifest_dict()) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index ba1c355a03..4242146cae 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -6797,8 +6797,16 @@ def __init__(self, data, url=None): self._data = data self._url = url or "https://api.github.com/repos/org/repo/releases/assets/42" - def read(self): - return self._data + def read(self, amt=None): + if not hasattr(self, "_pos"): + self._pos = 0 + if amt is None: + chunk = self._data[self._pos :] + self._pos = len(self._data) + return chunk + chunk = self._data[self._pos : self._pos + amt] + self._pos += len(chunk) + return chunk def geturl(self): return self._url @@ -6849,8 +6857,16 @@ def __init__(self, data, url=None): self._data = data self._url = url or "https://api.github.com/repos/org/repo/releases/assets/42" - def read(self): - return self._data + def read(self, amt=None): + if not hasattr(self, "_pos"): + self._pos = 0 + if amt is None: + chunk = self._data[self._pos :] + self._pos = len(self._data) + return chunk + chunk = self._data[self._pos : self._pos + amt] + self._pos += len(chunk) + return chunk def geturl(self): return self._url @@ -6892,8 +6908,16 @@ def __init__(self, data, url=None): self._data = data self._url = url or "https://api.github.com/repos/org/repo/releases/assets/55" - def read(self): - return self._data + def read(self, amt=None): + if not hasattr(self, "_pos"): + self._pos = 0 + if amt is None: + chunk = self._data[self._pos :] + self._pos = len(self._data) + return chunk + chunk = self._data[self._pos : self._pos + amt] + self._pos += len(chunk) + return chunk def geturl(self): return self._url @@ -6968,8 +6992,16 @@ def __init__(self, data, url=None): self._data = data self._url = url or "https://ghes.example/api/v3/repos/org/repo/releases/assets/42" - def read(self): - return self._data + def read(self, amt=None): + if not hasattr(self, "_pos"): + self._pos = 0 + if amt is None: + chunk = self._data[self._pos :] + self._pos = len(self._data) + return chunk + chunk = self._data[self._pos : self._pos + amt] + self._pos += len(chunk) + return chunk def geturl(self): return self._url @@ -7023,8 +7055,16 @@ def __init__(self, data, url=None): self._data = data self._url = url or "https://ghes.example/api/v3/repos/org/repo/releases/assets/55" - def read(self): - return self._data + def read(self, amt=None): + if not hasattr(self, "_pos"): + self._pos = 0 + if amt is None: + chunk = self._data[self._pos :] + self._pos = len(self._data) + return chunk + chunk = self._data[self._pos : self._pos + amt] + self._pos += len(chunk) + return chunk def geturl(self): return self._url @@ -7582,12 +7622,23 @@ def test_add_yaml_parse_error_escapes_rich_markup(self, project_dir, monkeypatch # -- add --from ---------------------------------------------------- class _FakeResponse: - def __init__(self, data, url="https://example.com/workflow.yml"): + def __init__(self, data, url="https://example.com/workflow.yml", headers=None): self._data = data self._url = url + self._pos = 0 + self._headers = headers or {} + + def read(self, amt=None): + if amt is None: + chunk = self._data[self._pos :] + self._pos = len(self._data) + return chunk + chunk = self._data[self._pos : self._pos + amt] + self._pos += len(chunk) + return chunk - def read(self): - return self._data + def getheader(self, name, default=None): + return self._headers.get(name, default) def geturl(self): return self._url @@ -7598,6 +7649,64 @@ def __enter__(self): def __exit__(self, *a): return False + def test_add_from_url_rejects_oversized_content_length(self, project_dir, monkeypatch): + """A --from download must not trust an advertised Content-Length + alone by reading the whole body first -- it must reject a response + that declares a size over the workflow YAML limit before reading + the (potentially huge) body into memory at all.""" + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands as wf_commands + + monkeypatch.chdir(project_dir) + monkeypatch.setattr(wf_commands, "_MAX_WORKFLOW_YAML_BYTES", 100) + small_body = b"id: align-wf\n" # small actual body; Content-Length lies + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + small_body, url, headers={"Content-Length": "1000"} + ), + ): + result = runner.invoke( + app, + ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + ) + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + + def test_add_from_url_rejects_oversized_streamed_body_without_content_length( + self, project_dir, monkeypatch + ): + """A chunked/no-Content-Length response must still be capped by + actually counting streamed bytes -- a malicious or misbehaving + server cannot bypass the limit merely by omitting or lying about + Content-Length.""" + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands as wf_commands + + monkeypatch.chdir(project_dir) + monkeypatch.setattr(wf_commands, "_MAX_WORKFLOW_YAML_BYTES", 100) + oversized_body = b"x" * 500 # no Content-Length header at all + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + oversized_body, url + ), + ): + result = runner.invoke( + app, + ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + ) + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + def test_add_from_url_installs(self, project_dir, monkeypatch): from unittest.mock import patch from typer.testing import CliRunner @@ -8061,6 +8170,91 @@ def boom(self): assert not dest_dir.exists() assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_catalog_rejects_oversized_content_length(self, project_dir, monkeypatch): + """Catalog installs must share the same size cap as --from: a + response that declares an oversized Content-Length is rejected + before its body is read into memory, and no orphan directory or + registry mutation is left behind.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands as wf_commands + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + monkeypatch.setattr(wf_commands, "_MAX_WORKFLOW_YAML_BYTES", 100) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + small_body = b"id: align-wf\n" # actual body is small; header lies + runner = CliRunner() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + small_body, url, headers={"Content-Length": "1000"} + ), + ) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + dest_dir = project_dir / ".specify" / "workflows" / "align-wf" + assert not dest_dir.exists() + assert not WorkflowRegistry(project_dir).is_installed("align-wf") + + def test_add_catalog_rejects_oversized_streamed_body_without_content_length( + self, project_dir, monkeypatch + ): + """Catalog installs must also cap actual streamed bytes when + Content-Length is absent or understated, leaving no orphan + directory or registry mutation behind.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands as wf_commands + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + monkeypatch.setattr(wf_commands, "_MAX_WORKFLOW_YAML_BYTES", 100) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + oversized_body = b"x" * 500 # no Content-Length header at all + runner = CliRunner() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + oversized_body, url + ), + ) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + dest_dir = project_dir / ".specify" / "workflows" / "align-wf" + assert not dest_dir.exists() + assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_catalog_reinstall_save_failure_restores_prior_file(self, project_dir, monkeypatch): """Re-adding an already-installed catalog workflow downloads the new version over the existing install directory. If registry.add() then From b8269c8635972dca585c7a2381a2c24dc8acb4a6 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:16:35 +0200 Subject: [PATCH 35/59] Fix temp-file leak in workflow add --from and strengthen size-limit test assertions workflow_add's --from download path opened a NamedTemporaryFile(delete=False) -- which creates the file on disk immediately -- then wrote the size-limited response body before assigning `tmp_path`. If `_read_response_within_limit` raised (oversized declared Content-Length, or an over-cap streamed body with no/understated Content-Length), the exception propagated out of the `with` block before `tmp_path` was ever set, so the outer except handler had no path to clean up: a 0-byte `.yml` temp file was left behind permanently on every rejected/failed --from download. Fixed by assigning `tmp_path` immediately after the file is opened (before the size-limited read/write), and unlinking it in the except branch when set. Normal post-download cleanup in the existing `finally: tmp_path.unlink(missing_ok=True)` is unchanged. Verified (not assumed) the catalog install path has no equivalent leak: it writes the response bytes directly to `workflow_file` inside `workflow_dir` (no separate temp file), and any read/size-limit failure is already caught by the existing `except Exception: _cleanup_failed_install()` handler, which correctly restores a reinstalled file or removes a freshly-created directory. While investigating, found the previous round's 4 size-limit tests were false positives: `_read_response_within_limit`'s `max_bytes` parameter had its default bound to `_MAX_WORKFLOW_YAML_BYTES` at function-definition time, so monkeypatching the module attribute in tests had no effect on the function's actual behavior -- the tests were passing because the oversized mock bodies failed downstream YAML/id validation instead of the size check. Fixed by resolving `max_bytes` from the module attribute at call time (default `None`, resolved inside the function body) so tests can actually override the effective limit, and strengthened all 4 tests' assertions to match the specific size-limit error text (whitespace-collapsed to tolerate Rich's line-wrapping), so they now prove the real code path fires. Tests: added 2 red-first regression tests (oversized-streamed-body and oversized-Content-Length --from downloads leave no leftover temp file, verified against a scratch tempfile.tempdir), confirmed red (real 0-byte file found) before the fix and green after. Strengthened the pre-existing 4 --from/catalog size-limit tests to assert on the actual error message instead of generic exit-code/non-empty-output checks. tests/test_workflows.py: 487 passed tests -k bundler: 186 passed tests -q: 3994 passed, 110 skipped ruff check: clean on all touched files Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 18 +++++- tests/test_workflows.py | 78 ++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index c3ba89157f..e075de0779 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -154,14 +154,21 @@ def _reject_insecure_download_redirect(old_url: str, new_url: str) -> None: _DOWNLOAD_CHUNK_SIZE = 65536 -def _read_response_within_limit(response, max_bytes: int = _MAX_WORKFLOW_YAML_BYTES) -> bytes: +def _read_response_within_limit(response, max_bytes: int | None = None) -> bytes: """Read *response* fully, enforcing *max_bytes* via bounded streaming. A ``Content-Length`` header is checked up front to fail fast, but it is never trusted alone: the actual bytes read are also counted as they stream in, so a chunked or ``Content-Length``-less response that lies about (or omits) its size still cannot exceed the limit. + + ``max_bytes`` defaults to ``None`` (resolved to the module-level + ``_MAX_WORKFLOW_YAML_BYTES`` at call time, not at function-definition + time) so tests can override the effective limit via monkeypatching the + module attribute. """ + if max_bytes is None: + max_bytes = _MAX_WORKFLOW_YAML_BYTES content_length = None getheader = getattr(response, "getheader", None) if callable(getheader): @@ -910,6 +917,7 @@ def _cleanup_failed_install() -> None: _wf_url_extra_headers = {"Accept": "application/octet-stream"} import tempfile + tmp_path: Path | None = None try: with _open_url( download_url, @@ -933,11 +941,17 @@ def _cleanup_failed_install() -> None: ) raise typer.Exit(1) with tempfile.NamedTemporaryFile(suffix=".yml", delete=False) as tmp: - tmp.write(_read_response_within_limit(resp)) + # Assign tmp_path immediately: NamedTemporaryFile(delete=False) + # creates the file on disk right away, before any bytes are + # written, so a failure in the size-limited read below must + # still be able to find and remove it. tmp_path = Path(tmp.name) + tmp.write(_read_response_within_limit(resp)) except typer.Exit: raise except Exception as exc: + if tmp_path is not None: + tmp_path.unlink(missing_ok=True) console.print(f"[red]Error:[/red] Failed to download workflow: {_escape_markup(str(exc))}") raise typer.Exit(1) try: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 4242146cae..85339b95be 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7675,7 +7675,7 @@ def test_add_from_url_rejects_oversized_content_length(self, project_dir, monkey ) assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) - assert result.output.strip() != "" + assert "exceedingthe100-byteworkflowsizelimit" in "".join(result.output.split()) def test_add_from_url_rejects_oversized_streamed_body_without_content_length( self, project_dir, monkeypatch @@ -7705,7 +7705,77 @@ def test_add_from_url_rejects_oversized_streamed_body_without_content_length( ) assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) - assert result.output.strip() != "" + assert "exceedsthe100-byteworkflowsizelimit" in "".join(result.output.split()) + + def test_add_from_url_oversized_streamed_body_leaves_no_temp_file( + self, project_dir, monkeypatch, tmp_path + ): + """A rejected --from download (oversized streamed body, no + Content-Length) must not leave the 0-byte NamedTemporaryFile behind: + the file is created on disk as soon as it is opened (delete=False), + before any bytes are written, so a failure inside the size-limit + check must still clean it up rather than merely erroring out.""" + import tempfile as tempfile_mod + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands as wf_commands + + monkeypatch.chdir(project_dir) + monkeypatch.setattr(wf_commands, "_MAX_WORKFLOW_YAML_BYTES", 100) + scratch_tmp = tmp_path / "scratch-tmp" + scratch_tmp.mkdir() + monkeypatch.setattr(tempfile_mod, "tempdir", str(scratch_tmp)) + oversized_body = b"x" * 500 # no Content-Length header at all + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + oversized_body, url + ), + ): + result = runner.invoke( + app, + ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + ) + assert result.exit_code != 0 + assert "exceedsthe100-byteworkflowsizelimit" in "".join(result.output.split()) + leaked = list(scratch_tmp.glob("*.yml")) + assert leaked == [], f"leaked temp files: {leaked}" + + def test_add_from_url_oversized_content_length_leaves_no_temp_file( + self, project_dir, monkeypatch, tmp_path + ): + """Same guarantee for the fail-fast Content-Length rejection path: + it must not even leave a 0-byte temp file behind.""" + import tempfile as tempfile_mod + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands as wf_commands + + monkeypatch.chdir(project_dir) + monkeypatch.setattr(wf_commands, "_MAX_WORKFLOW_YAML_BYTES", 100) + scratch_tmp = tmp_path / "scratch-tmp" + scratch_tmp.mkdir() + monkeypatch.setattr(tempfile_mod, "tempdir", str(scratch_tmp)) + small_body = b"id: align-wf\n" # small actual body; Content-Length lies + runner = CliRunner() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + small_body, url, headers={"Content-Length": "1000"} + ), + ): + result = runner.invoke( + app, + ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + ) + assert result.exit_code != 0 + assert "exceedingthe100-byteworkflowsizelimit" in "".join(result.output.split()) + leaked = list(scratch_tmp.glob("*.yml")) + assert leaked == [], f"leaked temp files: {leaked}" + def test_add_from_url_installs(self, project_dir, monkeypatch): from unittest.mock import patch @@ -8207,7 +8277,7 @@ def test_add_catalog_rejects_oversized_content_length(self, project_dir, monkeyp assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) - assert result.output.strip() != "" + assert "exceedingthe100-byteworkflowsizelimit" in "".join(result.output.split()) dest_dir = project_dir / ".specify" / "workflows" / "align-wf" assert not dest_dir.exists() assert not WorkflowRegistry(project_dir).is_installed("align-wf") @@ -8250,7 +8320,7 @@ def test_add_catalog_rejects_oversized_streamed_body_without_content_length( assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) - assert result.output.strip() != "" + assert "exceedsthe100-byteworkflowsizelimit" in "".join(result.output.split()) dest_dir = project_dir / ".specify" / "workflows" / "align-wf" assert not dest_dir.exists() assert not WorkflowRegistry(project_dir).is_installed("align-wf") From 00465c3645f3fcd9f2840b4e677be41be79cd6ab Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 12:44:01 +0200 Subject: [PATCH 36/59] Harden workflow install/remove transactions with atomic staging Addresses 5 Copilot review findings on HEAD b8269c8, all centered on transaction integrity around workflow install/remove/registry writes, following the atomic_write_json pattern already used in _utils.py: 1. WorkflowRegistry.save() now preserves the existing registry file's mode (e.g. 0640/0644) across a save instead of silently downgrading it to mkstemp's 0600 default; a brand-new registry still gets the secure 0600 default. 2. workflow_remove now stages the install directory out of the way via an atomic rename *before* the registry write, rather than deleting it directly with shutil.rmtree after the registry already claims it removed. This closes a real data-integrity gap: a partially-failed rmtree could no longer leave a damaged directory re-marked "installed" by the old manual restore-after-rmtree-failure code (now deleted -- it's structurally impossible to need it). A registry-write failure renames the staged directory back (guarded, with an explicit warning if the restore-back rename itself fails); a registry-write success is durable, so a later failure to delete the staged directory is now a warning (exit 0), not a contradictory "Error: Failed to remove" (exit 1) that used to claim failure while the registry already recorded success. 3. Local (--dev/--from/plain path) and catalog install/reinstall now write new content to a same-directory staging file and commit it onto the destination workflow.yml via a single atomic swap, instead of writing/downloading directly into the destination file. A prior file (reinstall) is renamed aside rather than overwritten in place, so it can be restored via rename -- never a content rewrite -- if registry.add() subsequently fails; a rollback failure is now explicitly reported as a warning instead of escaping unguarded and masking the original clean error. This also removes the need to read the prior file's bytes into memory before installing (that read-before-write step and its failure mode are now unreachable), and both local and catalog installs share the same four small helpers (_stage_workflow_file / _commit_workflow_file / _discard_staged_workflow_file / _rollback_committed_workflow_file, plus guarded wrappers) rather than duplicating the logic. 4. Updated a stale comment (workflow_run's ownership-guard rationale) that still described WorkflowRegistry._load() as silently substituting an empty registry; it now fails closed by raising OSError, which the comment now states plainly. Tests: rewrote the two workflow_remove tests whose assertions encoded the old (incoherent) rmtree-then-restore contract to instead prove the new stage-then-commit contract (post-registry-success cleanup failure is a warning+exit 0; pre-registry-success stage-restore failure is guarded and escapes markup correctly). Rewrote the local/catalog "backup read failure" tests, which tested a step the new design no longer performs, into "restore-rename failure" tests proving the new guarded rollback boundary. Added registry file-mode preservation tests. All other existing install/remove/reinstall tests (save-failure cleanup, pre-existing-empty-dir handling, early-failure-during- reinstall parametrized cases, Rich markup escaping) continue to pass unmodified against the new implementation. Verified via GraphQL that all 5 threads are current (not outdated/ resolved) before fixing. Full suite: 3996 passed, 110 skipped. Ruff clean on all touched files. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 289 +++++++++++++++++-------- src/specify_cli/workflows/catalog.py | 14 ++ tests/test_workflows.py | 269 +++++++++++++++-------- 3 files changed, 383 insertions(+), 189 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index e075de0779..db9faad5a2 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -262,6 +262,91 @@ def _safe_workflow_id_dir(workflows_dir: Path, workflow_id: str) -> Path: return dest_dir +def _stage_workflow_file(dest_dir: Path) -> Path: + """Reserve a same-directory staging file so new/updated workflow.yml + content can be written and validated without ever touching (and risking + truncating) an existing destination file before the final atomic swap. + Shared by the local-install and catalog-install paths.""" + import tempfile + + dest_dir.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(dir=dest_dir, prefix=".workflow.yml.", suffix=".tmp") + os.close(fd) + return Path(tmp_name) + + +def _commit_workflow_file(staged_file: Path, dest_file: Path, existed_before: bool) -> Path | None: + """Atomically swap ``staged_file`` onto ``dest_file``. If a prior file + existed, it is first renamed aside (path returned) so a later failure + (e.g. registry.add()) can restore it via rename instead of a content + rewrite -- the destination is never truncated/overwritten in place. If + the second rename fails after the first succeeded, the prior file is + put back immediately so dest_file is never left simply missing.""" + if existed_before and dest_file.exists(): + backup_file = dest_file.with_name(dest_file.name + ".bak") + os.replace(dest_file, backup_file) + try: + os.replace(staged_file, dest_file) + except OSError: + os.replace(backup_file, dest_file) + raise + return backup_file + os.replace(staged_file, dest_file) + return None + + +def _discard_staged_workflow_file(staged_file: Path, dest_dir: Path, existed_before: bool) -> None: + """Clean up after a pre-commit failure (staged_file was never swapped + onto dest_file): remove the staged file, and for a fresh install (no + prior directory) remove the now-orphaned dest_dir too.""" + staged_file.unlink(missing_ok=True) + if not existed_before: + import shutil + shutil.rmtree(dest_dir, ignore_errors=True) + + +def _rollback_committed_workflow_file( + dest_file: Path, dest_dir: Path, existed_before: bool, backup_file: Path | None +) -> None: + """Undo a successful _commit_workflow_file swap after a later failure + (registry.add()): restore the prior file via rename, remove the newly + committed file for a reinstall over a pre-existing empty directory + (no backup), or remove the whole directory for a fresh install.""" + if backup_file is not None: + os.replace(backup_file, dest_file) + elif existed_before: + dest_file.unlink(missing_ok=True) + else: + import shutil + shutil.rmtree(dest_dir, ignore_errors=True) + + +def _safe_discard_staged_workflow_file(staged_file: Path, dest_dir: Path, existed_before: bool) -> None: + """Guarded wrapper: a cleanup failure must be reported, never crash or + silently mask the original install error that triggered it.""" + try: + _discard_staged_workflow_file(staged_file, dest_dir, existed_before) + except OSError as exc: + console.print( + "[yellow]Warning:[/yellow] Failed to clean up incomplete workflow " + f"install: {_escape_markup(str(exc))}" + ) + + +def _safe_rollback_committed_workflow_file( + dest_file: Path, dest_dir: Path, existed_before: bool, backup_file: Path | None +) -> None: + """Guarded wrapper: a rollback failure must be reported, never crash or + silently claim the prior workflow file was restored when it wasn't.""" + try: + _rollback_committed_workflow_file(dest_file, dest_dir, existed_before, backup_file) + except OSError as exc: + console.print( + "[yellow]Warning:[/yellow] Failed to restore prior workflow file " + f"after registry update failure: {_escape_markup(str(exc))}" + ) + + # Root helper re-fetched at call time so test monkeypatching of # `specify_cli._require_specify_project` keeps working after the move. def _require_specify_project(*args, **kwargs): @@ -474,9 +559,11 @@ def workflow_run( # own .specify is itself a symlink to an attacker-controlled # tree) -- check it explicitly rather than trusting that # cwd-scoped guard, and don't rely on WorkflowRegistry's own - # symlinked-parent handling below (it silently substitutes - # an empty registry instead of raising, so a query against - # it can't be trusted as a safety signal here). + # symlinked-parent handling below as the safety signal here: + # it now fails closed by raising OSError at construction + # time (see catalog.py's _load), but that surfaces as an + # opaque exception rather than this guard's clean, specific + # CLI error for the actual owning project root. _reject_unsafe_dir(registry_root / ".specify", ".specify") _reject_unsafe_dir( registry_root / ".specify" / "workflows", ".specify/workflows" @@ -800,38 +887,39 @@ def _validate_and_install_local( dest_dir = _safe_workflow_id_dir(workflows_dir, definition.id) dest_file = dest_dir / "workflow.yml" existed_before = dest_dir.is_dir() + + import shutil + try: - backup_bytes = ( - dest_file.read_bytes() if existed_before and dest_file.is_file() else None - ) + staged_file = _stage_workflow_file(dest_dir) except OSError as exc: console.print( - f"[red]Error:[/red] Failed to read existing workflow " - f"'{_escape_markup(definition.id)}' before install: {_escape_markup(str(exc))}" + f"[red]Error:[/red] Failed to install workflow " + f"'{_escape_markup(definition.id)}': {_escape_markup(str(exc))}" ) raise typer.Exit(1) - import shutil - def _cleanup_failed_install() -> None: - # Don't leave an orphan directory behind for a fresh install; for - # a reinstall over an existing local workflow, restore the prior - # workflow.yml instead of clobbering it with the failed update. - # A pre-existing directory with no prior workflow.yml (no backup - # bytes) must have the newly written file removed instead of - # doing nothing, so it doesn't linger as an orphan. - if existed_before: - if backup_bytes is not None: - dest_file.write_bytes(backup_bytes) - else: - dest_file.unlink(missing_ok=True) - else: - shutil.rmtree(dest_dir, ignore_errors=True) + try: + # Copied into the staging file, never dest_file directly, so a + # reinstall's prior working copy is never touched until the + # atomic commit below runs. + shutil.copy2(yaml_path, staged_file) + except OSError as exc: + _safe_discard_staged_workflow_file(staged_file, dest_dir, existed_before) + console.print( + f"[red]Error:[/red] Failed to install workflow " + f"'{_escape_markup(definition.id)}': {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + # Commit the staged copy onto dest_file via an atomic swap. A prior + # file (reinstall) is renamed aside rather than overwritten in + # place, so it can be restored by rename (not a content rewrite) if + # registry.add() below fails. try: - dest_dir.mkdir(parents=True, exist_ok=True) - shutil.copy2(yaml_path, dest_file) + backup_file = _commit_workflow_file(staged_file, dest_file, existed_before) except OSError as exc: - _cleanup_failed_install() + _safe_discard_staged_workflow_file(staged_file, dest_dir, existed_before) console.print( f"[red]Error:[/red] Failed to install workflow " f"'{_escape_markup(definition.id)}': {_escape_markup(str(exc))}" @@ -845,7 +933,7 @@ def _cleanup_failed_install() -> None: "source": source_label, }) except OSError as exc: - _cleanup_failed_install() + _safe_rollback_committed_workflow_file(dest_file, dest_dir, existed_before, backup_file) console.print( f"[red]Error:[/red] Failed to update workflow registry for " f"'{_escape_markup(definition.id)}': {_escape_markup(str(exc))}" @@ -1065,40 +1153,18 @@ def _install_workflow_from_catalog( # Captured before any mkdir/download writes so every failure branch below # can tell a fresh install from a reinstall-over-an-existing-one, - # mirroring _validate_and_install_local's existed-before/backup-aware - # rollback. + # mirroring _validate_and_install_local's existed-before-aware cleanup. existed_before = workflow_dir.is_dir() + try: - prior_workflow_bytes = ( - workflow_file.read_bytes() if existed_before and workflow_file.is_file() else None - ) + staged_file = _stage_workflow_file(workflow_dir) except OSError as exc: console.print( - f"[red]Error:[/red] Failed to read existing workflow " - f"'{safe_wf_id}' before install: {_escape_markup(str(exc))}" + f"[red]Error:[/red] Failed to install workflow '{safe_wf_id}' from catalog: " + f"{_escape_markup(str(exc))}" ) raise typer.Exit(1) - def _cleanup_failed_install() -> None: - """Restore the prior workflow.yml on a reinstall, or remove the - directory entirely for a fresh install. Every failure branch that - runs after the mkdir/download step -- redirect rejection, download - exception, invalid YAML, ID mismatch, version mismatch, and - registry.add() failure -- must call this instead of rmtree'ing - directly, so none of them can destroy a working install that - predates this attempt. A pre-existing directory with no prior - workflow.yml (no backup bytes) must have the newly downloaded file - removed instead of doing nothing, so it doesn't linger as an - orphan.""" - if existed_before: - if prior_workflow_bytes is not None: - workflow_file.write_bytes(prior_workflow_bytes) - else: - workflow_file.unlink(missing_ok=True) - else: - import shutil - shutil.rmtree(workflow_dir, ignore_errors=True) - try: from specify_cli.authentication.http import open_url as _open_url from specify_cli.authentication.http import github_provider_hosts as _github_provider_hosts @@ -1112,7 +1178,6 @@ def _cleanup_failed_install() -> None: workflow_url = _resolved_workflow_url _wf_cat_extra_headers = {"Accept": "application/octet-stream"} - workflow_dir.mkdir(parents=True, exist_ok=True) with _open_url( workflow_url, timeout=30, @@ -1131,31 +1196,35 @@ def _cleanup_failed_install() -> None: # Host is not an IP literal (e.g., a regular hostname); treat as non-loopback. pass if final_parsed.scheme != "https" and not (final_parsed.scheme == "http" and final_loopback): - _cleanup_failed_install() + _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) console.print( f"[red]Error:[/red] Workflow '{safe_wf_id}' redirected to non-HTTPS URL: {_escape_markup(final_url)}" ) raise typer.Exit(1) - workflow_file.write_bytes(_read_response_within_limit(response)) + # Written to the staging file, never workflow_file directly, so a + # reinstall's prior working copy is never touched until the + # atomic commit below runs. + staged_file.write_bytes(_read_response_within_limit(response)) except typer.Exit: raise except Exception as exc: - _cleanup_failed_install() + _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) console.print(f"[red]Error:[/red] Failed to install workflow '{safe_wf_id}' from catalog: {_escape_markup(str(exc))}") raise typer.Exit(1) - # Validate the downloaded workflow before registering + # Validate the downloaded workflow (still staged, not yet committed) + # before registering. try: - definition = WorkflowDefinition.from_yaml(workflow_file) + definition = WorkflowDefinition.from_yaml(staged_file) except (ValueError, yaml.YAMLError) as exc: - _cleanup_failed_install() + _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) console.print(f"[red]Error:[/red] Downloaded workflow is invalid: {_escape_markup(str(exc))}") raise typer.Exit(1) from .engine import validate_workflow errors = validate_workflow(definition) if errors: - _cleanup_failed_install() + _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) console.print("[red]Error:[/red] Downloaded workflow validation failed:") for err in errors: console.print(f" \u2022 {_escape_markup(str(err))}") @@ -1163,7 +1232,7 @@ def _cleanup_failed_install() -> None: # Enforce that the workflow's internal ID matches the catalog key if definition.id and definition.id != workflow_id: - _cleanup_failed_install() + _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) console.print( f"[red]Error:[/red] Workflow ID in YAML ({_escape_markup(repr(definition.id))}) " f"does not match catalog key ({_escape_markup(repr(workflow_id))}). " @@ -1181,7 +1250,7 @@ def _cleanup_failed_install() -> None: except pkg_version.InvalidVersion: version_matches = str(definition.version) == expected_version if not version_matches: - _cleanup_failed_install() + _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) console.print( f"[red]Error:[/red] Downloaded workflow version ({_escape_markup(str(definition.version))}) " f"does not match the catalog version ({_escape_markup(expected_version)}). " @@ -1189,6 +1258,20 @@ def _cleanup_failed_install() -> None: ) raise typer.Exit(1) + # Commit the staged download onto workflow_file via an atomic swap. A + # prior file (reinstall) is renamed aside rather than overwritten in + # place, so it can be restored by rename (not a content rewrite) if + # registry.add() below fails. + try: + backup_file = _commit_workflow_file(staged_file, workflow_file, existed_before) + except OSError as exc: + _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) + console.print( + f"[red]Error:[/red] Failed to install workflow '{safe_wf_id}' from catalog: " + f"{_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + entry = { "name": definition.name or info.get("name", workflow_id), "version": definition.version or info.get("version", "0.0.0"), @@ -1204,7 +1287,7 @@ def _cleanup_failed_install() -> None: try: registry.add(workflow_id, entry) except OSError as exc: - _cleanup_failed_install() + _safe_rollback_committed_workflow_file(workflow_file, workflow_dir, existed_before, backup_file) console.print( f"[red]Error:[/red] Failed to update workflow registry for " f"'{_escape_markup(workflow_id)}': {_escape_markup(str(exc))}" @@ -1261,49 +1344,69 @@ def workflow_remove( ) raise typer.Exit(1) - # Captured before the registry write so a subsequent directory-removal - # failure can restore it verbatim (bypassing add(), which would stamp a - # new updated_at), mirroring workflow_step_remove's same restore pattern. - registry_metadata = registry.get(workflow_id) + import shutil + import tempfile + + # Stage the directory out of the way via an atomic rename *before* the + # registry write, so a mid-delete rmtree failure can never leave a + # partially-deleted directory that gets re-marked "installed". A rename + # is a metadata-only operation (unlike rmtree), so it either fully + # succeeds or leaves the original directory completely untouched. + staged_dir: Path | None = None + if workflow_dir.exists(): + try: + reserved = Path( + tempfile.mkdtemp(prefix=f".{workflow_id}.removing-", dir=workflows_dir) + ) + reserved.rmdir() + os.rename(workflow_dir, reserved) + staged_dir = reserved + except OSError as exc: + console.print( + f"[red]Error:[/red] Failed to stage workflow directory " + f"{_escape_markup(str(workflow_dir))} for removal: {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) - # Persist the registry removal before touching any files: if save() + # Persist the registry removal only after staging succeeded: if save() # fails, WorkflowRegistry.remove() rolls back its in-memory state and - # raises, so the workflow stays fully installed (files + registry) rather - # than being deleted while the registry still (or no longer) claims it. + # raises, so we rename the staged directory back to its original + # location, restoring the pre-command state exactly (files + registry + # both still claim the workflow installed). try: registry.remove(workflow_id) except OSError as exc: + if staged_dir is not None: + try: + os.rename(staged_dir, workflow_dir) + except OSError as restore_exc: + console.print( + f"[yellow]Warning:[/yellow] Failed to restore workflow directory " + f"after registry update failure; it remains staged at " + f"{_escape_markup(str(staged_dir))}: {_escape_markup(str(restore_exc))}" + ) console.print( f"[red]Error:[/red] Failed to update workflow registry for '{safe_id}': " f"{_escape_markup(str(exc))}" ) raise typer.Exit(1) - if workflow_dir.exists(): - import shutil + console.print(f"[green]✓[/green] Workflow '{workflow_id}' removed") + + # The registry has already durably committed the removal at this point, + # so it must stand regardless of what happens below: deleting the staged + # directory is now just cleanup, not a data-integrity concern, and a + # failure here is reported as a warning (not an error) to avoid + # contradicting the registry state that already succeeded. + if staged_dir is not None: try: - shutil.rmtree(workflow_dir) + shutil.rmtree(staged_dir) except OSError as exc: - # The registry removal already succeeded; restore the original - # entry verbatim so the registry doesn't claim this workflow is - # uninstalled while its directory is still sitting on disk. - try: - if registry_metadata is not None: - registry.data["workflows"][workflow_id] = registry_metadata - registry.save() - except Exception as restore_exc: # noqa: BLE001 - console.print( - f"[yellow]Warning:[/yellow] Failed to restore registry entry " - f"for '{safe_id}' after directory removal failure: " - f"{_escape_markup(str(restore_exc))}" - ) console.print( - f"[red]Error:[/red] Failed to remove workflow directory " - f"{_escape_markup(str(workflow_dir))}: {_escape_markup(str(exc))}" + f"[yellow]Warning:[/yellow] Workflow '{safe_id}' was removed, but its " + f"staged directory could not be deleted: {_escape_markup(str(exc))}. " + f"Remove it manually: {_escape_markup(str(staged_dir))}" ) - raise typer.Exit(1) - - console.print(f"[green]✓[/green] Workflow '{workflow_id}' removed") @workflow_app.command("update") diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 65b24e7265..2aa4764aed 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -13,6 +13,7 @@ import hashlib import json import os +import stat import tempfile import time from dataclasses import dataclass @@ -146,6 +147,19 @@ def save(self) -> None: try: with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(self.data, f, indent=2) + # mkstemp creates the temp file at 0600. A pre-existing registry + # may be shared more permissively (e.g. 0640/0644); preserve its + # mode across the replace so a save doesn't silently lock other + # project users out. A brand-new registry has no prior mode to + # preserve, so mkstemp's secure 0600 default stands. Mirrors + # _utils.py's atomic_write_json (best-effort; data safety over + # metadata preservation). + try: + if self.registry_path.exists(): + existing_mode = stat.S_IMODE(self.registry_path.stat().st_mode) + os.chmod(tmp, existing_mode) + except OSError: + pass os.replace(tmp, self.registry_path) except BaseException: try: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 85339b95be..46a393ed44 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -15,6 +15,7 @@ import json import os import shutil +import stat import sys import tempfile from pathlib import Path @@ -5985,18 +5986,28 @@ def boom(self): assert (workflow_dir / "workflow.yml").read_text(encoding="utf-8") == "keep-me" # The on-disk registry must still claim the workflow installed. assert WorkflowRegistry(project_dir).is_installed("test-wf") - - def test_remove_directory_failure_restores_registry_entry_verbatim( + # The directory must be restored to its exact original location, with + # no leftover staging directory from the stage/restore-on-failure + # sequence. + entries = [ + p.name + for p in (project_dir / ".specify" / "workflows").iterdir() + if p.name != "workflow-registry.json" + ] + assert entries == ["test-wf"] + + def test_remove_staged_cleanup_failure_reports_warning_not_error( self, project_dir, monkeypatch ): - """If the registry removal already persisted successfully but the - subsequent shutil.rmtree fails, the directory was never actually - deleted (rmtree raised before removing anything usable), so the - registry must not be left claiming the workflow uninstalled. The - restored entry must be byte-for-byte the original (same - installed_at/updated_at) -- calling add() again would stamp a new - updated_at, which is why workflow_step_remove restores directly via - registry.data and save() instead of add().""" + """The directory is staged (atomically renamed out of + .specify/workflows/) *before* the registry write, and the actual + deletion of the staged directory only happens *after* the registry + has already durably recorded the removal. If that final deletion + fails, the registry write already succeeded and must stand -- an + "Error: Failed to remove..." message at that point would contradict + the registry, which is exactly the incoherent state this staging + order exists to prevent. It must be reported as a cleanup warning, + and the command must still succeed.""" from typer.testing import CliRunner from specify_cli import app from specify_cli.workflows.catalog import WorkflowRegistry @@ -6007,8 +6018,6 @@ def test_remove_directory_failure_restores_registry_entry_verbatim( workflow_dir.mkdir(parents=True, exist_ok=True) (workflow_dir / "workflow.yml").write_text("keep-me", encoding="utf-8") - original_entry = WorkflowRegistry(project_dir).get("test-wf") - def boom(*args, **kwargs): raise OSError("permission denied") @@ -6017,26 +6026,31 @@ def boom(*args, **kwargs): mp.setattr("shutil.rmtree", boom) result = CliRunner().invoke(app, ["workflow", "remove", "test-wf"]) - assert result.exit_code != 0 - assert result.exception is None or isinstance(result.exception, SystemExit) - assert "Failed to remove workflow directory" in result.output - # rmtree raised, so nothing was actually deleted. - assert workflow_dir.exists() - assert (workflow_dir / "workflow.yml").read_text(encoding="utf-8") == "keep-me" - # The registry entry must come back exactly as it was, not re-added. - restored = WorkflowRegistry(project_dir).get("test-wf") - assert restored == original_entry - assert WorkflowRegistry(project_dir).is_installed("test-wf") - - def test_remove_directory_and_restore_failure_escapes_rich_markup( + assert result.exit_code == 0 + assert "Warning" in result.output + # The registry write already committed -- it must stand. + assert not WorkflowRegistry(project_dir).is_installed("test-wf") + # The original install path is gone (staged away before the registry + # write ever ran); only a leftover staged directory remains, never + # at the original path the registry/CLI would treat as installed. + assert not workflow_dir.exists() + leftovers = [ + p + for p in (project_dir / ".specify" / "workflows").iterdir() + if p.name != "workflow-registry.json" + ] + assert len(leftovers) == 1 + assert (leftovers[0] / "workflow.yml").read_text(encoding="utf-8") == "keep-me" + + def test_remove_stage_restore_failure_escapes_rich_markup( self, temp_dir, monkeypatch ): - """The project path (workflow_dir) and the rmtree/restore-save - exceptions interpolated into these new Rich error/warning messages - must be escaped like every other error path here -- unescaped Rich - markup characters (e.g. brackets) in a project directory name or an - OS/registry error message could otherwise be parsed as markup and - hide or corrupt the displayed text instead of showing it verbatim.""" + """When the registry write fails (already rolled back in-memory by + WorkflowRegistry.remove()) and the attempt to rename the staged + directory back to its original location also fails, both the + restore exception and the registry-update exception interpolated + into these warning/error messages must be escaped like every other + error path here.""" from typer.testing import CliRunner from specify_cli import app from specify_cli.workflows.catalog import WorkflowRegistry @@ -6052,37 +6066,31 @@ def test_remove_directory_and_restore_failure_escapes_rich_markup( workflow_dir.mkdir(parents=True, exist_ok=True) (workflow_dir / "workflow.yml").write_text("keep-me", encoding="utf-8") - def rmtree_boom(*args, **kwargs): - raise OSError("[disk] permission denied") + def save_boom(self): + raise OSError("[reg] disk full") - real_save = WorkflowRegistry.save - call_count = {"n": 0} + real_rename = os.rename + rename_calls = {"n": 0} - def save_boom(self): - # The first save() call is registry.remove()'s own persist, - # which must succeed so we reach the rmtree failure below; only - # the second call (the post-rmtree-failure restore attempt) - # should fail, to exercise the restore-failure warning path. - call_count["n"] += 1 - if call_count["n"] == 1: - return real_save(self) - raise Exception("[warn] save exploded") + def rename_boom(src, dst): + rename_calls["n"] += 1 + if rename_calls["n"] == 1: + # Allow the initial stage-out rename to succeed so the + # restore-back rename (the second call) is what fails. + return real_rename(src, dst) + raise OSError("[stage] permission denied") monkeypatch.chdir(project_dir) with pytest.MonkeyPatch.context() as mp: - mp.setattr("shutil.rmtree", rmtree_boom) mp.setattr(WorkflowRegistry, "save", save_boom) + mp.setattr(os, "rename", rename_boom) result = CliRunner().invoke(app, ["workflow", "remove", "test-wf"]) assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) - # Rich may soft-wrap the long path across lines; compare with - # whitespace collapsed so the wrap position doesn't affect the check. output_compact = "".join(result.output.split()) - assert "".join(str(workflow_dir).split()) in output_compact - assert "[disk]permissiondenied" in output_compact - assert "[warn]saveexploded" in output_compact - + assert "[stage]permissiondenied" in output_compact + assert "[reg]diskfull" in output_compact class TestWorkflowAddSymlinkGuard: def test_add_malformed_ipv6_url_exits_cleanly(self, temp_dir, monkeypatch): @@ -8034,6 +8042,33 @@ def test_registry_save_refuses_symlinked_parent(self, project_dir, tmp_path): WorkflowRegistry(project_dir) assert not (outside / "workflows").exists() + def test_registry_save_preserves_existing_file_mode(self, project_dir): + """A registry shared as 0640/0644 must keep that mode after a save, + not be silently replaced by mkstemp's 0600 default -- otherwise + every add/remove locks other project users out of a previously + shared registry file.""" + from specify_cli.workflows.catalog import WorkflowRegistry + + registry = WorkflowRegistry(project_dir) + registry.add("first-wf", {"name": "First"}) + registry.registry_path.chmod(0o644) + + registry.add("second-wf", {"name": "Second"}) + + mode = stat.S_IMODE(registry.registry_path.stat().st_mode) + assert mode == 0o644, f"expected 0644, got {oct(mode)}" + + def test_registry_save_on_new_registry_uses_secure_default_mode(self, project_dir): + """A brand-new registry file (no prior mode to preserve) should keep + mkstemp's secure 0600 default rather than something more permissive.""" + from specify_cli.workflows.catalog import WorkflowRegistry + + registry = WorkflowRegistry(project_dir) + registry.add("first-wf", {"name": "First"}) + + mode = stat.S_IMODE(registry.registry_path.stat().st_mode) + assert mode == 0o600, f"expected 0600, got {oct(mode)}" + def test_add_dev_dir_with_workflow_yml_directory_errors_cleanly(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app @@ -8089,13 +8124,16 @@ def boom(self): assert not dest_dir.exists() assert not WorkflowRegistry(project_dir).is_installed("align-wf") - def test_add_dev_reinstall_copy_failure_restores_prior_file(self, project_dir, monkeypatch): - """_validate_and_install_local's copy2 call currently runs *before* the - try/except block that protects registry.add(): a copy2 failure (e.g. a - truncating partial write on a reinstall) is not caught at all, so the - existing backup-restore cleanup never runs and the prior working - workflow.yml is left corrupted. copy2 must be covered by the same - rollback-protected section as registry.add().""" + def test_add_dev_reinstall_copy_failure_leaves_prior_file_untouched( + self, project_dir, monkeypatch + ): + """copy2 now writes into a same-directory staging file, never + dest_file directly, so a copy2 failure (even one that partially + writes before raising, mirroring a real disk-full/interrupted-copy + failure) can no longer touch -- let alone truncate -- the prior + working workflow.yml: dest_file is only ever touched by the final + atomic commit swap, which never runs if copy2 raises. No leftover + staging file may remain either.""" from typer.testing import CliRunner from specify_cli import app from specify_cli.workflows.catalog import WorkflowRegistry @@ -8113,10 +8151,11 @@ def test_add_dev_reinstall_copy_failure_restores_prior_file(self, project_dir, m self.WORKFLOW_YAML.format(version="2.0.0"), encoding="utf-8" ) - def boom(*args, **kwargs): - # Simulate a truncating partial write followed by an OSError, - # mirroring a real disk-full/interrupted-copy failure. - installed_yaml.write_bytes(b"") + def boom(src_path, dst_path, *args, **kwargs): + # Simulate a truncating partial write followed by an OSError on + # the *staging* file -- the only file copy2 is now allowed to + # touch -- mirroring a real disk-full/interrupted-copy failure. + Path(dst_path).write_bytes(b"") raise OSError("disk full") with pytest.MonkeyPatch.context() as mp: @@ -8128,44 +8167,58 @@ def boom(*args, **kwargs): assert result.output.strip() != "" assert installed_yaml.read_bytes() == original_bytes assert WorkflowRegistry(project_dir).get("align-wf") == original_registry_entry + # No orphaned staging file left behind in the workflow directory. + leftovers = [p.name for p in installed_yaml.parent.iterdir() if p.name != "workflow.yml"] + assert leftovers == [] - def test_add_dev_reinstall_backup_read_failure_gives_clean_error( + def test_add_dev_reinstall_restore_failure_reports_warning_and_original_error( self, project_dir, monkeypatch ): - """The prior-file backup read (used to restore on a later install - failure) ran before any guarded section: a read failure on the - existing workflow.yml (e.g. a transient permission/FS issue) leaked - a raw OSError instead of the clean escaped CLI error used by every - other failure branch here, and left the destination untouched since - it happens before any write.""" + """The prior file is now restored via an atomic rename (not a + content rewrite) when registry.add() fails on a reinstall. If that + restore rename itself also fails (e.g. a transient FS issue), it + must not silently claim success or crash with a raw traceback: it + must report a clear warning about the restore failure in addition + to the original clean registry error.""" from typer.testing import CliRunner from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry monkeypatch.chdir(project_dir) runner = CliRunner() src = self._install_dev(runner, app, project_dir) - installed_yaml = project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" - original_bytes = installed_yaml.read_bytes() (src / "workflow.yml").write_text( self.WORKFLOW_YAML.format(version="2.0.0"), encoding="utf-8" ) - real_read_bytes = Path.read_bytes + def save_boom(self): + raise OSError("disk full") - def boom(self_path, *args, **kwargs): - if self_path.resolve() == installed_yaml.resolve(): - raise OSError("permission denied") - return real_read_bytes(self_path, *args, **kwargs) + real_replace = os.replace + calls = {"n": 0} + + def replace_boom(src_path, dst_path): + # The commit swap for a reinstall makes exactly two os.replace + # calls (backup-aside, then staged-into-dest); let both succeed + # and only fail the third call -- the post-registry-failure + # restore-back rename. + calls["n"] += 1 + if calls["n"] <= 2: + return real_replace(src_path, dst_path) + raise OSError("permission denied") with pytest.MonkeyPatch.context() as mp: - mp.setattr(Path, "read_bytes", boom) + mp.setattr(WorkflowRegistry, "save", save_boom) + mp.setattr(os, "replace", replace_boom) result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) - assert result.output.strip() != "" - assert installed_yaml.read_bytes() == original_bytes + output_compact = "".join(result.output.split()) + assert "Warning" in result.output + assert "diskfull" in output_compact + assert "permissiondenied" in output_compact def test_add_dev_fresh_install_into_preexisting_empty_dir_cleans_new_file( self, project_dir, monkeypatch @@ -8390,18 +8443,18 @@ def boom(self): assert registry.is_installed("align-wf") assert registry.get("align-wf")["version"] == "1.0.0" - def test_add_catalog_reinstall_backup_read_failure_gives_clean_error( + def test_add_catalog_reinstall_restore_failure_reports_warning_and_original_error( self, project_dir, monkeypatch ): - """Same backup-read boundary gap as the local-install path: the - prior-file read used to seed the reinstall's rollback ran before - the download/validation error boundary, so a read failure on the - existing workflow.yml (e.g. a transient permission/FS issue) leaked - a raw OSError instead of a clean escaped CLI error, and must be - caught before any download/write is attempted.""" + """Same restore-rename boundary as the local-install path: the + prior file is restored via an atomic rename (not a content rewrite) + when registry.add() fails on a reinstall. If that restore rename + itself also fails, it must report a clear warning in addition to + the original clean registry error, never crash or silently claim + success.""" from typer.testing import CliRunner from specify_cli import app - from specify_cli.workflows.catalog import WorkflowCatalog + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry monkeypatch.chdir(project_dir) monkeypatch.setattr( @@ -8428,22 +8481,41 @@ def test_add_catalog_reinstall_backup_read_failure_gives_clean_error( result = runner.invoke(app, ["workflow", "add", "align-wf"]) assert result.exit_code == 0, result.output - dest_file = project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" - real_read_bytes = Path.read_bytes + new_data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + + def save_boom(self): + raise OSError("disk full") - def boom(self_path, *args, **kwargs): - if self_path.resolve() == dest_file.resolve(): - raise OSError("permission denied") - return real_read_bytes(self_path, *args, **kwargs) + real_replace = os.replace + calls = {"n": 0} + + def replace_boom(src_path, dst_path): + # The commit swap for a reinstall makes exactly two os.replace + # calls (backup-aside, then staged-into-dest); let both succeed + # and only fail the third call -- the post-registry-failure + # restore-back rename. + calls["n"] += 1 + if calls["n"] <= 2: + return real_replace(src_path, dst_path) + raise OSError("permission denied") with pytest.MonkeyPatch.context() as mp: - mp.setattr(Path, "read_bytes", boom) + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + new_data, url + ), + ) + mp.setattr(WorkflowRegistry, "save", save_boom) + mp.setattr(os, "replace", replace_boom) result = runner.invoke(app, ["workflow", "add", "align-wf"]) assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) - assert result.output.strip() != "" - assert dest_file.read_bytes() == original_data + output_compact = "".join(result.output.split()) + assert "Warning" in result.output + assert "diskfull" in output_compact + assert "permissiondenied" in output_compact def test_add_catalog_fresh_install_into_preexisting_empty_dir_cleans_new_file( self, project_dir, monkeypatch @@ -8612,11 +8684,15 @@ def fake_open(url, timeout=None, extra_headers=None, redirect_validator=None): assert seen["validator"] is _reject_insecure_download_redirect def test_registry_save_failure_preserves_file_on_disk(self, project_dir, monkeypatch): - """A failed dump must not truncate the persisted registry.""" + """A failed dump must not truncate the persisted registry, and must + not alter its on-disk mode either -- the chmod-to-match-existing-mode + step operates on the temp file, never the target, so a failed save + (which never reaches os.replace) cannot touch the original's mode.""" from specify_cli.workflows.catalog import WorkflowRegistry registry = WorkflowRegistry(project_dir) registry.add("align-wf", {"version": "1.0.0", "source": "catalog"}) + registry.registry_path.chmod(0o644) import specify_cli.workflows.catalog as catalog_mod @@ -8630,6 +8706,7 @@ def boom(*args, **kwargs): fresh = WorkflowRegistry(project_dir) assert fresh.get("align-wf")["version"] == "1.0.0" + assert stat.S_IMODE(registry.registry_path.stat().st_mode) == 0o644 assert not list(registry.workflows_dir.glob("*.tmp")) def test_update_mixed_targets_does_not_claim_all_up_to_date(self, project_dir, monkeypatch): From 7c6bc902ba06ee42194e1bb06c43e8128286c27f Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:03:11 +0200 Subject: [PATCH 37/59] Discard reinstall backup file after registry.add() succeeds _commit_workflow_file() renames a prior workflow.yml aside to workflow.yml.bak so it can be restored if registry.add() subsequently fails. Neither the local install/reinstall path nor the catalog install/reinstall path ever cleaned up that backup after a successful registry.add() -- every successful reinstall permanently left a workflow.yml.bak sibling, which later reinstalls would silently overwrite/re-orphan. Add a shared _discard_committed_backup_file() helper, called from both success paths right after registry.add() durably succeeds (and before the final "installed" message, preserving output ordering). A fresh install (backup_file is None) is a no-op. A cleanup failure is reported as a warning (exit 0), not a failure, since the install itself already succeeded -- consistent with workflow_remove's post-commit cleanup warning semantics. Add red-first regression tests proving: (1) successful local reinstall leaves no workflow.yml.bak sibling, (2) successful catalog reinstall leaves no workflow.yml.bak sibling, (3) a cleanup failure on the backup file after a successful reinstall reports a warning and still exits 0 with the registry correctly reflecting the new install. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 27 ++++++ tests/test_workflows.py | 126 +++++++++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index db9faad5a2..3eedd7108d 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -347,6 +347,27 @@ def _safe_rollback_committed_workflow_file( ) +def _discard_committed_backup_file(backup_file: Path | None) -> None: + """Once registry.add()/registry.remove() has durably succeeded after a + _commit_workflow_file() swap, the renamed-aside prior file is no longer + needed for rollback -- it must be discarded, not left as a permanent + orphan sibling that every future reinstall would silently accumulate or + clobber. A cleanup failure here must not turn an already-successful + install into a reported failure; it's reported as a warning, consistent + with workflow_remove's post-commit cleanup semantics. A fresh install + (backup_file is None) is a no-op.""" + if backup_file is None: + return + try: + backup_file.unlink(missing_ok=True) + except OSError as exc: + console.print( + "[yellow]Warning:[/yellow] Workflow installed, but its backup file " + f"could not be cleaned up: {_escape_markup(str(exc))}. Remove it " + f"manually: {_escape_markup(str(backup_file))}" + ) + + # Root helper re-fetched at call time so test monkeypatching of # `specify_cli._require_specify_project` keeps working after the move. def _require_specify_project(*args, **kwargs): @@ -939,6 +960,9 @@ def _validate_and_install_local( f"'{_escape_markup(definition.id)}': {_escape_markup(str(exc))}" ) raise typer.Exit(1) + # registry.add() durably succeeded -- the renamed-aside backup is no + # longer needed for rollback. + _discard_committed_backup_file(backup_file) console.print( f"[green]✓[/green] Workflow '{_escape_markup(definition.name)}' " f"({_escape_markup(definition.id)}) installed" @@ -1293,6 +1317,9 @@ def _install_workflow_from_catalog( f"'{_escape_markup(workflow_id)}': {_escape_markup(str(exc))}" ) raise typer.Exit(1) + # registry.add() durably succeeded -- the renamed-aside backup is no + # longer needed for rollback. + _discard_committed_backup_file(backup_file) console.print( f"[green]✓[/green] Workflow '{_escape_markup(str(info.get('name', workflow_id)))}' " "installed from catalog" diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 46a393ed44..c7dd4a9876 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8171,6 +8171,77 @@ def boom(src_path, dst_path, *args, **kwargs): leftovers = [p.name for p in installed_yaml.parent.iterdir() if p.name != "workflow.yml"] assert leftovers == [] + def test_add_dev_successful_reinstall_leaves_no_backup_file( + self, project_dir, monkeypatch + ): + """_commit_workflow_file() renames the prior workflow.yml aside to + workflow.yml.bak so it can be restored if registry.add() fails. Once + registry.add() durably succeeds, that backup is no longer needed -- + it must be discarded, not left behind as a permanent orphan sibling + that every future reinstall would silently accumulate/overwrite.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + src = self._install_dev(runner, app, project_dir) + workflow_dir = project_dir / ".specify" / "workflows" / "align-wf" + + # Reinstall (overwrite) with a new version -- a successful reinstall, + # not a failure path. + (src / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="2.0.0"), encoding="utf-8" + ) + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + + assert result.exit_code == 0, result.output + registry = WorkflowRegistry(project_dir) + assert registry.is_installed("align-wf") + assert registry.get("align-wf")["version"] == "2.0.0" + assert (workflow_dir / "workflow.yml").read_text(encoding="utf-8") == ( + self.WORKFLOW_YAML.format(version="2.0.0") + ) + leftovers = [p.name for p in workflow_dir.iterdir() if p.name != "workflow.yml"] + assert leftovers == [], f"orphan sibling(s) left behind: {leftovers}" + + def test_add_dev_successful_reinstall_backup_cleanup_failure_still_succeeds( + self, project_dir, monkeypatch + ): + """A failure to clean up the now-unneeded backup file after a + successful registry.add() must not turn the already-successful + install into a reported failure: it must be a warning (exit 0), + consistent with workflow_remove's post-commit cleanup semantics.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + src = self._install_dev(runner, app, project_dir) + + (src / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="2.0.0"), encoding="utf-8" + ) + + real_unlink = Path.unlink + + def unlink_boom(self_path, *args, **kwargs): + if self_path.name.endswith(".bak"): + raise OSError("permission denied") + return real_unlink(self_path, *args, **kwargs) + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(Path, "unlink", unlink_boom) + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + + assert result.exit_code == 0, result.output + assert "Warning" in result.output + assert "permissiondenied" in "".join(result.output.split()) + registry = WorkflowRegistry(project_dir) + assert registry.is_installed("align-wf") + assert registry.get("align-wf")["version"] == "2.0.0" + def test_add_dev_reinstall_restore_failure_reports_warning_and_original_error( self, project_dir, monkeypatch ): @@ -8213,6 +8284,7 @@ def replace_boom(src_path, dst_path): mp.setattr(os, "replace", replace_boom) result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) output_compact = "".join(result.output.split()) @@ -8443,6 +8515,60 @@ def boom(self): assert registry.is_installed("align-wf") assert registry.get("align-wf")["version"] == "1.0.0" + def test_add_catalog_successful_reinstall_leaves_no_backup_file( + self, project_dir, monkeypatch + ): + """Same orphan-backup gap as the local-install path: a successful + catalog reinstall must not leave workflow.yml.bak behind once + registry.add() durably succeeds.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + original_data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + runner = CliRunner() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + original_data, url + ), + ) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + assert result.exit_code == 0, result.output + + new_data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + new_data, url + ), + ) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + + assert result.exit_code == 0, result.output + workflow_dir = project_dir / ".specify" / "workflows" / "align-wf" + registry = WorkflowRegistry(project_dir) + assert registry.is_installed("align-wf") + assert registry.get("align-wf")["version"] == "2.0.0" + assert (workflow_dir / "workflow.yml").read_bytes() == new_data + leftovers = [p.name for p in workflow_dir.iterdir() if p.name != "workflow.yml"] + assert leftovers == [], f"orphan sibling(s) left behind: {leftovers}" + def test_add_catalog_reinstall_restore_failure_reports_warning_and_original_error( self, project_dir, monkeypatch ): From 915caf8da9b21512d9f88efc603f1750d481f02b Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 13:07:10 +0200 Subject: [PATCH 38/59] Clean up freshly-created dest_dir when staging mkstemp fails _stage_workflow_file() does dest_dir.mkdir(parents=True, exist_ok=True) then tempfile.mkstemp(dir=dest_dir, ...). For a fresh install (no prior directory), if mkdir succeeds but mkstemp then raises (disk full/EMFILE/quota), the exception previously propagated straight past both the local-install and catalog-install call sites without any cleanup, leaving the newly-created empty workflow directory orphaned on disk with no error indicating why. Fix at the shared _stage_workflow_file() boundary instead of duplicating cleanup at each call site: track whether this call created dest_dir: on a mkstemp failure, remove that directory via a guarded rmdir (never a broad rmtree, so any concurrently written content would be left untouched) before re-raising the original OSError unchanged. A pre-existing (reinstall) dest_dir is never touched by this cleanup, and a cleanup failure is reported as its own warning without masking the original error. Add red-first regression tests proving: a fresh local install (--dev, plain local path, --from) and a fresh catalog install both clean up the orphaned directory on a simulated mkstemp failure, and a reinstall over a pre-existing directory is left untouched by the same failure. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 23 ++++- tests/test_workflows.py | 124 +++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 3eedd7108d..5824d2ad86 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -266,11 +266,30 @@ def _stage_workflow_file(dest_dir: Path) -> Path: """Reserve a same-directory staging file so new/updated workflow.yml content can be written and validated without ever touching (and risking truncating) an existing destination file before the final atomic swap. - Shared by the local-install and catalog-install paths.""" + Shared by the local-install and catalog-install paths. + + If dest_dir did not already exist, this call creates it; if mkstemp then + fails (disk full/EMFILE/quota), the freshly-created directory is removed + again via a guarded rmdir (never a broad rmtree, so any concurrently + written content is left untouched) before the original OSError is + re-raised unchanged. A pre-existing dest_dir (reinstall) is never + touched by this cleanup.""" import tempfile + created_dir = not dest_dir.exists() dest_dir.mkdir(parents=True, exist_ok=True) - fd, tmp_name = tempfile.mkstemp(dir=dest_dir, prefix=".workflow.yml.", suffix=".tmp") + try: + fd, tmp_name = tempfile.mkstemp(dir=dest_dir, prefix=".workflow.yml.", suffix=".tmp") + except OSError: + if created_dir: + try: + dest_dir.rmdir() + except OSError as cleanup_exc: + console.print( + "[yellow]Warning:[/yellow] Failed to remove incomplete " + f"workflow directory: {_escape_markup(str(cleanup_exc))}" + ) + raise os.close(fd) return Path(tmp_name) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index c7dd4a9876..4e7a8a5f70 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8124,6 +8124,85 @@ def boom(self): assert not dest_dir.exists() assert not WorkflowRegistry(project_dir).is_installed("align-wf") + @pytest.mark.parametrize("mode", ["dev", "local", "from_url"]) + def test_add_fresh_install_mkstemp_failure_leaves_no_orphan_directory( + self, project_dir, monkeypatch, mode + ): + """_stage_workflow_file() does mkdir(dest_dir) then mkstemp() inside + it. For a fresh install (no prior directory), if mkdir succeeds but + mkstemp then fails (disk full/EMFILE/quota), the freshly-created + empty dest_dir must not be left orphaned -- it must be removed, and + the original mkstemp error must still be reported cleanly.""" + import contextlib + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + + def boom(*args, **kwargs): + raise OSError("disk full") + + if mode == "from_url": + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + args = ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"] + url_patch = patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), + ) + else: + src = self._write_workflow_dir(project_dir) + args = ["workflow", "add", str(src)] + (["--dev"] if mode == "dev" else []) + url_patch = contextlib.nullcontext() + + with url_patch, pytest.MonkeyPatch.context() as mp: + mp.setattr("tempfile.mkstemp", boom) + result = runner.invoke(app, args) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + dest_dir = project_dir / ".specify" / "workflows" / "align-wf" + assert not dest_dir.exists(), "fresh-install dest_dir left orphaned after mkstemp failure" + assert not WorkflowRegistry(project_dir).is_installed("align-wf") + + def test_add_reinstall_mkstemp_failure_preserves_preexisting_directory( + self, project_dir, monkeypatch + ): + """A pre-existing (reinstall) dest_dir must never be removed by the + mkstemp-failure cleanup -- only a directory _stage_workflow_file + itself just created.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + src = self._install_dev(runner, app, project_dir) + installed_yaml = project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" + original_bytes = installed_yaml.read_bytes() + original_registry_entry = WorkflowRegistry(project_dir).get("align-wf") + + (src / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="2.0.0"), encoding="utf-8" + ) + + def boom(*args, **kwargs): + raise OSError("disk full") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr("tempfile.mkstemp", boom) + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + assert installed_yaml.parent.is_dir() + assert installed_yaml.read_bytes() == original_bytes + assert WorkflowRegistry(project_dir).get("align-wf") == original_registry_entry + def test_add_dev_reinstall_copy_failure_leaves_prior_file_untouched( self, project_dir, monkeypatch ): @@ -8365,6 +8444,51 @@ def boom(self): assert not dest_dir.exists() assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_catalog_fresh_install_mkstemp_failure_leaves_no_orphan_directory( + self, project_dir, monkeypatch + ): + """Same guarantee as the local-install fresh-install case, but for a + fresh catalog install: if _stage_workflow_file's mkdir succeeds but + its mkstemp then fails, the freshly-created empty directory must not + be left orphaned.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + + def boom(*args, **kwargs): + raise OSError("disk full") + + runner = CliRunner() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), + ) + mp.setattr("tempfile.mkstemp", boom) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert result.output.strip() != "" + dest_dir = project_dir / ".specify" / "workflows" / "align-wf" + assert not dest_dir.exists(), "fresh-install dest_dir left orphaned after mkstemp failure" + assert not WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_catalog_rejects_oversized_content_length(self, project_dir, monkeypatch): """Catalog installs must share the same size cap as --from: a response that declares an oversized Content-Length is rejected From 792450e259c9be0b346b7efbe4126bb4db07066f Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:47:21 +0200 Subject: [PATCH 39/59] Fix installed-workflow ownership/disabled bypass and resume enforcement Address 3 current Copilot review findings on the disabled-workflow guard in `workflow run`/`workflow resume`: - The lexical `.specify/workflows/` ownership scan stopped at the first match scanning from the start of the path. A nested project living beneath an outer installed workflow's own directory tree (reusing the same segment names) was attributed to the wrong (outer) workflow and ID, gating the run on an unrelated workflow's disabled state. `_scan_for_workflow_owner` now scans from the end so the nearest (innermost) owner always wins. - A path with no `.specify/workflows` segments of its own (e.g. `/tmp/alias.yml`) that is itself a symlink resolving *into* installed storage bypassed the disabled check entirely, since only the raw lexical path was inspected. `_resolve_installed_workflow_ownership` now additionally resolves the real path when the lexical scan finds no owner and re-runs the same scan against it, so an outward-pointing alias into a disabled workflow is caught too. Genuinely standalone external files (no symlink anywhere on the path) are unaffected. - `workflow resume` bypassed the disabled check altogether: engine.resume() replays a persisted run directly from disk with no registry awareness. RunState now optionally persists `installed_workflow_id` and `installed_registry_root` at run start (set by workflow_run when the source resolved to an installed ID); `workflow_resume` pre-loads the run state and re-checks the registry's *current* disabled state before calling engine.resume(), mirroring workflow_run's own guard. Both new fields default to None via RunState.load()'s `.get()`, so runs from a direct/non-installed source, and any run persisted before this schema addition, resume exactly as before. The ownership-mapping logic (previously inlined in workflow_run) is extracted into `_resolve_installed_workflow_ownership` / `_scan_for_workflow_owner` so both the lexical and resolved-path cases share the same scan and the existing inward-symlink-component refusal. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 193 +++++++++++++++++++------ src/specify_cli/workflows/engine.py | 29 ++++ tests/test_workflows.py | 160 +++++++++++++++----- 3 files changed, 300 insertions(+), 82 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 5824d2ad86..9310aa4026 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -123,6 +123,109 @@ def _reject_unsafe_workflow_storage(project_root: Path) -> None: ) +def _scan_for_workflow_owner(parts: tuple[str, ...]) -> tuple[int, str] | None: + """Find the *nearest* (innermost) ``.specify/workflows/`` owner in + *parts*, scanning from the end of the path. + + Scanning from the end (rather than stopping at the first match from the + start) matters for a project nested beneath an unrelated outer path that + happens to reuse the same ``.specify``/``workflows`` segment names: the + first-from-start match would pick the outer directory and the wrong + workflow ID, silently missing the real (inner) owner's disabled check. + + Returns ``(i, workflow_id)`` where ``i`` is the index of the owning + ``.specify`` segment, or ``None`` if no owner segment is present. + """ + for i in range(len(parts) - 3, -1, -1): + if parts[i] == ".specify" and parts[i + 1] == "workflows": + return i, parts[i + 2] + return None + + +def _resolve_installed_workflow_ownership( + source_path: Path, err +) -> tuple[Path | None, str | None]: + """Map a direct ``workflow.yml`` *source_path* back to the installed + workflow (``registry_root``, ``registered_id``) it belongs to, if any. + + A path can point at installed storage two ways, both of which must + receive the same registry disabled-check: + + 1. Lexically: the path's own (symlink-preserving) segments literally + contain ``.specify/workflows/`` -- collapsing ``..``/``.`` but + never resolving symlinks, so a symlinked ``workflow.yml`` leaf (or + symlinked ```` directory) inside the owned tree is caught by the + inward-symlink refusal below rather than silently followed. + 2. Via an outward-pointing alias: *source_path* (or one of its parent + directories) is itself a symlink whose *resolved* target lands + inside some project's ``.specify/workflows//workflow.yml``, even + though the raw path used to invoke the command has no such segments + at all (e.g. ``/tmp/alias.yml`` -> a disabled installed workflow's + real file). Only the lexical case also runs the inward-symlink + refusal: the resolved case's segments are real by construction (an + already-fully-resolved path cannot itself contain a symlink + component), so that check would be a vacuous no-op there. + + Returns ``(None, None)`` when neither applies -- a genuinely standalone + external workflow file, which is allowed to run unchecked. + """ + lexical = Path(os.path.normpath(str(source_path.absolute()))) + parts = lexical.parts + match = _scan_for_workflow_owner(parts) + if match is not None: + i, registered_id = match + registry_root = Path(*parts[:i]) if i else Path(lexical.anchor or ".") + # The path-derived registry_root here may differ from the cwd's + # project_root already checked by _reject_unsafe_workflow_storage + # (e.g. this path points into another project entirely, or this + # project's own .specify is itself a symlink to an + # attacker-controlled tree) -- check it explicitly rather than + # trusting that cwd-scoped guard, and don't rely on + # WorkflowRegistry's own symlinked-parent handling as the safety + # signal here: it fails closed by raising OSError at construction + # time (see catalog.py's _load), but that surfaces as an opaque + # exception rather than this guard's clean, specific CLI error for + # the actual owning project root. + _reject_unsafe_dir(registry_root / ".specify", ".specify") + _reject_unsafe_dir( + registry_root / ".specify" / "workflows", ".specify/workflows" + ) + # A legitimately installed workflow's own directory tree never + # contains a symlink (workflow add/remove both refuse one at + # install time); one appearing here means the file actually loaded + # below would not be the file this ownership match is based on, so + # refuse rather than silently mismatch. + for k in range(i + 2, len(parts) + 1): + if Path(*parts[:k]).is_symlink(): + err.print( + "[red]Error:[/red] Refusing to run: " + f".specify/workflows/{_escape_markup(registered_id)} " + "contains a symlinked path component" + ) + raise typer.Exit(1) + return registry_root, registered_id + + # No lexical owner segments. source_path (or a parent directory) might + # still be a symlink whose resolved target lands inside some project's + # installed workflow storage under a completely unrelated-looking path + # -- check that too so an alias into a disabled workflow can't dodge + # the registry guard. + try: + resolved = source_path.resolve(strict=False) + except OSError: + return None, None + if resolved == lexical: + # Nothing on this path is a symlink; already covered above. + return None, None + resolved_parts = resolved.parts + resolved_match = _scan_for_workflow_owner(resolved_parts) + if resolved_match is None: + return None, None + i, registered_id = resolved_match + registry_root = Path(*resolved_parts[:i]) if i else Path(resolved.anchor or ".") + return registry_root, registered_id + + _WORKFLOW_ID_PATTERN = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") _RESERVED_WORKFLOW_IDS: frozenset[str] = frozenset({"runs", "steps"}) @@ -580,48 +683,13 @@ def workflow_run( registered_id = source else: # A direct YAML path may still point at an installed workflow's own - # file; map it back to its owning project and ID from the *lexical* - # path (collapsing .. / . without resolving symlinks) rather than - # resolve(): resolving first would follow a symlinked workflow.yml - # out of .specify/workflows, fail to find an owner, and let - # engine.load_workflow below run the symlink target unchecked -- - # silently bypassing a disabled workflow's guard. - lexical = Path(os.path.normpath(str(source_path.absolute()))) - parts = lexical.parts - for i in range(len(parts) - 2): - if parts[i] == ".specify" and parts[i + 1] == "workflows": - registry_root = Path(*parts[:i]) if i else Path(lexical.anchor or ".") - registered_id = parts[i + 2] - # The path-derived registry_root here may differ from the - # cwd's project_root already checked by - # _reject_unsafe_workflow_storage above (e.g. this path - # points into another project entirely, or this project's - # own .specify is itself a symlink to an attacker-controlled - # tree) -- check it explicitly rather than trusting that - # cwd-scoped guard, and don't rely on WorkflowRegistry's own - # symlinked-parent handling below as the safety signal here: - # it now fails closed by raising OSError at construction - # time (see catalog.py's _load), but that surfaces as an - # opaque exception rather than this guard's clean, specific - # CLI error for the actual owning project root. - _reject_unsafe_dir(registry_root / ".specify", ".specify") - _reject_unsafe_dir( - registry_root / ".specify" / "workflows", ".specify/workflows" - ) - # A legitimately installed workflow's own directory tree - # never contains a symlink (workflow add/remove both refuse - # one at install time); one appearing here means the file - # actually loaded below would not be the file this ownership - # match is based on, so refuse rather than silently mismatch. - for k in range(i + 2, len(parts) + 1): - if Path(*parts[:k]).is_symlink(): - err.print( - "[red]Error:[/red] Refusing to run: " - f".specify/workflows/{_escape_markup(registered_id)} " - "contains a symlinked path component" - ) - raise typer.Exit(1) - break + # file (lexically, or via a symlinked alias pointing into installed + # storage); map it back to its owning project and ID so the + # disabled check below can't be silently bypassed. + owner_root, owner_id = _resolve_installed_workflow_ownership(source_path, err) + if owner_id is not None: + registry_root = owner_root + registered_id = owner_id if registered_id is not None: installed_meta = _open_workflow_registry(registry_root, err).get(registered_id) @@ -658,7 +726,12 @@ def workflow_run( try: with _stdout_to_stderr_when(json_output): - state = engine.execute(definition, inputs) + state = engine.execute( + definition, + inputs, + installed_workflow_id=registered_id, + installed_registry_root=registry_root if registered_id else None, + ) except ValueError as exc: err.print(f"[red]Error:[/red] {exc}") raise typer.Exit(1) @@ -700,7 +773,7 @@ def workflow_resume( ): """Resume a paused or failed workflow run.""" from . import load_custom_steps - from .engine import WorkflowEngine + from .engine import RunState, WorkflowEngine project_root = _require_specify_project() load_custom_steps(project_root) @@ -711,6 +784,38 @@ def workflow_resume( inputs = _parse_input_values(input_values, json_output=json_output) err = _error_console(json_output) + # Pre-load the persisted run state so a run started from an installed + # workflow that has since been disabled cannot resume unchecked -- + # engine.resume() replays the run directly from disk with no registry + # awareness at all, which would otherwise bypass the same disabled + # guard `workflow run` enforces. Runs without installed_workflow_id + # (a direct/non-installed source, or a run persisted before this field + # existed) are unaffected and resume exactly as before. + try: + pre_state = RunState.load(run_id, project_root) + except FileNotFoundError: + err.print(f"[red]Error:[/red] Run not found: {run_id}") + raise typer.Exit(1) + except ValueError as exc: + err.print(f"[red]Error:[/red] {exc}") + raise typer.Exit(1) + + if pre_state.installed_workflow_id is not None: + owner_root = ( + Path(pre_state.installed_registry_root) + if pre_state.installed_registry_root + else project_root + ) + installed_meta = _open_workflow_registry(owner_root, err).get( + pre_state.installed_workflow_id + ) + if isinstance(installed_meta, dict) and not installed_meta.get("enabled", True): + err.print( + f"[red]Error:[/red] Workflow '{_escape_markup(pre_state.installed_workflow_id)}' is disabled. " + f"Enable with: specify workflow enable {_escape_markup(pre_state.installed_workflow_id)}" + ) + raise typer.Exit(1) + try: with _stdout_to_stderr_when(json_output): state = engine.resume(run_id, inputs or None) diff --git a/src/specify_cli/workflows/engine.py b/src/specify_cli/workflows/engine.py index 6025c30c5c..da3e2b483a 100644 --- a/src/specify_cli/workflows/engine.py +++ b/src/specify_cli/workflows/engine.py @@ -428,6 +428,8 @@ def __init__( run_id: str | None = None, workflow_id: str = "", project_root: Path | None = None, + installed_workflow_id: str | None = None, + installed_registry_root: str | None = None, ) -> None: # ``run_id is None`` (omitted) → auto-generate. An explicit empty # string is *not* the same as "omitted" and must be validated like @@ -441,6 +443,15 @@ def __init__( self._validate_run_id(self.run_id) self.workflow_id = workflow_id self.project_root = project_root or Path(".") + # Identifies the installed workflow (if any) this run was started + # from, and the project root that owns its registry — set by + # execute() when the source was resolved to an installed ID (see + # workflow_run's ownership mapping). None for a direct/non-installed + # YAML source, and for any run persisted before this field existed + # (load() defaults it to None), preserving old runs' existing + # resume behavior unchanged. + self.installed_workflow_id = installed_workflow_id + self.installed_registry_root = installed_registry_root self.status = RunStatus.CREATED self.current_step_index = 0 self.current_step_id: str | None = None @@ -503,6 +514,8 @@ def save(self) -> None: state_data = { "run_id": self.run_id, "workflow_id": self.workflow_id, + "installed_workflow_id": self.installed_workflow_id, + "installed_registry_root": self.installed_registry_root, "status": self.status.value, "current_step_index": self.current_step_index, "current_step_id": self.current_step_id, @@ -559,6 +572,8 @@ def load(cls, run_id: str, project_root: Path) -> RunState: run_id=state_data["run_id"], workflow_id=state_data["workflow_id"], project_root=project_root, + installed_workflow_id=state_data.get("installed_workflow_id"), + installed_registry_root=state_data.get("installed_registry_root"), ) state.status = RunStatus(state_data["status"]) state.current_step_index = state_data.get("current_step_index", 0) @@ -654,6 +669,8 @@ def execute( definition: WorkflowDefinition, inputs: dict[str, Any] | None = None, run_id: str | None = None, + installed_workflow_id: str | None = None, + installed_registry_root: Path | None = None, ) -> RunState: """Execute a workflow definition. @@ -665,6 +682,12 @@ def execute( User-provided input values. run_id: Optional run ID (uses SPECKIT_WORKFLOW_RUN_ID when set, otherwise auto-generated). + installed_workflow_id, installed_registry_root: + When the run was started from an installed workflow (as opposed + to a direct/non-installed YAML source), identifies it and its + owning registry root so a later ``resume`` can re-check the + registry's current disabled state before continuing — see + ``workflow_resume``. Returns ------- @@ -682,6 +705,12 @@ def execute( run_id=effective_run_id, workflow_id=definition.id, project_root=self.project_root, + installed_workflow_id=installed_workflow_id, + installed_registry_root=( + str(installed_registry_root) + if installed_registry_root is not None + else None + ), ) # Persist a copy of the workflow definition so resume can diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 4e7a8a5f70..9904762234 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7805,6 +7805,46 @@ def test_add_from_url_installs(self, project_dir, monkeypatch): assert result.exit_code == 0, result.output assert WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_from_url_temp_cleanup_failure_after_success_still_exits_zero( + self, project_dir, monkeypatch + ): + """An OSError while deleting the --from download's temp file after + _validate_and_install_local() has already committed the file and + registry entry must not surface as an unhandled failure for an + install that already succeeded -- it must be a warning, exit 0.""" + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + runner = CliRunner() + + import tempfile + + real_unlink = Path.unlink + + def unlink_boom(self_path, *args, **kwargs): + if self_path.suffix == ".yml" and self_path.parent == Path(tempfile.gettempdir()): + raise OSError("permission denied") + return real_unlink(self_path, *args, **kwargs) + + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(data, url), + ), pytest.MonkeyPatch.context() as mp: + mp.setattr(Path, "unlink", unlink_boom) + result = runner.invoke( + app, + ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + ) + + assert result.exit_code == 0, result.output + assert "Warning" in result.output + assert "permissiondenied" in "".join(result.output.split()) + assert WorkflowRegistry(project_dir).is_installed("align-wf") + def test_add_from_url_id_mismatch_errors(self, project_dir, monkeypatch): from unittest.mock import patch from typer.testing import CliRunner @@ -8042,6 +8082,7 @@ def test_registry_save_refuses_symlinked_parent(self, project_dir, tmp_path): WorkflowRegistry(project_dir) assert not (outside / "workflows").exists() + @pytest.mark.skipif(sys.platform == "win32", reason="chmod mode bits not reliable on Windows") def test_registry_save_preserves_existing_file_mode(self, project_dir): """A registry shared as 0640/0644 must keep that mode after a save, not be silently replaced by mkstemp's 0600 default -- otherwise @@ -8058,6 +8099,7 @@ def test_registry_save_preserves_existing_file_mode(self, project_dir): mode = stat.S_IMODE(registry.registry_path.stat().st_mode) assert mode == 0o644, f"expected 0644, got {oct(mode)}" + @pytest.mark.skipif(sys.platform == "win32", reason="chmod mode bits not reliable on Windows") def test_registry_save_on_new_registry_uses_secure_default_mode(self, project_dir): """A brand-new registry file (no prior mode to preserve) should keep mkstemp's secure 0600 default rather than something more permissive.""" @@ -8933,6 +8975,7 @@ def fake_open(url, timeout=None, extra_headers=None, redirect_validator=None): assert seen["validator"] is _reject_insecure_download_redirect + @pytest.mark.skipif(sys.platform == "win32", reason="chmod mode bits not reliable on Windows") def test_registry_save_failure_preserves_file_on_disk(self, project_dir, monkeypatch): """A failed dump must not truncate the persisted registry, and must not alter its on-disk mode either -- the chmod-to-match-existing-mode @@ -9463,55 +9506,96 @@ def test_update_reports_unsafe_registry_id_per_workflow(self, project_dir, monke assert result.exit_code != 0 assert "Failed to update" in result.output - def test_update_survives_oserror_from_backup_read(self, project_dir, monkeypatch): - """OSError while reading the backup for one workflow must not abort the whole update.""" - import json - from pathlib import Path + def test_update_registry_save_failure_restores_prior_file_without_redundant_write( + self, project_dir, monkeypatch + ): + """A registry.add() save failure during `workflow update` must be + fully restored by _install_workflow_from_catalog's own atomic + rollback (rename-based, not a byte-level rewrite). The outer + workflow_update loop must not perform any redundant write of its + own onto the destination file -- that write happened only after + typer.Exit already unwound, could itself fail/truncate the safely + preserved file, and is provably unnecessary here since the inner + transaction already restored it via rename.""" from typer.testing import CliRunner from specify_cli import app - from specify_cli.workflows.catalog import WorkflowRegistry, WorkflowCatalog + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry monkeypatch.chdir(project_dir) - registry_path = WorkflowRegistry(project_dir).registry_path - registry_path.parent.mkdir(parents=True, exist_ok=True) - registry_path.write_text( - json.dumps( - { - "schema_version": "1.0", - "workflows": { - "wobbly": { - "name": "Wobbly", - "version": "0.0.1", - "source": "catalog", - "url": "https://example.com/wobbly.yml", - }, - }, - } - ), - encoding="utf-8", - ) - # An existing installed file whose read_bytes will raise OSError. - wf_file = project_dir / ".specify" / "workflows" / "wobbly" / "workflow.yml" - wf_file.parent.mkdir(parents=True, exist_ok=True) - wf_file.write_bytes(b"schema_version: '1.0'\nworkflow:\n id: wobbly\n name: Wobbly\n version: 0.0.1\nsteps: []\n") monkeypatch.setattr( WorkflowCatalog, "get_workflow_info", - lambda self, wid: {"version": "9.9.9", "url": "https://example.com/wobbly.yml", "_install_allowed": True}, + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, ) - real_read = Path.read_bytes + original_data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + runner = CliRunner() + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + original_data, url + ), + ) + result = runner.invoke(app, ["workflow", "add", "align-wf"]) + assert result.exit_code == 0, result.output - def _boom(self, *args, **kwargs): - if self.name == "workflow.yml" and "wobbly" in str(self): - raise OSError("simulated permission denied") - return real_read(self, *args, **kwargs) + dest_file = project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" + assert dest_file.read_bytes() == original_data - monkeypatch.setattr(Path, "read_bytes", _boom) - runner = CliRunner() - result = runner.invoke(app, ["workflow", "update"], input="y\n") - assert result.exit_code != 0, result.output - assert "Filesystem error" in result.output + new_data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + + def boom_save(self): + raise OSError("disk full") + + dest_writes: list[bytes] = [] + real_write_bytes = Path.write_bytes + resolved_dest_file = dest_file.resolve() + + def tracking_write_bytes(self_path, data, *args, **kwargs): + if self_path.resolve() == resolved_dest_file: + dest_writes.append(data) + return real_write_bytes(self_path, data, *args, **kwargs) + + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + new_data, url + ), + ) + mp.setattr(WorkflowRegistry, "save", boom_save) + mp.setattr(Path, "write_bytes", tracking_write_bytes) + result = runner.invoke(app, ["workflow", "update"], input="y\n") + + assert result.exit_code != 0 assert "Failed to update" in result.output + # No redundant/second write of the destination file was attempted -- + # the inner atomic commit/rollback (rename-based) is the only thing + # that ever touches it. + assert dest_writes == [] + assert dest_file.read_bytes() == original_data + registry = WorkflowRegistry(project_dir) + assert registry.is_installed("align-wf") + assert registry.get("align-wf")["version"] == "1.0.0" def test_enable_disable_corrupted_registry_entry_errors(self, project_dir, monkeypatch): import json From 800b962feaced8d9b5dbcaa67bafbc5222284418 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:47:43 +0200 Subject: [PATCH 40/59] Guard --from temp cleanup; drop redundant update rollback; mark POSIX-only tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two more current Copilot review findings, both in workflow_add/update: - `workflow add --from`'s `finally: tmp_path.unlink(missing_ok=True)` ran unguarded after `_validate_and_install_local` had already committed the file and registry entry (success) or already raised its own clean `typer.Exit` (failure). An OSError from that cleanup unlink would surface as an unhandled failure even though the install itself succeeded. It is now wrapped in try/except OSError, printing a neutral warning that doesn't claim success or failure (the finally runs on both outcomes) instead of propagating. - `workflow_update`'s per-item loop performed its own outer backup (`wf_file.read_bytes()`) and restore (`wf_file.write_bytes(backup)`) around `_install_workflow_from_catalog`, which is itself fully transactional (staged download, atomic rename-based commit, its own rollback on registry failure) and never leaves a raw OSError or a partially-written workflow.yml. The outer restore was therefore dead weight for its stated purpose, and — being an unguarded byte-level write — was itself an unnecessary place a second failure could truncate an already-safely-preserved file. Removed; the loop now only records success/failure. Also marks 3 registry-save file-mode tests (`test_registry_save_preserves_existing_file_mode`, `test_registry_save_on_new_registry_uses_secure_default_mode`, `test_registry_save_failure_preserves_file_on_disk`) as POSIX-only via the repo's existing `skipif(sys.platform == "win32", ...)` pattern, since they assert exact POSIX permission bits that don't hold on Windows. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 37 ++--- tests/test_workflows.py | 203 +++++++++++++++++++++++++ 2 files changed, 222 insertions(+), 18 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 9310aa4026..4bdb8f16e0 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -1199,7 +1199,20 @@ def _validate_and_install_local( expected_id=source if from_url else None, ) finally: - tmp_path.unlink(missing_ok=True) + # Best-effort: _validate_and_install_local may already have + # committed the file + registry entry (success) or already + # raised its own clean typer.Exit (failure) by this point -- + # either way, a cleanup OSError here must never mask that + # outcome or surface as its own unhandled failure. Warn instead, + # same as the committed-backup cleanup above. + try: + tmp_path.unlink(missing_ok=True) + except OSError as exc: + console.print( + "[yellow]Warning:[/yellow] Could not remove temporary " + f"download file {_escape_markup(str(tmp_path))}: " + f"{_escape_markup(str(exc))}" + ) return # Try as a local file/directory @@ -1663,29 +1676,17 @@ def workflow_update( console.print() failed: list[str] = [] for update in updates_available: - # Installed workflows are a single workflow.yml — back it up so a - # failed download/validation doesn't destroy the working copy. - wf_dir: Path | None = None - wf_file: Path | None = None - backup: bytes | None = None + # _install_workflow_from_catalog is fully transactional (staged + # download, atomic commit, rename-based rollback on registry + # failure): it never leaves a partially-written workflow.yml, so + # this loop only needs to record success/failure, not perform its + # own backup/restore. try: - wf_dir = _safe_workflow_id_dir(workflows_dir, update["id"]) - wf_file = wf_dir / "workflow.yml" - backup = wf_file.read_bytes() if wf_file.is_file() else None _install_workflow_from_catalog( project_root, registry, workflows_dir, update["id"], expected_version=update["available"], ) except (typer.Exit, OSError) as exc: - if backup is not None and wf_dir is not None and wf_file is not None: - try: - wf_dir.mkdir(parents=True, exist_ok=True) - wf_file.write_bytes(backup) - except OSError as restore_exc: - console.print( - f"[yellow]Warning:[/yellow] Could not restore backup for " - f"'{_escape_markup(update['id'])}': {_escape_markup(str(restore_exc))}" - ) if isinstance(exc, OSError): console.print( f"[red]Error:[/red] Filesystem error updating " diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 9904762234..8c94142f40 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -9835,6 +9835,209 @@ def test_run_refuses_symlinked_specify_dir_hiding_disabled_workflow( assert result.exception is None or isinstance(result.exception, SystemExit) assert "symlink" in result.output.lower() + def test_run_nested_installed_paths_uses_nearest_owner( + self, temp_dir, monkeypatch + ): + """A direct workflow.yml path whose lexical segments contain + .specify/workflows more than once (an unrelated nested project + happens to live beneath an outer installed workflow's own + directory tree, reusing the same segment names) must be attributed + to its *nearest* (innermost) owning project/ID -- scanning from the + start of the path and stopping at the first match would pick the + outer project and the wrong workflow ID, gating the run on an + unrelated workflow's disabled state instead of the real owner's.""" + from typer.testing import CliRunner + from specify_cli import app + + def _write_registry(workflows_dir, workflow_id, enabled): + workflows_dir.mkdir(parents=True, exist_ok=True) + (workflows_dir / "workflow-registry.json").write_text( + json.dumps( + { + "schema_version": "1.0", + "workflows": { + workflow_id: { + "name": workflow_id, + "version": "1.0.0", + "source": "dev", + "enabled": enabled, + } + }, + } + ), + encoding="utf-8", + ) + + outer_workflows = temp_dir / "outer-proj" / ".specify" / "workflows" + outer_wf_dir = outer_workflows / "outer-wf" + outer_wf_dir.mkdir(parents=True) + (outer_wf_dir / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + _write_registry(outer_workflows, "outer-wf", enabled=False) + + # An unrelated nested project lives inside the outer workflow's own + # directory tree, with its own separate installed workflow. + inner_workflows = outer_wf_dir / "nested-proj" / ".specify" / "workflows" + inner_wf_dir = inner_workflows / "inner-wf" + inner_wf_dir.mkdir(parents=True) + (inner_wf_dir / "workflow.yml").write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + _write_registry(inner_workflows, "inner-wf", enabled=True) + + unrelated_cwd = temp_dir / "unrelated-cwd" + unrelated_cwd.mkdir() + monkeypatch.chdir(unrelated_cwd) + + runner = CliRunner() + target = inner_wf_dir / "workflow.yml" + result = runner.invoke(app, ["workflow", "run", str(target)]) + # inner-wf (the actual nearest owner) is enabled -- must run, not + # be blocked by the unrelated outer-wf's disabled state. + assert result.exit_code == 0, result.output + + # The inverse proves this isn't just ignoring nesting: disabling + # the true (nearest) owner must actually block this exact path. + _write_registry(inner_workflows, "inner-wf", enabled=False) + result = runner.invoke(app, ["workflow", "run", str(target)]) + assert result.exit_code != 0 + assert "disabled" in result.output + + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") + def test_run_blocks_disabled_workflow_via_outward_alias_symlink( + self, project_dir, monkeypatch + ): + """The inverse of the existing inward-symlink case: a path with no + .specify/workflows segments at all (e.g. /tmp/alias.yml) that is + itself a symlink resolving *into* installed storage must still + receive the disabled check. Only checking the lexical path's own + segments misses this alias entirely, since it has no such segments + to begin with, and would let engine.load_workflow follow the + symlink to the disabled workflow's real content unchecked.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + + result = runner.invoke(app, ["workflow", "disable", "align-wf"]) + assert result.exit_code == 0, result.output + + installed_yaml = ( + project_dir / ".specify" / "workflows" / "align-wf" / "workflow.yml" + ) + external_dir = project_dir / "outside-alias" + external_dir.mkdir() + alias = external_dir / "alias.yml" + alias.symlink_to(installed_yaml) + + result = runner.invoke(app, ["workflow", "run", str(alias)]) + assert result.exit_code != 0 + assert "disabled" in result.output + + result = runner.invoke(app, ["workflow", "enable", "align-wf"]) + assert result.exit_code == 0, result.output + result = runner.invoke(app, ["workflow", "run", str(alias)]) + assert result.exit_code == 0, result.output + + _GATED_WORKFLOW_YAML = """ +schema_version: "1.0" +workflow: + id: "gated-wf" + name: "Gated Workflow" + version: "1.0.0" +steps: + - id: ask + type: gate + message: "Review" + options: [approve, reject] +""" + + def _install_and_run_gated(self, runner, app, project_dir): + """Install a gate-step workflow and run it to a paused state. + + Returns the run_id. The gate step pauses without any interactive + input, giving a resumable run tied to an installed workflow ID. + """ + src = project_dir / "gated-src" + src.mkdir(exist_ok=True) + (src / "workflow.yml").write_text(self._GATED_WORKFLOW_YAML, encoding="utf-8") + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + assert result.exit_code == 0, result.output + + result = runner.invoke(app, ["workflow", "run", "gated-wf", "--json"]) + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["status"] == "paused" + return payload["run_id"] + + def test_resume_blocks_when_installed_workflow_disabled( + self, project_dir, monkeypatch + ): + """A run started from an installed workflow must not resume once + that workflow is disabled. engine.resume() replays the persisted + run directly from disk with no registry awareness at all, so the + installed workflow's origin (id + owning registry root) is + persisted at run start and re-checked against the registry's + *current* state before resuming, mirroring `workflow run`'s + disabled guard.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + run_id = self._install_and_run_gated(runner, app, project_dir) + + result = runner.invoke(app, ["workflow", "disable", "gated-wf"]) + assert result.exit_code == 0, result.output + + result = runner.invoke(app, ["workflow", "resume", run_id]) + assert result.exit_code != 0 + assert "disabled" in result.output + + # Re-enabling must unblock the exact same run. + result = runner.invoke(app, ["workflow", "enable", "gated-wf"]) + assert result.exit_code == 0, result.output + result = runner.invoke(app, ["workflow", "resume", run_id, "--json"]) + assert result.exit_code == 0, result.output + resumed = json.loads(result.stdout) + assert resumed["run_id"] == run_id + + def test_resume_backward_compatible_with_run_state_missing_new_fields( + self, project_dir, monkeypatch + ): + """A run's state.json persisted before installed-origin tracking + existed (missing the installed_workflow_id/installed_registry_root + keys entirely) must still resume normally: RunState.load's + defaults must not raise, and the absent origin must skip the + disabled check rather than block or error -- old runs are + unaffected by this guard.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + run_id = self._install_and_run_gated(runner, app, project_dir) + + state_path = ( + project_dir / ".specify" / "workflows" / "runs" / run_id / "state.json" + ) + data = json.loads(state_path.read_text(encoding="utf-8")) + data.pop("installed_workflow_id", None) + data.pop("installed_registry_root", None) + state_path.write_text(json.dumps(data), encoding="utf-8") + + result = runner.invoke(app, ["workflow", "disable", "gated-wf"]) + assert result.exit_code == 0, result.output + + # Origin metadata is absent from disk -- disabling the (now + # unrelated, from this run's perspective) installed workflow must + # not block resuming this legacy run. + result = runner.invoke(app, ["workflow", "resume", run_id, "--json"]) + assert result.exit_code == 0, result.output + def test_disable_shows_marker_in_list(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app From 616f72742591bff868b9a841a86275181ef5fb26 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:47:51 +0200 Subject: [PATCH 41/59] Report possible partial changes on zero-removed bundle removal failure The final Copilot review finding: `remove_bundle`'s zero-removed-components error message claimed "No components were removed." even when the failing installer component may have deleted files before raising -- prior review rounds already established that DefaultPrimitiveInstaller's removal paths are not atomic and can leave partial filesystem changes despite raising before `result.uninstalled` is populated. The zero-count message is now a conservative caution ("...but the failing component may have made partial changes before raising, so the project may be partially uninstalled.") instead of an unconditional claim of no side effects. The >0-removed path (which already reports the confirmed partial list) is unchanged. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/bundler/services/installer.py | 6 +++- .../integration/test_bundler_install_flow.py | 33 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/specify_cli/bundler/services/installer.py b/src/specify_cli/bundler/services/installer.py index 61c882d1f4..6882c0052f 100644 --- a/src/specify_cli/bundler/services/installer.py +++ b/src/specify_cli/bundler/services/installer.py @@ -207,7 +207,11 @@ def remove_bundle( "so the project may be partially uninstalled." ) else: - detail = "No components were removed." + detail = ( + "No components were removed, but the failing component may " + "have made partial changes before raising, so the project " + "may be partially uninstalled." + ) raise BundlerError( f"Failed to remove bundle '{bundle_id}': {exc}. {detail}" ) from exc diff --git a/tests/integration/test_bundler_install_flow.py b/tests/integration/test_bundler_install_flow.py index a2e34797fd..9a4163d044 100644 --- a/tests/integration/test_bundler_install_flow.py +++ b/tests/integration/test_bundler_install_flow.py @@ -213,6 +213,39 @@ def boom(project_root, component): assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} +def test_remove_zero_completed_removals_still_cautions_about_partial_changes( + tmp_path: Path, +): + """`result.uninstalled` only records a component after its `remove()` + call returns successfully. If the very first `remove()` call itself + raises after already deleting some files, zero completed removals are + recorded even though the project may already be partially uninstalled -- + the zero-count message must not claim "No components were removed" as + an unqualified fact; it must caution that the failing component may + have made partial changes before raising.""" + make_project(tmp_path) + manifest = BundleManifest.from_dict(valid_manifest_dict()) + installer = FakeInstaller() + install_bundle(tmp_path, _plan(manifest), installer, manifest=manifest) + + def boom(project_root, component): + # Simulates a remove() that deletes some files before raising -- + # from the caller's perspective this component was never recorded + # as completed, but disk state may already be partially changed. + raise OSError("disk full partway through removal") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(installer, "remove", boom) + with pytest.raises(BundlerError) as exc_info: + remove_bundle(tmp_path, "demo-bundle", installer) + + message = str(exc_info.value) + assert "no components were removed" in message.lower() + assert "partial" in message.lower() + assert "partially uninstalled" in message.lower() + assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} + + def test_remove_reports_uninstalled_not_installed(tmp_path: Path): make_project(tmp_path) manifest = BundleManifest.from_dict(valid_manifest_dict()) From 4f247356d00026e1ce4fe8812151be03351f49ba Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:25:18 +0200 Subject: [PATCH 42/59] Fix workflow resume disabled-check bypass after project move/rename RunState.installed_registry_root previously persisted the creation-time absolute project path unconditionally whenever a run belonged to an installed workflow. After the whole project directory was renamed or moved, workflow_resume would open a WorkflowRegistry at that now nonexistent path, get back an empty/default registry, and silently skip the disabled-workflow check -- a paused run for a disabled workflow could be resumed successfully from the new location. Fix persists installed_registry_root only when the owning root genuinely differs from the current project_root (true cross-project direct-file- source invocations). The common same-project case now persists None and is re-derived from the live project_root at resume time via a new _resolve_run_owner_root() helper, which also falls back to project_root if a stored root no longer exists on disk -- covering both the common case transparently surviving project moves and the cross-project case degrading safely if its owner project vanishes, rather than silently skipping the disabled check. Backward compatible: state files missing the new fields, and states with a still-existing distinct cross-project root, behave unchanged. Added regression tests: - resume blocked after project moved then disabled at new location - resume still works after project moved while workflow stays enabled - cross-project registry root is still correctly honored when it exists - resume falls back to current project's registry when a stored cross-project root no longer exists Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 48 +++++++- tests/test_workflows.py | 150 +++++++++++++++++++++++++ 2 files changed, 193 insertions(+), 5 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 4bdb8f16e0..0d95a4767b 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -77,6 +77,33 @@ def _open_workflow_registry(project_root: Path, out=None): raise typer.Exit(1) +def _resolve_run_owner_root( + installed_registry_root: str | None, project_root: Path +) -> Path: + """Determine which project's registry gates resuming a run. + + ``installed_registry_root`` is only ever persisted when the run's + installed workflow genuinely belongs to a *different* project than the + one whose ``runs/`` directory holds this run's own state (a direct + external workflow-file invocation) -- see ``workflow_run``. The common + case (an installed workflow run from its own project) stores ``None``, + so a later project rename/move is transparently picked up here by + falling back to the *current* ``project_root`` instead of a stale + absolute path baked in at run start. + + A persisted cross-project root that itself no longer exists (that + other project moved/was deleted) is not trusted either: silently + skipping the disabled check because a stored path merely happens not + to resolve would defeat the guard's purpose, so this also falls back + to ``project_root`` rather than risk that. + """ + if installed_registry_root: + candidate = Path(installed_registry_root) + if candidate.is_dir(): + return candidate + return project_root + + def _parse_input_values( input_values: list[str] | None, *, json_output: bool = False ) -> dict[str, Any]: @@ -730,7 +757,20 @@ def workflow_run( definition, inputs, installed_workflow_id=registered_id, - installed_registry_root=registry_root if registered_id else None, + # Only persist an explicit root when the installed workflow + # genuinely belongs to a *different* project than the one + # whose runs/ directory holds this run's own state (a + # direct external workflow-file invocation) -- the common + # case (an installed workflow run from its own project) + # leaves this None so resume re-derives the owning root + # from wherever the project currently is, transparently + # surviving a project rename/move instead of baking in a + # stale absolute path at run start. + installed_registry_root=( + registry_root + if registered_id and registry_root != project_root + else None + ), ) except ValueError as exc: err.print(f"[red]Error:[/red] {exc}") @@ -801,10 +841,8 @@ def workflow_resume( raise typer.Exit(1) if pre_state.installed_workflow_id is not None: - owner_root = ( - Path(pre_state.installed_registry_root) - if pre_state.installed_registry_root - else project_root + owner_root = _resolve_run_owner_root( + pre_state.installed_registry_root, project_root ) installed_meta = _open_workflow_registry(owner_root, err).get( pre_state.installed_workflow_id diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 8c94142f40..6dd2582524 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -10038,6 +10038,156 @@ def test_resume_backward_compatible_with_run_state_missing_new_fields( result = runner.invoke(app, ["workflow", "resume", run_id, "--json"]) assert result.exit_code == 0, result.output + def test_resume_blocks_after_project_moved_following_disable( + self, temp_dir, monkeypatch + ): + """Renaming/moving the entire project after starting a run must not + let a subsequent disable-then-resume bypass the guard. Persisting + the run's *creation-time absolute* project path would make resume + open a now-nonexistent old root (WorkflowRegistry falls back to an + empty default there), missing the disabled entry that actually + lives in the *current* (moved) project's registry. The common, + same-project case must instead re-derive the owning root from the + project's current location on every resume.""" + from typer.testing import CliRunner + from specify_cli import app + import shutil + + project_v1 = temp_dir / "project-v1" + (project_v1 / ".specify" / "workflows").mkdir(parents=True) + monkeypatch.chdir(project_v1) + runner = CliRunner() + run_id = self._install_and_run_gated(runner, app, project_v1) + + project_v2 = temp_dir / "project-v2" + shutil.move(str(project_v1), str(project_v2)) + monkeypatch.chdir(project_v2) + + result = runner.invoke(app, ["workflow", "disable", "gated-wf"]) + assert result.exit_code == 0, result.output + + result = runner.invoke(app, ["workflow", "resume", run_id]) + assert result.exit_code != 0 + assert "disabled" in result.output + + def test_resume_after_project_moved_still_works_when_enabled( + self, temp_dir, monkeypatch + ): + """The inverse of the move regression: an enabled workflow's run + must still resume normally after the project is moved -- the + current-project fallback must not itself block legitimate + resumes.""" + from typer.testing import CliRunner + from specify_cli import app + import shutil + + project_v1 = temp_dir / "project-v1-ok" + (project_v1 / ".specify" / "workflows").mkdir(parents=True) + monkeypatch.chdir(project_v1) + runner = CliRunner() + run_id = self._install_and_run_gated(runner, app, project_v1) + + project_v2 = temp_dir / "project-v2-ok" + shutil.move(str(project_v1), str(project_v2)) + monkeypatch.chdir(project_v2) + + result = runner.invoke(app, ["workflow", "resume", run_id, "--json"]) + assert result.exit_code == 0, result.output + + def test_resume_respects_cross_project_registry_root( + self, temp_dir, monkeypatch + ): + """A run started via a direct workflow.yml path belonging to a + different project than the cwd used for `workflow run`/`workflow + resume` must still gate resuming on *that* owning project's + registry, not the cwd project's (which has no entry for this ID + at all). This is the genuine cross-project case that must remain + unaffected by only special-casing the common same-project one.""" + from typer.testing import CliRunner + from specify_cli import app + + owner_project = temp_dir / "owner-project" + (owner_project / ".specify" / "workflows").mkdir(parents=True) + monkeypatch.chdir(owner_project) + runner = CliRunner() + src = owner_project / "gated-src" + src.mkdir() + (src / "workflow.yml").write_text(self._GATED_WORKFLOW_YAML, encoding="utf-8") + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + assert result.exit_code == 0, result.output + + unrelated_cwd = temp_dir / "unrelated-cwd" + unrelated_cwd.mkdir() + monkeypatch.chdir(unrelated_cwd) + + target = owner_project / ".specify" / "workflows" / "gated-wf" / "workflow.yml" + result = runner.invoke(app, ["workflow", "run", str(target), "--json"]) + assert result.exit_code == 0, result.output + run_id = json.loads(result.stdout)["run_id"] + + monkeypatch.chdir(owner_project) + result = runner.invoke(app, ["workflow", "disable", "gated-wf"]) + assert result.exit_code == 0, result.output + + # Resume must run from unrelated_cwd (where this run's own + # state.json actually lives) yet still be blocked by the owner + # project's disabled entry. + monkeypatch.chdir(unrelated_cwd) + result = runner.invoke(app, ["workflow", "resume", run_id]) + assert result.exit_code != 0 + assert "disabled" in result.output + + def test_resume_falls_back_to_current_project_when_cross_project_root_vanishes( + self, temp_dir, monkeypatch + ): + """A persisted cross-project owning root that no longer exists (that + other project was itself deleted/moved away) must not be trusted as + a safety signal -- silently skipping the disabled check just + because the stored path happens not to resolve would defeat the + guard. Falls back to the current project's own registry instead.""" + from typer.testing import CliRunner + from specify_cli import app + import shutil + + owner_project = temp_dir / "owner-project-2" + (owner_project / ".specify" / "workflows").mkdir(parents=True) + monkeypatch.chdir(owner_project) + runner = CliRunner() + src = owner_project / "gated-src" + src.mkdir() + (src / "workflow.yml").write_text(self._GATED_WORKFLOW_YAML, encoding="utf-8") + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + assert result.exit_code == 0, result.output + + unrelated_cwd = temp_dir / "unrelated-cwd-2" + unrelated_cwd.mkdir() + monkeypatch.chdir(unrelated_cwd) + + target = owner_project / ".specify" / "workflows" / "gated-wf" / "workflow.yml" + result = runner.invoke(app, ["workflow", "run", str(target), "--json"]) + assert result.exit_code == 0, result.output + run_id = json.loads(result.stdout)["run_id"] + + # owner_project vanishes entirely -- its persisted absolute root + # is now dangling. + shutil.rmtree(owner_project) + + # unrelated_cwd (where this run's own state.json lives) has its own + # separately-installed, disabled workflow under the same ID. + src2 = unrelated_cwd / "gated-src" + src2.mkdir() + (src2 / "workflow.yml").write_text( + self._GATED_WORKFLOW_YAML, encoding="utf-8" + ) + result = runner.invoke(app, ["workflow", "add", str(src2), "--dev"]) + assert result.exit_code == 0, result.output + result = runner.invoke(app, ["workflow", "disable", "gated-wf"]) + assert result.exit_code == 0, result.output + + result = runner.invoke(app, ["workflow", "resume", run_id]) + assert result.exit_code != 0 + assert "disabled" in result.output + def test_disable_shows_marker_in_list(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app From fa410d3bf959aac765d6d0fd09e3260d1ca74115 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:00:24 +0200 Subject: [PATCH 43/59] Fix silenced cleanup failures and malformed run-state type validation Four fixes from Copilot review on HEAD 4f24735: 1. _discard_staged_workflow_file's fresh-install directory removal used shutil.rmtree(dest_dir, ignore_errors=True), so a genuine cleanup failure there could never reach _safe_discard_staged_workflow_file's warning -- an orphaned directory was left behind with zero report. Now removes only if dest_dir still exists and lets a real OSError propagate to the existing safe wrapper, which warns while the already-printed original install error remains primary. 2. _rollback_committed_workflow_file's fresh-install directory removal (post registry.add() failure) had the same ignore_errors=True gap; fixed identically so _safe_rollback_committed_workflow_file's warning can actually fire. 3. In the --from download-failure branch, tmp_path.unlink(missing_ok= True) was unguarded: if it raised (e.g. read-only tempdir), it replaced the original "Failed to download workflow" error with a raw unhandled OSError instead of a clean typer.Exit. Now guarded exactly like the later post-install finally cleanup: a cleanup failure prints a warning and the original download error is still reported cleanly. 4. RunState.load() trusted installed_workflow_id/installed_registry_root straight out of state.json with no type validation. A malformed value (int/list/dict/bool instead of str-or-null) would crash deep inside _resolve_run_owner_root or the registry lookup (TypeError building a Path, unhashable dict/list as a mapping key) instead of failing cleanly. Both fields are now validated as str | None during load, raising a clear ValueError that workflow_resume's existing ValueError boundary already converts into a clean CLI error with no traceback. Valid values (including the empty-string fallback already handled by _resolve_run_owner_root) continue to load unchanged. Added red-first regression tests for each: staged-discard cleanup warning, rollback cleanup warning, download-failure cleanup-vs-original- error precedence, and parameterized malformed/valid run-state field coverage. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 31 +++- src/specify_cli/workflows/engine.py | 17 +- tests/test_workflows.py | 213 +++++++++++++++++++++++++ 3 files changed, 252 insertions(+), 9 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 0d95a4767b..2e57d9be12 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -447,11 +447,14 @@ def _commit_workflow_file(staged_file: Path, dest_file: Path, existed_before: bo def _discard_staged_workflow_file(staged_file: Path, dest_dir: Path, existed_before: bool) -> None: """Clean up after a pre-commit failure (staged_file was never swapped onto dest_file): remove the staged file, and for a fresh install (no - prior directory) remove the now-orphaned dest_dir too.""" + prior directory) remove the now-orphaned dest_dir too. A genuine + removal failure must propagate (not be swallowed) so the safe wrapper + below can warn instead of silently leaving an orphan; a dest_dir + already absent is not itself an error.""" staged_file.unlink(missing_ok=True) - if not existed_before: + if not existed_before and dest_dir.exists(): import shutil - shutil.rmtree(dest_dir, ignore_errors=True) + shutil.rmtree(dest_dir) def _rollback_committed_workflow_file( @@ -460,14 +463,17 @@ def _rollback_committed_workflow_file( """Undo a successful _commit_workflow_file swap after a later failure (registry.add()): restore the prior file via rename, remove the newly committed file for a reinstall over a pre-existing empty directory - (no backup), or remove the whole directory for a fresh install.""" + (no backup), or remove the whole directory for a fresh install. A + genuine removal failure must propagate (not be swallowed) so the safe + wrapper below can warn instead of silently leaving an orphan; a + dest_dir already absent is not itself an error.""" if backup_file is not None: os.replace(backup_file, dest_file) elif existed_before: dest_file.unlink(missing_ok=True) - else: + elif dest_dir.exists(): import shutil - shutil.rmtree(dest_dir, ignore_errors=True) + shutil.rmtree(dest_dir) def _safe_discard_staged_workflow_file(staged_file: Path, dest_dir: Path, existed_before: bool) -> None: @@ -1225,7 +1231,18 @@ def _validate_and_install_local( raise except Exception as exc: if tmp_path is not None: - tmp_path.unlink(missing_ok=True) + # A cleanup failure here must never replace/mask the + # original download error below with a raw, unhandled + # OSError -- warn about it and keep going, exactly like the + # later post-install finally cleanup does. + try: + tmp_path.unlink(missing_ok=True) + except OSError as cleanup_exc: + console.print( + "[yellow]Warning:[/yellow] Could not remove temporary " + f"download file {_escape_markup(str(tmp_path))}: " + f"{_escape_markup(str(cleanup_exc))}" + ) console.print(f"[red]Error:[/red] Failed to download workflow: {_escape_markup(str(exc))}") raise typer.Exit(1) try: diff --git a/src/specify_cli/workflows/engine.py b/src/specify_cli/workflows/engine.py index da3e2b483a..969d2b664d 100644 --- a/src/specify_cli/workflows/engine.py +++ b/src/specify_cli/workflows/engine.py @@ -568,12 +568,25 @@ def load(cls, run_id: str, project_root: Path) -> RunState: with open(state_path, encoding="utf-8") as f: state_data = json.load(f) + installed_workflow_id = state_data.get("installed_workflow_id") + if installed_workflow_id is not None and not isinstance(installed_workflow_id, str): + raise ValueError( + "Invalid run state: 'installed_workflow_id' must be a " + f"string or null, got {type(installed_workflow_id).__name__}" + ) + installed_registry_root = state_data.get("installed_registry_root") + if installed_registry_root is not None and not isinstance(installed_registry_root, str): + raise ValueError( + "Invalid run state: 'installed_registry_root' must be a " + f"string or null, got {type(installed_registry_root).__name__}" + ) + state = cls( run_id=state_data["run_id"], workflow_id=state_data["workflow_id"], project_root=project_root, - installed_workflow_id=state_data.get("installed_workflow_id"), - installed_registry_root=state_data.get("installed_registry_root"), + installed_workflow_id=installed_workflow_id, + installed_registry_root=installed_registry_root, ) state.status = RunStatus(state_data["status"]) state.current_step_index = state_data.get("current_step_index", 0) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 6dd2582524..6283bd2b71 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7784,6 +7784,58 @@ def test_add_from_url_oversized_content_length_leaves_no_temp_file( leaked = list(scratch_tmp.glob("*.yml")) assert leaked == [], f"leaked temp files: {leaked}" + def test_add_from_url_download_failure_cleanup_error_preserves_original_error( + self, project_dir, monkeypatch, tmp_path + ): + """The --from download-failure branch's `tmp_path.unlink(missing_ok= + True)` can itself raise (e.g. read-only tempdir) before the clean + "Failed to download workflow" message is ever printed, replacing it + with a raw unhandled OSError. A cleanup failure there must be + guarded exactly like the later post-install finally cleanup: warn + about the cleanup failure, then still preserve/report the original + download error via a clean typer.Exit, never a raw traceback.""" + import tempfile as tempfile_mod + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands as wf_commands + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + monkeypatch.setattr(wf_commands, "_MAX_WORKFLOW_YAML_BYTES", 100) + scratch_tmp = tmp_path / "scratch-tmp" + scratch_tmp.mkdir() + monkeypatch.setattr(tempfile_mod, "tempdir", str(scratch_tmp)) + oversized_body = b"x" * 500 # no Content-Length header at all + runner = CliRunner() + + real_unlink = Path.unlink + + def unlink_boom(self_path, *args, **kwargs): + if self_path.suffix == ".yml" and self_path.parent == scratch_tmp: + raise OSError("cleanup denied") + return real_unlink(self_path, *args, **kwargs) + + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse( + oversized_body, url + ), + ), pytest.MonkeyPatch.context() as mp: + mp.setattr(Path, "unlink", unlink_boom) + result = runner.invoke( + app, + ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + ) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + # Original download error remains present. + assert "exceedsthe100-byteworkflowsizelimit" in "".join(result.output.split()) + # Cleanup failure is reported too, not silently swallowed / crashing. + assert "cleanup denied" in result.output + assert "Warning" in result.output + assert not WorkflowRegistry(project_dir).is_installed("align-wf") def test_add_from_url_installs(self, project_dir, monkeypatch): from unittest.mock import patch @@ -8245,6 +8297,90 @@ def boom(*args, **kwargs): assert installed_yaml.read_bytes() == original_bytes assert WorkflowRegistry(project_dir).get("align-wf") == original_registry_entry + def test_add_fresh_install_staged_discard_cleanup_failure_reports_warning( + self, project_dir, monkeypatch + ): + """_discard_staged_workflow_file's fresh-install rmtree(dest_dir) was + previously called with ignore_errors=True, so a genuine cleanup + failure there could never reach _safe_discard_staged_workflow_file's + warning -- an orphaned directory was left behind with no report at + all. The cleanup failure must propagate so the safe wrapper can + warn, while the original copy2 install error remains the primary, + already-printed failure.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + src = self._write_workflow_dir(project_dir) + + def copy_boom(*args, **kwargs): + raise OSError("disk full") + + def rmtree_boom(path, ignore_errors=False, onerror=None, **kwargs): + # Mirrors real shutil.rmtree's ignore_errors contract: a caller + # that still passes ignore_errors=True must have any failure + # here swallowed exactly like the buggy prior behavior, so this + # fake only serves as a true red/green discriminator once the + # production code drops that flag. + if ignore_errors: + return + raise OSError("cleanup denied") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr("shutil.copy2", copy_boom) + mp.setattr("shutil.rmtree", rmtree_boom) + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + # Original install error remains present and primary. + assert "disk full" in result.output + # Cleanup failure is now reported, not silently swallowed. + assert "cleanup denied" in result.output + assert "Warning" in result.output + assert not WorkflowRegistry(project_dir).is_installed("align-wf") + + def test_add_fresh_install_registry_rollback_cleanup_failure_reports_warning( + self, project_dir, monkeypatch + ): + """_rollback_committed_workflow_file's fresh-install rmtree(dest_dir) + after a registry.add() failure had the same ignore_errors=True gap: + a genuine directory-removal failure there was silently swallowed, + leaving an orphan with no warning. The cleanup failure must + propagate so the safe wrapper can warn, while the original + registry-update error remains the primary, already-printed + failure.""" + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + src = self._write_workflow_dir(project_dir) + + def save_boom(self): + raise OSError("registry disk full") + + def rmtree_boom(path, ignore_errors=False, onerror=None, **kwargs): + if ignore_errors: + return + raise OSError("cleanup denied") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr(WorkflowRegistry, "save", save_boom) + mp.setattr("shutil.rmtree", rmtree_boom) + result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + # Original registry-update error remains present and primary. + assert "registry disk full" in result.output + # Cleanup failure is now reported, not silently swallowed. + assert "cleanup denied" in result.output + assert "Warning" in result.output + def test_add_dev_reinstall_copy_failure_leaves_prior_file_untouched( self, project_dir, monkeypatch ): @@ -10188,6 +10324,83 @@ def test_resume_falls_back_to_current_project_when_cross_project_root_vanishes( assert result.exit_code != 0 assert "disabled" in result.output + @pytest.mark.parametrize( + "field, bad_value", + [ + ("installed_workflow_id", 123), + ("installed_workflow_id", ["gated-wf"]), + ("installed_workflow_id", {"id": "gated-wf"}), + ("installed_workflow_id", True), + ("installed_registry_root", 123), + ("installed_registry_root", ["."]), + ("installed_registry_root", {"root": "."}), + ("installed_registry_root", False), + ], + ) + def test_resume_rejects_malformed_run_state_origin_fields( + self, project_dir, monkeypatch, field, bad_value + ): + """RunState.load() trusts installed_workflow_id/installed_registry_root + straight out of the JSON state file with no type validation. A + malformed value (int/list/dict/bool instead of str-or-null) -- from + disk corruption or an externally-crafted state.json -- would + otherwise crash deep inside `_resolve_run_owner_root`/registry + lookups (TypeError constructing a Path, unhashable dict/list key) + rather than failing cleanly. `load()` must validate each field as + `str | None` and raise a clear ValueError, which `workflow resume`'s + existing ValueError boundary already converts into a clean CLI + error with no traceback.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + run_id = self._install_and_run_gated(runner, app, project_dir) + + state_path = ( + project_dir / ".specify" / "workflows" / "runs" / run_id / "state.json" + ) + data = json.loads(state_path.read_text(encoding="utf-8")) + data[field] = bad_value + state_path.write_text(json.dumps(data), encoding="utf-8") + + result = runner.invoke(app, ["workflow", "resume", run_id]) + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "Error" in result.output + + @pytest.mark.parametrize( + "installed_workflow_id, installed_registry_root", + [ + (None, None), + ("gated-wf", None), + ("", ""), + ], + ) + def test_resume_accepts_valid_run_state_origin_fields( + self, project_dir, monkeypatch, installed_workflow_id, installed_registry_root + ): + """Valid str-or-null values (including the empty-string fallback + policy `_resolve_run_owner_root` already treats as "no root + stored") must continue to load and resume without error.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + run_id = self._install_and_run_gated(runner, app, project_dir) + + state_path = ( + project_dir / ".specify" / "workflows" / "runs" / run_id / "state.json" + ) + data = json.loads(state_path.read_text(encoding="utf-8")) + data["installed_workflow_id"] = installed_workflow_id + data["installed_registry_root"] = installed_registry_root + state_path.write_text(json.dumps(data), encoding="utf-8") + + result = runner.invoke(app, ["workflow", "resume", run_id, "--json"]) + assert result.exit_code == 0, result.output + def test_disable_shows_marker_in_list(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app From 8e69966818465e4ccc8a8b6634db2b93e7150f97 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 16:08:23 +0200 Subject: [PATCH 44/59] Add ValueError boundary to workflow status single-run lookup QUALITY re-review flagged that fa410d3's new RunState.load() type validation (malformed installed_workflow_id/installed_registry_root raising ValueError) leaked as a raw unhandled traceback through `workflow status `, which only caught FileNotFoundError. `workflow resume` already had the matching ValueError boundary. Adds an `except ValueError as exc: console.print(f"[red]Error:[/red] {exc}"); raise typer.Exit(1)` clause mirroring resume's exact pattern (unescaped interpolation, consistent with the existing convention at every other ValueError boundary in this file). FileNotFoundError behavior and the no-run-id list-all-runs path are unchanged. Added parametrized regression covering malformed installed_workflow_id/ installed_registry_root (int/list) via `workflow status`, plus regressions locking in the unaffected FileNotFoundError and no-run-id list-path behaviors. Assisted-by: GitHub Copilot (model: Claude Sonnet 5, autonomous) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 3 ++ tests/test_workflows.py | 63 ++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 2e57d9be12..a2a5d5e3ba 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -911,6 +911,9 @@ def workflow_status( except FileNotFoundError: console.print(f"[red]Error:[/red] Run not found: {run_id}") raise typer.Exit(1) + except ValueError as exc: + console.print(f"[red]Error:[/red] {exc}") + raise typer.Exit(1) if json_output: # Build on the shared run/resume payload so the common fields diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 6283bd2b71..6b4c15e6e9 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -10401,6 +10401,69 @@ def test_resume_accepts_valid_run_state_origin_fields( result = runner.invoke(app, ["workflow", "resume", run_id, "--json"]) assert result.exit_code == 0, result.output + @pytest.mark.parametrize( + "field, bad_value", + [ + ("installed_workflow_id", 123), + ("installed_workflow_id", ["gated-wf"]), + ("installed_registry_root", 123), + ("installed_registry_root", ["."]), + ], + ) + def test_status_rejects_malformed_run_state_origin_fields( + self, project_dir, monkeypatch, field, bad_value + ): + """`workflow status ` calls RunState.load() same as resume, + but only caught FileNotFoundError -- the new type validation there + (int/list instead of str-or-null) raises ValueError, which leaked + as a raw unhandled traceback instead of `workflow resume`'s clean + `[red]Error:[/red] {exc}` + exit 1. Must get the identical clean + boundary, leaving the no-run-id list path (and FileNotFoundError + behavior) unchanged.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + run_id = self._install_and_run_gated(runner, app, project_dir) + + state_path = ( + project_dir / ".specify" / "workflows" / "runs" / run_id / "state.json" + ) + data = json.loads(state_path.read_text(encoding="utf-8")) + data[field] = bad_value + state_path.write_text(json.dumps(data), encoding="utf-8") + + result = runner.invoke(app, ["workflow", "status", run_id]) + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "Error" in result.output + + def test_status_run_not_found_unchanged(self, project_dir, monkeypatch): + """FileNotFoundError behavior for a nonexistent run_id must remain + exactly as before this fix.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + (project_dir / ".specify" / "workflows").mkdir(parents=True, exist_ok=True) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "status", "nonexistent-run"]) + assert result.exit_code != 0 + assert "Run not found: nonexistent-run" in result.output + + def test_status_no_run_id_list_path_unaffected(self, project_dir, monkeypatch): + """The no-run-id list-all-runs path must remain unaffected by the + new single-run ValueError boundary.""" + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + (project_dir / ".specify" / "workflows").mkdir(parents=True, exist_ok=True) + runner = CliRunner() + result = runner.invoke(app, ["workflow", "status"]) + assert result.exit_code == 0, result.output + def test_disable_shows_marker_in_list(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app From a837fb55d17c1d44998fd4ae7f0e87e82d753ce0 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:07:17 +0200 Subject: [PATCH 45/59] fix: bound workflow step downloads Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 2 +- tests/test_workflows.py | 80 +++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index a2a5d5e3ba..cc24cd0c54 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -2200,7 +2200,7 @@ def _safe_fetch(url: str) -> bytes: raise ValueError(f"Redirect to non-HTTPS URL: {final_url}") if not final_parsed.hostname: raise ValueError(f"Redirect to URL with no hostname: {final_url}") - return resp.read() + return _read_response_within_limit(resp) _validate_step_id_or_exit(step_id) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 6b4c15e6e9..ca5f6b9be4 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -6351,6 +6351,71 @@ def _fake_get_step_info(self, step_id): assert result.exit_code != 0 assert "Refusing to use symlinked step directory" in result.output + def test_add_rejects_oversized_step_response(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands as wf_commands + from specify_cli.workflows.catalog import StepCatalog + from specify_cli.authentication import http as auth_http + + monkeypatch.chdir(project_dir) + monkeypatch.setattr(wf_commands, "_MAX_WORKFLOW_YAML_BYTES", 100) + monkeypatch.setattr( + StepCatalog, + "get_step_info", + lambda self, step_id: { + "id": step_id, + "name": "Test Step", + "url": "https://example.com/step.yml", + "init_url": "https://example.com/__init__.py", + "_install_allowed": True, + }, + ) + + class _FakeResponse: + def __init__(self, url): + self.url = url + self.body = b"x" * 500 + self.offset = 0 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def getheader(self, name): + return None + + def geturl(self): + return self.url + + def read(self, size=-1): + if size < 0: + size = len(self.body) - self.offset + chunk = self.body[self.offset : self.offset + size] + self.offset += len(chunk) + return chunk + + monkeypatch.setattr( + auth_http, + "open_url", + lambda url, timeout=30, redirect_validator=None: _FakeResponse(url), + ) + + result = CliRunner().invoke( + app, ["workflow", "step", "add", "my-step"] + ) + + assert result.exit_code != 0 + assert ( + "responseexceedsthe100-byteworkflowsizelimit" + in "".join(result.output.split()) + ) + assert not ( + project_dir / ".specify" / "workflows" / "steps" / "my-step" + ).exists() + def test_add_rejects_non_string_extra_files_key(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app @@ -6381,7 +6446,10 @@ def __enter__(self): def __exit__(self, exc_type, exc, tb): return False - def read(self): + def read(self, size=-1): + if getattr(self, "_read", False): + return b"" + self._read = True if self.url.endswith("/step.yml"): return b"step:\n type_key: my-step\n" return b"" @@ -6440,7 +6508,10 @@ def __enter__(self): def __exit__(self, exc_type, exc, tb): return False - def read(self): + def read(self, size=-1): + if getattr(self, "_read", False): + return b"" + self._read = True if self.url.endswith("/step.yml"): return b"step:\n type_key: my-step\n" return b"" @@ -6488,7 +6559,10 @@ def __enter__(self): def __exit__(self, exc_type, exc, tb): return False - def read(self): + def read(self, size=-1): + if getattr(self, "_read", False): + return b"" + self._read = True if self.url.endswith("/step.yml"): return b"step:\n type_key: my-step\n" return b"" From db67e962e084277d395f52a0d168c41b34d576cd Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:18:11 +0200 Subject: [PATCH 46/59] fix: preserve workflow reinstall state Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 12 ++- tests/test_workflows.py | 112 +++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index cc24cd0c54..726e430acb 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -432,6 +432,7 @@ def _commit_workflow_file(staged_file: Path, dest_file: Path, existed_before: bo the second rename fails after the first succeeded, the prior file is put back immediately so dest_file is never left simply missing.""" if existed_before and dest_file.exists(): + staged_file.chmod(dest_file.stat(follow_symlinks=False).st_mode & 0o7777) backup_file = dest_file.with_name(dest_file.name + ".bak") os.replace(dest_file, backup_file) try: @@ -845,6 +846,9 @@ def workflow_resume( except ValueError as exc: err.print(f"[red]Error:[/red] {exc}") raise typer.Exit(1) + except OSError as exc: + err.print(f"[red]Resume failed:[/red] {exc}") + raise typer.Exit(1) if pre_state.installed_workflow_id is not None: owner_root = _resolve_run_owner_root( @@ -1118,12 +1122,16 @@ def _validate_and_install_local( ) raise typer.Exit(1) try: - registry.add(definition.id, { + entry = { "name": definition.name, "version": definition.version, "description": definition.description, "source": source_label, - }) + } + existing = registry.get(definition.id) + if isinstance(existing, dict) and not existing.get("enabled", True): + entry["enabled"] = False + registry.add(definition.id, entry) except OSError as exc: _safe_rollback_committed_workflow_file(dest_file, dest_dir, existed_before, backup_file) console.print( diff --git a/tests/test_workflows.py b/tests/test_workflows.py index ca5f6b9be4..4169d1a9eb 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7731,6 +7731,45 @@ def __enter__(self): def __exit__(self, *a): return False + @pytest.mark.parametrize("mode", ["dev", "local", "from"]) + def test_reinstall_preserves_disabled_state( + self, project_dir, monkeypatch, mode + ): + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + src = self._install_dev(runner, app, project_dir) + result = runner.invoke(app, ["workflow", "disable", "align-wf"]) + assert result.exit_code == 0, result.output + + if mode == "dev": + result = runner.invoke( + app, ["workflow", "add", str(src), "--dev"] + ) + elif mode == "local": + result = runner.invoke(app, ["workflow", "add", str(src)]) + else: + data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse(data, url), + ): + result = runner.invoke( + app, + [ + "workflow", "add", "align-wf", + "--from", "https://example.com/workflow.yml", + ], + ) + + assert result.exit_code == 0, result.output + assert WorkflowRegistry(project_dir).get("align-wf")["enabled"] is False + def test_add_from_url_rejects_oversized_content_length(self, project_dir, monkeypatch): """A --from download must not trust an advertised Content-Length alone by reading the whole body first -- it must reject a response @@ -9573,6 +9612,58 @@ def test_update_preserves_disabled_state(self, project_dir, monkeypatch): assert meta["version"] == "2.0.0" assert meta["enabled"] is False + @pytest.mark.skipif(os.name == "nt", reason="POSIX permission bits") + def test_update_preserves_workflow_file_mode(self, project_dir, monkeypatch): + import stat + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + WorkflowRegistry(project_dir).add("align-wf", { + "name": "Align Workflow", + "version": "1.0.0", + "description": "", + "source": "catalog", + "url": "https://example.com/workflow.yml", + }) + workflow_file = ( + project_dir + / ".specify" + / "workflows" + / "align-wf" + / "workflow.yml" + ) + workflow_file.parent.mkdir(parents=True) + workflow_file.write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + workflow_file.chmod(0o640) + + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + }, + ) + data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse(data, url), + ): + result = CliRunner().invoke( + app, ["workflow", "update"], input="y\n" + ) + + assert result.exit_code == 0, result.output + assert stat.S_IMODE(workflow_file.stat().st_mode) == 0o640 + def test_update_skips_corrupted_registry_entry(self, project_dir, monkeypatch): import json from typer.testing import CliRunner @@ -10215,6 +10306,27 @@ def test_resume_blocks_when_installed_workflow_disabled( resumed = json.loads(result.stdout) assert resumed["run_id"] == run_id + def test_resume_preload_io_error_is_reported_cleanly( + self, project_dir, monkeypatch + ): + from unittest.mock import patch + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.engine import RunState + + monkeypatch.chdir(project_dir) + with patch.object( + RunState, "load", side_effect=OSError("permission denied") + ): + result = CliRunner().invoke( + app, ["workflow", "resume", "unreadable-run"] + ) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "Resume failed" in result.output + assert "permission denied" in result.output + def test_resume_backward_compatible_with_run_state_missing_new_fields( self, project_dir, monkeypatch ): From a874b971b6211799f4862920d9077ef13cb2382d Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:32:34 +0200 Subject: [PATCH 47/59] fix: fail closed on workflow registry state Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 56 +++++++++++++---------- tests/test_workflows.py | 61 ++++++++++++++++++-------- 2 files changed, 76 insertions(+), 41 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 726e430acb..ab95a41c66 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -77,6 +77,25 @@ def _open_workflow_registry(project_root: Path, out=None): raise typer.Exit(1) +def _require_enabled_workflow( + registry_root: Path, workflow_id: str, out: Any +) -> None: + """Fail closed for corrupted or explicitly disabled registry entries.""" + metadata = _open_workflow_registry(registry_root, out).get(workflow_id) + if metadata is not None and not isinstance(metadata, dict): + out.print( + f"[red]Error:[/red] Registry entry for " + f"'{_escape_markup(workflow_id)}' is corrupted" + ) + raise typer.Exit(1) + if isinstance(metadata, dict) and not metadata.get("enabled", True): + out.print( + f"[red]Error:[/red] Workflow '{_escape_markup(workflow_id)}' is disabled. " + f"Enable with: specify workflow enable {_escape_markup(workflow_id)}" + ) + raise typer.Exit(1) + + def _resolve_run_owner_root( installed_registry_root: str | None, project_root: Path ) -> Path: @@ -91,16 +110,17 @@ def _resolve_run_owner_root( falling back to the *current* ``project_root`` instead of a stale absolute path baked in at run start. - A persisted cross-project root that itself no longer exists (that - other project moved/was deleted) is not trusted either: silently - skipping the disabled check because a stored path merely happens not - to resolve would defeat the guard's purpose, so this also falls back - to ``project_root`` rather than risk that. + A persisted cross-project root that no longer exists cannot be safely + rediscovered and must fail closed instead of consulting the unrelated + project that happens to store the run state. """ if installed_registry_root: candidate = Path(installed_registry_root) if candidate.is_dir(): return candidate + raise ValueError( + "Installed workflow owner is unavailable; cannot safely resume" + ) return project_root @@ -726,13 +746,7 @@ def workflow_run( registered_id = owner_id if registered_id is not None: - installed_meta = _open_workflow_registry(registry_root, err).get(registered_id) - if isinstance(installed_meta, dict) and not installed_meta.get("enabled", True): - err.print( - f"[red]Error:[/red] Workflow '{_escape_markup(registered_id)}' is disabled. " - f"Enable with: specify workflow enable {_escape_markup(registered_id)}" - ) - raise typer.Exit(1) + _require_enabled_workflow(registry_root, registered_id, err) try: definition = engine.load_workflow(source_path if is_file_source else source) @@ -851,18 +865,16 @@ def workflow_resume( raise typer.Exit(1) if pre_state.installed_workflow_id is not None: - owner_root = _resolve_run_owner_root( - pre_state.installed_registry_root, project_root - ) - installed_meta = _open_workflow_registry(owner_root, err).get( - pre_state.installed_workflow_id - ) - if isinstance(installed_meta, dict) and not installed_meta.get("enabled", True): - err.print( - f"[red]Error:[/red] Workflow '{_escape_markup(pre_state.installed_workflow_id)}' is disabled. " - f"Enable with: specify workflow enable {_escape_markup(pre_state.installed_workflow_id)}" + try: + owner_root = _resolve_run_owner_root( + pre_state.installed_registry_root, project_root ) + except ValueError as exc: + err.print(f"[red]Error:[/red] {_escape_markup(str(exc))}") raise typer.Exit(1) + _require_enabled_workflow( + owner_root, pre_state.installed_workflow_id, err + ) try: with _stdout_to_stderr_when(json_output): diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 4169d1a9eb..30bc3026b7 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -10014,6 +10014,23 @@ def test_disable_blocks_run_enable_restores(self, project_dir, monkeypatch): result = runner.invoke(app, ["workflow", "run", "align-wf"]) assert result.exit_code == 0, result.output + def test_run_rejects_corrupted_registry_entry(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + + registry = WorkflowRegistry(project_dir) + registry.data["workflows"]["align-wf"] = "corrupted" + registry.save() + + result = runner.invoke(app, ["workflow", "run", "align-wf"]) + assert result.exit_code != 0 + assert "corrupted" in result.output + def test_disable_blocks_run_via_path_equivalent_id(self, project_dir, monkeypatch): """Path spelling "align-wf/" must not run a disabled workflow by dodging the registry lookup.""" from typer.testing import CliRunner @@ -10306,6 +10323,25 @@ def test_resume_blocks_when_installed_workflow_disabled( resumed = json.loads(result.stdout) assert resumed["run_id"] == run_id + def test_resume_rejects_corrupted_registry_entry( + self, project_dir, monkeypatch + ): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + run_id = self._install_and_run_gated(runner, app, project_dir) + + registry = WorkflowRegistry(project_dir) + registry.data["workflows"]["gated-wf"] = "corrupted" + registry.save() + + result = runner.invoke(app, ["workflow", "resume", run_id]) + assert result.exit_code != 0 + assert "corrupted" in result.output + def test_resume_preload_io_error_is_reported_cleanly( self, project_dir, monkeypatch ): @@ -10459,14 +10495,12 @@ def test_resume_respects_cross_project_registry_root( assert result.exit_code != 0 assert "disabled" in result.output - def test_resume_falls_back_to_current_project_when_cross_project_root_vanishes( + def test_resume_rejects_missing_cross_project_owner_root( self, temp_dir, monkeypatch ): - """A persisted cross-project owning root that no longer exists (that - other project was itself deleted/moved away) must not be trusted as - a safety signal -- silently skipping the disabled check just - because the stored path happens not to resolve would defeat the - guard. Falls back to the current project's own registry instead.""" + """A vanished explicit cross-project owner cannot be safely + rediscovered, so resume must fail closed instead of consulting the + unrelated project that stores the run state.""" from typer.testing import CliRunner from specify_cli import app import shutil @@ -10494,21 +10528,10 @@ def test_resume_falls_back_to_current_project_when_cross_project_root_vanishes( # is now dangling. shutil.rmtree(owner_project) - # unrelated_cwd (where this run's own state.json lives) has its own - # separately-installed, disabled workflow under the same ID. - src2 = unrelated_cwd / "gated-src" - src2.mkdir() - (src2 / "workflow.yml").write_text( - self._GATED_WORKFLOW_YAML, encoding="utf-8" - ) - result = runner.invoke(app, ["workflow", "add", str(src2), "--dev"]) - assert result.exit_code == 0, result.output - result = runner.invoke(app, ["workflow", "disable", "gated-wf"]) - assert result.exit_code == 0, result.output - result = runner.invoke(app, ["workflow", "resume", run_id]) assert result.exit_code != 0 - assert "disabled" in result.output + assert "owner" in result.output.lower() + assert "unavailable" in result.output.lower() @pytest.mark.parametrize( "field, bad_value", From c098d8623ae9a05b0ca8aeb90af7c18a1e022512 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 17:45:00 +0200 Subject: [PATCH 48/59] fix: make workflow installs transactional Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 249 +++++++++++++++++++------ tests/test_workflows.py | 158 ++++++++++++++++ 2 files changed, 345 insertions(+), 62 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index ab95a41c66..35d0ef6d36 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -412,7 +412,9 @@ def _safe_workflow_id_dir(workflows_dir: Path, workflow_id: str) -> Path: return dest_dir -def _stage_workflow_file(dest_dir: Path) -> Path: +def _stage_workflow_file( + dest_dir: Path, *, use_project_file_mode: bool = False +) -> Path: """Reserve a same-directory staging file so new/updated workflow.yml content can be written and validated without ever touching (and risking truncating) an existing destination file before the final atomic swap. @@ -423,14 +425,36 @@ def _stage_workflow_file(dest_dir: Path) -> Path: again via a guarded rmdir (never a broad rmtree, so any concurrently written content is left untouched) before the original OSError is re-raised unchanged. A pre-existing dest_dir (reinstall) is never - touched by this cleanup.""" + touched by this cleanup. For catalog-created files, + ``use_project_file_mode`` recreates the reserved path exclusively with + mode 0666 so the process umask supplies the normal project-file mode.""" import tempfile created_dir = not dest_dir.exists() dest_dir.mkdir(parents=True, exist_ok=True) + fd = -1 + staged_file: Path | None = None try: fd, tmp_name = tempfile.mkstemp(dir=dest_dir, prefix=".workflow.yml.", suffix=".tmp") + staged_file = Path(tmp_name) + if use_project_file_mode: + os.close(fd) + fd = -1 + staged_file.unlink() + flags = os.O_RDWR | os.O_CREAT | os.O_EXCL + flags |= getattr(os, "O_NOFOLLOW", 0) + fd = os.open(staged_file, flags, 0o666) except OSError: + if fd >= 0: + try: + os.close(fd) + except OSError: + pass + if staged_file is not None: + try: + staged_file.unlink(missing_ok=True) + except OSError: + pass if created_dir: try: dest_dir.rmdir() @@ -441,7 +465,58 @@ def _stage_workflow_file(dest_dir: Path) -> Path: ) raise os.close(fd) - return Path(tmp_name) + assert staged_file is not None + return staged_file + + +@contextlib.contextmanager +def _workflow_install_transaction(project_root: Path): + """Serialize workflow file swaps with their registry updates.""" + from ..shared_infra import _ensure_safe_shared_directory + + lock_dir = project_root / ".specify" + try: + _ensure_safe_shared_directory( + project_root, lock_dir, context="workflow install lock directory" + ) + except ValueError as exc: + raise OSError(str(exc)) from exc + lock_file = lock_dir / ".workflow-install.lock" + if lock_file.is_symlink(): + raise OSError(f"Refusing to use symlinked workflow install lock: {lock_file}") + + flags = os.O_RDWR | os.O_CREAT + flags |= getattr(os, "O_NOFOLLOW", 0) + flags |= getattr(os, "O_CLOEXEC", 0) + fd = os.open(lock_file, flags, 0o600) + try: + if lock_file.is_symlink(): + raise OSError( + f"Refusing to use symlinked workflow install lock: {lock_file}" + ) + if os.name == "nt": + import errno + import msvcrt + import time + + if os.fstat(fd).st_size == 0: + os.write(fd, b"\0") + while True: + os.lseek(fd, 0, os.SEEK_SET) + try: + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + break + except OSError as exc: + if exc.errno not in (errno.EACCES, errno.EDEADLK): + raise + time.sleep(0.05) + else: + import fcntl + + fcntl.flock(fd, fcntl.LOCK_EX) + yield + finally: + os.close(fd) def _commit_workflow_file(staged_file: Path, dest_file: Path, existed_before: bool) -> Path | None: @@ -1120,40 +1195,62 @@ def _validate_and_install_local( ) raise typer.Exit(1) - # Commit the staged copy onto dest_file via an atomic swap. A prior - # file (reinstall) is renamed aside rather than overwritten in - # place, so it can be restored by rename (not a content rewrite) if - # registry.add() below fails. try: - backup_file = _commit_workflow_file(staged_file, dest_file, existed_before) + transaction = _workflow_install_transaction(project_root) + with transaction: + transaction_existed_before = existed_before or dest_file.exists() + transaction_registry = _open_workflow_registry(project_root) + # Commit the staged copy onto dest_file via an atomic swap. A + # prior file is renamed aside so registry failure can restore it. + try: + backup_file = _commit_workflow_file( + staged_file, dest_file, transaction_existed_before + ) + except OSError as exc: + _safe_discard_staged_workflow_file( + staged_file, dest_dir, existed_before + ) + console.print( + f"[red]Error:[/red] Failed to install workflow " + f"'{_escape_markup(definition.id)}': " + f"{_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + try: + entry = { + "name": definition.name, + "version": definition.version, + "description": definition.description, + "source": source_label, + } + existing = transaction_registry.get(definition.id) + if isinstance(existing, dict) and not existing.get( + "enabled", True + ): + entry["enabled"] = False + transaction_registry.add(definition.id, entry) + except OSError as exc: + _safe_rollback_committed_workflow_file( + dest_file, + dest_dir, + transaction_existed_before, + backup_file, + ) + console.print( + f"[red]Error:[/red] Failed to update workflow registry for " + f"'{_escape_markup(definition.id)}': " + f"{_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + # Registry update succeeded while the transaction lock is held. + _discard_committed_backup_file(backup_file) except OSError as exc: _safe_discard_staged_workflow_file(staged_file, dest_dir, existed_before) console.print( - f"[red]Error:[/red] Failed to install workflow " - f"'{_escape_markup(definition.id)}': {_escape_markup(str(exc))}" - ) - raise typer.Exit(1) - try: - entry = { - "name": definition.name, - "version": definition.version, - "description": definition.description, - "source": source_label, - } - existing = registry.get(definition.id) - if isinstance(existing, dict) and not existing.get("enabled", True): - entry["enabled"] = False - registry.add(definition.id, entry) - except OSError as exc: - _safe_rollback_committed_workflow_file(dest_file, dest_dir, existed_before, backup_file) - console.print( - f"[red]Error:[/red] Failed to update workflow registry for " + f"[red]Error:[/red] Failed to lock workflow install " f"'{_escape_markup(definition.id)}': {_escape_markup(str(exc))}" ) raise typer.Exit(1) - # registry.add() durably succeeded -- the renamed-aside backup is no - # longer needed for rollback. - _discard_committed_backup_file(backup_file) console.print( f"[green]✓[/green] Workflow '{_escape_markup(definition.name)}' " f"({_escape_markup(definition.id)}) installed" @@ -1396,7 +1493,10 @@ def _install_workflow_from_catalog( existed_before = workflow_dir.is_dir() try: - staged_file = _stage_workflow_file(workflow_dir) + staged_file = _stage_workflow_file( + workflow_dir, + use_project_file_mode=not workflow_file.exists(), + ) except OSError as exc: console.print( f"[red]Error:[/red] Failed to install workflow '{safe_wf_id}' from catalog: " @@ -1497,44 +1597,69 @@ def _install_workflow_from_catalog( ) raise typer.Exit(1) - # Commit the staged download onto workflow_file via an atomic swap. A - # prior file (reinstall) is renamed aside rather than overwritten in - # place, so it can be restored by rename (not a content rewrite) if - # registry.add() below fails. try: - backup_file = _commit_workflow_file(staged_file, workflow_file, existed_before) + transaction = _workflow_install_transaction(project_root) + with transaction: + transaction_existed_before = ( + existed_before or workflow_file.exists() + ) + transaction_registry = _open_workflow_registry(project_root) + # Commit the staged download onto workflow_file via an atomic + # swap. A prior file is renamed aside for registry rollback. + try: + backup_file = _commit_workflow_file( + staged_file, workflow_file, transaction_existed_before + ) + except OSError as exc: + _safe_discard_staged_workflow_file( + staged_file, workflow_dir, existed_before + ) + console.print( + f"[red]Error:[/red] Failed to install workflow " + f"'{safe_wf_id}' from catalog: {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + + entry = { + "name": definition.name or info.get("name", workflow_id), + "version": definition.version or info.get("version", "0.0.0"), + "description": definition.description + or info.get("description", ""), + "source": "catalog", + "catalog_name": info.get("_catalog_name", ""), + "url": workflow_url, + } + # Preserve a prior disabled state across updates/reinstalls. + existing = transaction_registry.get(workflow_id) + if isinstance(existing, dict) and not existing.get( + "enabled", True + ): + entry["enabled"] = False + try: + transaction_registry.add(workflow_id, entry) + except OSError as exc: + _safe_rollback_committed_workflow_file( + workflow_file, + workflow_dir, + transaction_existed_before, + backup_file, + ) + console.print( + f"[red]Error:[/red] Failed to update workflow registry for " + f"'{_escape_markup(workflow_id)}': " + f"{_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + # Registry update succeeded while the transaction lock is held. + _discard_committed_backup_file(backup_file) except OSError as exc: _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) console.print( - f"[red]Error:[/red] Failed to install workflow '{safe_wf_id}' from catalog: " + f"[red]Error:[/red] Failed to lock workflow install " + f"'{safe_wf_id}': " f"{_escape_markup(str(exc))}" ) raise typer.Exit(1) - - entry = { - "name": definition.name or info.get("name", workflow_id), - "version": definition.version or info.get("version", "0.0.0"), - "description": definition.description or info.get("description", ""), - "source": "catalog", - "catalog_name": info.get("_catalog_name", ""), - "url": workflow_url, - } - # Preserve a prior disabled state across updates/reinstalls. - existing = registry.get(workflow_id) - if isinstance(existing, dict) and not existing.get("enabled", True): - entry["enabled"] = False - try: - registry.add(workflow_id, entry) - except OSError as exc: - _safe_rollback_committed_workflow_file(workflow_file, workflow_dir, existed_before, backup_file) - console.print( - f"[red]Error:[/red] Failed to update workflow registry for " - f"'{_escape_markup(workflow_id)}': {_escape_markup(str(exc))}" - ) - raise typer.Exit(1) - # registry.add() durably succeeded -- the renamed-aside backup is no - # longer needed for rollback. - _discard_committed_backup_file(backup_file) console.print( f"[green]✓[/green] Workflow '{_escape_markup(str(info.get('name', workflow_id)))}' " "installed from catalog" diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 30bc3026b7..99d3846763 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8984,6 +8984,164 @@ def test_add_catalog_successful_reinstall_leaves_no_backup_file( leftovers = [p.name for p in workflow_dir.iterdir() if p.name != "workflow.yml"] assert leftovers == [], f"orphan sibling(s) left behind: {leftovers}" + @pytest.mark.skipif(os.name == "nt", reason="POSIX permission bits") + def test_add_catalog_fresh_install_uses_project_file_mode( + self, project_dir, monkeypatch + ): + import stat + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog + + monkeypatch.chdir(project_dir) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + + previous_umask = os.umask(0o022) + try: + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse(data, url), + ) + result = CliRunner().invoke( + app, ["workflow", "add", "align-wf"] + ) + finally: + os.umask(previous_umask) + + assert result.exit_code == 0, result.output + workflow_file = ( + project_dir + / ".specify" + / "workflows" + / "align-wf" + / "workflow.yml" + ) + assert stat.S_IMODE(workflow_file.stat().st_mode) == 0o644 + + def test_concurrent_catalog_reinstalls_keep_file_and_registry_aligned( + self, project_dir, monkeypatch + ): + import threading + from specify_cli.workflows import _commands + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + from specify_cli.workflows.engine import WorkflowDefinition + + workflows_dir = project_dir / ".specify" / "workflows" + workflow_file = workflows_dir / "align-wf" / "workflow.yml" + workflow_file.parent.mkdir(parents=True) + workflow_file.write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + WorkflowRegistry(project_dir).add( + "align-wf", + { + "name": "Align Workflow", + "version": "1.0.0", + "source": "catalog", + }, + ) + + versions = {"install-a": "2.0.0", "install-b": "3.0.0"} + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": versions[threading.current_thread().name], + "url": ( + "https://example.com/" + f"{versions[threading.current_thread().name]}.yml" + ), + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + monkeypatch.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse( + self.WORKFLOW_YAML.format( + version=url.rsplit("/", 1)[-1].removesuffix(".yml") + ).encode(), + url, + ), + ) + + a_committed = threading.Event() + b_committed = threading.Event() + a_saving = threading.Event() + b_saved = threading.Event() + real_commit = _commands._commit_workflow_file + real_save = WorkflowRegistry.save + + def coordinated_commit(*args, **kwargs): + backup = real_commit(*args, **kwargs) + if threading.current_thread().name == "install-a": + a_committed.set() + b_committed.wait(0.5) + else: + b_committed.set() + return backup + + def coordinated_save(registry): + if threading.current_thread().name == "install-a": + a_saving.set() + b_saved.wait(0.5) + return real_save(registry) + assert a_saving.wait(2) + real_save(registry) + b_saved.set() + + monkeypatch.setattr( + _commands, "_commit_workflow_file", coordinated_commit + ) + monkeypatch.setattr(WorkflowRegistry, "save", coordinated_save) + + errors = [] + + def install(): + try: + _commands._install_workflow_from_catalog( + project_dir, + WorkflowRegistry(project_dir), + workflows_dir, + "align-wf", + ) + except BaseException as exc: + errors.append(exc) + + first = threading.Thread(target=install, name="install-a") + second = threading.Thread(target=install, name="install-b") + first.start() + assert a_committed.wait(2) + second.start() + first.join(5) + second.join(5) + + assert not first.is_alive() + assert not second.is_alive() + assert errors == [] + file_version = WorkflowDefinition.from_yaml(workflow_file).version + registry_version = WorkflowRegistry(project_dir).get("align-wf")[ + "version" + ] + assert file_version == registry_version + def test_add_catalog_reinstall_restore_failure_reports_warning_and_original_error( self, project_dir, monkeypatch ): From e799957444214af00f0de40497f4bf6123e7e849 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:03:31 +0200 Subject: [PATCH 49/59] fix: close workflow transaction races Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 275 ++++++++++++++---------- tests/test_workflows.py | 276 +++++++++++++++++++++++-- 2 files changed, 420 insertions(+), 131 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 35d0ef6d36..7f4db1a7bd 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -189,13 +189,33 @@ def _scan_for_workflow_owner(parts: tuple[str, ...]) -> tuple[int, str] | None: return None +def _expand_first_symlink_target(path: Path) -> Path | None: + """Expand one symlink component while preserving the remaining path.""" + parts = path.parts + current = Path(path.anchor) if path.is_absolute() else Path() + start = 1 if path.is_absolute() else 0 + for index in range(start, len(parts)): + current = current / parts[index] + if not current.is_symlink(): + continue + try: + target = Path(os.readlink(current)) + except OSError: + return None + if not target.is_absolute(): + target = current.parent / target + expanded = target.joinpath(*parts[index + 1 :]) + return Path(os.path.normpath(str(expanded.absolute()))) + return None + + def _resolve_installed_workflow_ownership( source_path: Path, err ) -> tuple[Path | None, str | None]: """Map a direct ``workflow.yml`` *source_path* back to the installed workflow (``registry_root``, ``registered_id``) it belongs to, if any. - A path can point at installed storage two ways, both of which must + A path can point at installed storage three ways, all of which must receive the same registry disabled-check: 1. Lexically: the path's own (symlink-preserving) segments literally @@ -203,25 +223,25 @@ def _resolve_installed_workflow_ownership( never resolving symlinks, so a symlinked ``workflow.yml`` leaf (or symlinked ```` directory) inside the owned tree is caught by the inward-symlink refusal below rather than silently followed. - 2. Via an outward-pointing alias: *source_path* (or one of its parent - directories) is itself a symlink whose *resolved* target lands - inside some project's ``.specify/workflows//workflow.yml``, even - though the raw path used to invoke the command has no such segments - at all (e.g. ``/tmp/alias.yml`` -> a disabled installed workflow's - real file). Only the lexical case also runs the inward-symlink - refusal: the resolved case's segments are real by construction (an - already-fully-resolved path cannot itself contain a symlink - component), so that check would be a vacuous no-op there. + 2. Via an intermediate alias target whose lexical path identifies + ``.specify/workflows/`` before a symlinked storage ancestor is + resolved away. + 3. Via an outward-pointing alias whose fully resolved target lands + inside legitimate installed storage, even though the raw invocation + path has no ownership segments. Returns ``(None, None)`` when neither applies -- a genuinely standalone external workflow file, which is allowed to run unchecked. """ - lexical = Path(os.path.normpath(str(source_path.absolute()))) - parts = lexical.parts - match = _scan_for_workflow_owner(parts) - if match is not None: + def ownership_for(candidate: Path) -> tuple[Path, str] | None: + parts = candidate.parts + match = _scan_for_workflow_owner(parts) + if match is None: + return None i, registered_id = match - registry_root = Path(*parts[:i]) if i else Path(lexical.anchor or ".") + registry_root = ( + Path(*parts[:i]) if i else Path(candidate.anchor or ".") + ) # The path-derived registry_root here may differ from the cwd's # project_root already checked by _reject_unsafe_workflow_storage # (e.g. this path points into another project entirely, or this @@ -252,25 +272,37 @@ def _resolve_installed_workflow_ownership( raise typer.Exit(1) return registry_root, registered_id - # No lexical owner segments. source_path (or a parent directory) might - # still be a symlink whose resolved target lands inside some project's - # installed workflow storage under a completely unrelated-looking path - # -- check that too so an alias into a disabled workflow can't dodge - # the registry guard. + lexical = Path(os.path.normpath(str(source_path.absolute()))) + ownership = ownership_for(lexical) + if ownership is not None: + return ownership + + # Inspect each intermediate symlink target before fully resolving it. + # Full resolution can erase .specify/workflows ownership segments when + # one of those storage directories is itself a symlink. + candidate = lexical + seen = {candidate} + for _ in range(40): + expanded = _expand_first_symlink_target(candidate) + if expanded is None or expanded in seen: + break + ownership = ownership_for(expanded) + if ownership is not None: + return ownership + seen.add(expanded) + candidate = expanded + + # A fully resolved target may still land in legitimate installed + # storage through an unrelated-looking alias. try: resolved = source_path.resolve(strict=False) - except OSError: + except (OSError, RuntimeError): return None, None if resolved == lexical: # Nothing on this path is a symlink; already covered above. return None, None - resolved_parts = resolved.parts - resolved_match = _scan_for_workflow_owner(resolved_parts) - if resolved_match is None: - return None, None - i, registered_id = resolved_match - registry_root = Path(*resolved_parts[:i]) if i else Path(resolved.anchor or ".") - return registry_root, registered_id + ownership = ownership_for(resolved) + return ownership if ownership is not None else (None, None) _WORKFLOW_ID_PATTERN = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") @@ -549,8 +581,15 @@ def _discard_staged_workflow_file(staged_file: Path, dest_dir: Path, existed_bef already absent is not itself an error.""" staged_file.unlink(missing_ok=True) if not existed_before and dest_dir.exists(): - import shutil - shutil.rmtree(dest_dir) + import errno + + try: + dest_dir.rmdir() + except OSError as exc: + # Another concurrent install may already have committed content + # into this once-fresh directory. Never recursively delete it. + if exc.errno not in (errno.ENOTEMPTY, errno.EEXIST): + raise def _rollback_committed_workflow_file( @@ -1666,24 +1705,19 @@ def _install_workflow_from_catalog( ) -@workflow_app.command("remove") -def workflow_remove( - workflow_id: str = typer.Argument(..., help="Workflow ID to uninstall"), -): - """Uninstall a workflow.""" - project_root = _require_specify_project() - workflows_dir = project_root / ".specify" / "workflows" - _validate_workflow_id_or_exit(workflow_id) - +def _remove_workflow_locked( + project_root: Path, workflows_dir: Path, workflow_id: str +) -> Path | None: + """Stage a workflow directory and persist removal while locked.""" registry = _open_workflow_registry(project_root) - + safe_id = _escape_markup(workflow_id) if not registry.is_installed(workflow_id): - console.print(f"[red]Error:[/red] Workflow '{workflow_id}' is not installed") + console.print( + f"[red]Error:[/red] Workflow '{safe_id}' is not installed" + ) raise typer.Exit(1) - # Remove workflow files workflow_dir_unresolved = workflows_dir / workflow_id - safe_id = _escape_markup(workflow_id) if workflow_dir_unresolved.is_symlink(): console.print( f"[red]Error:[/red] Refusing to remove symlinked " @@ -1696,34 +1730,33 @@ def workflow_remove( rel_parts = workflow_dir.relative_to(workflows_dir.resolve()).parts except ValueError: console.print( - f"[red]Error:[/red] Invalid workflow ID: {_escape_markup(repr(workflow_id))}" + f"[red]Error:[/red] Invalid workflow ID: " + f"{_escape_markup(repr(workflow_id))}" ) raise typer.Exit(1) if rel_parts != (workflow_id,): console.print( - f"[red]Error:[/red] Invalid workflow ID: {_escape_markup(repr(workflow_id))}" + f"[red]Error:[/red] Invalid workflow ID: " + f"{_escape_markup(repr(workflow_id))}" ) raise typer.Exit(1) if workflow_dir.exists() and not workflow_dir.is_dir(): console.print( - f"[red]Error:[/red] .specify/workflows/{safe_id} exists but is not a directory" + f"[red]Error:[/red] .specify/workflows/{safe_id} exists " + "but is not a directory" ) raise typer.Exit(1) - import shutil import tempfile - # Stage the directory out of the way via an atomic rename *before* the - # registry write, so a mid-delete rmtree failure can never leave a - # partially-deleted directory that gets re-marked "installed". A rename - # is a metadata-only operation (unlike rmtree), so it either fully - # succeeds or leaves the original directory completely untouched. staged_dir: Path | None = None if workflow_dir.exists(): try: reserved = Path( - tempfile.mkdtemp(prefix=f".{workflow_id}.removing-", dir=workflows_dir) + tempfile.mkdtemp( + prefix=f".{workflow_id}.removing-", dir=workflows_dir + ) ) reserved.rmdir() os.rename(workflow_dir, reserved) @@ -1731,15 +1764,11 @@ def workflow_remove( except OSError as exc: console.print( f"[red]Error:[/red] Failed to stage workflow directory " - f"{_escape_markup(str(workflow_dir))} for removal: {_escape_markup(str(exc))}" + f"{_escape_markup(str(workflow_dir))} for removal: " + f"{_escape_markup(str(exc))}" ) raise typer.Exit(1) - # Persist the registry removal only after staging succeeded: if save() - # fails, WorkflowRegistry.remove() rolls back its in-memory state and - # raises, so we rename the staged directory back to its original - # location, restoring the pre-command state exactly (files + registry - # both still claim the workflow installed). try: registry.remove(workflow_id) except OSError as exc: @@ -1748,13 +1777,38 @@ def workflow_remove( os.rename(staged_dir, workflow_dir) except OSError as restore_exc: console.print( - f"[yellow]Warning:[/yellow] Failed to restore workflow directory " - f"after registry update failure; it remains staged at " - f"{_escape_markup(str(staged_dir))}: {_escape_markup(str(restore_exc))}" + f"[yellow]Warning:[/yellow] Failed to restore workflow " + "directory after registry update failure; it remains " + f"staged at {_escape_markup(str(staged_dir))}: " + f"{_escape_markup(str(restore_exc))}" ) console.print( - f"[red]Error:[/red] Failed to update workflow registry for '{safe_id}': " - f"{_escape_markup(str(exc))}" + f"[red]Error:[/red] Failed to update workflow registry for " + f"'{safe_id}': {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + return staged_dir + + +@workflow_app.command("remove") +def workflow_remove( + workflow_id: str = typer.Argument(..., help="Workflow ID to uninstall"), +): + """Uninstall a workflow.""" + project_root = _require_specify_project() + workflows_dir = project_root / ".specify" / "workflows" + _validate_workflow_id_or_exit(workflow_id) + safe_id = _escape_markup(workflow_id) + import shutil + try: + with _workflow_install_transaction(project_root): + staged_dir = _remove_workflow_locked( + project_root, workflows_dir, workflow_id + ) + except OSError as exc: + console.print( + f"[red]Error:[/red] Failed to lock workflow removal " + f"'{safe_id}': {_escape_markup(str(exc))}" ) raise typer.Exit(1) @@ -1904,36 +1958,56 @@ def workflow_update( raise typer.Exit(1) -@workflow_app.command("enable") -def workflow_enable( - workflow_id: str = typer.Argument(..., help="Workflow ID to enable"), -): - """Enable a disabled workflow.""" +def _set_workflow_enabled(workflow_id: str, enabled: bool) -> None: + """Update enabled state from a fresh registry snapshot while locked.""" project_root = _require_specify_project() - registry = _open_workflow_registry(project_root) - metadata = registry.get(workflow_id) - if metadata is None: - console.print(f"[red]Error:[/red] Workflow '{_escape_markup(workflow_id)}' is not installed") - raise typer.Exit(1) - if not isinstance(metadata, dict): - console.print( - f"[red]Error:[/red] Registry entry for '{_escape_markup(workflow_id)}' is corrupted" - ) - raise typer.Exit(1) - if metadata.get("enabled", True): - console.print(f"[yellow]Workflow '{_escape_markup(workflow_id)}' is already enabled[/yellow]") - raise typer.Exit(0) - # Fresh mapping: registry.get() returns the live entry, and mutating it - # in place would defeat WorkflowRegistry.add's rollback-on-save-failure. + safe_id = _escape_markup(workflow_id) try: - registry.add(workflow_id, {**metadata, "enabled": True}) + with _workflow_install_transaction(project_root): + registry = _open_workflow_registry(project_root) + metadata = registry.get(workflow_id) + if metadata is None: + console.print( + f"[red]Error:[/red] Workflow '{safe_id}' is not installed" + ) + raise typer.Exit(1) + if not isinstance(metadata, dict): + console.print( + f"[red]Error:[/red] Registry entry for '{safe_id}' " + "is corrupted" + ) + raise typer.Exit(1) + current = bool(metadata.get("enabled", True)) + state = "enabled" if enabled else "disabled" + if current is enabled: + console.print( + f"[yellow]Workflow '{safe_id}' is already {state}[/yellow]" + ) + raise typer.Exit(0) + try: + registry.add(workflow_id, {**metadata, "enabled": enabled}) + except OSError as exc: + console.print( + f"[red]Error:[/red] Failed to update workflow registry " + f"for '{safe_id}': {_escape_markup(str(exc))}" + ) + raise typer.Exit(1) except OSError as exc: console.print( - f"[red]Error:[/red] Failed to update workflow registry for " - f"'{_escape_markup(workflow_id)}': {_escape_markup(str(exc))}" + f"[red]Error:[/red] Failed to lock workflow registry for " + f"'{safe_id}': {_escape_markup(str(exc))}" ) raise typer.Exit(1) - console.print(f"[green]✓[/green] Workflow '{_escape_markup(workflow_id)}' enabled") + state = "enabled" if enabled else "disabled" + console.print(f"[green]✓[/green] Workflow '{safe_id}' {state}") + + +@workflow_app.command("enable") +def workflow_enable( + workflow_id: str = typer.Argument(..., help="Workflow ID to enable"), +): + """Enable a disabled workflow.""" + _set_workflow_enabled(workflow_id, True) @workflow_app.command("disable") @@ -1941,30 +2015,7 @@ def workflow_disable( workflow_id: str = typer.Argument(..., help="Workflow ID to disable"), ): """Disable a workflow without removing it.""" - project_root = _require_specify_project() - registry = _open_workflow_registry(project_root) - metadata = registry.get(workflow_id) - if metadata is None: - console.print(f"[red]Error:[/red] Workflow '{_escape_markup(workflow_id)}' is not installed") - raise typer.Exit(1) - if not isinstance(metadata, dict): - console.print( - f"[red]Error:[/red] Registry entry for '{_escape_markup(workflow_id)}' is corrupted" - ) - raise typer.Exit(1) - if not metadata.get("enabled", True): - console.print(f"[yellow]Workflow '{_escape_markup(workflow_id)}' is already disabled[/yellow]") - raise typer.Exit(0) - # Fresh mapping for the same rollback reason as workflow_enable. - try: - registry.add(workflow_id, {**metadata, "enabled": False}) - except OSError as exc: - console.print( - f"[red]Error:[/red] Failed to update workflow registry for " - f"'{_escape_markup(workflow_id)}': {_escape_markup(str(exc))}" - ) - raise typer.Exit(1) - console.print(f"[green]✓[/green] Workflow '{_escape_markup(workflow_id)}' disabled") + _set_workflow_enabled(workflow_id, False) console.print(f"To re-enable: specify workflow enable {_escape_markup(workflow_id)}") diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 99d3846763..ffc0358dce 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8413,13 +8413,8 @@ def boom(*args, **kwargs): def test_add_fresh_install_staged_discard_cleanup_failure_reports_warning( self, project_dir, monkeypatch ): - """_discard_staged_workflow_file's fresh-install rmtree(dest_dir) was - previously called with ignore_errors=True, so a genuine cleanup - failure there could never reach _safe_discard_staged_workflow_file's - warning -- an orphaned directory was left behind with no report at - all. The cleanup failure must propagate so the safe wrapper can - warn, while the original copy2 install error remains the primary, - already-printed failure.""" + """A genuine fresh-directory rmdir failure must be reported while + the original copy failure remains the primary error.""" from typer.testing import CliRunner from specify_cli import app from specify_cli.workflows.catalog import WorkflowRegistry @@ -8431,19 +8426,16 @@ def test_add_fresh_install_staged_discard_cleanup_failure_reports_warning( def copy_boom(*args, **kwargs): raise OSError("disk full") - def rmtree_boom(path, ignore_errors=False, onerror=None, **kwargs): - # Mirrors real shutil.rmtree's ignore_errors contract: a caller - # that still passes ignore_errors=True must have any failure - # here swallowed exactly like the buggy prior behavior, so this - # fake only serves as a true red/green discriminator once the - # production code drops that flag. - if ignore_errors: - return - raise OSError("cleanup denied") + real_rmdir = Path.rmdir + + def rmdir_boom(path): + if path.name == "align-wf": + raise OSError("cleanup denied") + return real_rmdir(path) with pytest.MonkeyPatch.context() as mp: mp.setattr("shutil.copy2", copy_boom) - mp.setattr("shutil.rmtree", rmtree_boom) + mp.setattr(Path, "rmdir", rmdir_boom) result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) assert result.exit_code != 0 @@ -9142,6 +9134,221 @@ def install(): ] assert file_version == registry_version + def test_precommit_discard_preserves_concurrent_install(self, project_dir): + from specify_cli.workflows import _commands + + workflow_dir = ( + project_dir / ".specify" / "workflows" / "concurrent-wf" + ) + workflow_dir.mkdir(parents=True) + staged_file = workflow_dir / ".workflow.yml.staged.tmp" + staged_file.write_text("staged", encoding="utf-8") + committed_file = workflow_dir / "workflow.yml" + committed_file.write_text("committed", encoding="utf-8") + + _commands._discard_staged_workflow_file( + staged_file, workflow_dir, existed_before=False + ) + + assert committed_file.read_text(encoding="utf-8") == "committed" + assert not staged_file.exists() + + def test_remove_serializes_with_concurrent_catalog_install( + self, project_dir, monkeypatch + ): + import threading + from specify_cli.workflows import _commands + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + from specify_cli.workflows.engine import WorkflowDefinition + + workflows_dir = project_dir / ".specify" / "workflows" + workflow_file = workflows_dir / "align-wf" / "workflow.yml" + workflow_file.parent.mkdir(parents=True) + workflow_file.write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + WorkflowRegistry(project_dir).add( + "align-wf", + { + "name": "Align Workflow", + "version": "1.0.0", + "source": "catalog", + }, + ) + monkeypatch.setattr( + _commands, "_require_specify_project", lambda: project_dir + ) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + new_data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + monkeypatch.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse(new_data, url), + ) + + removal_ready = threading.Event() + install_done = threading.Event() + real_remove = WorkflowRegistry.remove + + def coordinated_remove(registry, workflow_id): + if threading.current_thread().name == "remove": + removal_ready.set() + install_done.wait(0.5) + return real_remove(registry, workflow_id) + + monkeypatch.setattr(WorkflowRegistry, "remove", coordinated_remove) + errors = [] + + def remove(): + try: + _commands.workflow_remove("align-wf") + except BaseException as exc: + errors.append(exc) + + def install(): + try: + _commands._install_workflow_from_catalog( + project_dir, + WorkflowRegistry(project_dir), + workflows_dir, + "align-wf", + ) + except BaseException as exc: + errors.append(exc) + finally: + install_done.set() + + remove_thread = threading.Thread(target=remove, name="remove") + install_thread = threading.Thread(target=install, name="install") + remove_thread.start() + assert removal_ready.wait(2) + install_thread.start() + remove_thread.join(5) + install_thread.join(5) + + assert not remove_thread.is_alive() + assert not install_thread.is_alive() + assert errors == [] + assert WorkflowDefinition.from_yaml(workflow_file).version == "2.0.0" + metadata = WorkflowRegistry(project_dir).get("align-wf") + assert metadata["version"] == "2.0.0" + + @pytest.mark.parametrize( + ("command_name", "initial_enabled", "expected_enabled"), + [ + ("enable", False, True), + ("disable", True, False), + ], + ) + def test_toggle_serializes_with_concurrent_catalog_update( + self, + project_dir, + monkeypatch, + command_name, + initial_enabled, + expected_enabled, + ): + import threading + from specify_cli.workflows import _commands + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + from specify_cli.workflows.engine import WorkflowDefinition + + workflows_dir = project_dir / ".specify" / "workflows" + workflow_file = workflows_dir / "align-wf" / "workflow.yml" + workflow_file.parent.mkdir(parents=True) + workflow_file.write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + WorkflowRegistry(project_dir).add( + "align-wf", + { + "name": "Align Workflow", + "version": "1.0.0", + "source": "catalog", + "enabled": initial_enabled, + }, + ) + monkeypatch.setattr( + _commands, "_require_specify_project", lambda: project_dir + ) + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + new_data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + monkeypatch.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse(new_data, url), + ) + + toggle_ready = threading.Event() + update_done = threading.Event() + real_add = WorkflowRegistry.add + + def coordinated_add(registry, workflow_id, metadata): + if threading.current_thread().name == "toggle": + toggle_ready.set() + update_done.wait(0.5) + return real_add(registry, workflow_id, metadata) + + monkeypatch.setattr(WorkflowRegistry, "add", coordinated_add) + errors = [] + + def toggle(): + try: + getattr(_commands, f"workflow_{command_name}")("align-wf") + except BaseException as exc: + errors.append(exc) + + def update(): + try: + _commands._install_workflow_from_catalog( + project_dir, + WorkflowRegistry(project_dir), + workflows_dir, + "align-wf", + ) + except BaseException as exc: + errors.append(exc) + finally: + update_done.set() + + toggle_thread = threading.Thread(target=toggle, name="toggle") + update_thread = threading.Thread(target=update, name="update") + toggle_thread.start() + assert toggle_ready.wait(2) + update_thread.start() + toggle_thread.join(5) + update_thread.join(5) + + assert not toggle_thread.is_alive() + assert not update_thread.is_alive() + assert errors == [] + assert WorkflowDefinition.from_yaml(workflow_file).version == "2.0.0" + metadata = WorkflowRegistry(project_dir).get("align-wf") + assert metadata["version"] == "2.0.0" + assert metadata.get("enabled", True) is expected_enabled + def test_add_catalog_reinstall_restore_failure_reports_warning_and_original_error( self, project_dir, monkeypatch ): @@ -10257,6 +10464,37 @@ def test_disable_blocks_run_when_installed_yaml_is_symlinked( assert result.exception is None or isinstance(result.exception, SystemExit) assert "disabled" in result.output or "symlink" in result.output.lower() + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") + def test_alias_rejects_symlinked_workflow_storage_before_resolve( + self, project_dir, temp_dir + ): + import shutil + import typer + from specify_cli.workflows import _commands + + specify_dir = project_dir / ".specify" + shutil.rmtree(specify_dir) + redirected = temp_dir / "redirected-storage" + workflow_file = redirected / "workflows" / "evil" / "workflow.yml" + workflow_file.parent.mkdir(parents=True) + workflow_file.write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + specify_dir.symlink_to(redirected, target_is_directory=True) + alias = temp_dir / "workflow-alias.yml" + alias.symlink_to( + project_dir + / ".specify" + / "workflows" + / "evil" + / "workflow.yml" + ) + + with pytest.raises(typer.Exit): + _commands._resolve_installed_workflow_ownership( + alias, _commands.err_console + ) + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") def test_run_refuses_symlinked_specify_dir_hiding_disabled_workflow( self, temp_dir, monkeypatch @@ -10270,8 +10508,8 @@ def test_run_refuses_symlinked_specify_dir_hiding_disabled_workflow( unsafe .specify/.specify-workflows for the actual path-derived registry root before ever consulting the registry -- it must not rely on WorkflowRegistry's own symlinked-parent handling, which - silently returns an empty registry instead of raising and so is not - a safety signal a caller can depend on.""" + raises a generic OSError; the ownership guard should surface the + specific unsafe-storage error before registry construction.""" from typer.testing import CliRunner from specify_cli import app From d964c9754595dd891bbb22eda7f091b096d57614 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:18:34 +0200 Subject: [PATCH 50/59] fix: clean up failed workflow transactions Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/bundler/services/installer.py | 2 +- src/specify_cli/workflows/_commands.py | 10 +++ .../integration/test_bundler_install_flow.py | 24 ++++++ tests/test_workflows.py | 74 +++++++++++++++++++ 4 files changed, 109 insertions(+), 1 deletion(-) diff --git a/src/specify_cli/bundler/services/installer.py b/src/specify_cli/bundler/services/installer.py index 6882c0052f..6513abf588 100644 --- a/src/specify_cli/bundler/services/installer.py +++ b/src/specify_cli/bundler/services/installer.py @@ -199,6 +199,7 @@ def remove_bundle( result.uninstalled.append(component) else: result.skipped.append(component) + save_records(project_root, remove_record(records, bundle_id)) except Exception as exc: # noqa: BLE001 if result.uninstalled: detail = ( @@ -216,7 +217,6 @@ def remove_bundle( f"Failed to remove bundle '{bundle_id}': {exc}. {detail}" ) from exc - save_records(project_root, remove_record(records, bundle_id)) return result diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 7f4db1a7bd..9448ed323c 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -1283,6 +1283,11 @@ def _validate_and_install_local( raise typer.Exit(1) # Registry update succeeded while the transaction lock is held. _discard_committed_backup_file(backup_file) + except typer.Exit: + _safe_discard_staged_workflow_file( + staged_file, dest_dir, existed_before + ) + raise except OSError as exc: _safe_discard_staged_workflow_file(staged_file, dest_dir, existed_before) console.print( @@ -1691,6 +1696,11 @@ def _install_workflow_from_catalog( raise typer.Exit(1) # Registry update succeeded while the transaction lock is held. _discard_committed_backup_file(backup_file) + except typer.Exit: + _safe_discard_staged_workflow_file( + staged_file, workflow_dir, existed_before + ) + raise except OSError as exc: _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) console.print( diff --git a/tests/integration/test_bundler_install_flow.py b/tests/integration/test_bundler_install_flow.py index 9a4163d044..8158fab396 100644 --- a/tests/integration/test_bundler_install_flow.py +++ b/tests/integration/test_bundler_install_flow.py @@ -246,6 +246,30 @@ def boom(project_root, component): assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} +def test_remove_record_save_failure_reports_partial_state(tmp_path: Path): + make_project(tmp_path) + manifest = BundleManifest.from_dict(valid_manifest_dict()) + installer = FakeInstaller() + install_bundle(tmp_path, _plan(manifest), installer, manifest=manifest) + + def fail_save(*_args, **_kwargs): + raise OSError("disk full") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.bundler.services.installer.save_records", + fail_save, + ) + with pytest.raises(BundlerError) as exc_info: + remove_bundle(tmp_path, "demo-bundle", installer) + + message = str(exc_info.value) + assert "disk full" in message + assert "partially uninstalled" in message.lower() + assert installer.installed == set() + assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} + + def test_remove_reports_uninstalled_not_installed(tmp_path: Path): make_project(tmp_path) manifest = BundleManifest.from_dict(valid_manifest_dict()) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index ffc0358dce..d084955ca6 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -9153,6 +9153,80 @@ def test_precommit_discard_preserves_concurrent_install(self, project_dir): assert committed_file.read_text(encoding="utf-8") == "committed" assert not staged_file.exists() + def test_add_dev_registry_reopen_exit_discards_staged_file( + self, project_dir, monkeypatch + ): + import typer + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands + + monkeypatch.chdir(project_dir) + source_dir = self._write_workflow_dir(project_dir) + real_open_registry = _commands._open_workflow_registry + calls = 0 + + def fail_transaction_reopen(root): + nonlocal calls + calls += 1 + if calls == 2: + raise typer.Exit(1) + return real_open_registry(root) + + monkeypatch.setattr( + _commands, "_open_workflow_registry", fail_transaction_reopen + ) + result = CliRunner().invoke( + app, ["workflow", "add", str(source_dir), "--dev"] + ) + + assert result.exit_code != 0 + assert not ( + project_dir / ".specify" / "workflows" / "align-wf" + ).exists() + + def test_add_catalog_registry_reopen_exit_discards_staged_file( + self, project_dir, monkeypatch + ): + import typer + from specify_cli.workflows import _commands + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + workflows_dir = project_dir / ".specify" / "workflows" + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + monkeypatch.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse(data, url), + ) + monkeypatch.setattr( + _commands, + "_open_workflow_registry", + lambda _root: (_ for _ in ()).throw(typer.Exit(1)), + ) + + with pytest.raises(typer.Exit): + _commands._install_workflow_from_catalog( + project_dir, + WorkflowRegistry(project_dir), + workflows_dir, + "align-wf", + ) + + assert not (workflows_dir / "align-wf").exists() + def test_remove_serializes_with_concurrent_catalog_install( self, project_dir, monkeypatch ): From 6b968ef5d8e4bb5b19b4debcf9e52761224eabe5 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:33:05 +0200 Subject: [PATCH 51/59] fix: clean up workflow removal state Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/bundler/services/installer.py | 2 -- src/specify_cli/workflows/_commands.py | 7 +++---- tests/integration/test_bundler_install_flow.py | 6 +++--- tests/test_workflows.py | 6 +----- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/specify_cli/bundler/services/installer.py b/src/specify_cli/bundler/services/installer.py index 6513abf588..c6c1f12aed 100644 --- a/src/specify_cli/bundler/services/installer.py +++ b/src/specify_cli/bundler/services/installer.py @@ -197,8 +197,6 @@ def remove_bundle( if installer.is_installed(project_root, component): installer.remove(project_root, component) result.uninstalled.append(component) - else: - result.skipped.append(component) save_records(project_root, remove_record(records, bundle_id)) except Exception as exc: # noqa: BLE001 if result.uninstalled: diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 9448ed323c..fa4e71367c 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -1158,7 +1158,7 @@ def workflow_add( from .engine import WorkflowDefinition project_root = _require_specify_project() - registry = _open_workflow_registry(project_root) + _open_workflow_registry(project_root) workflows_dir = project_root / ".specify" / "workflows" # With --from, source names the expected workflow ID: validate it up # front so a URL/path/typo fails without a network fetch. @@ -1449,12 +1449,11 @@ def _validate_and_install_local( return # Try from catalog - _install_workflow_from_catalog(project_root, registry, workflows_dir, source) + _install_workflow_from_catalog(project_root, workflows_dir, source) def _install_workflow_from_catalog( project_root: Path, - registry: Any, workflows_dir: Path, workflow_id: str, expected_version: str | None = None, @@ -1950,7 +1949,7 @@ def workflow_update( # own backup/restore. try: _install_workflow_from_catalog( - project_root, registry, workflows_dir, update["id"], + project_root, workflows_dir, update["id"], expected_version=update["available"], ) except (typer.Exit, OSError) as exc: diff --git a/tests/integration/test_bundler_install_flow.py b/tests/integration/test_bundler_install_flow.py index 8158fab396..63b94c67c8 100644 --- a/tests/integration/test_bundler_install_flow.py +++ b/tests/integration/test_bundler_install_flow.py @@ -153,7 +153,7 @@ def remove_then_fail(project_root, component): assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} -def test_remove_bundlerror_from_installer_after_partial_removal_reports_partial_state( +def test_remove_bundler_error_from_installer_after_partial_removal_reports_partial_state( tmp_path: Path, ): """If the primitive installer itself raises BundlerError (not a raw/ @@ -189,7 +189,7 @@ def remove_then_raise_bundler_error(project_root, component): assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} -def test_remove_bundlerror_from_installer_with_zero_removed_reports_no_changes( +def test_remove_bundler_error_from_installer_with_zero_removed_reports_no_changes( tmp_path: Path, ): """When the installer raises BundlerError before anything was actually @@ -301,7 +301,7 @@ def test_remove_counts_only_components_actually_removed(tmp_path: Path): assert len(result.uninstalled) == 3 assert (gone.kind, gone.id) not in installer.remove_calls - assert gone in result.skipped + assert gone not in result.skipped make_project(tmp_path) manifest = BundleManifest.from_dict(valid_manifest_dict()) installer = FakeInstaller() diff --git a/tests/test_workflows.py b/tests/test_workflows.py index d084955ca6..3aff894fed 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -9110,7 +9110,6 @@ def install(): try: _commands._install_workflow_from_catalog( project_dir, - WorkflowRegistry(project_dir), workflows_dir, "align-wf", ) @@ -9190,7 +9189,7 @@ def test_add_catalog_registry_reopen_exit_discards_staged_file( ): import typer from specify_cli.workflows import _commands - from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + from specify_cli.workflows.catalog import WorkflowCatalog workflows_dir = project_dir / ".specify" / "workflows" monkeypatch.setattr( @@ -9220,7 +9219,6 @@ def test_add_catalog_registry_reopen_exit_discards_staged_file( with pytest.raises(typer.Exit): _commands._install_workflow_from_catalog( project_dir, - WorkflowRegistry(project_dir), workflows_dir, "align-wf", ) @@ -9294,7 +9292,6 @@ def install(): try: _commands._install_workflow_from_catalog( project_dir, - WorkflowRegistry(project_dir), workflows_dir, "align-wf", ) @@ -9398,7 +9395,6 @@ def update(): try: _commands._install_workflow_from_catalog( project_dir, - WorkflowRegistry(project_dir), workflows_dir, "align-wf", ) From c82e2144edd51d75f99866d03d7abc92e85b4a2d Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:53:05 +0200 Subject: [PATCH 52/59] fix: guard workflow update transactions Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 40 ++++- src/specify_cli/workflows/catalog.py | 2 +- tests/test_workflows.py | 225 +++++++++++++++++++++++++ 3 files changed, 258 insertions(+), 9 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index fa4e71367c..45d62d9a8f 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -1268,7 +1268,7 @@ def _validate_and_install_local( ): entry["enabled"] = False transaction_registry.add(definition.id, entry) - except OSError as exc: + except (OSError, TypeError, ValueError) as exc: _safe_rollback_committed_workflow_file( dest_file, dest_dir, @@ -1457,6 +1457,7 @@ def _install_workflow_from_catalog( workflows_dir: Path, workflow_id: str, expected_version: str | None = None, + expected_installed_version: str | None = None, ) -> None: """Download, validate, and register a catalog workflow. @@ -1464,10 +1465,22 @@ def _install_workflow_from_catalog( on any failure; the registry entry is only written on full success. ``expected_version``, when given, rejects a downloaded workflow whose version does not match the catalog version that triggered the install. + ``expected_installed_version``, when given by ``workflow update``, aborts + if another process changes the installed source or version before commit. """ from .catalog import WorkflowCatalog, WorkflowCatalogError from .engine import WorkflowDefinition + def versions_match(actual: object, expected: str) -> bool: + from packaging import version as pkg_version + + try: + return pkg_version.Version(str(actual)) == pkg_version.Version( + expected + ) + except pkg_version.InvalidVersion: + return str(actual) == expected + safe_wf_id = _escape_markup(workflow_id) catalog = WorkflowCatalog(project_root) @@ -1626,12 +1639,7 @@ def _install_workflow_from_catalog( # catalog advertised; without this check `update` would report success # while leaving the old version installed (or even downgrading). if expected_version is not None: - from packaging import version as pkg_version - try: - version_matches = pkg_version.Version(str(definition.version)) == pkg_version.Version(expected_version) - except pkg_version.InvalidVersion: - version_matches = str(definition.version) == expected_version - if not version_matches: + if not versions_match(definition.version, expected_version): _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) console.print( f"[red]Error:[/red] Downloaded workflow version ({_escape_markup(str(definition.version))}) " @@ -1647,6 +1655,21 @@ def _install_workflow_from_catalog( existed_before or workflow_file.exists() ) transaction_registry = _open_workflow_registry(project_root) + if expected_installed_version is not None: + current = transaction_registry.get(workflow_id) + if ( + not isinstance(current, dict) + or current.get("source") != "catalog" + or not versions_match( + current.get("version"), expected_installed_version + ) + ): + console.print( + f"[yellow]Warning:[/yellow] Workflow '{safe_wf_id}' " + "changed during update; rerun the command to use its " + "current source and version." + ) + raise typer.Exit(1) # Commit the staged download onto workflow_file via an atomic # swap. A prior file is renamed aside for registry rollback. try: @@ -1680,7 +1703,7 @@ def _install_workflow_from_catalog( entry["enabled"] = False try: transaction_registry.add(workflow_id, entry) - except OSError as exc: + except (OSError, TypeError, ValueError) as exc: _safe_rollback_committed_workflow_file( workflow_file, workflow_dir, @@ -1951,6 +1974,7 @@ def workflow_update( _install_workflow_from_catalog( project_root, workflows_dir, update["id"], expected_version=update["available"], + expected_installed_version=update["installed"], ) except (typer.Exit, OSError) as exc: if isinstance(exc, OSError): diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 2aa4764aed..adf55e4140 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -183,7 +183,7 @@ def add(self, workflow_id: str, metadata: dict[str, Any]) -> None: self.data["workflows"][workflow_id] = metadata try: self.save() - except OSError: + except (OSError, TypeError, ValueError): # Roll back the in-memory mutation so a later successful save # cannot persist metadata for a write that failed. if had_entry: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 3aff894fed..4f97d6116e 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8212,6 +8212,27 @@ def boom(): registry.add("other-wf", {"version": "1.0.0", "source": "catalog"}) assert registry.get("other-wf") is None + @pytest.mark.parametrize("error_type", [TypeError, ValueError]) + def test_registry_add_rolls_back_memory_on_serialization_failure( + self, project_dir, monkeypatch, error_type + ): + from specify_cli.workflows.catalog import WorkflowRegistry + + registry = WorkflowRegistry(project_dir) + registry.add("align-wf", {"version": "1.0.0", "source": "catalog"}) + + def boom(): + raise error_type("not JSON serializable") + + monkeypatch.setattr(registry, "save", boom) + with pytest.raises(error_type): + registry.add("align-wf", {"version": "2.0.0", "source": "catalog"}) + assert registry.get("align-wf")["version"] == "1.0.0" + + with pytest.raises(error_type): + registry.add("other-wf", {"version": "1.0.0", "source": "catalog"}) + assert registry.get("other-wf") is None + def test_registry_add_survives_non_dict_existing_entry(self, project_dir): from specify_cli.workflows.catalog import WorkflowRegistry @@ -8331,6 +8352,59 @@ def boom(self): assert not dest_dir.exists() assert not WorkflowRegistry(project_dir).is_installed("align-wf") + @pytest.mark.parametrize("mode", ["dev", "catalog"]) + def test_add_non_json_description_rolls_back_transaction( + self, project_dir, monkeypatch, mode + ): + import contextlib + from unittest.mock import patch + + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + data = self.WORKFLOW_YAML.format(version="1.0.0").replace( + 'description: "CLI alignment test workflow"', + "description: 2026-01-02", + ).encode() + + if mode == "dev": + source = project_dir / "dated-description" + source.mkdir() + (source / "workflow.yml").write_bytes(data) + args = ["workflow", "add", str(source), "--dev"] + download = contextlib.nullcontext() + else: + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + args = ["workflow", "add", "align-wf"] + download = patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse(data, url), + ) + + with download: + result = CliRunner().invoke(app, args) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "Failed to update workflow registry" in result.output + dest_dir = project_dir / ".specify" / "workflows" / "align-wf" + assert not dest_dir.exists() + assert not WorkflowRegistry(project_dir).is_installed("align-wf") + @pytest.mark.parametrize("mode", ["dev", "local", "from_url"]) def test_add_fresh_install_mkstemp_failure_leaves_no_orphan_directory( self, project_dir, monkeypatch, mode @@ -10333,6 +10407,157 @@ def tracking_write_bytes(self_path, data, *args, **kwargs): assert registry.is_installed("align-wf") assert registry.get("align-wf")["version"] == "1.0.0" + def test_update_non_json_description_restores_prior_file_and_registry( + self, project_dir, monkeypatch + ): + from unittest.mock import patch + + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + registry = WorkflowRegistry(project_dir) + registry.add( + "align-wf", + { + "name": "Align Workflow", + "version": "1.0.0", + "description": "", + "source": "catalog", + "url": "https://example.com/workflow.yml", + }, + ) + workflow_file = ( + project_dir + / ".specify" + / "workflows" + / "align-wf" + / "workflow.yml" + ) + workflow_file.parent.mkdir(parents=True) + original_data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + workflow_file.write_bytes(original_data) + + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + invalid_data = self.WORKFLOW_YAML.format(version="2.0.0").replace( + 'description: "CLI alignment test workflow"', + "description: 2026-01-02", + ).encode() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse(invalid_data, url), + ): + result = CliRunner().invoke( + app, ["workflow", "update", "align-wf"], input="y\n" + ) + + assert result.exit_code != 0 + assert result.exception is None or isinstance(result.exception, SystemExit) + assert "Failed to update workflow registry" in result.output + assert workflow_file.read_bytes() == original_data + current = WorkflowRegistry(project_dir).get("align-wf") + assert current["version"] == "1.0.0" + assert not workflow_file.with_name("workflow.yml.bak").exists() + + @pytest.mark.parametrize( + ("replacement_source", "replacement_version"), + [("local", "1.0.0"), ("catalog", "1.5.0")], + ) + def test_update_rechecks_registry_after_confirmation( + self, project_dir, monkeypatch, replacement_source, replacement_version + ): + from unittest.mock import patch + + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands + from specify_cli.workflows.catalog import WorkflowCatalog, WorkflowRegistry + + monkeypatch.chdir(project_dir) + registry = WorkflowRegistry(project_dir) + registry.add( + "align-wf", + { + "name": "Align Workflow", + "version": "1.0.0", + "description": "", + "source": "catalog", + "url": "https://example.com/workflow.yml", + }, + ) + workflow_file = ( + project_dir + / ".specify" + / "workflows" + / "align-wf" + / "workflow.yml" + ) + workflow_file.parent.mkdir(parents=True) + workflow_file.write_text( + self.WORKFLOW_YAML.format(version="1.0.0"), encoding="utf-8" + ) + replacement_data = self.WORKFLOW_YAML.format( + version=replacement_version + ).replace( + 'description: "CLI alignment test workflow"', + 'description: "concurrent replacement"', + ).encode() + + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "2.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + + def replace_while_confirming(*args, **kwargs): + WorkflowRegistry(project_dir).add( + "align-wf", + { + "name": "Concurrent replacement", + "version": replacement_version, + "description": "", + "source": replacement_source, + }, + ) + workflow_file.write_bytes(replacement_data) + return True + + monkeypatch.setattr(_commands.typer, "confirm", replace_while_confirming) + catalog_data = self.WORKFLOW_YAML.format(version="2.0.0").encode() + with patch( + "specify_cli.authentication.http.open_url", + side_effect=lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse(catalog_data, url), + ): + result = CliRunner().invoke(app, ["workflow", "update", "align-wf"]) + + assert result.exit_code != 0 + assert "changed during update" in result.output + assert workflow_file.read_bytes() == replacement_data + current = WorkflowRegistry(project_dir).get("align-wf") + assert current["source"] == replacement_source + assert current["version"] == replacement_version + def test_enable_disable_corrupted_registry_entry_errors(self, project_dir, monkeypatch): import json from typer.testing import CliRunner From 26551aecbce07a28273ae7dbbffb555e7748b28d Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:06:52 +0200 Subject: [PATCH 53/59] fix: harden workflow lifecycle edge cases Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/bundler/services/installer.py | 9 +- src/specify_cli/workflows/_commands.py | 36 ++++- src/specify_cli/workflows/catalog.py | 11 +- .../integration/test_bundler_install_flow.py | 28 ++++ tests/test_workflows.py | 125 +++++++++++++++++- 5 files changed, 201 insertions(+), 8 deletions(-) diff --git a/src/specify_cli/bundler/services/installer.py b/src/specify_cli/bundler/services/installer.py index c6c1f12aed..7c5abf84db 100644 --- a/src/specify_cli/bundler/services/installer.py +++ b/src/specify_cli/bundler/services/installer.py @@ -187,6 +187,7 @@ def remove_bundle( still_needed = components_still_needed(records, exclude_bundle_id=bundle_id) result = InstallResult(bundle_id=bundle_id) + remove_attempted = False try: for component in target.contributed_components: @@ -195,6 +196,7 @@ def remove_bundle( result.skipped.append(component) continue if installer.is_installed(project_root, component): + remove_attempted = True installer.remove(project_root, component) result.uninstalled.append(component) save_records(project_root, remove_record(records, bundle_id)) @@ -205,12 +207,17 @@ def remove_bundle( "before this failure; the bundle record was left unchanged, " "so the project may be partially uninstalled." ) - else: + elif remove_attempted: detail = ( "No components were removed, but the failing component may " "have made partial changes before raising, so the project " "may be partially uninstalled." ) + else: + detail = ( + "No components were removed and no removal was attempted; " + "the bundle record was left unchanged." + ) raise BundlerError( f"Failed to remove bundle '{bundle_id}': {exc}. {detail}" ) from exc diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 45d62d9a8f..f1bbcf1700 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -116,7 +116,7 @@ def _resolve_run_owner_root( """ if installed_registry_root: candidate = Path(installed_registry_root) - if candidate.is_dir(): + if not candidate.is_symlink() and candidate.is_dir(): return candidate raise ValueError( "Installed workflow owner is unavailable; cannot safely resume" @@ -564,8 +564,16 @@ def _commit_workflow_file(staged_file: Path, dest_file: Path, existed_before: bo os.replace(dest_file, backup_file) try: os.replace(staged_file, dest_file) - except OSError: - os.replace(backup_file, dest_file) + except OSError as commit_exc: + try: + os.replace(backup_file, dest_file) + except OSError as restore_exc: + raise OSError( + f"Failed to commit workflow file ({commit_exc}); failed " + f"to restore the prior workflow from {backup_file} " + f"({restore_exc}). The prior workflow remains at " + f"{backup_file}." + ) from restore_exc raise return backup_file os.replace(staged_file, dest_file) @@ -1349,6 +1357,28 @@ def _validate_and_install_local( console.print("[red]Error:[/red] Only HTTPS URLs are allowed, except HTTP for localhost.") raise typer.Exit(1) + if from_url is not None: + from rich.panel import Panel + + safe_url = _escape_markup(from_url) + console.print() + console.print( + Panel( + "[bold]You are installing a workflow from an external URL " + "that is not\nlisted in any of your configured workflow " + "catalogs.[/bold]\n\n" + f"URL: {safe_url}\n\n" + "Only install workflows from sources you trust.", + title="[bold yellow]⚠ Untrusted Source[/bold yellow]", + border_style="yellow", + padding=(1, 2), + ) + ) + console.print() + if not typer.confirm("Continue with installation?", default=False): + console.print("Cancelled") + raise typer.Exit(0) + from specify_cli._github_http import resolve_github_release_asset_api_url as _resolve_gh_asset from specify_cli.authentication.http import github_provider_hosts as _github_provider_hosts diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index adf55e4140..29237d534e 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -156,8 +156,15 @@ def save(self) -> None: # metadata preservation). try: if self.registry_path.exists(): - existing_mode = stat.S_IMODE(self.registry_path.stat().st_mode) - os.chmod(tmp, existing_mode) + existing_stat = self.registry_path.stat() + os.chmod(tmp, stat.S_IMODE(existing_stat.st_mode)) + if hasattr(os, "chown"): + try: + os.chown( + tmp, existing_stat.st_uid, existing_stat.st_gid + ) + except PermissionError: + pass except OSError: pass os.replace(tmp, self.registry_path) diff --git a/tests/integration/test_bundler_install_flow.py b/tests/integration/test_bundler_install_flow.py index 63b94c67c8..b4d91ec05b 100644 --- a/tests/integration/test_bundler_install_flow.py +++ b/tests/integration/test_bundler_install_flow.py @@ -209,6 +209,8 @@ def boom(project_root, component): message = str(exc_info.value) assert "no components were removed" in message.lower() + assert "no removal was attempted" in message.lower() + assert "partially uninstalled" not in message.lower() assert "kind manager unavailable" in message assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} @@ -270,6 +272,32 @@ def fail_save(*_args, **_kwargs): assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} +def test_remove_record_save_failure_without_remove_attempt_is_not_partial( + tmp_path: Path, +): + make_project(tmp_path) + manifest = BundleManifest.from_dict(valid_manifest_dict()) + installer = FakeInstaller() + install_bundle(tmp_path, _plan(manifest), installer, manifest=manifest) + installer.installed.clear() + + def fail_save(*_args, **_kwargs): + raise OSError("disk full") + + with pytest.MonkeyPatch.context() as mp: + mp.setattr( + "specify_cli.bundler.services.installer.save_records", + fail_save, + ) + with pytest.raises(BundlerError) as exc_info: + remove_bundle(tmp_path, "demo-bundle", installer) + + message = str(exc_info.value) + assert "no removal was attempted" in message.lower() + assert "partially uninstalled" not in message.lower() + assert {r.bundle_id for r in load_records(tmp_path)} == {"demo-bundle"} + + def test_remove_reports_uninstalled_not_installed(tmp_path: Path): make_project(tmp_path) manifest = BundleManifest.from_dict(valid_manifest_dict()) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 4f97d6116e..3bf7264c41 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7765,6 +7765,7 @@ def test_reinstall_preserves_disabled_state( "workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml", ], + input="y\n", ) assert result.exit_code == 0, result.output @@ -7793,11 +7794,41 @@ def test_add_from_url_rejects_oversized_content_length(self, project_dir, monkey result = runner.invoke( app, ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + input="y\n", ) assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) assert "exceedingthe100-byteworkflowsizelimit" in "".join(result.output.split()) + def test_add_from_url_requires_default_deny_confirmation( + self, project_dir, monkeypatch + ): + from unittest.mock import patch + + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + with patch( + "specify_cli.authentication.http.open_url", + side_effect=AssertionError("download should not start"), + ): + result = CliRunner().invoke( + app, + [ + "workflow", + "add", + "align-wf", + "--from", + "https://example.com/workflow.yml", + ], + input="n\n", + ) + + assert result.exit_code == 0, result.output + assert "Untrusted Source" in result.output + assert "Cancelled" in result.output + def test_add_from_url_rejects_oversized_streamed_body_without_content_length( self, project_dir, monkeypatch ): @@ -7823,6 +7854,7 @@ def test_add_from_url_rejects_oversized_streamed_body_without_content_length( result = runner.invoke( app, ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + input="y\n", ) assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) @@ -7858,6 +7890,7 @@ def test_add_from_url_oversized_streamed_body_leaves_no_temp_file( result = runner.invoke( app, ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + input="y\n", ) assert result.exit_code != 0 assert "exceedsthe100-byteworkflowsizelimit" in "".join(result.output.split()) @@ -7891,6 +7924,7 @@ def test_add_from_url_oversized_content_length_leaves_no_temp_file( result = runner.invoke( app, ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + input="y\n", ) assert result.exit_code != 0 assert "exceedingthe100-byteworkflowsizelimit" in "".join(result.output.split()) @@ -7939,6 +7973,7 @@ def unlink_boom(self_path, *args, **kwargs): result = runner.invoke( app, ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + input="y\n", ) assert result.exit_code != 0 @@ -7966,6 +8001,7 @@ def test_add_from_url_installs(self, project_dir, monkeypatch): result = runner.invoke( app, ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + input="y\n", ) assert result.exit_code == 0, result.output assert WorkflowRegistry(project_dir).is_installed("align-wf") @@ -8003,6 +8039,7 @@ def unlink_boom(self_path, *args, **kwargs): result = runner.invoke( app, ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + input="y\n", ) assert result.exit_code == 0, result.output @@ -8026,6 +8063,7 @@ def test_add_from_url_id_mismatch_errors(self, project_dir, monkeypatch): result = runner.invoke( app, ["workflow", "add", "other-id", "--from", "https://example.com/workflow.yml"], + input="y\n", ) assert result.exit_code != 0 assert "does not match" in result.output @@ -8060,6 +8098,7 @@ def test_add_from_url_non_https_redirect_escapes_rich_markup(self, project_dir, result = runner.invoke( app, ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + input="y\n", ) assert result.exit_code != 0 assert redirected_url in result.output @@ -8285,6 +8324,31 @@ def test_registry_save_preserves_existing_file_mode(self, project_dir): mode = stat.S_IMODE(registry.registry_path.stat().st_mode) assert mode == 0o644, f"expected 0644, got {oct(mode)}" + @pytest.mark.skipif(not hasattr(os, "chown"), reason="os.chown is unavailable") + def test_registry_save_preserves_existing_owner_group( + self, project_dir, monkeypatch + ): + from specify_cli.workflows.catalog import WorkflowRegistry + import specify_cli.workflows.catalog as catalog_mod + + registry = WorkflowRegistry(project_dir) + registry.add("first-wf", {"name": "First"}) + existing = registry.registry_path.stat() + calls: list[tuple[Path, int, int]] = [] + + monkeypatch.setattr( + catalog_mod.os, + "chown", + lambda path, uid, gid: calls.append((Path(path), uid, gid)), + ) + registry.add("second-wf", {"name": "Second"}) + + assert len(calls) == 1 + temp_path, uid, gid = calls[0] + assert temp_path.parent == registry.registry_path.parent + assert uid == existing.st_uid + assert gid == existing.st_gid + @pytest.mark.skipif(sys.platform == "win32", reason="chmod mode bits not reliable on Windows") def test_registry_save_on_new_registry_uses_secure_default_mode(self, project_dir): """A brand-new registry file (no prior mode to preserve) should keep @@ -8343,7 +8407,9 @@ def boom(self): with url_patch, pytest.MonkeyPatch.context() as mp: mp.setattr(WorkflowRegistry, "save", boom) - result = runner.invoke(app, args) + result = runner.invoke( + app, args, input="y\n" if mode == "from_url" else None + ) assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) @@ -8440,7 +8506,9 @@ def boom(*args, **kwargs): with url_patch, pytest.MonkeyPatch.context() as mp: mp.setattr("tempfile.mkstemp", boom) - result = runner.invoke(app, args) + result = runner.invoke( + app, args, input="y\n" if mode == "from_url" else None + ) assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) @@ -9727,6 +9795,7 @@ def fake_open(url, timeout=None, extra_headers=None, redirect_validator=None): result = runner.invoke( app, ["workflow", "add", "align-wf", "--from", "https://example.com/workflow.yml"], + input="y\n", ) assert result.exit_code == 0, result.output from specify_cli.workflows._commands import _reject_insecure_download_redirect @@ -10472,6 +10541,44 @@ def test_update_non_json_description_restores_prior_file_and_registry( assert current["version"] == "1.0.0" assert not workflow_file.with_name("workflow.yml.bak").exists() + def test_commit_failure_reports_unrestored_backup_location( + self, tmp_path, monkeypatch + ): + from specify_cli.workflows import _commands + + dest_dir = tmp_path / "align-wf" + dest_dir.mkdir() + dest_file = dest_dir / "workflow.yml" + staged_file = dest_dir / ".workflow.yml.staged" + backup_file = dest_dir / "workflow.yml.bak" + dest_file.write_text("original", encoding="utf-8") + staged_file.write_text("replacement", encoding="utf-8") + + real_replace = os.replace + calls = 0 + + def fail_commit_and_restore(src, dst): + nonlocal calls + calls += 1 + if calls == 1: + return real_replace(src, dst) + if calls == 2: + raise OSError("commit denied") + raise OSError("restore denied") + + monkeypatch.setattr(os, "replace", fail_commit_and_restore) + with pytest.raises(OSError) as exc_info: + _commands._commit_workflow_file( + staged_file, dest_file, existed_before=True + ) + + message = str(exc_info.value) + assert "commit denied" in message + assert "restore denied" in message + assert str(backup_file) in message + assert not dest_file.exists() + assert backup_file.read_text(encoding="utf-8") == "original" + @pytest.mark.parametrize( ("replacement_source", "replacement_version"), [("local", "1.0.0"), ("catalog", "1.5.0")], @@ -10558,6 +10665,20 @@ def replace_while_confirming(*args, **kwargs): assert current["source"] == replacement_source assert current["version"] == replacement_version + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") + def test_resume_rejects_symlinked_cross_project_owner_root( + self, project_dir, tmp_path + ): + from specify_cli.workflows import _commands + + real_owner = tmp_path / "real-owner" + real_owner.mkdir() + owner_link = tmp_path / "owner-link" + owner_link.symlink_to(real_owner, target_is_directory=True) + + with pytest.raises(ValueError, match="unavailable"): + _commands._resolve_run_owner_root(str(owner_link), project_dir) + def test_enable_disable_corrupted_registry_entry_errors(self, project_dir, monkeypatch): import json from typer.testing import CliRunner From ec53251e567f26f1530213e5adcc28c1f24b36b4 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:17:06 +0200 Subject: [PATCH 54/59] fix: verify installed workflow ownership Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 8 +++- tests/test_workflows.py | 66 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index f1bbcf1700..90c607bda4 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -215,8 +215,8 @@ def _resolve_installed_workflow_ownership( """Map a direct ``workflow.yml`` *source_path* back to the installed workflow (``registry_root``, ``registered_id``) it belongs to, if any. - A path can point at installed storage three ways, all of which must - receive the same registry disabled-check: + A registered path can point at installed storage three ways, all of + which must receive the same registry disabled-check: 1. Lexically: the path's own (symlink-preserving) segments literally contain ``.specify/workflows/`` -- collapsing ``..``/``.`` but @@ -257,6 +257,10 @@ def ownership_for(candidate: Path) -> tuple[Path, str] | None: _reject_unsafe_dir( registry_root / ".specify" / "workflows", ".specify/workflows" ) + if not _open_workflow_registry(registry_root, err).is_installed( + registered_id + ): + return None # A legitimately installed workflow's own directory tree never # contains a symlink (workflow add/remove both refuse one at # install time); one appearing here means the file actually loaded diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 3bf7264c41..cf5885431a 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -11103,6 +11103,72 @@ def _install_and_run_gated(self, runner, app, project_dir): assert payload["status"] == "paused" return payload["run_id"] + def test_unregistered_workflow_shaped_path_is_not_persisted_as_owner( + self, project_dir, temp_dir, monkeypatch + ): + """A direct file is not installed merely because its path resembles + installed storage; only registry membership establishes ownership.""" + import shutil + from typer.testing import CliRunner + from specify_cli import app + + standalone_root = temp_dir / "standalone-project" + workflows_dir = standalone_root / ".specify" / "workflows" + workflow_file = workflows_dir / "gated-wf" / "workflow.yml" + workflow_file.parent.mkdir(parents=True) + workflow_file.write_text(self._GATED_WORKFLOW_YAML, encoding="utf-8") + + monkeypatch.chdir(project_dir) + runner = CliRunner() + run_ids = [] + for _ in range(2): + result = runner.invoke( + app, ["workflow", "run", str(workflow_file), "--json"] + ) + assert result.exit_code == 0, result.output + run_ids.append(json.loads(result.stdout)["run_id"]) + + for run_id in run_ids: + state_path = ( + project_dir + / ".specify" + / "workflows" + / "runs" + / run_id + / "state.json" + ) + state = json.loads(state_path.read_text(encoding="utf-8")) + assert state["installed_workflow_id"] is None + assert state["installed_registry_root"] is None + + shutil.rmtree(standalone_root) + result = runner.invoke( + app, ["workflow", "resume", run_ids[0], "--json"] + ) + assert result.exit_code == 0, result.output + + workflows_dir.mkdir(parents=True) + (workflows_dir / "workflow-registry.json").write_text( + json.dumps( + { + "schema_version": "1.0", + "workflows": { + "gated-wf": { + "name": "Unrelated workflow", + "version": "9.9.9", + "source": "dev", + "enabled": False, + } + }, + } + ), + encoding="utf-8", + ) + result = runner.invoke( + app, ["workflow", "resume", run_ids[1], "--json"] + ) + assert result.exit_code == 0, result.output + def test_resume_blocks_when_installed_workflow_disabled( self, project_dir, monkeypatch ): From 77116a1ade028b98f506aa3593a58b849d583d0b Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:26:24 +0200 Subject: [PATCH 55/59] fix: isolate workflow rollback cleanup Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 28 ++++++---- tests/test_workflows.py | 71 +++++++++++++++++++++----- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 90c607bda4..ade74d3565 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -610,17 +610,25 @@ def _rollback_committed_workflow_file( """Undo a successful _commit_workflow_file swap after a later failure (registry.add()): restore the prior file via rename, remove the newly committed file for a reinstall over a pre-existing empty directory - (no backup), or remove the whole directory for a fresh install. A - genuine removal failure must propagate (not be swallowed) so the safe - wrapper below can warn instead of silently leaving an orphan; a - dest_dir already absent is not itself an error.""" + (no backup), or remove the new file and then its directory when empty + for a fresh install. A genuine removal failure must propagate (not be + swallowed) so the safe wrapper below can warn instead of silently + leaving an orphan; a dest_dir already absent is not itself an error.""" if backup_file is not None: os.replace(backup_file, dest_file) - elif existed_before: + else: dest_file.unlink(missing_ok=True) - elif dest_dir.exists(): - import shutil - shutil.rmtree(dest_dir) + if not existed_before and dest_dir.exists(): + import errno + + try: + dest_dir.rmdir() + except OSError as exc: + # Another installer may have staged a sibling before taking + # the transaction lock. Preserve it rather than recursively + # deleting the shared directory during this rollback. + if exc.errno not in (errno.ENOTEMPTY, errno.EEXIST): + raise def _safe_discard_staged_workflow_file(staged_file: Path, dest_dir: Path, existed_before: bool) -> None: @@ -984,7 +992,7 @@ def workflow_resume( err.print(f"[red]Error:[/red] Run not found: {run_id}") raise typer.Exit(1) except ValueError as exc: - err.print(f"[red]Error:[/red] {exc}") + err.print(f"[red]Error:[/red] {_escape_markup(str(exc))}") raise typer.Exit(1) except OSError as exc: err.print(f"[red]Resume failed:[/red] {exc}") @@ -1054,7 +1062,7 @@ def workflow_status( console.print(f"[red]Error:[/red] Run not found: {run_id}") raise typer.Exit(1) except ValueError as exc: - console.print(f"[red]Error:[/red] {exc}") + console.print(f"[red]Error:[/red] {_escape_markup(str(exc))}") raise typer.Exit(1) if json_output: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index cf5885431a..dcf97a6f1b 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8592,13 +8592,8 @@ def rmdir_boom(path): def test_add_fresh_install_registry_rollback_cleanup_failure_reports_warning( self, project_dir, monkeypatch ): - """_rollback_committed_workflow_file's fresh-install rmtree(dest_dir) - after a registry.add() failure had the same ignore_errors=True gap: - a genuine directory-removal failure there was silently swallowed, - leaving an orphan with no warning. The cleanup failure must - propagate so the safe wrapper can warn, while the original - registry-update error remains the primary, already-printed - failure.""" + """A fresh-install rollback directory-removal failure must be + reported while the registry-update error remains primary.""" from typer.testing import CliRunner from specify_cli import app from specify_cli.workflows.catalog import WorkflowRegistry @@ -8610,14 +8605,16 @@ def test_add_fresh_install_registry_rollback_cleanup_failure_reports_warning( def save_boom(self): raise OSError("registry disk full") - def rmtree_boom(path, ignore_errors=False, onerror=None, **kwargs): - if ignore_errors: - return - raise OSError("cleanup denied") + real_rmdir = Path.rmdir + + def rmdir_boom(path): + if path.name == "align-wf": + raise OSError("cleanup denied") + return real_rmdir(path) with pytest.MonkeyPatch.context() as mp: mp.setattr(WorkflowRegistry, "save", save_boom) - mp.setattr("shutil.rmtree", rmtree_boom) + mp.setattr(Path, "rmdir", rmdir_boom) result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) assert result.exit_code != 0 @@ -9294,6 +9291,32 @@ def test_precommit_discard_preserves_concurrent_install(self, project_dir): assert committed_file.read_text(encoding="utf-8") == "committed" assert not staged_file.exists() + def test_fresh_install_rollback_preserves_concurrent_staged_file( + self, project_dir + ): + """A second installer stages before taking the transaction lock, so + the first installer's rollback must not recursively remove siblings.""" + from specify_cli.workflows import _commands + + workflow_dir = ( + project_dir / ".specify" / "workflows" / "concurrent-wf" + ) + workflow_dir.mkdir(parents=True) + committed_file = workflow_dir / "workflow.yml" + committed_file.write_text("failed install", encoding="utf-8") + concurrent_stage = workflow_dir / ".workflow.yml.concurrent.tmp" + concurrent_stage.write_text("next install", encoding="utf-8") + + _commands._rollback_committed_workflow_file( + committed_file, + workflow_dir, + existed_before=False, + backup_file=None, + ) + + assert not committed_file.exists() + assert concurrent_stage.read_text(encoding="utf-8") == "next install" + def test_add_dev_registry_reopen_exit_discards_staged_file( self, project_dir, monkeypatch ): @@ -11456,6 +11479,30 @@ def test_resume_rejects_malformed_run_state_origin_fields( assert result.exception is None or isinstance(result.exception, SystemExit) assert "Error" in result.output + @pytest.mark.parametrize("command", ["resume", "status"]) + def test_state_load_errors_escape_rich_markup( + self, project_dir, monkeypatch, command + ): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + run_id = self._install_and_run_gated(runner, app, project_dir) + + state_path = ( + project_dir / ".specify" / "workflows" / "runs" / run_id / "state.json" + ) + data = json.loads(state_path.read_text(encoding="utf-8")) + malicious_status = "[bold red]forged[/bold red]" + data["status"] = malicious_status + state_path.write_text(json.dumps(data), encoding="utf-8") + + result = runner.invoke(app, ["workflow", command, run_id]) + + assert result.exit_code != 0 + assert malicious_status in result.output + @pytest.mark.parametrize( "installed_workflow_id, installed_registry_root", [ From 8f795a6f2260dd23564b031dac8063ee98f3246b Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:37:56 +0200 Subject: [PATCH 56/59] fix: preserve unique workflow backups Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 30 +++++++++++++---- tests/test_workflows.py | 46 +++++++++++++++++++++----- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index ade74d3565..5ed6ec5577 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -557,15 +557,31 @@ def _workflow_install_transaction(project_root: Path): def _commit_workflow_file(staged_file: Path, dest_file: Path, existed_before: bool) -> Path | None: """Atomically swap ``staged_file`` onto ``dest_file``. If a prior file - existed, it is first renamed aside (path returned) so a later failure - (e.g. registry.add()) can restore it via rename instead of a content - rewrite -- the destination is never truncated/overwritten in place. If - the second rename fails after the first succeeded, the prior file is - put back immediately so dest_file is never left simply missing.""" + existed, it is first renamed to a unique sibling (path returned) so a + later failure (e.g. registry.add()) can restore it via rename instead + of a content rewrite -- the destination is never truncated/overwritten + in place. If the second rename fails after the first succeeded, the + prior file is put back immediately so dest_file is never left simply + missing.""" if existed_before and dest_file.exists(): + import tempfile + staged_file.chmod(dest_file.stat(follow_symlinks=False).st_mode & 0o7777) - backup_file = dest_file.with_name(dest_file.name + ".bak") - os.replace(dest_file, backup_file) + fd, backup_name = tempfile.mkstemp( + dir=dest_file.parent, + prefix=f".{dest_file.name}.", + suffix=".bak", + ) + os.close(fd) + backup_file = Path(backup_name) + try: + os.replace(dest_file, backup_file) + except BaseException: + try: + backup_file.unlink(missing_ok=True) + except OSError: + pass + raise try: os.replace(staged_file, dest_file) except OSError as commit_exc: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index dcf97a6f1b..5de94a0b21 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8675,11 +8675,8 @@ def boom(src_path, dst_path, *args, **kwargs): def test_add_dev_successful_reinstall_leaves_no_backup_file( self, project_dir, monkeypatch ): - """_commit_workflow_file() renames the prior workflow.yml aside to - workflow.yml.bak so it can be restored if registry.add() fails. Once - registry.add() durably succeeds, that backup is no longer needed -- - it must be discarded, not left behind as a permanent orphan sibling - that every future reinstall would silently accumulate/overwrite.""" + """Once registry.add() succeeds, the unique rollback backup must be + discarded rather than left as a permanent orphan sibling.""" from typer.testing import CliRunner from specify_cli import app from specify_cli.workflows.catalog import WorkflowRegistry @@ -9065,7 +9062,7 @@ def test_add_catalog_successful_reinstall_leaves_no_backup_file( self, project_dir, monkeypatch ): """Same orphan-backup gap as the local-install path: a successful - catalog reinstall must not leave workflow.yml.bak behind once + catalog reinstall must not leave its unique backup behind once registry.add() durably succeeds.""" from typer.testing import CliRunner from specify_cli import app @@ -10562,7 +10559,12 @@ def test_update_non_json_description_restores_prior_file_and_registry( assert workflow_file.read_bytes() == original_data current = WorkflowRegistry(project_dir).get("align-wf") assert current["version"] == "1.0.0" - assert not workflow_file.with_name("workflow.yml.bak").exists() + leftovers = [ + path.name + for path in workflow_file.parent.iterdir() + if path.name != "workflow.yml" + ] + assert leftovers == [] def test_commit_failure_reports_unrestored_backup_location( self, tmp_path, monkeypatch @@ -10573,17 +10575,18 @@ def test_commit_failure_reports_unrestored_backup_location( dest_dir.mkdir() dest_file = dest_dir / "workflow.yml" staged_file = dest_dir / ".workflow.yml.staged" - backup_file = dest_dir / "workflow.yml.bak" dest_file.write_text("original", encoding="utf-8") staged_file.write_text("replacement", encoding="utf-8") real_replace = os.replace calls = 0 + backup_file = None def fail_commit_and_restore(src, dst): - nonlocal calls + nonlocal backup_file, calls calls += 1 if calls == 1: + backup_file = Path(dst) return real_replace(src, dst) if calls == 2: raise OSError("commit denied") @@ -10598,10 +10601,35 @@ def fail_commit_and_restore(src, dst): message = str(exc_info.value) assert "commit denied" in message assert "restore denied" in message + assert backup_file is not None assert str(backup_file) in message assert not dest_file.exists() assert backup_file.read_text(encoding="utf-8") == "original" + def test_commit_uses_unique_backup_without_overwriting_existing_sibling( + self, tmp_path + ): + from specify_cli.workflows import _commands + + dest_dir = tmp_path / "align-wf" + dest_dir.mkdir() + dest_file = dest_dir / "workflow.yml" + staged_file = dest_dir / ".workflow.yml.staged" + fixed_backup = dest_dir / "workflow.yml.bak" + dest_file.write_text("original", encoding="utf-8") + staged_file.write_text("replacement", encoding="utf-8") + fixed_backup.write_text("diagnostic copy", encoding="utf-8") + + backup_file = _commands._commit_workflow_file( + staged_file, dest_file, existed_before=True + ) + + assert backup_file is not None + assert backup_file != fixed_backup + assert backup_file.read_text(encoding="utf-8") == "original" + assert fixed_backup.read_text(encoding="utf-8") == "diagnostic copy" + assert dest_file.read_text(encoding="utf-8") == "replacement" + @pytest.mark.parametrize( ("replacement_source", "replacement_version"), [("local", "1.0.0"), ("catalog", "1.5.0")], From da86926b209d054db5d27fc5991c6c5d5f8ec21c Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 19:54:36 +0200 Subject: [PATCH 57/59] fix: bind workflow staging to file descriptors Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 133 ++++++++++++++++++++---- src/specify_cli/workflows/catalog.py | 39 +++++-- tests/test_workflows.py | 135 +++++++++++++++++++++---- 3 files changed, 263 insertions(+), 44 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 5ed6ec5577..fe2f5c3c73 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -448,9 +448,70 @@ def _safe_workflow_id_dir(workflows_dir: Path, workflow_id: str) -> Path: return dest_dir +class _StagedWorkflowFile: + """Exclusive staging inode kept open until its atomic commit.""" + + def __init__(self, path: Path, fd: int) -> None: + self.path = path + self.fd = fd + + def _write(self, chunks) -> None: + os.lseek(self.fd, 0, os.SEEK_SET) + os.ftruncate(self.fd, 0) + for chunk in chunks: + view = memoryview(chunk) + while view: + written = os.write(self.fd, view) + if written <= 0: + raise OSError("Failed to write staged workflow file") + view = view[written:] + + def write_bytes(self, data: bytes) -> None: + self._write((data,)) + + def copy_from(self, source: Path) -> None: + with source.open("rb") as src: + self._write(iter(lambda: src.read(1024 * 1024), b"")) + if hasattr(os, "fchmod"): + os.fchmod( + self.fd, + source.stat(follow_symlinks=True).st_mode & 0o7777, + ) + + def verify_path(self) -> None: + import stat + + try: + path_stat = self.path.stat(follow_symlinks=False) + open_stat = os.fstat(self.fd) + except OSError as exc: + raise OSError( + "Staged workflow file changed before commit" + ) from exc + if ( + not stat.S_ISREG(path_stat.st_mode) + or path_stat.st_dev != open_stat.st_dev + or path_stat.st_ino != open_stat.st_ino + ): + raise OSError("Staged workflow file changed before commit") + + def set_mode(self, mode: int) -> None: + if hasattr(os, "fchmod"): + os.fchmod(self.fd, mode) + + def close(self) -> None: + if self.fd < 0: + return + fd, self.fd = self.fd, -1 + try: + os.close(fd) + except OSError: + pass + + def _stage_workflow_file( dest_dir: Path, *, use_project_file_mode: bool = False -) -> Path: +) -> _StagedWorkflowFile: """Reserve a same-directory staging file so new/updated workflow.yml content can be written and validated without ever touching (and risking truncating) an existing destination file before the final atomic swap. @@ -463,7 +524,9 @@ def _stage_workflow_file( re-raised unchanged. A pre-existing dest_dir (reinstall) is never touched by this cleanup. For catalog-created files, ``use_project_file_mode`` recreates the reserved path exclusively with - mode 0666 so the process umask supplies the normal project-file mode.""" + mode 0666 so the process umask supplies the normal project-file mode. + The final descriptor remains open so callers write to and verify the + reserved inode rather than reopening a replaceable pathname.""" import tempfile created_dir = not dest_dir.exists() @@ -500,9 +563,8 @@ def _stage_workflow_file( f"workflow directory: {_escape_markup(str(cleanup_exc))}" ) raise - os.close(fd) assert staged_file is not None - return staged_file + return _StagedWorkflowFile(staged_file, fd) @contextlib.contextmanager @@ -555,7 +617,11 @@ def _workflow_install_transaction(project_root: Path): os.close(fd) -def _commit_workflow_file(staged_file: Path, dest_file: Path, existed_before: bool) -> Path | None: +def _commit_workflow_file( + staged_file: Path | _StagedWorkflowFile, + dest_file: Path, + existed_before: bool, +) -> Path | None: """Atomically swap ``staged_file`` onto ``dest_file``. If a prior file existed, it is first renamed to a unique sibling (path returned) so a later failure (e.g. registry.add()) can restore it via rename instead @@ -563,10 +629,21 @@ def _commit_workflow_file(staged_file: Path, dest_file: Path, existed_before: bo in place. If the second rename fails after the first succeeded, the prior file is put back immediately so dest_file is never left simply missing.""" + staged_path = ( + staged_file.path + if isinstance(staged_file, _StagedWorkflowFile) + else staged_file + ) + if isinstance(staged_file, _StagedWorkflowFile): + staged_file.verify_path() if existed_before and dest_file.exists(): import tempfile - staged_file.chmod(dest_file.stat(follow_symlinks=False).st_mode & 0o7777) + mode = dest_file.stat(follow_symlinks=False).st_mode & 0o7777 + if isinstance(staged_file, _StagedWorkflowFile): + staged_file.set_mode(mode) + else: + staged_path.chmod(mode) fd, backup_name = tempfile.mkstemp( dir=dest_file.parent, prefix=f".{dest_file.name}.", @@ -583,7 +660,9 @@ def _commit_workflow_file(staged_file: Path, dest_file: Path, existed_before: bo pass raise try: - os.replace(staged_file, dest_file) + if isinstance(staged_file, _StagedWorkflowFile): + staged_file.verify_path() + os.replace(staged_path, dest_file) except OSError as commit_exc: try: os.replace(backup_file, dest_file) @@ -595,19 +674,34 @@ def _commit_workflow_file(staged_file: Path, dest_file: Path, existed_before: bo f"{backup_file}." ) from restore_exc raise + if isinstance(staged_file, _StagedWorkflowFile): + staged_file.close() return backup_file - os.replace(staged_file, dest_file) + os.replace(staged_path, dest_file) + if isinstance(staged_file, _StagedWorkflowFile): + staged_file.close() return None -def _discard_staged_workflow_file(staged_file: Path, dest_dir: Path, existed_before: bool) -> None: +def _discard_staged_workflow_file( + staged_file: Path | _StagedWorkflowFile, + dest_dir: Path, + existed_before: bool, +) -> None: """Clean up after a pre-commit failure (staged_file was never swapped onto dest_file): remove the staged file, and for a fresh install (no prior directory) remove the now-orphaned dest_dir too. A genuine removal failure must propagate (not be swallowed) so the safe wrapper below can warn instead of silently leaving an orphan; a dest_dir already absent is not itself an error.""" - staged_file.unlink(missing_ok=True) + staged_path = ( + staged_file.path + if isinstance(staged_file, _StagedWorkflowFile) + else staged_file + ) + if isinstance(staged_file, _StagedWorkflowFile): + staged_file.close() + staged_path.unlink(missing_ok=True) if not existed_before and dest_dir.exists(): import errno @@ -647,7 +741,11 @@ def _rollback_committed_workflow_file( raise -def _safe_discard_staged_workflow_file(staged_file: Path, dest_dir: Path, existed_before: bool) -> None: +def _safe_discard_staged_workflow_file( + staged_file: Path | _StagedWorkflowFile, + dest_dir: Path, + existed_before: bool, +) -> None: """Guarded wrapper: a cleanup failure must be reported, never crash or silently mask the original install error that triggered it.""" try: @@ -1246,8 +1344,6 @@ def _validate_and_install_local( dest_file = dest_dir / "workflow.yml" existed_before = dest_dir.is_dir() - import shutil - try: staged_file = _stage_workflow_file(dest_dir) except OSError as exc: @@ -1261,7 +1357,7 @@ def _validate_and_install_local( # Copied into the staging file, never dest_file directly, so a # reinstall's prior working copy is never touched until the # atomic commit below runs. - shutil.copy2(yaml_path, staged_file) + staged_file.copy_from(yaml_path) except OSError as exc: _safe_discard_staged_workflow_file(staged_file, dest_dir, existed_before) console.print( @@ -1657,7 +1753,8 @@ def versions_match(actual: object, expected: str) -> bool: # Written to the staging file, never workflow_file directly, so a # reinstall's prior working copy is never touched until the # atomic commit below runs. - staged_file.write_bytes(_read_response_within_limit(response)) + downloaded_content = _read_response_within_limit(response) + staged_file.write_bytes(downloaded_content) except typer.Exit: raise except Exception as exc: @@ -1668,8 +1765,10 @@ def versions_match(actual: object, expected: str) -> bool: # Validate the downloaded workflow (still staged, not yet committed) # before registering. try: - definition = WorkflowDefinition.from_yaml(staged_file) - except (ValueError, yaml.YAMLError) as exc: + definition = WorkflowDefinition.from_string( + downloaded_content.decode("utf-8") + ) + except (UnicodeDecodeError, ValueError, yaml.YAMLError) as exc: _safe_discard_staged_workflow_file(staged_file, workflow_dir, existed_before) console.print(f"[red]Error:[/red] Downloaded workflow is invalid: {_escape_markup(str(exc))}") raise typer.Exit(1) diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 29237d534e..027a6a03b5 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -145,7 +145,9 @@ def save(self) -> None: suffix=".tmp", ) try: - with os.fdopen(fd, "w", encoding="utf-8") as f: + # Write through a duplicate so the exclusive mkstemp descriptor + # stays open for fd-based metadata updates and inode verification. + with os.fdopen(os.dup(fd), "w", encoding="utf-8") as f: json.dump(self.data, f, indent=2) # mkstemp creates the temp file at 0600. A pre-existing registry # may be shared more permissively (e.g. 0640/0644); preserve its @@ -156,19 +158,44 @@ def save(self) -> None: # metadata preservation). try: if self.registry_path.exists(): - existing_stat = self.registry_path.stat() - os.chmod(tmp, stat.S_IMODE(existing_stat.st_mode)) - if hasattr(os, "chown"): + existing_stat = self.registry_path.stat( + follow_symlinks=False + ) + if stat.S_ISREG(existing_stat.st_mode) and hasattr( + os, "fchmod" + ): + os.fchmod(fd, stat.S_IMODE(existing_stat.st_mode)) + if stat.S_ISREG(existing_stat.st_mode) and hasattr( + os, "fchown" + ): try: - os.chown( - tmp, existing_stat.st_uid, existing_stat.st_gid + os.fchown( + fd, existing_stat.st_uid, existing_stat.st_gid ) except PermissionError: pass except OSError: pass + staged_stat = os.stat(tmp, follow_symlinks=False) + open_stat = os.fstat(fd) + if ( + not stat.S_ISREG(staged_stat.st_mode) + or staged_stat.st_dev != open_stat.st_dev + or staged_stat.st_ino != open_stat.st_ino + ): + raise OSError( + "Refusing to replace workflow registry: " + "staged file changed before commit" + ) + os.close(fd) + fd = -1 os.replace(tmp, self.registry_path) except BaseException: + if fd >= 0: + try: + os.close(fd) + except OSError: + pass try: os.unlink(tmp) except OSError: diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 5de94a0b21..63b8995332 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8324,7 +8324,7 @@ def test_registry_save_preserves_existing_file_mode(self, project_dir): mode = stat.S_IMODE(registry.registry_path.stat().st_mode) assert mode == 0o644, f"expected 0644, got {oct(mode)}" - @pytest.mark.skipif(not hasattr(os, "chown"), reason="os.chown is unavailable") + @pytest.mark.skipif(not hasattr(os, "fchown"), reason="os.fchown is unavailable") def test_registry_save_preserves_existing_owner_group( self, project_dir, monkeypatch ): @@ -8334,18 +8334,18 @@ def test_registry_save_preserves_existing_owner_group( registry = WorkflowRegistry(project_dir) registry.add("first-wf", {"name": "First"}) existing = registry.registry_path.stat() - calls: list[tuple[Path, int, int]] = [] + calls: list[tuple[os.stat_result, int, int]] = [] monkeypatch.setattr( catalog_mod.os, - "chown", - lambda path, uid, gid: calls.append((Path(path), uid, gid)), + "fchown", + lambda fd, uid, gid: calls.append((os.fstat(fd), uid, gid)), ) registry.add("second-wf", {"name": "Second"}) assert len(calls) == 1 - temp_path, uid, gid = calls[0] - assert temp_path.parent == registry.registry_path.parent + temp_stat, uid, gid = calls[0] + assert stat.S_ISREG(temp_stat.st_mode) assert uid == existing.st_uid assert gid == existing.st_gid @@ -8361,6 +8361,49 @@ def test_registry_save_on_new_registry_uses_secure_default_mode(self, project_di mode = stat.S_IMODE(registry.registry_path.stat().st_mode) assert mode == 0o600, f"expected 0600, got {oct(mode)}" + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") + def test_registry_save_rejects_swapped_temp_without_touching_target( + self, project_dir, monkeypatch + ): + from specify_cli.workflows.catalog import WorkflowRegistry + import specify_cli.workflows.catalog as catalog_mod + + registry = WorkflowRegistry(project_dir) + registry.add("first-wf", {"name": "First"}) + registry.registry_path.chmod(0o640) + + victim = project_dir / "victim.txt" + victim.write_text("untouched", encoding="utf-8") + victim.chmod(0o600) + + tmp_path = None + real_mkstemp = catalog_mod.tempfile.mkstemp + real_dump = catalog_mod.json.dump + + def tracking_mkstemp(*args, **kwargs): + nonlocal tmp_path + fd, name = real_mkstemp(*args, **kwargs) + tmp_path = Path(name) + return fd, name + + def swap_after_dump(*args, **kwargs): + result = real_dump(*args, **kwargs) + assert tmp_path is not None + tmp_path.unlink() + tmp_path.symlink_to(victim) + return result + + monkeypatch.setattr(catalog_mod.tempfile, "mkstemp", tracking_mkstemp) + monkeypatch.setattr(catalog_mod.json, "dump", swap_after_dump) + + with pytest.raises(OSError): + registry.add("second-wf", {"name": "Second"}) + + assert victim.read_text(encoding="utf-8") == "untouched" + assert stat.S_IMODE(victim.stat().st_mode) == 0o600 + assert not registry.registry_path.is_symlink() + assert WorkflowRegistry(project_dir).is_installed("first-wf") + def test_add_dev_dir_with_workflow_yml_directory_errors_cleanly(self, project_dir, monkeypatch): from typer.testing import CliRunner from specify_cli import app @@ -8552,6 +8595,60 @@ def boom(*args, **kwargs): assert installed_yaml.read_bytes() == original_bytes assert WorkflowRegistry(project_dir).get("align-wf") == original_registry_entry + @pytest.mark.skipif(not hasattr(os, "symlink"), reason="symlinks are unavailable") + @pytest.mark.parametrize("mode", ["dev", "catalog"]) + def test_stage_write_rejects_swapped_symlink( + self, project_dir, monkeypatch, mode + ): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands + from specify_cli.workflows.catalog import WorkflowCatalog + + monkeypatch.chdir(project_dir) + victim = project_dir / "victim.txt" + victim.write_text("untouched", encoding="utf-8") + + real_stage = _commands._stage_workflow_file + + def raced_stage(*args, **kwargs): + staged = real_stage(*args, **kwargs) + staged_path = getattr(staged, "path", staged) + staged_path.unlink() + staged_path.symlink_to(victim) + return staged + + monkeypatch.setattr(_commands, "_stage_workflow_file", raced_stage) + + if mode == "dev": + source = self._write_workflow_dir(project_dir) + args = ["workflow", "add", str(source), "--dev"] + else: + monkeypatch.setattr( + WorkflowCatalog, + "get_workflow_info", + lambda self, wid: { + "id": wid, + "name": "Align Workflow", + "version": "1.0.0", + "url": "https://example.com/workflow.yml", + "_install_allowed": True, + "_catalog_name": "test-catalog", + }, + ) + data = self.WORKFLOW_YAML.format(version="1.0.0").encode() + monkeypatch.setattr( + "specify_cli.authentication.http.open_url", + lambda url, timeout=None, extra_headers=None, + redirect_validator=None: self._FakeResponse(data, url), + ) + args = ["workflow", "add", "align-wf"] + + result = CliRunner().invoke(app, args) + + assert result.exit_code != 0 + assert victim.read_text(encoding="utf-8") == "untouched" + def test_add_fresh_install_staged_discard_cleanup_failure_reports_warning( self, project_dir, monkeypatch ): @@ -8559,13 +8656,14 @@ def test_add_fresh_install_staged_discard_cleanup_failure_reports_warning( the original copy failure remains the primary error.""" from typer.testing import CliRunner from specify_cli import app + from specify_cli.workflows import _commands from specify_cli.workflows.catalog import WorkflowRegistry monkeypatch.chdir(project_dir) runner = CliRunner() src = self._write_workflow_dir(project_dir) - def copy_boom(*args, **kwargs): + def copy_boom(self, source): raise OSError("disk full") real_rmdir = Path.rmdir @@ -8576,7 +8674,7 @@ def rmdir_boom(path): return real_rmdir(path) with pytest.MonkeyPatch.context() as mp: - mp.setattr("shutil.copy2", copy_boom) + mp.setattr(_commands._StagedWorkflowFile, "copy_from", copy_boom) mp.setattr(Path, "rmdir", rmdir_boom) result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) @@ -8628,15 +8726,11 @@ def rmdir_boom(path): def test_add_dev_reinstall_copy_failure_leaves_prior_file_untouched( self, project_dir, monkeypatch ): - """copy2 now writes into a same-directory staging file, never - dest_file directly, so a copy2 failure (even one that partially - writes before raising, mirroring a real disk-full/interrupted-copy - failure) can no longer touch -- let alone truncate -- the prior - working workflow.yml: dest_file is only ever touched by the final - atomic commit swap, which never runs if copy2 raises. No leftover - staging file may remain either.""" + """A staged descriptor-copy failure cannot touch the prior installed + workflow or leave a staging file behind.""" from typer.testing import CliRunner from specify_cli import app + from specify_cli.workflows import _commands from specify_cli.workflows.catalog import WorkflowRegistry monkeypatch.chdir(project_dir) @@ -8652,15 +8746,14 @@ def test_add_dev_reinstall_copy_failure_leaves_prior_file_untouched( self.WORKFLOW_YAML.format(version="2.0.0"), encoding="utf-8" ) - def boom(src_path, dst_path, *args, **kwargs): + def boom(staged, source): # Simulate a truncating partial write followed by an OSError on - # the *staging* file -- the only file copy2 is now allowed to - # touch -- mirroring a real disk-full/interrupted-copy failure. - Path(dst_path).write_bytes(b"") + # the reserved staging inode, mirroring disk exhaustion. + staged.write_bytes(b"") raise OSError("disk full") with pytest.MonkeyPatch.context() as mp: - mp.setattr("shutil.copy2", boom) + mp.setattr(_commands._StagedWorkflowFile, "copy_from", boom) result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) assert result.exit_code != 0 @@ -9990,7 +10083,7 @@ def test_update_downloaded_invalid_yaml_escapes_rich_markup(self, project_dir, m side_effect=lambda url, timeout=None, extra_headers=None, redirect_validator=None: self._FakeResponse(b"", url), ), patch.object( WorkflowDefinition, - "from_yaml", + "from_string", side_effect=ValueError('bad snippet: "New [Feature]"'), ): result = runner.invoke(app, ["workflow", "update"], input="y\n") From 2c47c868190daa00c21e069991f40ae82d455e27 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:04:18 +0200 Subject: [PATCH 58/59] fix: fail closed on corrupt workflow registry Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 8 +++-- src/specify_cli/workflows/catalog.py | 32 ++++++++++++-------- tests/test_workflows.py | 42 ++++++++++++++++++++++---- 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index fe2f5c3c73..98f4bee92a 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -662,6 +662,9 @@ def _commit_workflow_file( try: if isinstance(staged_file, _StagedWorkflowFile): staged_file.verify_path() + # Windows cannot replace an open file. Verify through the + # exclusive descriptor, then close immediately before rename. + staged_file.close() os.replace(staged_path, dest_file) except OSError as commit_exc: try: @@ -674,12 +677,11 @@ def _commit_workflow_file( f"{backup_file}." ) from restore_exc raise - if isinstance(staged_file, _StagedWorkflowFile): - staged_file.close() return backup_file - os.replace(staged_path, dest_file) if isinstance(staged_file, _StagedWorkflowFile): + staged_file.verify_path() staged_file.close() + os.replace(staged_path, dest_file) return None diff --git a/src/specify_cli/workflows/catalog.py b/src/specify_cli/workflows/catalog.py index 027a6a03b5..7a156c92e4 100644 --- a/src/specify_cli/workflows/catalog.py +++ b/src/specify_cli/workflows/catalog.py @@ -106,24 +106,32 @@ def _load(self) -> dict[str, Any]: with open(self.registry_path, encoding="utf-8") as f: data = json.load(f) except OSError as exc: - # An I/O failure (e.g. permissions, transient FS issue) is not - # the same as a corrupted file: the real data may still be - # intact on disk. Fail closed here, at construction, rather - # than falling back to an empty registry -- a caller that - # only queries is_installed()/get()/list() before writing a - # file (never reaching save()) would otherwise mistake this - # for "nothing installed" and overwrite real data. + # The real data may still be intact on disk. Fail closed at + # construction rather than fabricating an empty registry that + # a read-only caller could mistake for "nothing installed." raise OSError( f"Failed to read workflow registry at {self.registry_path}: {exc}" ) from exc - except (json.JSONDecodeError, ValueError, UnicodeError): - # Corrupted registry file — reset to default - return default_registry + except ( + json.JSONDecodeError, + ValueError, + UnicodeError, + ) as exc: + raise OSError( + f"Workflow registry at {self.registry_path} is corrupted: " + f"{exc}" + ) from exc # Validate shape: must be a dict with a dict "workflows" field. if not isinstance(data, dict): - return default_registry + raise OSError( + f"Workflow registry at {self.registry_path} is corrupted: " + "top-level value must be an object" + ) if not isinstance(data.get("workflows"), dict): - data["workflows"] = {} + raise OSError( + f"Workflow registry at {self.registry_path} is corrupted: " + "'workflows' must be an object" + ) return data return default_registry diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 63b8995332..9096c73ab3 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -8280,16 +8280,28 @@ def test_registry_add_survives_non_dict_existing_entry(self, project_dir): registry.add("align-wf", {"version": "1.0.0", "source": "catalog"}) assert registry.get("align-wf")["version"] == "1.0.0" - def test_registry_load_normalizes_malformed_workflows_field(self, project_dir): + @pytest.mark.parametrize( + "contents", + [ + "not json", + "[]", + '{"schema_version": "1.0"}', + '{"schema_version": "1.0", "workflows": "broken"}', + ], + ) + def test_registry_load_rejects_corrupt_contents( + self, project_dir, contents + ): from specify_cli.workflows.catalog import WorkflowRegistry registry = WorkflowRegistry(project_dir) registry.workflows_dir.mkdir(parents=True, exist_ok=True) - registry.registry_path.write_text('{"workflows": "broken"}', encoding="utf-8") - fresh = WorkflowRegistry(project_dir) - assert fresh.get("anything") is None - fresh.add("align-wf", {"version": "1.0.0", "source": "catalog"}) - assert fresh.get("align-wf")["version"] == "1.0.0" + registry.registry_path.write_text(contents, encoding="utf-8") + + with pytest.raises(OSError, match="corrupt"): + WorkflowRegistry(project_dir) + + assert registry.registry_path.read_text(encoding="utf-8") == contents def test_registry_save_refuses_symlinked_parent(self, project_dir, tmp_path): """Construction now fails closed on a symlinked .specify just like @@ -10956,6 +10968,24 @@ def test_run_rejects_corrupted_registry_entry(self, project_dir, monkeypatch): assert result.exit_code != 0 assert "corrupted" in result.output + def test_run_rejects_corrupt_registry_file(self, project_dir, monkeypatch): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + + registry_path = WorkflowRegistry(project_dir).registry_path + registry_path.write_text("not json", encoding="utf-8") + + result = runner.invoke(app, ["workflow", "run", "align-wf"]) + + assert result.exit_code != 0 + assert "registry" in result.output.lower() + assert "corrupt" in result.output.lower() + def test_disable_blocks_run_via_path_equivalent_id(self, project_dir, monkeypatch): """Path spelling "align-wf/" must not run a disabled workflow by dodging the registry lookup.""" from typer.testing import CliRunner From 5163aa274a7a77f4320de52a90d4770da828b339 Mon Sep 17 00:00:00 2001 From: marcelsafin <179933638+marcelsafin@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:17:01 +0200 Subject: [PATCH 59/59] fix: bind workflow ownership and source identity Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/specify_cli/workflows/_commands.py | 104 ++++++++++++++++--------- tests/test_workflows.py | 88 +++++++++++++++++++-- 2 files changed, 149 insertions(+), 43 deletions(-) diff --git a/src/specify_cli/workflows/_commands.py b/src/specify_cli/workflows/_commands.py index 98f4bee92a..c9897569f9 100644 --- a/src/specify_cli/workflows/_commands.py +++ b/src/specify_cli/workflows/_commands.py @@ -170,7 +170,7 @@ def _reject_unsafe_workflow_storage(project_root: Path) -> None: ) -def _scan_for_workflow_owner(parts: tuple[str, ...]) -> tuple[int, str] | None: +def _scan_for_workflow_owner(parts: tuple[str, ...]) -> int | None: """Find the *nearest* (innermost) ``.specify/workflows/`` owner in *parts*, scanning from the end of the path. @@ -180,12 +180,15 @@ def _scan_for_workflow_owner(parts: tuple[str, ...]) -> tuple[int, str] | None: first-from-start match would pick the outer directory and the wrong workflow ID, silently missing the real (inner) owner's disabled check. - Returns ``(i, workflow_id)`` where ``i`` is the index of the owning - ``.specify`` segment, or ``None`` if no owner segment is present. + Returns the index of the owning ``.specify`` segment, or ``None`` if no + owner segment is present. """ for i in range(len(parts) - 3, -1, -1): - if parts[i] == ".specify" and parts[i + 1] == "workflows": - return i, parts[i + 2] + if ( + parts[i].casefold() == ".specify" + and parts[i + 1].casefold() == "workflows" + ): + return i return None @@ -218,8 +221,8 @@ def _resolve_installed_workflow_ownership( A registered path can point at installed storage three ways, all of which must receive the same registry disabled-check: - 1. Lexically: the path's own (symlink-preserving) segments literally - contain ``.specify/workflows/`` -- collapsing ``..``/``.`` but + 1. Lexically: the path's own (symlink-preserving) segments identify + ``.specify/workflows/`` -- collapsing ``..``/``.`` but never resolving symlinks, so a symlinked ``workflow.yml`` leaf (or symlinked ```` directory) inside the owned tree is caught by the inward-symlink refusal below rather than silently followed. @@ -235,13 +238,17 @@ def _resolve_installed_workflow_ownership( """ def ownership_for(candidate: Path) -> tuple[Path, str] | None: parts = candidate.parts - match = _scan_for_workflow_owner(parts) - if match is None: + i = _scan_for_workflow_owner(parts) + if i is None: return None - i, registered_id = match registry_root = ( Path(*parts[:i]) if i else Path(candidate.anchor or ".") ) + candidate_specify = Path(*parts[: i + 1]) + candidate_workflows = Path(*parts[: i + 2]) + candidate_id_dir = Path(*parts[: i + 3]) + canonical_specify = registry_root / ".specify" + canonical_workflows = canonical_specify / "workflows" # The path-derived registry_root here may differ from the cwd's # project_root already checked by _reject_unsafe_workflow_storage # (e.g. this path points into another project entirely, or this @@ -253,13 +260,38 @@ def ownership_for(candidate: Path) -> tuple[Path, str] | None: # time (see catalog.py's _load), but that surfaces as an opaque # exception rather than this guard's clean, specific CLI error for # the actual owning project root. - _reject_unsafe_dir(registry_root / ".specify", ".specify") - _reject_unsafe_dir( - registry_root / ".specify" / "workflows", ".specify/workflows" - ) - if not _open_workflow_registry(registry_root, err).is_installed( - registered_id - ): + _reject_unsafe_dir(canonical_specify, ".specify") + _reject_unsafe_dir(canonical_workflows, ".specify/workflows") + _reject_unsafe_dir(candidate_specify, ".specify") + _reject_unsafe_dir(candidate_workflows, ".specify/workflows") + try: + if not os.path.samefile(candidate_specify, canonical_specify): + return None + if not os.path.samefile( + candidate_workflows, canonical_workflows + ): + return None + except OSError: + return None + registry = _open_workflow_registry(registry_root, err) + registered_id = None + for workflow_id in registry.list(): + if ( + not isinstance(workflow_id, str) + or workflow_id in _RESERVED_WORKFLOW_IDS + or not _WORKFLOW_ID_PATTERN.fullmatch(workflow_id) + ): + continue + try: + if os.path.samefile( + candidate_id_dir, + canonical_workflows / workflow_id, + ): + registered_id = workflow_id + break + except OSError: + continue + if registered_id is None: return None # A legitimately installed workflow's own directory tree never # contains a symlink (workflow add/remove both refuse one at @@ -469,15 +501,6 @@ def _write(self, chunks) -> None: def write_bytes(self, data: bytes) -> None: self._write((data,)) - def copy_from(self, source: Path) -> None: - with source.open("rb") as src: - self._write(iter(lambda: src.read(1024 * 1024), b"")) - if hasattr(os, "fchmod"): - os.fchmod( - self.fd, - source.stat(follow_symlinks=True).st_mode & 0o7777, - ) - def verify_path(self) -> None: import stat @@ -1111,7 +1134,7 @@ def workflow_resume( err.print(f"[red]Error:[/red] {_escape_markup(str(exc))}") raise typer.Exit(1) except OSError as exc: - err.print(f"[red]Resume failed:[/red] {exc}") + err.print(f"[red]Resume failed:[/red] {_escape_markup(str(exc))}") raise typer.Exit(1) if pre_state.installed_workflow_id is not None: @@ -1133,10 +1156,10 @@ def workflow_resume( err.print(f"[red]Error:[/red] Run not found: {run_id}") raise typer.Exit(1) except ValueError as exc: - err.print(f"[red]Error:[/red] {exc}") + err.print(f"[red]Error:[/red] {_escape_markup(str(exc))}") raise typer.Exit(1) except Exception as exc: - err.print(f"[red]Resume failed:[/red] {exc}") + err.print(f"[red]Resume failed:[/red] {_escape_markup(str(exc))}") raise typer.Exit(1) if json_output: @@ -1311,8 +1334,19 @@ def _validate_and_install_local( ) -> None: """Validate and install a workflow from a local YAML file.""" try: - definition = WorkflowDefinition.from_yaml(yaml_path) - except (ValueError, yaml.YAMLError) as exc: + with yaml_path.open("rb") as source_file: + source_mode = os.fstat(source_file.fileno()).st_mode & 0o7777 + source_content = source_file.read() + definition = WorkflowDefinition.from_string( + source_content.decode("utf-8") + ) + except OSError as exc: + console.print( + f"[red]Error:[/red] Failed to read workflow YAML: " + f"{_escape_markup(str(exc))}" + ) + raise typer.Exit(1) + except (UnicodeDecodeError, ValueError, yaml.YAMLError) as exc: console.print(f"[red]Error:[/red] Invalid workflow YAML: {_escape_markup(str(exc))}") raise typer.Exit(1) # Non-string ids (e.g. unquoted ``id: 123`` or ``id: 0``) fall through @@ -1356,10 +1390,10 @@ def _validate_and_install_local( raise typer.Exit(1) try: - # Copied into the staging file, never dest_file directly, so a - # reinstall's prior working copy is never touched until the - # atomic commit below runs. - staged_file.copy_from(yaml_path) + # Write the exact bytes parsed above so a concurrent source edit + # cannot desynchronize installed content from validated metadata. + staged_file.write_bytes(source_content) + staged_file.set_mode(source_mode) except OSError as exc: _safe_discard_staged_workflow_file(staged_file, dest_dir, existed_before) console.print( diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 9096c73ab3..9a6d4424af 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -7694,7 +7694,7 @@ def test_add_yaml_parse_error_escapes_rich_markup(self, project_dir, monkeypatch runner = CliRunner() with patch.object( WorkflowDefinition, - "from_yaml", + "from_string", side_effect=ValueError('bad snippet: "New [Feature]"'), ): result = runner.invoke(app, ["workflow", "add", str(bad)]) @@ -8661,6 +8661,48 @@ def raced_stage(*args, **kwargs): assert result.exit_code != 0 assert victim.read_text(encoding="utf-8") == "untouched" + def test_local_install_writes_the_same_bytes_it_validates( + self, project_dir, monkeypatch + ): + from typer.testing import CliRunner + from specify_cli import app + from specify_cli.workflows import _commands + from specify_cli.workflows.catalog import WorkflowRegistry + + monkeypatch.chdir(project_dir) + source = self._write_workflow_dir(project_dir) + source_file = source / "workflow.yml" + validated_content = source_file.read_text(encoding="utf-8") + replacement_content = self.WORKFLOW_YAML.format(version="9.9.9") + + real_stage = _commands._stage_workflow_file + + def replace_source_after_validation(*args, **kwargs): + staged = real_stage(*args, **kwargs) + source_file.write_text(replacement_content, encoding="utf-8") + return staged + + monkeypatch.setattr( + _commands, + "_stage_workflow_file", + replace_source_after_validation, + ) + + result = CliRunner().invoke( + app, ["workflow", "add", str(source), "--dev"] + ) + + assert result.exit_code == 0, result.output + installed_file = ( + project_dir + / ".specify" + / "workflows" + / "align-wf" + / "workflow.yml" + ) + assert installed_file.read_text(encoding="utf-8") == validated_content + assert WorkflowRegistry(project_dir).get("align-wf")["version"] == "1.0.0" + def test_add_fresh_install_staged_discard_cleanup_failure_reports_warning( self, project_dir, monkeypatch ): @@ -8675,7 +8717,7 @@ def test_add_fresh_install_staged_discard_cleanup_failure_reports_warning( runner = CliRunner() src = self._write_workflow_dir(project_dir) - def copy_boom(self, source): + def copy_boom(self, data): raise OSError("disk full") real_rmdir = Path.rmdir @@ -8686,7 +8728,7 @@ def rmdir_boom(path): return real_rmdir(path) with pytest.MonkeyPatch.context() as mp: - mp.setattr(_commands._StagedWorkflowFile, "copy_from", copy_boom) + mp.setattr(_commands._StagedWorkflowFile, "write_bytes", copy_boom) mp.setattr(Path, "rmdir", rmdir_boom) result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) @@ -8758,14 +8800,14 @@ def test_add_dev_reinstall_copy_failure_leaves_prior_file_untouched( self.WORKFLOW_YAML.format(version="2.0.0"), encoding="utf-8" ) - def boom(staged, source): + def boom(staged, data): # Simulate a truncating partial write followed by an OSError on # the reserved staging inode, mirroring disk exhaustion. - staged.write_bytes(b"") + os.ftruncate(staged.fd, 0) raise OSError("disk full") with pytest.MonkeyPatch.context() as mp: - mp.setattr(_commands._StagedWorkflowFile, "copy_from", boom) + mp.setattr(_commands._StagedWorkflowFile, "write_bytes", boom) result = runner.invoke(app, ["workflow", "add", str(src), "--dev"]) assert result.exit_code != 0 @@ -10986,6 +11028,36 @@ def test_run_rejects_corrupt_registry_file(self, project_dir, monkeypatch): assert "registry" in result.output.lower() assert "corrupt" in result.output.lower() + def test_disable_blocks_case_variant_installed_path( + self, project_dir, monkeypatch + ): + from typer.testing import CliRunner + from specify_cli import app + + monkeypatch.chdir(project_dir) + runner = CliRunner() + self._install_dev(runner, app, project_dir) + + result = runner.invoke(app, ["workflow", "disable", "align-wf"]) + assert result.exit_code == 0, result.output + + case_variant = ( + project_dir + / ".SPECIFY" + / "WORKFLOWS" + / "ALIGN-WF" + / "workflow.yml" + ) + if not case_variant.is_file(): + pytest.skip("filesystem is case-sensitive") + + result = runner.invoke( + app, ["workflow", "run", str(case_variant)] + ) + + assert result.exit_code != 0 + assert "disabled" in result.output + def test_disable_blocks_run_via_path_equivalent_id(self, project_dir, monkeypatch): """Path spelling "align-wf/" must not run a disabled workflow by dodging the registry lookup.""" from typer.testing import CliRunner @@ -11404,7 +11476,7 @@ def test_resume_preload_io_error_is_reported_cleanly( monkeypatch.chdir(project_dir) with patch.object( - RunState, "load", side_effect=OSError("permission denied") + RunState, "load", side_effect=OSError("permission [denied]") ): result = CliRunner().invoke( app, ["workflow", "resume", "unreadable-run"] @@ -11413,7 +11485,7 @@ def test_resume_preload_io_error_is_reported_cleanly( assert result.exit_code != 0 assert result.exception is None or isinstance(result.exception, SystemExit) assert "Resume failed" in result.output - assert "permission denied" in result.output + assert "permission [denied]" in result.output def test_resume_backward_compatible_with_run_state_missing_new_fields( self, project_dir, monkeypatch